diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c0e7a55b8428..0de9385111d5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -100,7 +100,10 @@ jobs: ./joern --src /tmp/foo --run scan ./joern-scan /tmp/foo ./joern-scan --dump - ./joern-slice data-flow -o target/slice + - run: | + mkdir /tmp/slice + ./joern-slice data-flow tests/code/javasrc/SliceTest.java -o /tmp/slice/dataflow-slice-javasrc.json + ./joern --script "./test-dataflow-slice.sc" --param sliceFile=/tmp/slice/dataflow-slice-javasrc.json | grep -q 'List(boolean b, b, this, s, "MALICIOUS", s, new Foo("MALICIOUS"), s, s, "SAFE", s, b, this, this, b, s, System.out)' - run: | cd joern-cli/target/universal/stage ./schema-extender/test.sh diff --git a/Dockerfile b/Dockerfile index 9c82db69dc19..72207da3ea3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM alpine:3.17.3 +FROM alpine:3.20 # dependencies RUN apk update && apk upgrade && apk add --no-cache openjdk17-jdk python3 git curl gnupg bash nss ncurses php RUN ln -sf python3 /usr/bin/python # sbt -ENV SBT_VERSION 1.8.0 +ENV SBT_VERSION 1.10.0 ENV SBT_HOME /usr/local/sbt ENV PATH ${PATH}:${SBT_HOME}/bin RUN curl -sL "https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz" | gunzip | tar -x -C /usr/local diff --git a/README.md b/README.md index c12c8cab1544..77627b4192af 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Specification: https://cpg.joern.io ## News / Changelog +- Joern v4.0.0 [migrates from overflowdb to flatgraph](changelog/4.0.0-flatgraph.md) - Joern v2.0.0 [upgrades from Scala2 to Scala3](changelog/2.0.0-scala3.md) - Joern v1.2.0 removes the `overflowdb.traversal.Traversal` class. This change is not completely backwards compatible. See [here](changelog/traversal_removal.md) for a detailed writeup. diff --git a/build.sbt b/build.sbt index 897fa0fb8d67..5bbfbb1a9ac7 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "joern" ThisBuild / organization := "io.joern" ThisBuild / scalaVersion := "3.4.2" -val cpgVersion = "1.6.16" +val cpgVersion = "1.7.11" lazy val joerncli = Projects.joerncli lazy val querydb = Projects.querydb @@ -45,7 +45,8 @@ ThisBuild / compile / javacOptions ++= Seq( ThisBuild / scalacOptions ++= Seq( "-deprecation", // Emit warning and location for usages of deprecated APIs. "--release", - "11" + "11", + "-Wshadow:type-parameter-shadow", ) lazy val createDistribution = taskKey[File]("Create a complete Joern distribution") diff --git a/changelog/4.0.0-flatgraph.md b/changelog/4.0.0-flatgraph.md new file mode 100644 index 000000000000..2e3de7ed0de7 --- /dev/null +++ b/changelog/4.0.0-flatgraph.md @@ -0,0 +1,43 @@ +# 4.0.x: Migration to flatgraph + +Joern uses the domain-specific classes from codepropertygraph, which (up to joern 2.x) were generated by overflowdb (specifically https://github.com/ShiftLeftSecurity/overflowdb and https://github.com/ShiftLeftSecurity/overflowdb-codegen). +As of joern 4.0.x we replaced overflowdb with it's successor, [flatgraph](https://github.com/joernio/flatgraph). The most important PRs paving the way for flatgraph are https://github.com/ShiftLeftSecurity/codepropertygraph/pull/1769 and https://github.com/joernio/joern/pull/4630. + +### Why the change? +Most importantly, flatgraph brings us about 40% less memory usage as well as faster traversals. The reduced memory footprint is achieved by flatgraph's efficient columnar layout, essentially we hold everything in few (albeit very large) arrays. +The faster traversals account for about 40% performance improvement for many joern use-cases, e.g. running the default passes while importing a large cpg into joern. Some numbers for Linux 4, as an example for a very large codebase. Numbers are based on my workstation and just rough measurements. + +Linux 4.1.16, cpg created with c2cpg after `importCpg` into joern: +* 48M nodes with 630M properties (mostly `String` and `Integer`) +* 431M edges with 115M properties (all `String`) + +| | joern 2 (overflowdb) | joern 4 (flatgraph) | +| --------------------------------------------|----------------------|-------------------- | +| heap after import (after garbage collection)| 33g | 20g | +| minimum required heap (Xmx) for import | 80g | 30g | +| time for importCpg | 18 minutes | 11 minutes | +| file size on disk | 2600M | 400m | + +Linux 5 and 6 are considerably larger, so I wasn't able to import them into joern 2 on my workstation (which has 128G physical memory). With joern 4 it works just fine with `./joern -J-Xmx90g` for linux6 😀 + +Also worth noting: one of overflowdb's features was the overflowing-to-disk mechanism. While it sounds nice to be able to handle graphs larger than the available memory, in practice it was too slow to be useful, so we didn't reimplement it in flatgraph. + +### API changes / upgrade guide +We tried to minimise the joern-user-facing API changes, and depending on your usage you may not notice anything at all. That being said, if your code makes use of the `overflowdb` namespace then you will have to make some changes. In most cases, it's simply a namespace change to `flatgraph`. Since hopefully no joern user used the overflowdb api (with one exception listed below), I won't list the changes here, instead please look at the [joern migration PR](https://github.com/joernio/joern/pull/4630/files) and/or ask us on [discord](https://discord.com/channels/832209896089976854/842699104383533076). + +Most relevant changes: +1) `overflowdb.BatchedUpdate.applyDiff` -> `flatgraph.DiffGraphApplier.applyDiff` +1) `io.shiftleft.passes.IntervalKeyPool` -> `io.joern.x2cpg.utils.IntervalKeyPool` + +1) `StoredNode.propertyOption` now returns an `Option` rather than a `java.util.Optional` - the API is almost identical, and there's builtin conversions both ways (`.toScala|.toJava` via `import scala.jdk.OptionConverters.*`). + +1) the arrow syntax for quickly constructing graphs, e.g. `v0 --- "CFG" --> v1`, quite useful for testing, doesn't exist in flatgraph yet. You'll need to create a diffgraph instead. There's plenty of examples in the [joern migration PR](https://github.com/joernio/joern/pull/4630/files). + +1) Edges can only have zero or one properties. Since the codepropertygraph schema never defined more than one property per edge type, this should not affect you as a joern user, unless you've extended the cpg schema... + +### Credits and kudos +Flatgraph is based on [@bbrehm](https://github.com/bbrehm)'s great ideas for a memory efficient columnar layout on the jvm. He built a working prototype with very promising benchmarks that convinced us that the effort to migrate is worth-while, and that turned out to be true. + +### Why did we leave out version 3? +I'm glad you asked! Version 3 is typically a source for trouble, you know... just look at Gnome 3, Python 3 and many more. The only exception is Scala 3, of course - ymmv :) + diff --git a/console/src/main/scala/io/joern/console/Commit.scala b/console/src/main/scala/io/joern/console/Commit.scala index 402b16d4c619..acd53ddbe24a 100644 --- a/console/src/main/scala/io/joern/console/Commit.scala +++ b/console/src/main/scala/io/joern/console/Commit.scala @@ -3,7 +3,7 @@ package io.joern.console import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder object Commit { val overlayName: String = "commit" @@ -26,7 +26,7 @@ class Commit(opts: CommitOptions) extends LayerCreator { builder.absorb(opts.diffGraphBuilder) } } - runPass(pass, context) + pass.createAndApply() opts.diffGraphBuilder = Cpg.newDiffGraphBuilder } diff --git a/console/src/main/scala/io/joern/console/Console.scala b/console/src/main/scala/io/joern/console/Console.scala index 9f8ac55e2b17..63236e5de1e2 100644 --- a/console/src/main/scala/io/joern/console/Console.scala +++ b/console/src/main/scala/io/joern/console/Console.scala @@ -12,8 +12,8 @@ import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.dotextension.ImageViewer import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} -import overflowdb.traversal.help.Doc -import overflowdb.traversal.help.Table.AvailableWidthProvider +import io.shiftleft.codepropertygraph.generated.help.Doc +import flatgraph.help.Table.AvailableWidthProvider import scala.sys.process.Process import scala.util.control.NoStackTrace @@ -349,10 +349,14 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur val cpgDestinationPath = cpgDestinationPathOpt.get - if (CpgLoader.isLegacyCpg(cpgFile)) { - report("You have provided a legacy proto CPG. Attempting conversion.") + val isProtoFormat = CpgLoader.isProtoFormat(cpgFile.path) + val isOverflowDbFormat = CpgLoader.isOverflowDbFormat(cpgFile.path) + if (isProtoFormat || isOverflowDbFormat) { + if (isProtoFormat) report("You have provided a legacy proto CPG. Attempting conversion.") + else if (isOverflowDbFormat) report("You have provided a legacy overflowdb CPG. Attempting conversion.") try { - CpgConverter.convertProtoCpgToOverflowDb(cpgFile.path.toString, cpgDestinationPath.toString) + val cpg = CpgLoader.load(cpgFile.path, cpgDestinationPath) + cpg.close() } catch { case exc: Exception => report("Error converting legacy CPG: " + exc.getMessage) diff --git a/console/src/main/scala/io/joern/console/ConsoleConfig.scala b/console/src/main/scala/io/joern/console/ConsoleConfig.scala index a5b4ea738f95..fc86ca54d857 100644 --- a/console/src/main/scala/io/joern/console/ConsoleConfig.scala +++ b/console/src/main/scala/io/joern/console/ConsoleConfig.scala @@ -1,6 +1,6 @@ package io.joern.console -import better.files._ +import better.files.* import scala.annotation.tailrec import scala.collection.mutable diff --git a/console/src/main/scala/io/joern/console/CpgConverter.scala b/console/src/main/scala/io/joern/console/CpgConverter.scala index 3019e65ab816..7ce22c6bb670 100644 --- a/console/src/main/scala/io/joern/console/CpgConverter.scala +++ b/console/src/main/scala/io/joern/console/CpgConverter.scala @@ -1,14 +1,18 @@ package io.joern.console -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} -import overflowdb.Config +import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, ProtoCpgLoader} + +import java.nio.file.Paths object CpgConverter { - def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = { - val odbConfig = Config.withDefaults.withStorageLocation(dstFilename) - val config = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - CpgLoader.load(srcFilename, config).close + def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = { + val cpg = ProtoCpgLoader.loadFromProtoZip(srcFilename, Option(Paths.get(dstFilename))) + cpg.close() } + @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3") + def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = + convertProtoCpgToFlatgraph(srcFilename, dstFilename) + } diff --git a/console/src/main/scala/io/joern/console/Help.scala b/console/src/main/scala/io/joern/console/Help.scala index a1b3e017709b..b45739f88f42 100644 --- a/console/src/main/scala/io/joern/console/Help.scala +++ b/console/src/main/scala/io/joern/console/Help.scala @@ -1,8 +1,8 @@ package io.joern.console -import overflowdb.traversal.help.DocFinder.* -import overflowdb.traversal.help.Table.AvailableWidthProvider -import overflowdb.traversal.help.{DocFinder, Table} +import flatgraph.help.DocFinder.* +import flatgraph.help.Table.AvailableWidthProvider +import flatgraph.help.{DocFinder, Table} object Help { diff --git a/console/src/main/scala/io/joern/console/PluginManager.scala b/console/src/main/scala/io/joern/console/PluginManager.scala index 0d049a1ab640..0666355bc429 100644 --- a/console/src/main/scala/io/joern/console/PluginManager.scala +++ b/console/src/main/scala/io/joern/console/PluginManager.scala @@ -1,5 +1,5 @@ package io.joern.console -import better.files.Dsl._ +import better.files.Dsl.* import better.files.File import better.files.File.apply diff --git a/console/src/main/scala/io/joern/console/Run.scala b/console/src/main/scala/io/joern/console/Run.scala index 5dfe92451f25..30ed0f72d505 100644 --- a/console/src/main/scala/io/joern/console/Run.scala +++ b/console/src/main/scala/io/joern/console/Run.scala @@ -6,7 +6,7 @@ import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import org.reflections8.Reflections import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* object Run { @@ -22,7 +22,7 @@ object Run { query.store()(builder) } } - runPass(pass, context) + pass.createAndApply() } }) } @@ -64,7 +64,7 @@ object Run { | |val opts = new OptsDynamic() | - | import _root_.overflowdb.BatchedUpdate.DiffGraphBuilder + | import _root_.io.shiftleft.codepropertygraph.generated.DiffGraphBuilder | implicit def _diffGraph: DiffGraphBuilder = opts.commit.diffGraphBuilder | def diffGraph = _diffGraph |""".stripMargin @@ -75,7 +75,7 @@ object Run { val toStringCode = s""" - | import overflowdb.traversal.help.Table + | import flatgraph.help.Table | override def toString() : String = { | val columnNames = List("name", "description") | val rows = diff --git a/console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala index e429124dc67a..6a816135227f 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala @@ -3,7 +3,7 @@ package io.joern.console.cpgcreation import better.files.File import io.shiftleft.codepropertygraph.generated.Cpg -import scala.sys.process._ +import scala.sys.process.* import scala.util.Try /** A CpgGenerator generates Code Property Graphs from code. Each supported language implements a Generator, e.g., diff --git a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala index 710ba95363dc..37362dfad732 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala @@ -1,11 +1,10 @@ package io.joern.console.cpgcreation -import better.files.Dsl._ +import better.files.Dsl.* import better.files.File -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.codepropertygraph.generated.Languages -import io.joern.console.ConsoleConfig -import overflowdb.Config +import io.joern.console.{ConsoleConfig, CpgConverter} import java.nio.file.Path import scala.util.Try @@ -60,12 +59,12 @@ class CpgGeneratorFactory(config: ConsoleConfig) { generator.generate(inputPath, outputPath).map(File(_)) outputFileOpt.map { outFile => val parentPath = outFile.parent.path.toAbsolutePath - if (isZipFile(outFile)) { + if (CpgLoader.isProtoFormat(outFile.path)) { report("Creating database from bin.zip") val srcFilename = outFile.path.toAbsolutePath.toString val dstFilename = parentPath.resolve("cpg.bin").toAbsolutePath.toString // MemoryHelper.hintForInsufficientMemory(srcFilename).map(report) - convertProtoCpgToOverflowDb(srcFilename, dstFilename) + convertProtoCpgToFlatgraph(srcFilename, dstFilename) } else { report("moving cpg.bin.zip to cpg.bin because it is already a database file") val srcPath = parentPath.resolve("cpg.bin.zip") @@ -77,18 +76,13 @@ class CpgGeneratorFactory(config: ConsoleConfig) { } } - def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = { - val odbConfig = Config.withDefaults.withStorageLocation(dstFilename) - val config = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - CpgLoader.load(srcFilename, config).close - File(srcFilename).delete() - } + @deprecated("method got renamed to `convertProtoCpgToFlatgraph, please use that instead", "joern v3") + def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = + convertProtoCpgToFlatgraph(srcFilename, dstFilename) - def isZipFile(file: File): Boolean = { - val bytes = file.bytes - Try { - bytes.next() == 'P' && bytes.next() == 'K' - }.getOrElse(false) + def convertProtoCpgToFlatgraph(srcFilename: String, dstFilename: String): Unit = { + CpgConverter.convertProtoCpgToFlatgraph(srcFilename, dstFilename) + File(srcFilename).delete() } private def report(str: String): Unit = System.err.println(str) diff --git a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala index 1f683d49a741..d31023ba646d 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala @@ -5,8 +5,8 @@ import io.joern.console.workspacehandling.Project import io.joern.console.{ConsoleException, FrontendConfig, Reporting} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.util.{Failure, Success, Try} @@ -52,7 +52,7 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T])(implicit new BinaryFrontend("jvm", Languages.JAVA, "Java/Dalvik Bytecode Frontend (based on SOOT's jimple)") def ghidra: Frontend = new BinaryFrontend("ghidra", Languages.GHIDRA, "ghidra reverse engineering frontend") def kotlin: SourceBasedFrontend = - new SourceBasedFrontend("kotlin", Languages.KOTLIN, "Kotlin Source Frontend", "kotlin") + new SourceBasedFrontend("kotlin", Languages.KOTLIN, "Kotlin Source Frontend", "kt") def python: SourceBasedFrontend = new SourceBasedFrontend("python", Languages.PYTHONSRC, "Python Source Frontend", "py") def golang: SourceBasedFrontend = new SourceBasedFrontend("golang", Languages.GOLANG, "Golang Source Frontend", "go") diff --git a/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala index 1d9277902471..307a4d1793b7 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala @@ -3,7 +3,7 @@ package io.joern.console.cpgcreation import io.joern.console.FrontendConfig import java.nio.file.Path -import scala.sys.process._ +import scala.sys.process.* import scala.util.{Failure, Try} /** Language frontend for Java archives (JAR files). Translates Java archives into code property graphs. diff --git a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala index bf73f66be772..a17effed438d 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala @@ -1,17 +1,14 @@ package io.joern.console.cpgcreation import io.joern.console.FrontendConfig -import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.frontendspecific.pysrc2cpg import io.joern.x2cpg.frontendspecific.pysrc2cpg.* -import io.joern.x2cpg.passes.base.AstLinkerPass -import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.shiftleft.codepropertygraph.generated.Cpg import java.nio.file.Path -import scala.util.Try import scala.compiletime.uninitialized +import scala.util.Try case class PythonSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { private lazy val command: Path = if (isWin) rootPath.resolve("pysrc2cpg.bat") else rootPath.resolve("pysrc2cpg") diff --git a/console/src/main/scala/io/joern/console/cpgcreation/RubyCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/RubyCpgGenerator.scala index 72fd425b9fb7..82f945090e89 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/RubyCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/RubyCpgGenerator.scala @@ -21,9 +21,7 @@ case class RubyCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgG override def isJvmBased = true override def applyPostProcessingPasses(cpg: Cpg): Cpg = { - // TODO: here we need a Ruby-specific `Config`, which we shall build from the existing `FrontendConfig`. We only - // care for `--useDeprecatedFrontend` though, for now. Nevertheless, there should be a better way of handling this. - val rubyConfig = Config().withUseDeprecatedFrontend(config.cmdLineParams.exists(_ == "--useDeprecatedFrontend")) + val rubyConfig = Config() RubySrc2Cpg.postProcessingPasses(cpg, rubyConfig).foreach(_.createAndApply()) cpg } diff --git a/console/src/main/scala/io/joern/console/package.scala b/console/src/main/scala/io/joern/console/package.scala index a29435afba29..958f26202bde 100644 --- a/console/src/main/scala/io/joern/console/package.scala +++ b/console/src/main/scala/io/joern/console/package.scala @@ -1,6 +1,6 @@ package io.joern -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider import replpp.Operators.* import replpp.Colors diff --git a/console/src/main/scala/io/joern/console/scan/package.scala b/console/src/main/scala/io/joern/console/scan/package.scala index 41e01d2e32e4..31230076bb8d 100644 --- a/console/src/main/scala/io/joern/console/scan/package.scala +++ b/console/src/main/scala/io/joern/console/scan/package.scala @@ -11,11 +11,6 @@ package object scan { private val logger: Logger = LoggerFactory.getLogger(this.getClass) - implicit class ScannerStarters(val cpg: Cpg) extends AnyVal { - def finding: Iterator[Finding] = - overflowdb.traversal.InitialTraversal.from[Finding](cpg.graph, NodeTypes.FINDING) - } - implicit class QueryWrapper(q: Query) { /** Obtain list of findings by running query on CPG diff --git a/console/src/main/scala/io/joern/console/workspacehandling/Project.scala b/console/src/main/scala/io/joern/console/workspacehandling/Project.scala index 8de086cd693b..68d2ddc8780e 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/Project.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/Project.scala @@ -1,6 +1,6 @@ package io.joern.console.workspacehandling -import better.files.Dsl._ +import better.files.Dsl.* import better.files.File import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.Overlays diff --git a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala index 0d3a068d4bc2..82caaff7897b 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/Workspace.scala @@ -1,8 +1,8 @@ package io.joern.console.workspacehandling import io.joern.console.defaultAvailableWidthProvider -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider import scala.collection.mutable.ListBuffer diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala index c0297fb43551..4d7d943db5e1 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala @@ -2,7 +2,7 @@ package io.joern.console.workspacehandling import better.files.Dsl.mkdirs import better.files.File -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider import java.nio.file.Path import scala.collection.mutable.ListBuffer diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala index 723a058617dc..3beb2345060a 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala @@ -1,18 +1,17 @@ package io.joern.console.workspacehandling -import better.files.Dsl._ -import better.files._ +import better.files.Dsl.* +import better.files.* import io.joern.console import io.joern.console.defaultAvailableWidthProvider import io.joern.console.Reporting import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import org.json4s.DefaultFormats -import org.json4s.native.Serialization.{write => jsonWrite} -import overflowdb.Config +import org.json4s.native.Serialization.write as jsonWrite import java.net.URLEncoder -import java.nio.file.Path +import java.nio.file.{Path, Paths} import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} @@ -305,17 +304,12 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo private def loadCpgRaw(cpgFilename: String): Option[Cpg] = { Try { - val odbConfig = Config.withDefaults.withStorageLocation(cpgFilename) - val config = - CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig) - val newCpg = CpgLoader.loadFromOverflowDb(config) - CpgLoader.createIndexes(newCpg) - newCpg + CpgLoader.load(cpgFilename) } match { case Success(v) => Some(v) case Failure(ex) => System.err.println("Error loading CPG") - System.err.println(ex) + ex.printStackTrace() None } } diff --git a/console/src/test/scala/io/joern/console/ConsoleTests.scala b/console/src/test/scala/io/joern/console/ConsoleTests.scala index 1114558d56fb..728a83d55331 100644 --- a/console/src/test/scala/io/joern/console/ConsoleTests.scala +++ b/console/src/test/scala/io/joern/console/ConsoleTests.scala @@ -1,11 +1,11 @@ package io.joern.console -import better.files.Dsl._ -import better.files._ -import io.joern.console.testing._ +import better.files.Dsl.* +import better.files.* +import io.joern.console.testing.* import io.joern.x2cpg.X2Cpg.defaultOverlayCreators import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -428,7 +428,7 @@ class ConsoleTests extends AnyWordSpec with Matchers { "cpg" should { "provide .help command" in ConsoleFixture() { (console, codeDir) => // part of Predefined.shared, which makes the below work in the repl without separate import - import io.shiftleft.codepropertygraph.Cpg.docSearchPackages + import io.shiftleft.semanticcpg.language.docSearchPackages import io.joern.console.testing.availableWidthProvider console.importCode(codeDir.toString) diff --git a/console/src/test/scala/io/joern/console/LanguageHelperTests.scala b/console/src/test/scala/io/joern/console/LanguageHelperTests.scala index 80b145767aea..afe860453215 100644 --- a/console/src/test/scala/io/joern/console/LanguageHelperTests.scala +++ b/console/src/test/scala/io/joern/console/LanguageHelperTests.scala @@ -1,7 +1,7 @@ package io.joern.console -import better.files.Dsl._ -import better.files._ +import better.files.Dsl.* +import better.files.* import io.shiftleft.codepropertygraph.generated.Languages import io.joern.console.cpgcreation.{guessLanguage, LlvmCpgGenerator} import org.scalatest.matchers.should.Matchers diff --git a/console/src/test/scala/io/joern/console/PluginManagerTests.scala b/console/src/test/scala/io/joern/console/PluginManagerTests.scala index 9a0957769c27..63eb5d2b933c 100644 --- a/console/src/test/scala/io/joern/console/PluginManagerTests.scala +++ b/console/src/test/scala/io/joern/console/PluginManagerTests.scala @@ -1,7 +1,7 @@ package io.joern.console -import better.files.Dsl._ -import better.files._ +import better.files.Dsl.* +import better.files.* import io.shiftleft.utils.ProjectRoot import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/console/src/test/scala/io/joern/console/testing/package.scala b/console/src/test/scala/io/joern/console/testing/package.scala index 897bb9e9633c..1b2053d82a7c 100644 --- a/console/src/test/scala/io/joern/console/testing/package.scala +++ b/console/src/test/scala/io/joern/console/testing/package.scala @@ -3,7 +3,7 @@ package io.joern.console import better.files.Dsl.* import better.files.* import io.joern.console.workspacehandling.Project -import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} +import flatgraph.help.Table.{AvailableWidthProvider, ConstantWidth} import scala.util.Try diff --git a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala index 4d1ddb669d49..867f2e4cfd06 100644 --- a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala +++ b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala @@ -1,6 +1,6 @@ package io.joern.console.workspacehandling -import better.files._ +import better.files.* import io.shiftleft.codepropertygraph.generated.Cpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala index 0abe2f0887d2..9d1b5f48b3ef 100644 --- a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala +++ b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala @@ -1,6 +1,6 @@ package io.joern.console.workspacehandling -import better.files.Dsl._ +import better.files.Dsl.* import better.files.File import io.joern.console.testing.availableWidthProvider import io.shiftleft.semanticcpg.testing.MockCpg diff --git a/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala b/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala deleted file mode 100644 index 7649f012eccd..000000000000 --- a/console/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/TestProtoCpg.scala +++ /dev/null @@ -1,45 +0,0 @@ -package io.shiftleft.codepropertygraph.cpgloading - -import better.files.File -import io.shiftleft.proto.cpg.Cpg -import io.shiftleft.proto.cpg.Cpg.CpgStruct - -import java.io.FileOutputStream - -object TestProtoCpg { - - def createTestProtoCpg: File = { - val outDir = better.files.File.newTemporaryDirectory("cpgloadertests") - val outStream = new FileOutputStream((outDir / "1.proto").pathAsString) - CpgStruct - .newBuilder() - .addNode( - CpgStruct.Node - .newBuilder() - .setKey(1) - .setType(CpgStruct.Node.NodeType.valueOf("METHOD")) - .addProperty( - CpgStruct.Node.Property - .newBuilder() - .setName(Cpg.NodePropertyName.valueOf("FULL_NAME")) - .setValue( - Cpg.PropertyValue - .newBuilder() - .setStringValue("foo") - .build() - ) - .build - ) - .build() - ) - .build() - .writeTo(outStream) - outStream.close() - - val zipFile = better.files.File.newTemporaryFile("cpgloadertests", ".bin.zip") - outDir.zipTo(zipFile) - outDir.delete() - zipFile - } - -} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/DefaultSemantics.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/DefaultSemantics.scala index fd8a1df0b264..33312f054950 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/DefaultSemantics.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/DefaultSemantics.scala @@ -1,6 +1,6 @@ package io.joern.dataflowengineoss -import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, PassThroughMapping, Semantics} +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, PassThroughMapping, FullNameSemantics} import io.shiftleft.codepropertygraph.generated.Operators import scala.annotation.unused @@ -10,9 +10,9 @@ object DefaultSemantics { /** @return * a default set of common external procedure calls for all languages. */ - def apply(): Semantics = { + def apply(): FullNameSemantics = { val list = operatorFlows ++ cFlows ++ javaFlows - Semantics.fromList(list) + FullNameSemantics.fromList(list) } private def F = (x: String, y: List[(Int, Int)]) => FlowSemantic.from(x, y) @@ -60,18 +60,6 @@ object DefaultSemantics { F(Operators.preIncrement, List((1, 1), (1, -1))), F(Operators.sizeOf, List.empty[(Int, Int)]), - // some of those operators have duplicate mappings due to a typo - // - see https://github.com/ShiftLeftSecurity/codepropertygraph/pull/1630 - - F(".assignmentExponentiation", List((2, 1), (1, 1))), - F(".assignmentModulo", List((2, 1), (1, 1))), - F(".assignmentShiftLeft", List((2, 1), (1, 1))), - F(".assignmentLogicalShiftRight", List((2, 1), (1, 1))), - F(".assignmentArithmeticShiftRight", List((2, 1), (1, 1))), - F(".assignmentAnd", List((2, 1), (1, 1))), - F(".assignmentOr", List((2, 1), (1, 1))), - F(".assignmentXor", List((2, 1), (1, 1))), - // Language specific operators PTF(".tupleLiteral"), PTF(".dictLiteral"), @@ -157,6 +145,6 @@ object DefaultSemantics { * procedure semantics for operators and common external Java calls only. */ @unused - def javaSemantics(): Semantics = Semantics.fromList(operatorFlows ++ javaFlows) + def javaSemantics(): FullNameSemantics = FullNameSemantics.fromList(operatorFlows ++ javaFlows) } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala index 61c55435add3..ca007968914d 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala @@ -1,15 +1,13 @@ package io.joern.dataflowengineoss.dotgenerator import io.joern.dataflowengineoss.DefaultSemantics -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties} -import io.joern.dataflowengineoss.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.utils.MemberAccess.isGenericMemberAccessName -import overflowdb.Node -import overflowdb.traversal.jIteratortoTraversal import scala.collection.mutable @@ -59,7 +57,7 @@ class DdgGenerator { } } - private def shouldBeDisplayed(v: Node): Boolean = !( + private def shouldBeDisplayed(v: StoredNode): Boolean = !( v.isInstanceOf[ControlStructure] || v.isInstanceOf[JumpTarget] ) @@ -91,7 +89,16 @@ class DdgGenerator { val allInEdges = v .inE(EdgeTypes.REACHING_DEF) .map(x => - Edge(x.outNode.asInstanceOf[StoredNode], v, srcVisible = true, x.property(Properties.Variable), edgeType) + // note: this looks strange, but let me explain... + // in overflowdb, edges were allowed multiple properties and this used to be `x.property(Properties.VARIABLE)` + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variablePropertyMaybe = x.property match { + case null => null + case variableProperty: String => variableProperty + case _ => null + } + Edge(x.src.asInstanceOf[StoredNode], v, srcVisible = true, variablePropertyMaybe, edgeType) ) v match { diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala index 241a3d95750d..981b054ec657 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/Path.scala @@ -3,8 +3,8 @@ package io.joern.dataflowengineoss.language import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member, MethodParameterIn} import io.shiftleft.semanticcpg import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Table -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider case class Path(elements: List[AstNode]) { def resultPairs(): List[(String, Option[Int])] = { diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExpressionMethods.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExpressionMethods.scala index 52a66ffecbfa..362cefee6d46 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExpressionMethods.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExpressionMethods.scala @@ -64,8 +64,8 @@ class ExpressionMethods[NodeType <: Expression](val node: NodeType) extends AnyV srcIndex == node.argumentIndex && dstName == tgt.argumentName.get case FlowMapping(ParameterNode(srcIndex, _), ParameterNode(dstIndex, _)) => srcIndex == node.argumentIndex && dstIndex == tgt.argumentIndex - case PassThroughMapping if tgt.argumentIndex == node.argumentIndex || tgt.argumentIndex == -1 => true - case _ => false + case PassThroughMapping => node.argumentIndex == tgt.argumentIndex && node.argumentName == tgt.argumentName + case _ => false } } } @@ -73,9 +73,7 @@ class ExpressionMethods[NodeType <: Expression](val node: NodeType) extends AnyV /** Retrieve flow semantic for the call this argument is a part of. */ def semanticsForCallByArg(implicit semantics: Semantics): Iterator[FlowSemantic] = { - argToMethods(node).flatMap { method => - semantics.forMethod(method.fullName) - } + argToMethods(node).flatMap(semantics.forMethod) } private def argToMethods(arg: Expression): Iterator[Method] = { diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExtendedCfgNodeMethods.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExtendedCfgNodeMethods.scala index fef9acf3c1bd..7a914187e6b6 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExtendedCfgNodeMethods.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/nodemethods/ExtendedCfgNodeMethods.scala @@ -10,7 +10,7 @@ import io.shiftleft.semanticcpg.language.* import scala.collection.mutable import scala.jdk.CollectionConverters.* -class ExtendedCfgNodeMethods[NodeType <: CfgNode](val node: NodeType) extends AnyVal { +class ExtendedCfgNodeMethods[CfgNodeType <: CfgNode](val node: CfgNodeType) extends AnyVal { /** Convert to nearest AST node */ diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/package.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/package.scala index 842ce65d0d74..7101c872177f 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/package.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/package.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.joern.dataflowengineoss.language.dotextension.DdgNodeDot import io.joern.dataflowengineoss.language.nodemethods.{ExpressionMethods, ExtendedCfgNodeMethods} -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc import scala.language.implicitConversions diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpCpg14.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpCpg14.scala index 16ea1b420cb0..4575fba5ab9b 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpCpg14.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpCpg14.scala @@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows import better.files.File import io.joern.dataflowengineoss.DefaultSemantics -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class Cpg14DumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpDdg.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpDdg.scala index ec5db89d16c6..c76e5b9275d5 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpDdg.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpDdg.scala @@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows import better.files.File import io.joern.dataflowengineoss.DefaultSemantics -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class DdgDumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpPdg.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpPdg.scala index 2221a5372c8d..3b3dc6d24d30 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpPdg.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpPdg.scala @@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows import better.files.File import io.joern.dataflowengineoss.DefaultSemantics -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class PdgDumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/OssDataFlow.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/OssDataFlow.scala index ce0fa3800eef..60dbeef7a526 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/OssDataFlow.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/OssDataFlow.scala @@ -2,7 +2,7 @@ package io.joern.dataflowengineoss.layers.dataflows import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.passes.reachingdef.ReachingDefPass -import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, FullNameSemantics, Semantics} import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} object OssDataFlow { @@ -12,23 +12,15 @@ object OssDataFlow { def defaultOpts = new OssDataFlowOptions() } -class OssDataFlowOptions( - var maxNumberOfDefinitions: Int = 4000, - var extraFlows: List[FlowSemantic] = List.empty[FlowSemantic] -) extends LayerCreatorOptions {} +class OssDataFlowOptions(var maxNumberOfDefinitions: Int = 4000, var semantics: Semantics = DefaultSemantics()) + extends LayerCreatorOptions {} -class OssDataFlow(opts: OssDataFlowOptions)(implicit - s: Semantics = Semantics.fromList(DefaultSemantics().elements ++ opts.extraFlows) -) extends LayerCreator { +class OssDataFlow(opts: OssDataFlowOptions)(implicit val semantics: Semantics = opts.semantics) extends LayerCreator { override val overlayName: String = OssDataFlow.overlayName override val description: String = OssDataFlow.description override def create(context: LayerCreatorContext): Unit = { - val cpg = context.cpg - val enhancementExecList = Iterator(new ReachingDefPass(cpg, opts.maxNumberOfDefinitions)) - enhancementExecList.zipWithIndex.foreach { case (pass, index) => - runPass(pass, context, index) - } + ReachingDefPass(context.cpg, opts.maxNumberOfDefinitions).createAndApply() } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala index 887531648c7e..7693f28757ff 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala @@ -4,10 +4,10 @@ import io.joern.dataflowengineoss.{globalFromLiteral, identifierToFirstUsages} import io.joern.dataflowengineoss.queryengine.AccessPathUsage.toTrackedBaseAndAccessPathSimple import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators} import io.shiftleft.semanticcpg.accesspath.MatchResult import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import scala.collection.{Set, mutable} @@ -132,7 +132,7 @@ class DdgGenerator(semantics: Semantics) { // There is always an edge from the method input parameter // to the corresponding method output parameter as modifications // of the input parameter only affect a copy. - paramOut.paramIn.foreach { paramIn => + paramOut.start.paramIn.foreach { paramIn => addEdge(paramIn, paramOut, paramIn.name) } usageAnalyzer.usedIncomingDefs(paramOut).foreach { case (_, inElements) => @@ -224,7 +224,7 @@ class DdgGenerator(semantics: Semantics) { (fromNode, toNode) match { case (parentNode: CfgNode, childNode: CfgNode) if EdgeValidator.isValidEdge(childNode, parentNode) => - dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, variable) + dstGraph.addEdge(fromNode, toNode, EdgeTypes.REACHING_DEF, variable) case _ => } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/EdgeValidator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/EdgeValidator.scala index 9f8666fd6848..505bd4e3b7fc 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/EdgeValidator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/EdgeValidator.scala @@ -1,7 +1,7 @@ package io.joern.dataflowengineoss.passes.reachingdef import io.joern.dataflowengineoss.language.* -import io.joern.dataflowengineoss.queryengine.Engine.isOutputArgOfInternalMethod +import io.joern.dataflowengineoss.queryengine.Engine.{isOutputArgOfInternalMethod, semanticsForCall} import io.joern.dataflowengineoss.semanticsloader.{ FlowMapping, FlowPath, @@ -50,7 +50,7 @@ object EdgeValidator { */ private def isCallRetval(parentNode: StoredNode)(implicit semantics: Semantics): Boolean = parentNode match { - case call: Call => semantics.forMethod(call.methodFullName).exists(!explicitlyFlowsToReturnValue(_)) + case call: Call => semanticsForCall(call).exists(!explicitlyFlowsToReturnValue(_)) case _ => false } @@ -58,8 +58,10 @@ object EdgeValidator { flowSemantic.mappings.exists(explicitlyFlowsToReturnValue) private def explicitlyFlowsToReturnValue(flowPath: FlowPath): Boolean = flowPath match { - case FlowMapping(_, ParameterNode(dst, _)) => dst == -1 - case PassThroughMapping => true - case _ => false + // Some frontends (e.g. python) denote named arguments using `-1` as the argument index. As such + // `-1` denotes the return value only if there's no argument name. + case FlowMapping(_, ParameterNode(-1, None)) => true + case PassThroughMapping => true + case _ => false } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala index a5aab780ed00..a01796831c3c 100755 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala @@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.passes.reachingdef import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} import scala.collection.mutable @@ -16,7 +16,7 @@ class ReachingDefPass(cpg: Cpg, maxNumberOfDefinitions: Int = 4000)(implicit s: private val logger: Logger = LoggerFactory.getLogger(this.getClass) // If there are any regex method full names, load them early - s.loadRegexSemantics(cpg) + s.initialize(cpg) override def generateParts(): Array[Method] = cpg.method.toArray diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala index 936f43ef4b9f..e06bb34447e0 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala @@ -167,7 +167,7 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph) val gen: Map[StoredNode, mutable.BitSet] = initGen(method).withDefaultValue(mutable.BitSet()) - val kill: Map[StoredNode, Set[Definition]] = + val kill: Map[StoredNode, mutable.BitSet] = initKill(method, gen).withDefaultValue(mutable.BitSet()) /** For a given flow graph node `n` and set of definitions, apply the transfer function to obtain the updated set of @@ -224,7 +224,7 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph) * All operations in our graph are represented by calls and non-operations such as identifiers or field-identifiers * have empty gen and kill sets, meaning that they just pass on definitions unaltered. */ - private def initKill(method: Method, gen: Map[StoredNode, Set[Definition]]): Map[StoredNode, Set[Definition]] = { + private def initKill(method: Method, gen: Map[StoredNode, mutable.BitSet]): Map[StoredNode, mutable.BitSet] = { val allIdentifiers: Map[String, List[CfgNode]] = { val results = mutable.Map.empty[String, List[CfgNode]] @@ -259,42 +259,44 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph) * calculate kill(call) based on gen(call). */ private def killsForGens( - genOfCall: Set[Definition], + genOfCall: mutable.BitSet, allIdentifiers: Map[String, List[CfgNode]], allCalls: Map[String, List[Call]] - ): Set[Definition] = { + ): mutable.BitSet = { - def definitionsOfSameVariable(definition: Definition): Set[Definition] = { + def definitionsOfSameVariable(definition: Definition): Iterator[Definition] = { val definedNodes = flowGraph.numberToNode(definition) match { case param: MethodParameterIn => - allIdentifiers(param.name) + allIdentifiers(param.name).iterator .filter(x => x.id != param.id) case identifier: Identifier => - val sameIdentifiers = allIdentifiers(identifier.name) + val sameIdentifiers = allIdentifiers(identifier.name).iterator .filter(x => x.id != identifier.id) /** Killing an identifier should also kill field accesses on that identifier. For example, a reassignment `x = * new Box()` should kill any previous calls to `x.value`, `x.length()`, etc. */ - val sameObjects: Iterable[Call] = allCalls.values.flatten + val sameObjects: Iterator[Call] = allCalls.valuesIterator.flatten .filter(_.name == Operators.fieldAccess) .filter(_.ast.isIdentifier.nameExact(identifier.name).nonEmpty) sameIdentifiers ++ sameObjects case call: Call => - allCalls(call.code) + allCalls(call.code).iterator .filter(x => x.id != call.id) - case _ => Set() + case _ => Iterator.empty } definedNodes // It can happen that the CFG is broken and contains isolated nodes, // in which case they are not in `nodeToNumber`. Let's filter those. - .collect { case x if nodeToNumber.contains(x) => Definition.fromNode(x, nodeToNumber) }.toSet + .collect { case x if nodeToNumber.contains(x) => Definition.fromNode(x, nodeToNumber) } } - genOfCall.flatMap { definition => - definitionsOfSameVariable(definition) + val res = mutable.BitSet() + for (definition <- genOfCall) { + res.addAll(definitionsOfSameVariable(definition)) } + res } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala index 341f29621d53..534b8bb565b8 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala @@ -1,8 +1,8 @@ package io.joern.dataflowengineoss.queryengine -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.accesspath._ -import io.shiftleft.semanticcpg.language.{AccessPathHandling, toCallMethods} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.accesspath.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.utils.MemberAccess import org.slf4j.LoggerFactory diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala index 1c0c279a9758..0e963c9c8aaf 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala @@ -1,14 +1,14 @@ package io.joern.dataflowengineoss.queryengine +import flatgraph.Edge import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.passes.reachingdef.EdgeValidator import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties} +import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.Edge import java.util.concurrent.* import scala.collection.mutable @@ -205,9 +205,11 @@ object Engine { private def elemForEdge(e: Edge, callSiteStack: List[Call] = List())(implicit semantics: Semantics ): Option[PathElement] = { - val curNode = e.inNode().asInstanceOf[CfgNode] - val parNode = e.outNode().asInstanceOf[CfgNode] - val outLabel = Some(e.property(Properties.Variable)).getOrElse("") + val curNode = e.dst.asInstanceOf[CfgNode] + val parNode = e.src.asInstanceOf[CfgNode] + // note: flatgraph only allows at most one property per edge, and since we know :tm: that this is a ReachingDef edge it must be the Variable property... + val variablePropertyMaybe = Option(e.property).map(_.asInstanceOf[String]) + val outLabel = variablePropertyMaybe.getOrElse("") if (!EdgeValidator.isValidEdge(curNode, parNode)) { return None @@ -254,9 +256,8 @@ object Engine { private def ddgInE(node: CfgNode, path: Vector[PathElement], callSiteStack: List[Call] = List()): Vector[Edge] = { node .inE(EdgeTypes.REACHING_DEF) - .asScala .filter { e => - e.outNode() match { + e.src match { case srcNode: CfgNode => !srcNode.isInstanceOf[Method] && !path .map(x => x.node) @@ -291,9 +292,7 @@ object Engine { } def semanticsForCall(call: Call)(implicit semantics: Semantics): List[FlowSemantic] = { - Engine.methodsForCall(call).flatMap { method => - semantics.forMethod(method.fullName) - } + Engine.methodsForCall(call).flatMap(semantics.forMethod) } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/HeldTaskCompletion.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/HeldTaskCompletion.scala index f0b0cf5fc9e1..46326fbc0e6c 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/HeldTaskCompletion.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/HeldTaskCompletion.scala @@ -1,7 +1,7 @@ package io.joern.dataflowengineoss.queryengine import scala.collection.mutable -import scala.collection.parallel.CollectionConverters._ +import scala.collection.parallel.CollectionConverters.* /** Complete held tasks using the result table. The result table is modified in the process. * diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala index cac222fbd476..c686271ea0f6 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala @@ -32,7 +32,7 @@ object SourcesToStartingPoints { .map(src => { // We need to get Cpg wrapper from graph. Hence we are taking head element from source iterator. // This will also ensure if the source list is empty then these tasks are invoked. - val cpg = Cpg(src.graph()) + val cpg = Cpg(src.graph) val (startingPoints, methodTasks) = calculateStartingPoints(sources, executorService) val startingPointFromUsageInOtherClasses = calculateStatingPointsWithUsageInOtherClasses(methodTasks, cpg, executorService) @@ -189,7 +189,8 @@ abstract class BaseSourceToStartingPoints extends Callable[Unit] { protected def sourceToStartingPoints(src: StoredNode): (List[CfgNode], List[UsageInput]) = { src match { case methodReturn: MethodReturn => - (methodReturn.method.callIn.l, Nil) + // n.b. there's a generated `callIn` step that we really want to use, but it's shadowed by `MethodTraversal.callIn` + (methodReturn.method._callIn.cast[Call].l, Nil) case lit: Literal => val usageInput = targetsToClassIdentifierPair(literalToInitializedMembers(lit), src) val uses = usages(usageInput) diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala index f10f105d2409..5c8713e505c8 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala @@ -100,7 +100,7 @@ class TaskCreator(context: EngineContext) { */ private def paramToMethodRefCallReceivers(param: MethodParameterIn): List[Expression] = - new Cpg(param.graph()).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l + new Cpg(param.graph).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l /** Create new tasks from all results that end in an output argument, including return arguments. In this case, we * want to traverse to corresponding method output parameters and method return nodes respectively. diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskSolver.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskSolver.scala index 263251775694..784ab9ee45a5 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskSolver.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskSolver.scala @@ -2,8 +2,8 @@ package io.joern.dataflowengineoss.queryengine import io.joern.dataflowengineoss.queryengine.QueryEngineStatistics.{PATH_CACHE_HITS, PATH_CACHE_MISSES} import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language.{toCfgNodeMethods, toExpressionMethods, _} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import java.util.concurrent.Callable import scala.collection.mutable diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemantics.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemantics.scala new file mode 100644 index 000000000000..3a4196a9c09c --- /dev/null +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemantics.scala @@ -0,0 +1,79 @@ +package io.joern.dataflowengineoss.semanticsloader + +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.Method +import io.shiftleft.semanticcpg.language.* +import org.slf4j.LoggerFactory + +import scala.collection.mutable + +object FullNameSemantics { + + private val logger = LoggerFactory.getLogger(getClass) + + /** Builds FullNameSemantics given their constituent FlowSemantics. Same methodFullNamed FlowSemantic elements are + * combined into a single one with both of their FlowMappings. + */ + def fromList(elements: List[FlowSemantic]): FullNameSemantics = FullNameSemantics( + elements.groupBy(_.methodFullName).map { (fullName, semantics) => + val howMany = semantics.length + if (howMany > 1) { + logger.warn(s"$howMany competing FlowSemantics found for $fullName, merging them") + } + fullName -> FlowSemantic( + methodFullName = fullName, + mappings = semantics.flatMap(_.mappings), + regex = semantics.exists(_.regex) + ) + } + ) + + def empty: FullNameSemantics = fromList(List()) + +} + +class FullNameSemantics private (methodToSemantic: Map[String, FlowSemantic]) extends Semantics { + + /** The map below keeps a mapping between results of a regex and the regex string it matches. e.g. + * + * `path/to/file.py:.Foo.sink` -> `^path.*Foo\\.sink$` + */ + private val regexMatchedFullNames = mutable.HashMap.empty[String, String] + + /** Initialize all the method semantics that use regex with all their regex results before query time. + */ + override def initialize(cpg: Cpg): Unit = { + import io.shiftleft.semanticcpg.language._ + + methodToSemantic.filter(_._2.regex).foreach { case (regexString, _) => + cpg.method.fullName(regexString).fullName.foreach { methodMatch => + regexMatchedFullNames.put(methodMatch, regexString) + } + } + } + + def elements: List[FlowSemantic] = methodToSemantic.values.toList + + private def forMethod(fullName: String): Option[FlowSemantic] = regexMatchedFullNames.get(fullName) match { + case Some(matchedFullName) => methodToSemantic.get(matchedFullName) + case None => methodToSemantic.get(fullName) + } + + override def forMethod(method: Method): Option[FlowSemantic] = forMethod(method.fullName) + + def serialize: String = { + elements + .sortBy(_.methodFullName) + .map { elem => + s"\"${elem.methodFullName}\" " + elem.mappings + .collect { case FlowMapping(x, y) => s"$x -> $y" } + .mkString(" ") + } + .mkString("\n") + } + + /** Immutably extends the current `FullNameSemantics` with `extraFlows`. + */ + def plus(extraFlows: List[FlowSemantic]): FullNameSemantics = FullNameSemantics.fromList(elements ++ extraFlows) + +} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParser.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParser.scala new file mode 100644 index 000000000000..669495da8a39 --- /dev/null +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParser.scala @@ -0,0 +1,73 @@ +package io.joern.dataflowengineoss.semanticsloader + +import io.joern.dataflowengineoss.SemanticsParser.MappingContext +import io.joern.dataflowengineoss.{SemanticsBaseListener, SemanticsLexer, SemanticsParser} +import org.antlr.v4.runtime.tree.ParseTreeWalker +import org.antlr.v4.runtime.{CharStream, CharStreams, CommonTokenStream} + +import scala.collection.mutable +import scala.jdk.CollectionConverters.* + +class FullNameSemanticsParser { + + def parse(input: String): List[FlowSemantic] = { + val charStream = CharStreams.fromString(input) + parseCharStream(charStream) + } + + def parseFile(fileName: String): List[FlowSemantic] = { + val charStream = CharStreams.fromFileName(fileName) + parseCharStream(charStream) + } + + private def parseCharStream(charStream: CharStream): List[FlowSemantic] = { + val lexer = new SemanticsLexer(charStream) + val tokenStream = new CommonTokenStream(lexer) + val parser = new SemanticsParser(tokenStream) + val treeWalker = new ParseTreeWalker() + + val tree = parser.taintSemantics() + val listener = new Listener() + treeWalker.walk(listener, tree) + listener.result.toList + } + + implicit class AntlrFlowExtensions(val ctx: MappingContext) { + + def isPassThrough: Boolean = Option(ctx.PASSTHROUGH()).isDefined + + def srcIdx: Int = ctx.src().argIdx().NUMBER().getText.toInt + + def srcArgName: Option[String] = Option(ctx.src().argName()).map(_.name().getText) + + def dstIdx: Int = ctx.dst().argIdx().NUMBER().getText.toInt + + def dstArgName: Option[String] = Option(ctx.dst().argName()).map(_.name().getText) + + } + + private class Listener extends SemanticsBaseListener { + + val result: mutable.ListBuffer[FlowSemantic] = mutable.ListBuffer[FlowSemantic]() + + override def enterTaintSemantics(ctx: SemanticsParser.TaintSemanticsContext): Unit = { + ctx.singleSemantic().asScala.foreach { semantic => + val methodName = semantic.methodName().name().getText + val mappings = semantic.mapping().asScala.toList.map(ctxToParamMapping) + result.addOne(FlowSemantic(methodName, mappings)) + } + } + + private def ctxToParamMapping(ctx: MappingContext): FlowPath = + if (ctx.isPassThrough) { + PassThroughMapping + } else { + val src = ParameterNode(ctx.srcIdx, ctx.srcArgName) + val dst = ParameterNode(ctx.dstIdx, ctx.dstArgName) + + FlowMapping(src, dst) + } + + } + +} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Parser.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Parser.scala deleted file mode 100644 index f3e245d575c3..000000000000 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Parser.scala +++ /dev/null @@ -1,211 +0,0 @@ -package io.joern.dataflowengineoss.semanticsloader - -import io.joern.dataflowengineoss.SemanticsParser.MappingContext -import io.joern.dataflowengineoss.{SemanticsBaseListener, SemanticsLexer, SemanticsParser} -import io.shiftleft.codepropertygraph.generated.Cpg -import org.antlr.v4.runtime.tree.ParseTreeWalker -import org.antlr.v4.runtime.{CharStream, CharStreams, CommonTokenStream} - -import scala.collection.mutable -import scala.jdk.CollectionConverters._ - -object Semantics { - - def fromList(elements: List[FlowSemantic]): Semantics = { - new Semantics( - mutable.Map.newBuilder - .addAll(elements.map { e => - e.methodFullName -> e - }) - .result() - ) - } - - def empty: Semantics = fromList(List()) - -} - -class Semantics private (methodToSemantic: mutable.Map[String, FlowSemantic]) { - - /** The map below keeps a mapping between results of a regex and the regex string it matches. e.g. - * - * `path/to/file.py:.Foo.sink` -> `^path.*Foo\\.sink$` - */ - private val regexMatchedFullNames = mutable.HashMap.empty[String, String] - - /** Initialize all the method semantics that use regex with all their regex results before query time. - */ - def loadRegexSemantics(cpg: Cpg): Unit = { - import io.shiftleft.semanticcpg.language._ - - methodToSemantic.filter(_._2.regex).foreach { case (regexString, _) => - cpg.method.fullName(regexString).fullName.foreach { methodMatch => - regexMatchedFullNames.put(methodMatch, regexString) - } - } - } - - def elements: List[FlowSemantic] = methodToSemantic.values.toList - - def forMethod(fullName: String): Option[FlowSemantic] = regexMatchedFullNames.get(fullName) match { - case Some(matchedFullName) => methodToSemantic.get(matchedFullName) - case None => methodToSemantic.get(fullName) - } - - def serialize: String = { - elements - .sortBy(_.methodFullName) - .map { elem => - s"\"${elem.methodFullName}\" " + elem.mappings - .collect { case FlowMapping(x, y) => s"$x -> $y" } - .mkString(" ") - } - .mkString("\n") - } - -} -case class FlowSemantic(methodFullName: String, mappings: List[FlowPath] = List.empty, regex: Boolean = false) - -object FlowSemantic { - - def from(methodFullName: String, mappings: List[?], regex: Boolean = false): FlowSemantic = { - FlowSemantic( - methodFullName, - mappings.map { - case (src: Int, dst: Int) => FlowMapping(src, dst) - case (srcIdx: Int, src: String, dst: Int) => FlowMapping(srcIdx, src, dst) - case (src: Int, dstIdx: Int, dst: String) => FlowMapping(src, dstIdx, dst) - case (srcIdx: Int, src: String, dstIdx: Int, dst: String) => FlowMapping(srcIdx, src, dstIdx, dst) - case x: FlowMapping => x - }, - regex - ) - } - -} - -abstract class FlowNode - -/** Collects parameters and return nodes under a common trait. This trait acknowledges their argument index which is - * relevant when a caller wants to coordinate relevant tainted flows through specific arguments and the return flow. - */ -trait ParamOrRetNode extends FlowNode { - - /** Temporary backward compatible idx field. - * - * @return - * the argument index. - */ - def index: Int -} - -/** A parameter where the index of the argument matches the position of the parameter at the callee. The name is used to - * match named arguments if used instead of positional arguments. - * - * @param index - * the position or argument index. - * @param name - * the name of the parameter. - */ -case class ParameterNode(index: Int, name: Option[String] = None) extends ParamOrRetNode - -object ParameterNode { - def apply(index: Int, name: String): ParameterNode = ParameterNode(index, Option(name)) -} - -/** Represents explicit mappings or special cases. - */ -sealed trait FlowPath - -/** Maps flow between arguments based on how they interact as parameters at the callee. - * - * @param src - * source of the flow. - * @param dst - * destination of the flow. - */ -case class FlowMapping(src: FlowNode, dst: FlowNode) extends FlowPath - -object FlowMapping { - def apply(from: Int, to: Int): FlowMapping = FlowMapping(ParameterNode(from), ParameterNode(to)) - - def apply(fromIdx: Int, from: String, toIdx: Int, to: String): FlowMapping = - FlowMapping(ParameterNode(fromIdx, from), ParameterNode(toIdx, to)) - - def apply(fromIdx: Int, from: String, toIdx: Int): FlowMapping = - FlowMapping(ParameterNode(fromIdx, from), ParameterNode(toIdx)) - - def apply(from: Int, toIdx: Int, to: String): FlowMapping = FlowMapping(ParameterNode(from), ParameterNode(toIdx, to)) - -} - -/** Represents an instance where parameters are not sanitized, may affect the return value, and do not cross-taint. e.g. - * foo(1, 2) = 1 -> 1, 2 -> 2, 1 -> -1, 2 -> -1 - * - * The main benefit is that this works for unbounded parameters e.g. VARARGS. Note this does not taint 0 -> 0. - */ -object PassThroughMapping extends FlowPath - -class Parser() { - - def parse(input: String): List[FlowSemantic] = { - val charStream = CharStreams.fromString(input) - parseCharStream(charStream) - } - - def parseFile(fileName: String): List[FlowSemantic] = { - val charStream = CharStreams.fromFileName(fileName) - parseCharStream(charStream) - } - - private def parseCharStream(charStream: CharStream): List[FlowSemantic] = { - val lexer = new SemanticsLexer(charStream) - val tokenStream = new CommonTokenStream(lexer) - val parser = new SemanticsParser(tokenStream) - val treeWalker = new ParseTreeWalker() - - val tree = parser.taintSemantics() - val listener = new Listener() - treeWalker.walk(listener, tree) - listener.result.toList - } - - implicit class AntlrFlowExtensions(val ctx: MappingContext) { - - def isPassThrough: Boolean = Option(ctx.PASSTHROUGH()).isDefined - - def srcIdx: Int = ctx.src().argIdx().NUMBER().getText.toInt - - def srcArgName: Option[String] = Option(ctx.src().argName()).map(_.name().getText) - - def dstIdx: Int = ctx.dst().argIdx().NUMBER().getText.toInt - - def dstArgName: Option[String] = Option(ctx.dst().argName()).map(_.name().getText) - - } - - private class Listener extends SemanticsBaseListener { - - val result: mutable.ListBuffer[FlowSemantic] = mutable.ListBuffer[FlowSemantic]() - - override def enterTaintSemantics(ctx: SemanticsParser.TaintSemanticsContext): Unit = { - ctx.singleSemantic().asScala.foreach { semantic => - val methodName = semantic.methodName().name().getText - val mappings = semantic.mapping().asScala.toList.map(ctxToParamMapping) - result.addOne(FlowSemantic(methodName, mappings)) - } - } - - private def ctxToParamMapping(ctx: MappingContext): FlowPath = - if (ctx.isPassThrough) { - PassThroughMapping - } else { - val src = ParameterNode(ctx.srcIdx, ctx.srcArgName) - val dst = ParameterNode(ctx.dstIdx, ctx.dstArgName) - - FlowMapping(src, dst) - } - - } - -} diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Semantics.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Semantics.scala new file mode 100644 index 000000000000..1a67ec60c42a --- /dev/null +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Semantics.scala @@ -0,0 +1,167 @@ +package io.joern.dataflowengineoss.semanticsloader + +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.Method +import io.shiftleft.semanticcpg.language.* + +trait Semantics { + + /** Useful for `Semantics` that benefit from having some kind of internal state tailored to the current CPG. + */ + def initialize(cpg: Cpg): Unit = {} + + def forMethod(method: Method): Option[FlowSemantic] + + /** Builds a new `Semantics` whose `forMethod` behaviour first lookups in `other` and only if it fails (i.e. returns + * `None`) lookups in the current one. + */ + def after(other: Semantics): Semantics = Semantics.compose(this, other) +} + +object Semantics { + + private def compose(first: Semantics, second: Semantics): Semantics = new Semantics { + + override def initialize(cpg: Cpg): Unit = { + second.initialize(cpg) + first.initialize(cpg) + } + + override def forMethod(method: Method): Option[FlowSemantic] = + second.forMethod(method).orElse { first.forMethod(method) } + } +} + +/** The empty Semantics, whose `forMethod` always fails, i.e. the identity under `Semantics.after`. */ +object NoSemantics extends Semantics { + + override def forMethod(method: Method): Option[FlowSemantic] = None +} + +/** The nil Semantics, whose `forMethod` always succeeds but returns the empty (nil) mapping. */ +object NilSemantics { + + /** Builds a universal nil semantics. Beware this is right-absorbing under `Semantics.after`. */ + def apply(): Semantics = new Semantics { + override def forMethod(method: Method): Option[FlowSemantic] = Some(FlowSemantic(method.fullName, List.empty)) + } + + /** Extensionally builds a nil semantics. */ + def where(methodFullNames: List[String], regex: Boolean = false): Semantics = + FullNameSemantics.fromList(methodFullNames.map { + FlowSemantic(_, List.empty, regex) + }) + + /** Intensionally builds a nil semantics. */ + def where(predicate: Method => Boolean): Semantics = new Semantics { + override def forMethod(method: Method): Option[FlowSemantic] = Option.when(predicate(method)) { + FlowSemantic(method.fullName, List.empty) + } + } +} + +/** Semantics whose mappings are: 0->0, PassThroughMapping. */ +object NoCrossTaintSemantics { + + /** Builds a universal no-cross-taint semantics. Beware this is right-absorbing under `Semantics.after`. */ + def apply(): Semantics = new Semantics { + override def forMethod(method: Method): Option[FlowSemantic] = Some( + FlowSemantic(method.fullName, List(FlowMapping(0, 0), PassThroughMapping)) + ) + } + + /** Extensionally builds a no-cross-taint semantics. */ + def where(methodFullNames: List[String], regex: Boolean = false): Semantics = + FullNameSemantics.fromList(methodFullNames.map { + FlowSemantic(_, List(FlowMapping(0, 0), PassThroughMapping), regex) + }) + + /** Intensionally builds a no-cross-taint semantics. */ + def where(predicate: Method => Boolean): Semantics = new Semantics { + override def forMethod(method: Method): Option[FlowSemantic] = Option.when(predicate(method)) { + FlowSemantic(method.fullName, List(FlowMapping(0, 0), PassThroughMapping)) + } + } +} + +case class FlowSemantic(methodFullName: String, mappings: List[FlowPath] = List.empty, regex: Boolean = false) + +object FlowSemantic { + + def from(methodFullName: String, mappings: List[?], regex: Boolean = false): FlowSemantic = { + FlowSemantic( + methodFullName, + mappings.map { + case (src: Int, dst: Int) => FlowMapping(src, dst) + case (srcIdx: Int, src: String, dst: Int) => FlowMapping(srcIdx, src, dst) + case (src: Int, dstIdx: Int, dst: String) => FlowMapping(src, dstIdx, dst) + case (srcIdx: Int, src: String, dstIdx: Int, dst: String) => FlowMapping(srcIdx, src, dstIdx, dst) + case x: FlowMapping => x + }, + regex + ) + } + +} + +abstract class FlowNode + +/** Collects parameters and return nodes under a common trait. This trait acknowledges their argument index which is + * relevant when a caller wants to coordinate relevant tainted flows through specific arguments and the return flow. + */ +trait ParamOrRetNode extends FlowNode { + + /** Temporary backward compatible idx field. + * + * @return + * the argument index. + */ + def index: Int +} + +/** A parameter where the index of the argument matches the position of the parameter at the callee. The name is used to + * match named arguments if used instead of positional arguments. + * + * @param index + * the position or argument index. + * @param name + * the name of the parameter. + */ +case class ParameterNode(index: Int, name: Option[String] = None) extends ParamOrRetNode + +object ParameterNode { + def apply(index: Int, name: String): ParameterNode = ParameterNode(index, Option(name)) +} + +/** Represents explicit mappings or special cases. + */ +sealed trait FlowPath + +/** Maps flow between arguments based on how they interact as parameters at the callee. + * + * @param src + * source of the flow. + * @param dst + * destination of the flow. + */ +case class FlowMapping(src: FlowNode, dst: FlowNode) extends FlowPath + +object FlowMapping { + def apply(from: Int, to: Int): FlowMapping = FlowMapping(ParameterNode(from), ParameterNode(to)) + + def apply(fromIdx: Int, from: String, toIdx: Int, to: String): FlowMapping = + FlowMapping(ParameterNode(fromIdx, from), ParameterNode(toIdx, to)) + + def apply(fromIdx: Int, from: String, toIdx: Int): FlowMapping = + FlowMapping(ParameterNode(fromIdx, from), ParameterNode(toIdx)) + + def apply(from: Int, toIdx: Int, to: String): FlowMapping = FlowMapping(ParameterNode(from), ParameterNode(toIdx, to)) + +} + +/** Represents an instance where parameters are not sanitized, may affect the return value, and do not cross-taint. e.g. + * foo(1, 2) = 1 -> 1, 2 -> 2, 1 -> -1, 2 -> -1 + * + * The main benefit is that this works for unbounded parameters e.g. VARARGS. Note this does not taint 0 -> 0. + */ +object PassThroughMapping extends FlowPath diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala index bfcc4a4ad67b..e1c64e8a5694 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/DataFlowSlicing.scala @@ -2,8 +2,7 @@ package io.joern.dataflowengineoss.slicing import io.joern.dataflowengineoss.language.* import io.joern.x2cpg.utils.ConcurrentTaskUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, Properties} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory @@ -48,9 +47,9 @@ object DataFlowSlicing { val sliceNodesIdSet = sliceNodes.id.toSet // Lazily set up the rest if the filters are satisfied lazy val sliceEdges = sliceNodes - .flatMap(_.outE) - .filter(x => sliceNodesIdSet.contains(x.inNode().id())) - .map { e => SliceEdge(e.outNode().id(), e.inNode().id(), e.label()) } + .inE(EdgeTypes.REACHING_DEF) + .filter(x => sliceNodesIdSet.contains(x.src.id())) + .map { e => SliceEdge(e.src.id(), e.dst.id(), e.label) } .toSet lazy val slice = Option(DataFlowSlice(sliceNodes.map(cfgNodeToSliceNode).toSet, sliceEdges)) @@ -82,8 +81,8 @@ object DataFlowSlicing { case n: TypeRef => sliceNode.copy(name = n.typeFullName, code = n.code) case n => sliceNode.copy( - name = n.property(PropertyNames.NAME, ""), - typeFullName = n.property(PropertyNames.TYPE_FULL_NAME, "") + name = n.propertyOption(Properties.Name).getOrElse(""), + typeFullName = n.propertyOption(Properties.TypeFullName).getOrElse("") ) } } diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala index 2636af71d20a..a95974c2e57f 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/UsageSlicing.scala @@ -3,7 +3,7 @@ package io.joern.dataflowengineoss.slicing import io.joern.x2cpg.utils.ConcurrentTaskUtil import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties} import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory @@ -182,15 +182,12 @@ object UsageSlicing { .getOrElse(Iterator.empty) else baseCall.argument) .collect { case n: Expression if n.argumentIndex > 0 => n } - .flatMap { - case _: MethodRef => Option("LAMBDA") + .map { + case _: MethodRef => "LAMBDA" case x => - Option( - x.property( - PropertyNames.TYPE_FULL_NAME, - x.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq("ANY")).headOption - ) - ) + x.propertyOption(Properties.TypeFullName) + .orElse(x.property(Properties.DynamicTypeHintFullName).headOption) + .getOrElse("ANY") } .collect { case x: String => x } .toList diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala index e1b705041298..6025f8a83517 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala @@ -1,11 +1,10 @@ package io.joern.dataflowengineoss import better.files.File -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.Properties import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory -import overflowdb.PropertyKey import upickle.default.* import java.util.concurrent.{ExecutorService, Executors} @@ -332,13 +331,15 @@ package object slicing { * extracted. */ def fromNode(node: StoredNode, typeMap: Map[String, String] = Map.empty[String, String]): DefComponent = { - val nodeType = (node.property(PropertyNames.TYPE_FULL_NAME, "ANY") +: node.property( - PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, - Seq.empty[String] - )).filterNot(_.matches("(ANY|UNKNOWN)")).headOption.getOrElse("ANY") + val typeFullNameProperty = node.propertyOption(Properties.TypeFullName).getOrElse("ANY") + val dynamicTypeHintFullNamesProperty = node.property(Properties.DynamicTypeHintFullName) + val nodeType = (typeFullNameProperty +: dynamicTypeHintFullNamesProperty) + .filterNot(_.matches("(ANY|UNKNOWN)")) + .headOption + .getOrElse("ANY") val typeFullName = typeMap.getOrElse(nodeType, nodeType) - val lineNumber = Option(node.property(new PropertyKey[Integer](PropertyNames.LINE_NUMBER))).map(_.toInt) - val columnNumber = Option(node.property(new PropertyKey[Integer](PropertyNames.COLUMN_NUMBER))).map(_.toInt) + val lineNumber = node.propertyOption(Properties.LineNumber) + val columnNumber = node.propertyOption(Properties.ColumnNumber) node match { case x: MethodParameterIn => ParamDef(x.name, typeFullName, x.index, lineNumber, columnNumber) case x: Call if x.code.startsWith("new ") => diff --git a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala index a0bb2ea6a4bf..9749294a9984 100644 --- a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala +++ b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsageTests.scala @@ -1,13 +1,14 @@ package io.joern.dataflowengineoss.queryengine -import io.shiftleft.OverflowDbTestInstance -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Operators, Properties} +import flatgraph.{GNode, Graph} +import flatgraph.misc.TestUtils.* +import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, Operators} import io.joern.dataflowengineoss.queryengine.AccessPathUsage.toTrackedBaseAndAccessPathSimple -import io.shiftleft.semanticcpg.accesspath._ -import org.scalatest.matchers.should.Matchers._ +import io.shiftleft.semanticcpg.accesspath.* +import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ class AccessPathUsageTests extends AnyWordSpec { @@ -22,35 +23,28 @@ class AccessPathUsageTests extends AnyWordSpec { private val VS = VariablePointerShift private val S = PointerShift - private val g = OverflowDbTestInstance.create + private val g = Cpg.empty.graph - private def genCALL(graph: Graph, op: String, args: Node*): Call = { - val ret = graph + NodeTypes.CALL // (NodeTypes.CALL, Properties.NAME -> op) - ret.setProperty(Properties.Name, op) + private def genCALL(graph: Graph, op: String, args: GNode*): Call = { + val diffGraphBuilder = Cpg.newDiffGraphBuilder + val newCall = NewCall().name(op) + diffGraphBuilder.addNode(newCall) args.reverse.zipWithIndex.foreach { case (arg, idx) => - ret --- EdgeTypes.ARGUMENT --> arg - arg.setProperty(Properties.ArgumentIndex, idx + 1) + diffGraphBuilder.setNodeProperty(arg, PropertyNames.ARGUMENT_INDEX, idx + 1) + diffGraphBuilder.addEdge(newCall, arg, EdgeTypes.ARGUMENT) } - ret.asInstanceOf[Call] + diffGraphBuilder.apply(graph) + newCall.storedRef.get } - private def genLit(graph: Graph, payload: String): Literal = { - val ret = graph + NodeTypes.LITERAL - ret.setProperty(Properties.Code, payload) - ret.asInstanceOf[Literal] - } + private def genLit(graph: Graph, payload: String): Literal = + graph.addNode(NewLiteral().code(payload)) - private def genID(graph: Graph, payload: String): Identifier = { - val ret = graph + NodeTypes.IDENTIFIER - ret.setProperty(Properties.Name, payload) - ret.asInstanceOf[Identifier] - } + private def genID(graph: Graph, payload: String): Identifier = + graph.addNode(NewIdentifier().name(payload)) - private def genFID(graph: Graph, payload: String): FieldIdentifier = { - val ret = graph + NodeTypes.FIELD_IDENTIFIER - ret.setProperty(Properties.CanonicalName, payload) - ret.asInstanceOf[FieldIdentifier] - } + private def genFID(graph: Graph, payload: String): FieldIdentifier = + graph.addNode(NewFieldIdentifier().canonicalName(payload)) private def toTrackedAccessPath(node: StoredNode): AccessPath = toTrackedBaseAndAccessPathSimple(node)._2 diff --git a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/ParserTests.scala b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParserTests.scala similarity index 94% rename from dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/ParserTests.scala rename to dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParserTests.scala index 2c2e08c9fd18..2ed38059a378 100644 --- a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/ParserTests.scala +++ b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/semanticsloader/FullNameSemanticsParserTests.scala @@ -3,10 +3,10 @@ package io.joern.dataflowengineoss.semanticsloader import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -class ParserTests extends AnyWordSpec with Matchers { +class FullNameSemanticsParserTests extends AnyWordSpec with Matchers { class Fixture() { - val parser = new Parser() + val parser = new FullNameSemanticsParser() } "Parser" should { diff --git a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/testfixtures/SemanticTestCpg.scala b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/testfixtures/SemanticTestCpg.scala index f2a68aa35341..63c7061e6a62 100644 --- a/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/testfixtures/SemanticTestCpg.scala +++ b/dataflowengineoss/src/test/scala/io/joern/dataflowengineoss/testfixtures/SemanticTestCpg.scala @@ -3,7 +3,7 @@ package io.joern.dataflowengineoss.testfixtures import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, FullNameSemantics, Semantics} import io.joern.x2cpg.testfixtures.TestCpg import io.shiftleft.semanticcpg.layers.LayerCreatorContext @@ -12,7 +12,7 @@ import io.shiftleft.semanticcpg.layers.LayerCreatorContext trait SemanticTestCpg { this: TestCpg => protected var _withOssDataflow = false - protected var _extraFlows = List.empty[FlowSemantic] + protected var _semantics: Semantics = DefaultSemantics() protected implicit var context: EngineContext = EngineContext() /** Allows one to enable data-flow analysis capabilities to the TestCpg. @@ -22,10 +22,9 @@ trait SemanticTestCpg { this: TestCpg => this } - /** Allows one to add additional semantics to the engine context during PDG creation. - */ - def withExtraFlows(value: List[FlowSemantic] = List.empty): this.type = { - _extraFlows = value + /** Allows one to provide custom semantics to the TestCpg. */ + def withSemantics(value: Semantics): this.type = { + _semantics = value this } @@ -34,10 +33,11 @@ trait SemanticTestCpg { this: TestCpg => */ def applyOssDataFlow(): Unit = { if (_withOssDataflow) { - val context = new LayerCreatorContext(this) - val options = new OssDataFlowOptions(extraFlows = _extraFlows) - new OssDataFlow(options).run(context) - this.context = EngineContext(Semantics.fromList(DefaultSemantics().elements ++ _extraFlows)) + val context = new LayerCreatorContext(this) + val options = new OssDataFlowOptions(semantics = _semantics) + val dataflow = new OssDataFlow(options) + dataflow.run(context) + this.context = EngineContext(dataflow.semantics) } } @@ -45,8 +45,8 @@ trait SemanticTestCpg { this: TestCpg => /** Allows the tests to make use of the data-flow engine and any additional semantics. */ -trait SemanticCpgTestFixture(extraFlows: List[FlowSemantic] = List.empty) { +trait SemanticCpgTestFixture(semantics: Semantics = DefaultSemantics()) { - implicit val context: EngineContext = EngineContext(Semantics.fromList(DefaultSemantics().elements ++ extraFlows)) + implicit val context: EngineContext = EngineContext(semantics) } diff --git a/joern-cli/build.sbt b/joern-cli/build.sbt index 88b17ef355bd..2360137782b5 100644 --- a/joern-cli/build.sbt +++ b/joern-cli/build.sbt @@ -131,4 +131,26 @@ generateScaladocs := { Universal / packageBin / mappings ++= sbt.Path.directory(new File("joern-cli/src/main/resources/scripts")) +lazy val removeModuleInfoFromJars = taskKey[Unit]("remove module-info.class from dependency jars - a hacky workaround for a scala3 compiler bug https://github.com/scala/scala3/issues/20421") +removeModuleInfoFromJars := { + import java.nio.file.{Files, FileSystems} + val logger = streams.value.log + val libDir = (Universal/stagingDirectory).value / "lib" + + // remove all `/module-info.class` from all jars + Files.walk(libDir.toPath) + .filter(_.toString.endsWith(".jar")) + .forEach { jar => + val zipFs = FileSystems.newFileSystem(jar) + zipFs.getRootDirectories.forEach { zipRootDir => + Files.list(zipRootDir).filter(_.toString == "/module-info.class").forEach { moduleInfoClass => + logger.info(s"workaround for scala completion bug: deleting $moduleInfoClass from $jar") + Files.delete(moduleInfoClass) + } + } + zipFs.close() + } +} +removeModuleInfoFromJars := removeModuleInfoFromJars.triggeredBy(Universal/stage).value + maintainer := "fabs@shiftleft.io" diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala index b59c5d0b22a2..a22bfb150b34 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala @@ -1,26 +1,32 @@ package io.joern.c2cpg import io.joern.c2cpg.passes.{AstCreationPass, PreprocessorPass, TypeDeclNodePass} +import io.joern.c2cpg.passes.FunctionDeclNodePass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages import io.joern.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass} import io.joern.x2cpg.X2Cpg.withNewEmptyCpg import io.joern.x2cpg.X2CpgFrontend import io.joern.x2cpg.utils.Report +import org.slf4j.LoggerFactory import java.util.regex.Pattern +import scala.util.control.NonFatal import scala.util.Try import scala.util.matching.Regex class C2Cpg extends X2CpgFrontend[Config] { - private val report: Report = new Report() + private val logger = LoggerFactory.getLogger(classOf[C2Cpg]) def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => + val report = new Report() new MetaDataPass(cpg, Languages.NEWC, config.inputPath).createAndApply() val astCreationPass = new AstCreationPass(cpg, config, report) astCreationPass.createAndApply() + new FunctionDeclNodePass(cpg, astCreationPass.unhandledMethodDeclarations())(config.schemaValidation) + .createAndApply() TypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() new TypeDeclNodePass(cpg)(config.schemaValidation).createAndApply() report.print() @@ -28,8 +34,14 @@ class C2Cpg extends X2CpgFrontend[Config] { } def printIfDefsOnly(config: Config): Unit = { - val stmts = new PreprocessorPass(config).run().mkString(",") - println(stmts) + try { + val stmts = new PreprocessorPass(config).run().mkString(",") + println(stmts) + } catch { + case NonFatal(ex) => + logger.error("Failed to print preprocessor statements.", ex) + throw ex + } } } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/Main.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/Main.scala index 358a14f3c1c5..61b0675dcbc2 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/Main.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/Main.scala @@ -1,12 +1,12 @@ package io.joern.c2cpg -import io.joern.c2cpg.Frontend._ +import io.joern.c2cpg.Frontend.* import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer +import io.joern.x2cpg.SourceFiles import org.slf4j.LoggerFactory import scopt.OParser -import scala.util.control.NonFatal - final case class Config( includePaths: Set[String] = Set.empty, defines: Set[String] = Set.empty, @@ -17,7 +17,8 @@ final case class Config( includePathsAutoDiscovery: Boolean = false, skipFunctionBodies: Boolean = false, noImageLocations: Boolean = false, - withPreprocessedFiles: Boolean = false + withPreprocessedFiles: Boolean = false, + compilationDatabase: Option[String] = None ) extends X2CpgConfig[Config] { def withIncludePaths(includePaths: Set[String]): Config = { this.copy(includePaths = includePaths).withInheritedFields(this) @@ -58,6 +59,10 @@ final case class Config( def withPreprocessedFiles(value: Boolean): Config = { this.copy(withPreprocessedFiles = value).withInheritedFields(this) } + + def withCompilationDatabase(value: String): Config = { + this.copy(compilationDatabase = Some(value)).withInheritedFields(this) + } } private object Frontend { @@ -94,9 +99,9 @@ private object Frontend { .text("instructs the parser to skip function and method bodies.") .action((_, c) => c.withSkipFunctionBodies(true)), opt[Unit]("no-image-locations") - .text( - "performance optimization, allows the parser not to create image-locations. An image location explains how a name made it into the translation unit. Eg: via macro expansion or preprocessor." - ) + .text("""performance optimization, allows the parser not to create image-locations. + | An image location explains how a name made it into the translation unit. + | E.g., via macro expansion or preprocessor.""".stripMargin) .action((_, c) => c.withNoImageLocations(true)), opt[Unit]("with-preprocessed-files") .text("includes *.i files and gives them priority over their unprocessed origin source files.") @@ -104,27 +109,29 @@ private object Frontend { opt[String]("define") .unbounded() .text("define a name") - .action((d, c) => c.withDefines(c.defines + d)) + .action((d, c) => c.withDefines(c.defines + d)), + opt[String]("compilation-database") + .text("""enables the processing of compilation database files (e.g., compile_commands.json). + | This allows to automatically extract compiler options, source files, and other build information from the specified database + | and ensuring consistency with the build configuration. + | For a cmake based build such a file is generated with the environment variable CMAKE_EXPORT_COMPILE_COMMANDS being present. + | Clang based build are supported e.g., with https://github.com/rizsotto/Bear + | """.stripMargin) + .action((d, c) => c.withCompilationDatabase(SourceFiles.toAbsolutePath(d, c.inputPath))) ) } } -object Main extends X2CpgMain(cmdLineParser, new C2Cpg()) { - - private val logger = LoggerFactory.getLogger(classOf[C2Cpg]) - - def run(config: Config, c2cpg: C2Cpg): Unit = { - if (config.printIfDefsOnly) { - try { - c2cpg.printIfDefsOnly(config) - } catch { - case NonFatal(ex) => - logger.error("Failed to print preprocessor statements.", ex) - throw ex - } - } else { - c2cpg.run(config) +object Main extends X2CpgMain(cmdLineParser, new C2Cpg()) with FrontendHTTPServer[Config, C2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + + override def run(config: Config, c2cpg: C2Cpg): Unit = { + config match { + case c if c.serverMode => startup() + case c if c.printIfDefsOnly => c2cpg.printIfDefsOnly(config) + case _ => c2cpg.run(config) } } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala index 6f4ca5fbd34b..20fc35428ef6 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala @@ -4,13 +4,12 @@ import io.joern.c2cpg.Config import io.joern.x2cpg.datastructures.Scope import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} -import io.joern.x2cpg.datastructures.Global import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.eclipse.cdt.core.dom.ast.{IASTNode, IASTTranslationUnit} import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import java.util.concurrent.ConcurrentHashMap import scala.collection.mutable @@ -19,7 +18,7 @@ import scala.collection.mutable */ class AstCreator( val filename: String, - val global: Global, + val global: CGlobal, val config: Config, val cdtAst: IASTTranslationUnit, val file2OffsetTable: ConcurrentHashMap[String, Array[Int]] @@ -32,6 +31,7 @@ class AstCreator( with AstForExpressionsCreator with AstNodeBuilder with AstCreatorHelper + with FullNameProvider with MacroHandler with X2CpgAstNodeBuilder[IASTNode, AstCreator] { @@ -42,7 +42,7 @@ class AstCreator( protected val usingDeclarationMappings: mutable.Map[String, String] = mutable.HashMap.empty // TypeDecls with their bindings (with their refs) for lambdas and methods are not put in the AST - // where the respective nodes are defined. Instead we put them under the parent TYPE_DECL in which they are defined. + // where the respective nodes are defined. Instead, we put them under the parent TYPE_DECL in which they are defined. // To achieve this we need this extra stack. protected val methodAstParentStack: Stack[NewNode] = new Stack() @@ -82,12 +82,12 @@ class AstCreator( methodAstParentStack.push(fakeGlobalMethod) scope.pushNewScope(fakeGlobalMethod) - val blockNode_ = blockNode(iASTTranslationUnit, Defines.empty, registerType(Defines.anyTypeName)) + val blockNode_ = blockNode(iASTTranslationUnit) val declsAsts = allDecls.flatMap(astsForDeclaration) setArgumentIndices(declsAsts) - val methodReturn = newMethodReturnNode(iASTTranslationUnit, Defines.anyTypeName) + val methodReturn = methodReturnNode(iASTTranslationUnit, Defines.Any) Ast(fakeGlobalTypeDecl).withChild( methodAst(fakeGlobalMethod, Seq.empty, blockAst(blockNode_, declsAsts), methodReturn) ) @@ -100,7 +100,7 @@ class AstCreator( } override protected def lineEnd(node: IASTNode): Option[Int] = { - nullSafeFileLocation(node).map(_.getEndingLineNumber) + nullSafeFileLocationLast(node).map(_.getEndingLineNumber) } protected def column(node: IASTNode): Option[Int] = { diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala index a15e0f844637..f40f94954515 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala @@ -1,31 +1,36 @@ package io.joern.c2cpg.astcreation -import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewCall, NewNode} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.x2cpg.{Ast, SourceFiles, ValidationMode} +import io.joern.x2cpg.Ast +import io.joern.x2cpg.SourceFiles +import io.joern.x2cpg.ValidationMode +import io.joern.x2cpg.Defines as X2CpgDefines import io.joern.x2cpg.utils.NodeBuilders.newDependencyNode +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.ExpressionNew +import io.shiftleft.codepropertygraph.generated.nodes.NewCall +import io.shiftleft.codepropertygraph.generated.nodes.NewNode import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.utils.IOUtils import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast.* -import org.eclipse.cdt.core.dom.ast.c.{ICASTArrayDesignator, ICASTDesignatedInitializer, ICASTFieldDesignator} +import org.eclipse.cdt.core.dom.ast.c.ICASTArrayDesignator +import org.eclipse.cdt.core.dom.ast.c.ICASTDesignatedInitializer +import org.eclipse.cdt.core.dom.ast.c.ICASTFieldDesignator import org.eclipse.cdt.core.dom.ast.cpp.* import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator -import org.eclipse.cdt.internal.core.dom.parser.c.{CASTArrayRangeDesignator, CASTFunctionDeclarator} +import org.eclipse.cdt.internal.core.dom.parser.c.CASTArrayRangeDesignator +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTArrayRangeDesignator +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFieldReference +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTIdExpression +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPMethod +import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPEvaluation import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalBinding -import org.eclipse.cdt.internal.core.dom.parser.cpp.{ - CPPASTArrayRangeDesignator, - CPPASTFieldReference, - CPPASTFunctionDeclarator, - CPPASTIdExpression, - CPPFunction, - CPPMethod, - ICPPEvaluation -} import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalMemberAccess import org.eclipse.cdt.internal.core.model.ASTStringUtil -import java.nio.file.{Path, Paths} +import java.nio.file.Path +import java.nio.file.Paths import scala.annotation.nowarn import scala.collection.mutable import scala.util.Try @@ -46,8 +51,6 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As private var usedVariablePostfix: Int = 0 - private val IncludeKeyword = "include" - protected def isIncludedNode(node: IASTNode): Boolean = fileName(node) != filename protected def uniqueName(target: String, name: String, fullName: String): (String, String) = { @@ -80,9 +83,11 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def nullSafeFileLocation(node: IASTNode): Option[IASTFileLocation] = Option(cdtAst.flattenLocationsToFile(node.getNodeLocations)).map(_.asFileLocation()) + protected def nullSafeFileLocationLast(node: IASTNode): Option[IASTFileLocation] = + Option(cdtAst.flattenLocationsToFile(node.getNodeLocations.lastOption.toArray)).map(_.asFileLocation()) protected def fileName(node: IASTNode): String = { - val path = nullSafeFileLocation(node).map(_.getFileName).getOrElse(filename) + val path = Try(node.getContainingFilename).getOrElse(filename) SourceFiles.toRelativePath(path, config.inputPath) } @@ -105,12 +110,19 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As fixedTypeName } + protected def registerMethodDeclaration(fullName: String, methodInfo: CGlobal.MethodInfo): Unit = { + global.methodDeclarations.putIfAbsent(fullName, methodInfo) + } + + protected def registerMethodDefinition(fullName: String): Unit = { + global.methodDefinitions.putIfAbsent(fullName, true) + } + // Sadly, there is no predefined List / Enum of this within Eclipse CDT: - private val reservedTypeKeywords: List[String] = + private val ReservedTypeKeywords: List[String] = List( "const", "static", - "volatile", "restrict", "extern", "typedef", @@ -124,108 +136,143 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As "class" ) + private val KeepTypeKeywords: List[String] = List("unsigned", "volatile") + protected def cleanType(rawType: String, stripKeywords: Boolean = true): String = { + if (rawType == Defines.Any) return rawType val tpe = if (stripKeywords) { - reservedTypeKeywords.foldLeft(rawType) { (cur, repl) => - if (cur.contains(s"$repl ")) { - dereferenceTypeFullName(cur.replace(s"$repl ", "")) - } else { - cur - } + ReservedTypeKeywords.foldLeft(rawType) { (cur, repl) => + if (cur.contains(s"$repl ")) cur.replace(s"$repl ", "") else cur } } else { rawType } StringUtils.normalizeSpace(tpe) match { - case "" => Defines.anyTypeName - case t if t.contains("org.eclipse.cdt.internal.core.dom.parser.ProblemType") => Defines.anyTypeName - case t if t.contains(" ->") && t.contains("}::") => - fixQualifiedName(t.substring(t.indexOf("}::") + 3, t.indexOf(" ->"))) - case t if t.contains(" ->") => - fixQualifiedName(t.substring(0, t.indexOf(" ->"))) - case t if t.contains("( ") => - fixQualifiedName(t.substring(0, t.indexOf("( "))) - case t if t.contains("?") => Defines.anyTypeName - case t if t.contains("#") => Defines.anyTypeName - case t if t.contains("{") && t.contains("}") => - val anonType = - s"${uniqueName("type", "", "")._1}${t.substring(0, t.indexOf("{"))}${t.substring(t.indexOf("}") + 1)}" - anonType.replace(" ", "") - case t if t.startsWith("[") && t.endsWith("]") => Defines.anyTypeName - case t if t.contains(Defines.qualifiedNameSeparator) => fixQualifiedName(t) - case t if t.startsWith("unsigned ") => "unsigned " + t.substring(9).replace(" ", "") - case t if t.contains("[") && t.contains("]") => t.replace(" ", "") - case t if t.contains("*") => t.replace(" ", "") - case someType => someType + case "" => Defines.Any + case t if t.startsWith("[") && t.endsWith("]") => Defines.Array + case t if t.contains("->") => Defines.Function + case t if t.contains("?") => Defines.Any + case t if t.contains("#") => Defines.Any + case t if t.contains("::{") || t.contains("}::") => Defines.Any + case t if t.contains("{") || t.contains("}") => Defines.Any + case t if t.contains("org.eclipse.cdt.internal.core.dom.parser.ProblemType") => Defines.Any + case t if t.contains("( ") => replaceWhitespaceAfterTypeKeyword(fixQualifiedName(t.substring(0, t.indexOf("( ")))) + case t if t.contains(Defines.QualifiedNameSeparator) => replaceWhitespaceAfterTypeKeyword(fixQualifiedName(t)) + case t if KeepTypeKeywords.exists(k => t.startsWith(s"$k ")) => replaceWhitespaceAfterTypeKeyword(t) + case t if t.contains("[") && t.contains("]") => replaceWhitespaceAfterTypeKeyword(t) + case t if t.contains("*") => replaceWhitespaceAfterTypeKeyword(t) + case someType => someType } } - private def safeGetEvaluation(expr: ICPPASTExpression): Option[ICPPEvaluation] = { + private def replaceWhitespaceAfterTypeKeyword(tpe: String): String = { + if (KeepTypeKeywords.exists(k => tpe.startsWith(s"$k "))) { + KeepTypeKeywords.foldLeft(tpe) { (cur, repl) => + val prefix = s"$repl " + if (cur.startsWith(prefix)) { + prefix + cur.substring(prefix.length).replace(" ", "") + } else { + cur + } + } + } else { + tpe.replace(" ", "") + } + } + + protected def safeGetEvaluation(expr: ICPPASTExpression): Option[ICPPEvaluation] = { // In case of unresolved includes etc. this may fail throwing an unrecoverable exception Try(expr.getEvaluation).toOption } + protected def safeGetType(tpe: IType): String = { + // In case of unresolved includes etc. this may fail throwing an unrecoverable exception + Try(ASTTypeUtil.getType(tpe)).getOrElse(Defines.Any) + } + + private def safeGetNodeType(node: IASTNode): String = { + // In case of unresolved includes etc. this may fail throwing an unrecoverable exception + Try(ASTTypeUtil.getNodeType(node)).getOrElse(Defines.Any) + } + + private def typeForCPPASTFieldReference(f: CPPASTFieldReference, stripKeywords: Boolean = true): String = { + safeGetEvaluation(f.getFieldOwner) match { + case Some(evaluation: EvalBinding) => cleanType(evaluation.getType.toString, stripKeywords) + case _ => cleanType(safeGetType(f.getFieldOwner.getExpressionType), stripKeywords) + } + } + @nowarn - protected def typeFor(node: IASTNode, stripKeywords: Boolean = true): String = { + private def typeForIASTArrayDeclarator(a: IASTArrayDeclarator, stripKeywords: Boolean = true): String = { import org.eclipse.cdt.core.dom.ast.ASTSignatureUtil.getNodeSignature - node match { - case f: CPPASTFieldReference => - safeGetEvaluation(f.getFieldOwner) match { - case Some(evaluation: EvalBinding) => cleanType(evaluation.getType.toString, stripKeywords) - case _ => cleanType(ASTTypeUtil.getType(f.getFieldOwner.getExpressionType), stripKeywords) - } - case f: IASTFieldReference => - cleanType(ASTTypeUtil.getType(f.getFieldOwner.getExpressionType), stripKeywords) - case a: IASTArrayDeclarator if ASTTypeUtil.getNodeType(a).startsWith("? ") => - val tpe = getNodeSignature(a).replace("[]", "").strip() - val arr = ASTTypeUtil.getNodeType(a).replace("? ", "") - s"$tpe$arr" - case a: IASTArrayDeclarator - if ASTTypeUtil.getNodeType(a).contains("} ") || ASTTypeUtil.getNodeType(a).contains(" [") => - val tpe = getNodeSignature(a).replace("[]", "").strip() - val arr = a.getArrayModifiers.map { - case m if m.getConstantExpression != null => s"[${nodeSignature(m.getConstantExpression)}]" - case _ if a.getInitializer != null => - a.getInitializer match { - case l: IASTInitializerList => s"[${l.getSize}]" - case _ => "[]" - } - case _ => "[]" - }.mkString - s"$tpe$arr" - case s: CPPASTIdExpression => - safeGetEvaluation(s) match { - case Some(evaluation: EvalMemberAccess) => - cleanType(evaluation.getOwnerType.toString, stripKeywords) - case Some(evalBinding: EvalBinding) => - evalBinding.getBinding match { - case m: CPPMethod => cleanType(fullName(m.getDefinition)) - case _ => cleanType(ASTTypeUtil.getNodeType(s), stripKeywords) - } - case _ => cleanType(ASTTypeUtil.getNodeType(s), stripKeywords) + if (safeGetNodeType(a).startsWith("? ")) { + val tpe = getNodeSignature(a).replace("[]", "").strip() + val arr = safeGetNodeType(a).replace("? ", "") + s"$tpe$arr" + } else if (safeGetNodeType(a).contains("} ") || safeGetNodeType(a).contains(" [")) { + val tpe = getNodeSignature(a).replace("[]", "").strip() + val arr = a.getArrayModifiers.map { + case m if m.getConstantExpression != null => s"[${nodeSignature(m.getConstantExpression)}]" + case _ if a.getInitializer != null => + a.getInitializer match { + case l: IASTInitializerList => s"[${l.getSize}]" + case _ => "[]" + } + case _ => "[]" + }.mkString + s"$tpe$arr" + } else { + cleanType(safeGetNodeType(a), stripKeywords) + } + } + + private def typeForCPPASTIdExpression(s: CPPASTIdExpression, stripKeywords: Boolean = true): String = { + safeGetEvaluation(s) match { + case Some(evaluation: EvalMemberAccess) => + val deref = if (evaluation.isPointerDeref) "*" else "" + cleanType(evaluation.getOwnerType.toString + deref, stripKeywords) + case Some(evalBinding: EvalBinding) => + evalBinding.getBinding match { + case m: CPPMethod => cleanType(fullName(m.getDefinition)) + case _ => cleanType(safeGetNodeType(s), stripKeywords) } - case _: IASTIdExpression | _: IASTName | _: IASTDeclarator => - cleanType(ASTTypeUtil.getNodeType(node), stripKeywords) - case s: IASTNamedTypeSpecifier => - cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) - case s: IASTCompositeTypeSpecifier => - cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) - case s: IASTEnumerationSpecifier => - cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) - case s: IASTElaboratedTypeSpecifier => - cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) - case l: IASTLiteralExpression => - cleanType(ASTTypeUtil.getType(l.getExpressionType)) - case e: IASTExpression => - cleanType(ASTTypeUtil.getNodeType(e), stripKeywords) - case c: ICPPASTConstructorInitializer if c.getParent.isInstanceOf[ICPPASTConstructorChainInitializer] => - cleanType( - fullName(c.getParent.asInstanceOf[ICPPASTConstructorChainInitializer].getMemberInitializerId), - stripKeywords - ) + case _ => cleanType(safeGetNodeType(s), stripKeywords) + } + } + + @nowarn + private def typeForICPPASTConstructorInitializer( + c: ICPPASTConstructorInitializer, + stripKeywords: Boolean = true + ): String = { + import org.eclipse.cdt.core.dom.ast.ASTSignatureUtil.getNodeSignature + c.getParent match { + case initializer: ICPPASTConstructorChainInitializer => + val fullName_ = fullName(initializer.getMemberInitializerId) + cleanType(fullName_, stripKeywords) case _ => - cleanType(getNodeSignature(node), stripKeywords) + cleanType(getNodeSignature(c), stripKeywords) + } + } + + @nowarn + protected def typeFor(node: IASTNode, stripKeywords: Boolean = true): String = { + import org.eclipse.cdt.core.dom.ast.ASTSignatureUtil.getNodeSignature + node match { + case f: CPPASTFieldReference => typeForCPPASTFieldReference(f) + case f: IASTFieldReference => cleanType(safeGetType(f.getFieldOwner.getExpressionType), stripKeywords) + case a: IASTArrayDeclarator => typeForIASTArrayDeclarator(a) + case s: CPPASTIdExpression => typeForCPPASTIdExpression(s) + case _: IASTIdExpression | _: IASTName | _: IASTDeclarator => cleanType(safeGetNodeType(node), stripKeywords) + case s: IASTNamedTypeSpecifier => cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) + case s: IASTCompositeTypeSpecifier => cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) + case s: IASTEnumerationSpecifier => cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) + case s: IASTElaboratedTypeSpecifier => cleanType(ASTStringUtil.getReturnTypeString(s, null), stripKeywords) + case l: IASTLiteralExpression => cleanType(safeGetType(l.getExpressionType)) + case e: IASTExpression => cleanType(safeGetNodeType(e), stripKeywords) + case c: ICPPASTConstructorInitializer => typeForICPPASTConstructorInitializer(c) + case _ => cleanType(getNodeSignature(node), stripKeywords) } } @@ -268,148 +315,10 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As Option(node).map(astsForStatement(_, argIndex)).getOrElse(Seq.empty) } - protected def dereferenceTypeFullName(fullName: String): String = - fullName.replace("*", "") - - protected def fixQualifiedName(name: String): String = - name.stripPrefix(Defines.qualifiedNameSeparator).replace(Defines.qualifiedNameSeparator, ".") - - protected def isQualifiedName(name: String): Boolean = - name.startsWith(Defines.qualifiedNameSeparator) - - protected def lastNameOfQualifiedName(name: String): String = { - val cleanedName = if (name.contains("<") && name.contains(">")) { - name.substring(0, name.indexOf("<")) - } else { - name - } - cleanedName.split(Defines.qualifiedNameSeparator).lastOption.getOrElse(cleanedName) - } - protected def functionTypeToSignature(typ: IFunctionType): String = { - val returnType = ASTTypeUtil.getType(typ.getReturnType) - val parameterTypes = typ.getParameterTypes.map(ASTTypeUtil.getType) - s"$returnType(${parameterTypes.mkString(",")})" - } - - protected def fullName(node: IASTNode): String = { - node match { - case declarator: CPPASTFunctionDeclarator => - declarator.getName.resolveBinding() match { - case function: ICPPFunction => - val fullNameNoSig = function.getQualifiedName.mkString(".") - val fn = - if (function.isExternC) { - function.getName - } else { - s"$fullNameNoSig:${functionTypeToSignature(function.getType)}" - } - return fn - case field: ICPPField => - case _: IProblemBinding => - return "" - } - case declarator: CASTFunctionDeclarator => - val fn = declarator.getName.toString - return fn - case definition: ICPPASTFunctionDefinition => - return fullName(definition.getDeclarator) - case x => - } - - val qualifiedName: String = node match { - case d: CPPASTIdExpression => - safeGetEvaluation(d) match { - case Some(evalBinding: EvalBinding) => - evalBinding.getBinding match { - case f: CPPFunction if f.getDeclarations != null => - f.getDeclarations.headOption.map(n => s"${fullName(n)}").getOrElse(f.getName) - case f: CPPFunction if f.getDefinition != null => - s"${fullName(f.getDefinition)}" - case other => - other.getName - } - case _ => ASTStringUtil.getSimpleName(d.getName) - } - - case alias: ICPPASTNamespaceAlias => alias.getMappingName.toString - case namespace: ICPPASTNamespaceDefinition if ASTStringUtil.getSimpleName(namespace.getName).nonEmpty => - s"${fullName(namespace.getParent)}.${ASTStringUtil.getSimpleName(namespace.getName)}" - case namespace: ICPPASTNamespaceDefinition if ASTStringUtil.getSimpleName(namespace.getName).isEmpty => - s"${fullName(namespace.getParent)}.${uniqueName("namespace", "", "")._1}" - case compType: IASTCompositeTypeSpecifier if ASTStringUtil.getSimpleName(compType.getName).nonEmpty => - s"${fullName(compType.getParent)}.${ASTStringUtil.getSimpleName(compType.getName)}" - case compType: IASTCompositeTypeSpecifier if ASTStringUtil.getSimpleName(compType.getName).isEmpty => - val name = compType.getParent match { - case decl: IASTSimpleDeclaration => - decl.getDeclarators.headOption - .map(n => ASTStringUtil.getSimpleName(n.getName)) - .getOrElse(uniqueName("composite_type", "", "")._1) - case _ => uniqueName("composite_type", "", "")._1 - } - s"${fullName(compType.getParent)}.$name" - case enumSpecifier: IASTEnumerationSpecifier => - s"${fullName(enumSpecifier.getParent)}.${ASTStringUtil.getSimpleName(enumSpecifier.getName)}" - case f: ICPPASTLambdaExpression => - s"${fullName(f.getParent)}." - case f: IASTFunctionDefinition if f.getDeclarator != null => - s"${fullName(f.getParent)}.${ASTStringUtil.getQualifiedName(f.getDeclarator.getName)}" - case f: IASTFunctionDefinition => - s"${fullName(f.getParent)}.${shortName(f)}" - case e: IASTElaboratedTypeSpecifier => - s"${fullName(e.getParent)}.${ASTStringUtil.getSimpleName(e.getName)}" - case d: IASTIdExpression => ASTStringUtil.getSimpleName(d.getName) - case _: IASTTranslationUnit => "" - case u: IASTUnaryExpression => code(u.getOperand) - case x: ICPPASTQualifiedName => ASTStringUtil.getQualifiedName(x) - case other if other != null && other.getParent != null => fullName(other.getParent) - case other if other != null => notHandledYet(other); "" - case null => "" - } - fixQualifiedName(qualifiedName).stripPrefix(".") - } - - protected def shortName(node: IASTNode): String = { - val name = node match { - case d: IASTDeclarator if ASTStringUtil.getSimpleName(d.getName).isEmpty && d.getNestedDeclarator != null => - shortName(d.getNestedDeclarator) - case d: IASTDeclarator => ASTStringUtil.getSimpleName(d.getName) - case f: ICPPASTFunctionDefinition - if ASTStringUtil - .getSimpleName(f.getDeclarator.getName) - .isEmpty && f.getDeclarator.getNestedDeclarator != null => - shortName(f.getDeclarator.getNestedDeclarator) - case f: ICPPASTFunctionDefinition => lastNameOfQualifiedName(ASTStringUtil.getSimpleName(f.getDeclarator.getName)) - case f: IASTFunctionDefinition - if ASTStringUtil - .getSimpleName(f.getDeclarator.getName) - .isEmpty && f.getDeclarator.getNestedDeclarator != null => - shortName(f.getDeclarator.getNestedDeclarator) - case f: IASTFunctionDefinition => ASTStringUtil.getSimpleName(f.getDeclarator.getName) - case d: CPPASTIdExpression => - safeGetEvaluation(d) match { - case Some(evalBinding: EvalBinding) => - evalBinding.getBinding match { - case f: CPPFunction if f.getDeclarations != null => - f.getDeclarations.headOption.map(n => ASTStringUtil.getSimpleName(n.getName)).getOrElse(f.getName) - case f: CPPFunction if f.getDefinition != null => - ASTStringUtil.getSimpleName(f.getDefinition.getName) - case other => - other.getName - } - case _ => lastNameOfQualifiedName(ASTStringUtil.getSimpleName(d.getName)) - } - case d: IASTIdExpression => lastNameOfQualifiedName(ASTStringUtil.getSimpleName(d.getName)) - case u: IASTUnaryExpression => shortName(u.getOperand) - case c: IASTFunctionCallExpression => shortName(c.getFunctionNameExpression) - case s: IASTSimpleDeclSpecifier => s.getRawSignature - case e: IASTEnumerationSpecifier => ASTStringUtil.getSimpleName(e.getName) - case c: IASTCompositeTypeSpecifier => ASTStringUtil.getSimpleName(c.getName) - case e: IASTElaboratedTypeSpecifier => ASTStringUtil.getSimpleName(e.getName) - case s: IASTNamedTypeSpecifier => ASTStringUtil.getSimpleName(s.getName) - case other => notHandledYet(other); "" - } - name + val returnType = cleanType(safeGetType(typ.getReturnType)) + val parameterTypes = typ.getParameterTypes.map(t => cleanType(safeGetType(t))) + StringUtils.normalizeSpace(s"$returnType(${parameterTypes.mkString(",")})") } private def pointersAsString(spec: IASTDeclSpecifier, parentDecl: IASTDeclarator, stripKeywords: Boolean): String = { @@ -421,7 +330,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } if (pointers.isEmpty) { s"$tpe$arr" } else { - val refs = "*" * (pointers.length - pointers.count(_.isInstanceOf[ICPPASTReferenceOperator])) + val refs = pointers.map(_.getRawSignature).mkString("") s"$tpe$arr$refs".strip() } } @@ -430,7 +339,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As val allIncludes = iASTTranslationUnit.getIncludeDirectives.toList.filterNot(isIncludedNode) allIncludes.map { include => val name = include.getName.toString - val _dependencyNode = newDependencyNode(name, name, IncludeKeyword) + val _dependencyNode = newDependencyNode(name, name, "include") val importNode = newImportNode(code(include), name, name, include) diffGraph.addNode(_dependencyNode) diffGraph.addEdge(importNode, _dependencyNode, EdgeTypes.IMPORTS) @@ -447,19 +356,19 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } private def astForDecltypeSpecifier(decl: ICPPASTDecltypeSpecifier): Ast = { - val op = ".typeOf" - val cpgUnary = callNode(decl, code(decl), op, op, DispatchTypes.STATIC_DISPATCH) + val op = Defines.OperatorTypeOf + val cpgUnary = callNode(decl, code(decl), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val operand = nullSafeAst(decl.getDecltypeExpression) callAst(cpgUnary, List(operand)) } private def astForCASTDesignatedInitializer(d: ICASTDesignatedInitializer): Ast = { - val node = blockNode(d, Defines.empty, Defines.voidTypeName) + val node = blockNode(d, Defines.Empty, Defines.Void) scope.pushNewScope(node) val op = Operators.assignment val calls = withIndex(d.getDesignators) { (des, o) => val callNode_ = - callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH) + callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) .argumentIndex(o) val left = astForNode(des) val right = astForNode(d.getOperand) @@ -470,12 +379,12 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } private def astForCPPASTDesignatedInitializer(d: ICPPASTDesignatedInitializer): Ast = { - val node = blockNode(d, Defines.empty, Defines.voidTypeName) + val node = blockNode(d, Defines.Empty, Defines.Void) scope.pushNewScope(node) val op = Operators.assignment val calls = withIndex(d.getDesignators) { (des, o) => val callNode_ = - callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH) + callNode(d, code(d), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) .argumentIndex(o) val left = astForNode(des) val right = astForNode(d.getOperand) @@ -486,16 +395,15 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } private def astForCPPASTConstructorInitializer(c: ICPPASTConstructorInitializer): Ast = { - val name = ".constructorInitializer" - val callNode_ = - callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH) - val args = c.getArguments.toList.map(a => astForNode(a)) + val name = Defines.OperatorConstructorInitializer + val callNode_ = callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) + val args = c.getArguments.toList.map(a => astForNode(a)) callAst(callNode_, args) } private def astForCASTArrayRangeDesignator(des: CASTArrayRangeDesignator): Ast = { val op = Operators.arrayInitializer - val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH) + val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val floorAst = nullSafeAst(des.getRangeFloor) val ceilingAst = nullSafeAst(des.getRangeCeiling) callAst(callNode_, List(floorAst, ceilingAst)) @@ -503,7 +411,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As private def astForCPPASTArrayRangeDesignator(des: CPPASTArrayRangeDesignator): Ast = { val op = Operators.arrayInitializer - val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH) + val callNode_ = callNode(des, code(des), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val floorAst = nullSafeAst(des.getRangeFloor) val ceilingAst = nullSafeAst(des.getRangeCeiling) callAst(callNode_, List(floorAst, ceilingAst)) @@ -570,9 +478,9 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As pointersAsString(s, parentDecl, stripKeywords) case s: IASTElaboratedTypeSpecifier => ASTStringUtil.getSignatureString(s, null) // TODO: handle other types of IASTDeclSpecifier - case _ => Defines.anyTypeName + case _ => Defines.Any } - if (tpe.isEmpty) Defines.anyTypeName else tpe + if (tpe.isEmpty) Defines.Any else tpe } // We use our own call ast creation function since the version in x2cpg treats diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForExpressionsCreator.scala index 3d999c23d4cd..f4bfb89d328a 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForExpressionsCreator.scala @@ -1,31 +1,24 @@ package io.joern.c2cpg.astcreation -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier, NewMethodRef} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.x2cpg.{Ast, ValidationMode} +import io.joern.x2cpg.Ast +import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.Defines as X2CpgDefines +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} +import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.* import org.eclipse.cdt.core.dom.ast.gnu.IGNUASTCompoundStatementExpression -import org.eclipse.cdt.core.model.IMethod -import org.eclipse.cdt.internal.core.dom.parser.c.{ - CASTFieldReference, - CASTFunctionCallExpression, - CASTIdExpression, - CBasicType, - CFunctionType, - CPointerType -} -import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.{EvalBinding, EvalFunctionCall} -import org.eclipse.cdt.internal.core.dom.parser.cpp.{ - CPPASTIdExpression, - CPPASTQualifiedName, - CPPClosureType, - CPPField, - CPPFunction, - CPPFunctionType -} +import org.eclipse.cdt.internal.core.dom.parser.c.CASTFunctionCallExpression +import org.eclipse.cdt.internal.core.dom.parser.c.CASTIdExpression +import org.eclipse.cdt.internal.core.dom.parser.c.CFunctionType +import org.eclipse.cdt.internal.core.dom.parser.c.CPointerType +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTIdExpression +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTQualifiedName +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPClosureType +import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalFunctionCall + +import scala.util.Try trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => @@ -62,22 +55,22 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case IASTBinaryExpression.op_notequals => Operators.notEquals case IASTBinaryExpression.op_pmdot => Operators.indirectFieldAccess case IASTBinaryExpression.op_pmarrow => Operators.indirectFieldAccess - case IASTBinaryExpression.op_max => ".max" - case IASTBinaryExpression.op_min => ".min" - case IASTBinaryExpression.op_ellipses => ".op_ellipses" - case _ => ".unknown" + case IASTBinaryExpression.op_max => Defines.OperatorMax + case IASTBinaryExpression.op_min => Defines.OperatorMin + case IASTBinaryExpression.op_ellipses => Defines.OperatorEllipses + case _ => Defines.OperatorUnknown } - val callNode_ = callNode(bin, code(bin), op, op, DispatchTypes.STATIC_DISPATCH) + val callNode_ = callNode(bin, code(bin), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val left = nullSafeAst(bin.getOperand1) val right = nullSafeAst(bin.getOperand2) callAst(callNode_, List(left, right)) } private def astForExpressionList(exprList: IASTExpressionList): Ast = { - val name = ".expressionList" + val name = Defines.OperatorExpressionList val callNode_ = - callNode(exprList, code(exprList), name, name, DispatchTypes.STATIC_DISPATCH) + callNode(exprList, code(exprList), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val childAsts = exprList.getExpressions.map(nullSafeAst) callAst(callNode_, childAsts.toIndexedSeq) } @@ -87,10 +80,10 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val typ = functionNameExpr.getExpressionType typ match { case pointerType: IPointerType => - createPointerCallAst(call, cleanType(ASTTypeUtil.getType(call.getExpressionType))) + createPointerCallAst(call, cleanType(safeGetType(call.getExpressionType))) case functionType: ICPPFunctionType => functionNameExpr match { - case idExpr: CPPASTIdExpression => + case idExpr: CPPASTIdExpression if idExpr.getName.getBinding.isInstanceOf[ICPPFunction] => val function = idExpr.getName.getBinding.asInstanceOf[ICPPFunction] val name = idExpr.getName.getLastName.toString val signature = @@ -102,9 +95,9 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val fullName = if (function.isExternC) { - name + StringUtils.normalizeSpace(name) } else { - val fullNameNoSig = function.getQualifiedName.mkString(".") + val fullNameNoSig = StringUtils.normalizeSpace(function.getQualifiedName.mkString(".")) s"$fullNameNoSig:$signature" } @@ -117,7 +110,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { fullName, dispatchType, Some(signature), - Some(cleanType(ASTTypeUtil.getType(call.getExpressionType))) + Some(registerType(cleanType(safeGetType(call.getExpressionType)))) ) val args = call.getArguments.toList.map(a => astForNode(a)) @@ -130,11 +123,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val name = fieldRefExpr.getFieldName.toString val signature = functionTypeToSignature(functionType) - val classFullName = cleanType(ASTTypeUtil.getType(fieldRefExpr.getFieldOwnerType)) + val classFullName = cleanType(safeGetType(fieldRefExpr.getFieldOwnerType)) val fullName = s"$classFullName.$name:$signature" fieldRefExpr.getFieldName.resolveBinding() - val method = fieldRefExpr.getFieldName.getBinding().asInstanceOf[ICPPMethod] + val method = fieldRefExpr.getFieldName.getBinding.asInstanceOf[ICPPMethod] val (dispatchType, receiver) = if (method.isVirtual || method.isPureVirtual) { (DispatchTypes.DYNAMIC_DISPATCH, Some(instanceAst)) @@ -148,16 +141,18 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { fullName, dispatchType, Some(signature), - Some(cleanType(ASTTypeUtil.getType(call.getExpressionType))) + Some(registerType(cleanType(safeGetType(call.getExpressionType)))) ) - createCallAst(callCpgNode, args, base = Some(instanceAst), receiver) + case other => + astForCppCallExpressionUntyped(call) } case classType: ICPPClassType => - val evaluation = call.getEvaluation.asInstanceOf[EvalFunctionCall] - val functionType = evaluation.getOverload.getType - val signature = functionTypeToSignature(functionType) - val name = "()" + val evaluation = call.getEvaluation.asInstanceOf[EvalFunctionCall] + + val functionType = Try(evaluation.getOverload.getType).toOption + val signature = functionType.map(functionTypeToSignature).getOrElse(X2CpgDefines.UnresolvedSignature) + val name = Defines.OperatorCall classType match { case closureType: CPPClosureType => @@ -171,7 +166,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { fullName, dispatchType, Some(signature), - Some(cleanType(ASTTypeUtil.getType(call.getExpressionType))) + Some(registerType(cleanType(safeGetType(call.getExpressionType)))) ) val receiverAst = astForExpression(functionNameExpr) @@ -179,17 +174,19 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { createCallAst(callCpgNode, args, receiver = Some(receiverAst)) case _ => - val classFullName = cleanType(ASTTypeUtil.getType(classType)) + val classFullName = cleanType(safeGetType(classType)) val fullName = s"$classFullName.$name:$signature" - val method = evaluation.getOverload.asInstanceOf[ICPPMethod] - val dispatchType = - if (method.isVirtual || method.isPureVirtual) { - DispatchTypes.DYNAMIC_DISPATCH - } else { + val dispatchType = evaluation.getOverload match { + case method: ICPPMethod => + if (method.isVirtual || method.isPureVirtual) { + DispatchTypes.DYNAMIC_DISPATCH + } else { + DispatchTypes.STATIC_DISPATCH + } + case _ => DispatchTypes.STATIC_DISPATCH - } - + } val callCpgNode = callNode( call, code(call), @@ -197,18 +194,19 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { fullName, dispatchType, Some(signature), - Some(cleanType(ASTTypeUtil.getType(call.getExpressionType))) + Some(registerType(cleanType(safeGetType(call.getExpressionType)))) ) val instanceAst = astForExpression(functionNameExpr) val args = call.getArguments.toList.map(a => astForNode(a)) - createCallAst(callCpgNode, args, base = Some(instanceAst), receiver = Some(instanceAst)) } case _: IProblemType => astForCppCallExpressionUntyped(call) case _: IProblemBinding => astForCppCallExpressionUntyped(call) + case other => + astForCppCallExpressionUntyped(call) } } @@ -220,7 +218,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val instanceAst = astForExpression(fieldRefExpr.getFieldOwner) val args = call.getArguments.toList.map(a => astForNode(a)) - val name = fieldRefExpr.getFieldName.toString + val name = StringUtils.normalizeSpace(fieldRefExpr.getFieldName.toString) val signature = X2CpgDefines.UnresolvedSignature val fullName = s"${X2CpgDefines.UnresolvedNamespace}.$name:$signature(${args.size})" @@ -233,12 +231,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { Some(signature), Some(X2CpgDefines.Any) ) - createCallAst(callCpgNode, args, base = Some(instanceAst), receiver = Some(instanceAst)) case idExpr: CPPASTIdExpression => val args = call.getArguments.toList.map(a => astForNode(a)) - val name = idExpr.getName.getLastName.toString + val name = StringUtils.normalizeSpace(idExpr.getName.getLastName.toString) val signature = X2CpgDefines.UnresolvedSignature val fullName = s"${X2CpgDefines.UnresolvedNamespace}.$name:$signature(${args.size})" @@ -251,14 +248,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { Some(signature), Some(X2CpgDefines.Any) ) - createCallAst(callCpgNode, args) case other => - // This could either be a pointer or an operator() call we dont know at this point + // This could either be a pointer or an operator() call we do not know at this point // but since it is CPP we opt for the later. val args = call.getArguments.toList.map(a => astForNode(a)) - val name = "()" + val name = Defines.OperatorCall val signature = X2CpgDefines.UnresolvedSignature val fullName = s"${X2CpgDefines.UnresolvedNamespace}.$name:$signature(${args.size})" @@ -271,7 +267,6 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { Some(signature), Some(X2CpgDefines.Any) ) - val instanceAst = astForExpression(functionNameExpr) createCallAst(callCpgNode, args, base = Some(instanceAst), receiver = Some(instanceAst)) } @@ -282,13 +277,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val typ = functionNameExpr.getExpressionType typ match { case pointerType: CPointerType => - createPointerCallAst(call, cleanType(ASTTypeUtil.getType(call.getExpressionType))) + createPointerCallAst(call, cleanType(safeGetType(call.getExpressionType))) case functionType: CFunctionType => functionNameExpr match { case idExpr: CASTIdExpression => - createCFunctionCallAst(call, idExpr, cleanType(ASTTypeUtil.getType(call.getExpressionType))) + createCFunctionCallAst(call, idExpr, cleanType(safeGetType(call.getExpressionType))) case _ => - createPointerCallAst(call, cleanType(ASTTypeUtil.getType(call.getExpressionType))) + createPointerCallAst(call, cleanType(safeGetType(call.getExpressionType))) } case _ => astForCCallExpressionUntyped(call) @@ -300,25 +295,22 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { idExpr: CASTIdExpression, callTypeFullName: String ): Ast = { - val name = idExpr.getName.getLastName.toString - val signature = "" - + val name = idExpr.getName.getLastName.toString + val signature = "" val dispatchType = DispatchTypes.STATIC_DISPATCH - - val callCpgNode = callNode(call, code(call), name, name, dispatchType, Some(signature), Some(callTypeFullName)) - val args = call.getArguments.toList.map(a => astForNode(a)) - + val callCpgNode = + callNode(call, code(call), name, name, dispatchType, Some(signature), Some(registerType(callTypeFullName))) + val args = call.getArguments.toList.map(a => astForNode(a)) createCallAst(callCpgNode, args) } private def createPointerCallAst(call: IASTFunctionCallExpression, callTypeFullName: String): Ast = { val functionNameExpr = call.getFunctionNameExpression - val name = Defines.operatorPointerCall + val name = Defines.OperatorPointerCall val signature = "" - + val dispatchType = DispatchTypes.DYNAMIC_DISPATCH val callCpgNode = - callNode(call, code(call), name, name, DispatchTypes.DYNAMIC_DISPATCH, Some(signature), Some(callTypeFullName)) - + callNode(call, code(call), name, name, dispatchType, Some(signature), Some(registerType(callTypeFullName))) val args = call.getArguments.toList.map(a => astForNode(a)) val receiverAst = astForExpression(functionNameExpr) createCallAst(callCpgNode, args, receiver = Some(receiverAst)) @@ -326,24 +318,25 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForCCallExpressionUntyped(call: CASTFunctionCallExpression): Ast = { val functionNameExpr = call.getFunctionNameExpression - functionNameExpr match { - case idExpr: CASTIdExpression => - createCFunctionCallAst(call, idExpr, X2CpgDefines.Any) - case _ => - createPointerCallAst(call, X2CpgDefines.Any) + case idExpr: CASTIdExpression => createCFunctionCallAst(call, idExpr, X2CpgDefines.Any) + case _ => createPointerCallAst(call, X2CpgDefines.Any) } } private def astForCallExpression(call: IASTFunctionCallExpression): Ast = { call match { - case cppCall: ICPPASTFunctionCallExpression => - astForCppCallExpression(cppCall) - case cCall: CASTFunctionCallExpression => - astForCCallExpression(cCall) + case cppCall: ICPPASTFunctionCallExpression => astForCppCallExpression(cppCall) + case cCall: CASTFunctionCallExpression => astForCCallExpression(cCall) } } + private def astForThrowExpression(expression: IASTUnaryExpression): Ast = { + val operand = nullSafeAst(expression.getOperand) + Ast(controlStructureNode(expression, ControlStructureTypes.THROW, code(expression))) + .withChild(operand) + } + private def astForUnaryExpression(unary: IASTUnaryExpression): Ast = { val operatorMethod = unary.getOperator match { case IASTUnaryExpression.op_prefixIncr => Operators.preIncrement @@ -357,10 +350,9 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case IASTUnaryExpression.op_sizeof => Operators.sizeOf case IASTUnaryExpression.op_postFixIncr => Operators.postIncrement case IASTUnaryExpression.op_postFixDecr => Operators.postDecrement - case IASTUnaryExpression.op_throw => ".throw" - case IASTUnaryExpression.op_typeid => ".typeOf" - case IASTUnaryExpression.op_bracketedPrimary => ".bracketedPrimary" - case _ => ".unknown" + case IASTUnaryExpression.op_typeid => Defines.OperatorTypeOf + case IASTUnaryExpression.op_bracketedPrimary => Defines.OperatorBracketedPrimary + case _ => Defines.OperatorUnknown } if ( @@ -369,8 +361,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { ) { nullSafeAst(unary.getOperand) } else { - val cpgUnary = - callNode(unary, code(unary), operatorMethod, operatorMethod, DispatchTypes.STATIC_DISPATCH) + val cpgUnary = callNode( + unary, + code(unary), + operatorMethod, + operatorMethod, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val operand = nullSafeAst(unary.getOperand) callAst(cpgUnary, List(operand)) } @@ -385,7 +384,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { op == IASTTypeIdExpression.op_alignof || op == IASTTypeIdExpression.op_typeof => val call = - callNode(typeId, code(typeId), Operators.sizeOf, Operators.sizeOf, DispatchTypes.STATIC_DISPATCH) + callNode( + typeId, + code(typeId), + Operators.sizeOf, + Operators.sizeOf, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val arg = astForNode(typeId.getTypeId.getDeclSpecifier) callAst(call, List(arg)) case _ => notHandledYet(typeId) @@ -394,7 +401,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForConditionalExpression(expr: IASTConditionalExpression): Ast = { val name = Operators.conditional - val call = callNode(expr, code(expr), name, name, DispatchTypes.STATIC_DISPATCH) + val call = callNode(expr, code(expr), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val condAst = nullSafeAst(expr.getLogicalConditionExpression) val posAst = nullSafeAst(expr.getPositiveResultExpression) @@ -407,7 +414,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForArrayIndexExpression(arrayIndexExpression: IASTArraySubscriptExpression): Ast = { val name = Operators.indirectIndexAccess val cpgArrayIndexing = - callNode(arrayIndexExpression, code(arrayIndexExpression), name, name, DispatchTypes.STATIC_DISPATCH) + callNode( + arrayIndexExpression, + code(arrayIndexExpression), + name, + name, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val expr = astForExpression(arrayIndexExpression.getArrayExpression) val arg = astForNode(arrayIndexExpression.getArgument) @@ -416,7 +431,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForCastExpression(castExpression: IASTCastExpression): Ast = { val cpgCastExpression = - callNode(castExpression, code(castExpression), Operators.cast, Operators.cast, DispatchTypes.STATIC_DISPATCH) + callNode( + castExpression, + code(castExpression), + Operators.cast, + Operators.cast, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val expr = astForExpression(castExpression.getOperand) val argNode = castExpression.getTypeId @@ -438,9 +461,16 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } private def astForNewExpression(newExpression: ICPPASTNewExpression): Ast = { - val name = ".new" - val cpgNewExpression = - callNode(newExpression, code(newExpression), name, name, DispatchTypes.STATIC_DISPATCH) + val name = Defines.OperatorNew + val cpgNewExpression = callNode( + newExpression, + code(newExpression), + name, + name, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val typeId = newExpression.getTypeId if (newExpression.isArrayAllocation) { @@ -457,7 +487,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForDeleteExpression(delExpression: ICPPASTDeleteExpression): Ast = { val name = Operators.delete val cpgDeleteNode = - callNode(delExpression, code(delExpression), name, name, DispatchTypes.STATIC_DISPATCH) + callNode( + delExpression, + code(delExpression), + name, + name, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val arg = astForExpression(delExpression.getOperand) callAst(cpgDeleteNode, List(arg)) } @@ -465,7 +503,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForTypeIdInitExpression(typeIdInit: IASTTypeIdInitializerExpression): Ast = { val name = Operators.cast val cpgCastExpression = - callNode(typeIdInit, code(typeIdInit), name, name, DispatchTypes.STATIC_DISPATCH) + callNode(typeIdInit, code(typeIdInit), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val typeAst = unknownNode(typeIdInit.getTypeId, code(typeIdInit.getTypeId)) val expr = astForNode(typeIdInit.getInitializer) @@ -474,7 +512,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { private def astForConstructorExpression(c: ICPPASTSimpleTypeConstructorExpression): Ast = { val name = c.getDeclSpecifier.toString - val callNode_ = callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH) + val callNode_ = callNode(c, code(c), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val arg = astForNode(c.getInitializer) callAst(callNode_, List(arg)) } @@ -487,14 +525,15 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { protected def astForExpression(expression: IASTExpression): Ast = { val r = expression match { - case lit: IASTLiteralExpression => astForLiteral(lit) - case un: IASTUnaryExpression => astForUnaryExpression(un) - case bin: IASTBinaryExpression => astForBinaryExpression(bin) - case exprList: IASTExpressionList => astForExpressionList(exprList) - case idExpr: IASTIdExpression => astForIdExpression(idExpr) - case call: IASTFunctionCallExpression => astForCallExpression(call) - case typeId: IASTTypeIdExpression => astForTypeIdExpression(typeId) - case fieldRef: IASTFieldReference => astForFieldReference(fieldRef) + case lit: IASTLiteralExpression => astForLiteral(lit) + case un: IASTUnaryExpression if un.getOperator == IASTUnaryExpression.op_throw => astForThrowExpression(un) + case un: IASTUnaryExpression => astForUnaryExpression(un) + case bin: IASTBinaryExpression => astForBinaryExpression(bin) + case exprList: IASTExpressionList => astForExpressionList(exprList) + case idExpr: IASTIdExpression => astForIdExpression(idExpr) + case call: IASTFunctionCallExpression => astForCallExpression(call) + case typeId: IASTTypeIdExpression => astForTypeIdExpression(typeId) + case fieldRef: IASTFieldReference => astForFieldReference(fieldRef) case expr: IASTConditionalExpression => astForConditionalExpression(expr) case arr: IASTArraySubscriptExpression => astForArrayIndexExpression(arr) case castExpression: IASTCastExpression => astForCastExpression(castExpression) @@ -516,11 +555,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForStaticAssert(a: ICPPASTStaticAssertDeclaration): Ast = { - val name = "static_assert" - val call = callNode(a, code(a), name, name, DispatchTypes.STATIC_DISPATCH) - val cond = nullSafeAst(a.getCondition) - val messg = nullSafeAst(a.getMessage) - callAst(call, List(cond, messg)) + val name = "static_assert" + val call = callNode(a, code(a), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) + val cond = nullSafeAst(a.getCondition) + val message = nullSafeAst(a.getMessage) + callAst(call, List(cond, message)) } } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala index 1e7ac31e24f6..1a6797e59377 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForFunctionsCreator.scala @@ -1,29 +1,39 @@ package io.joern.c2cpg.astcreation -import io.joern.x2cpg.{Ast, ValidationMode} +import io.joern.x2cpg.Ast +import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.utils.NodeBuilders.newModifierNode +import io.shiftleft.codepropertygraph.generated.EvaluationStrategies +import io.shiftleft.codepropertygraph.generated.ModifierTypes import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes} import org.apache.commons.lang3.StringUtils import org.eclipse.cdt.core.dom.ast.* -import org.eclipse.cdt.core.dom.ast.cpp.{ICPPASTLambdaExpression, ICPPFunction} +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTFunctionDefinition +import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTLambdaExpression +import org.eclipse.cdt.core.dom.ast.cpp.ICPPBinding import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator -import org.eclipse.cdt.internal.core.dom.parser.c.{CASTFunctionDeclarator, CASTParameterDeclaration, CTypedef} -import org.eclipse.cdt.internal.core.dom.parser.cpp.{ - CPPASTFunctionDeclarator, - CPPASTFunctionDefinition, - CPPASTParameterDeclaration, - CPPFunction -} +import org.eclipse.cdt.internal.core.dom.parser.c.CASTFunctionDeclarator +import org.eclipse.cdt.internal.core.dom.parser.c.CASTParameterDeclaration +import org.eclipse.cdt.internal.core.dom.parser.c.CVariable +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFunctionDeclarator +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFunctionDefinition +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTParameterDeclaration +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPClassType +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPEnumeration +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPStructuredBindingComposite import org.eclipse.cdt.internal.core.model.ASTStringUtil import scala.annotation.tailrec -import scala.collection.mutable +import scala.util.Try trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - private val seenFunctionFullnames = mutable.HashSet.empty[String] + private def methodDeclarationParentInfo(): (String, String) = { + methodAstParentStack.collectFirst { case t: NewTypeDecl => (t.label, t.fullName) }.getOrElse { + (methodAstParentStack.head.label, methodAstParentStack.head.properties("FULL_NAME").toString) + } + } private def createFunctionTypeAndTypeDecl( node: IASTNode, @@ -57,7 +67,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th Ast(functionBinding).withBindsEdge(parentNode, functionBinding).withRefEdge(functionBinding, method) } - private def parameters(functionNode: IASTNode): Seq[IASTNode] = functionNode match { + final protected def parameters(functionNode: IASTNode): Seq[IASTNode] = functionNode match { case arr: IASTArrayDeclarator => parameters(arr.getNestedDeclarator) case decl: CPPASTFunctionDeclarator => decl.getParameters.toIndexedSeq ++ parameters(decl.getNestedDeclarator) case decl: CASTFunctionDeclarator => decl.getParameters.toIndexedSeq ++ parameters(decl.getNestedDeclarator) @@ -70,7 +80,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } @tailrec - private def isVariadic(functionNode: IASTNode): Boolean = functionNode match { + final protected def isVariadic(functionNode: IASTNode): Boolean = functionNode match { case decl: CPPASTFunctionDeclarator => decl.takesVarArgs() case decl: CASTFunctionDeclarator => decl.takesVarArgs() case defn: IASTFunctionDefinition => isVariadic(defn.getDeclarator) @@ -78,15 +88,6 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th case _ => false } - private def parameterListSignature(func: IASTNode): String = { - val variadic = if (isVariadic(func)) "..." else "" - val elements = parameters(func).map { - case p: IASTParameterDeclaration => typeForDeclSpecifier(p.getDeclSpecifier) - case other => typeForDeclSpecifier(other) - } - s"(${elements.mkString(",")}$variadic)" - } - private def setVariadic(parameterNodes: Seq[NewMethodParameterIn], func: IASTNode): Unit = { parameterNodes.lastOption.foreach { case p: NewMethodParameterIn if isVariadic(func) => @@ -96,22 +97,20 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } - protected def astForMethodRefForLambda(lambdaExpression: ICPPASTLambdaExpression): Ast = { - val filename = fileName(lambdaExpression) - - val returnType = lambdaExpression.getDeclarator match { - case declarator: IASTDeclarator => - declarator.getTrailingReturnType match { - case id: IASTTypeId => typeForDeclSpecifier(id.getDeclSpecifier) - case null => Defines.anyTypeName - } - case null => Defines.anyTypeName + private def setVariadicParameterInfo(parameterNodeInfos: Seq[CGlobal.ParameterInfo], func: IASTNode): Unit = { + parameterNodeInfos.lastOption.foreach { + case p: CGlobal.ParameterInfo if isVariadic(func) => + p.isVariadic = true + p.code = s"${p.code}..." + case _ => } - val name = nextClosureName() - val fullname = s"${fullName(lambdaExpression)}$name" - val signature = s"$returnType${parameterListSignature(lambdaExpression)}" - val codeString = code(lambdaExpression) - val methodNode_ = methodNode(lambdaExpression, name, codeString, fullname, Some(signature), filename) + } + + protected def astForMethodRefForLambda(lambdaExpression: ICPPASTLambdaExpression): Ast = { + val filename = fileName(lambdaExpression) + val MethodFullNameInfo(name, fullName, signature, returnType) = this.methodFullNameInfo(lambdaExpression) + val codeString = code(lambdaExpression) + val methodNode_ = methodNode(lambdaExpression, name, codeString, fullName, Some(signature), filename) scope.pushNewScope(methodNode_) val parameterNodes = withIndex(parameters(lambdaExpression.getDeclarator)) { (p, i) => @@ -125,50 +124,54 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th methodNode_, parameterNodes.map(Ast(_)), astForMethodBody(Option(lambdaExpression.getBody)), - newMethodReturnNode(lambdaExpression, registerType(returnType)), + methodReturnNode(lambdaExpression, registerType(returnType)), newModifierNode(ModifierTypes.LAMBDA) :: Nil ) - val typeDeclAst = createFunctionTypeAndTypeDecl(lambdaExpression, methodNode_, name, fullname, signature) + val typeDeclAst = createFunctionTypeAndTypeDecl(lambdaExpression, methodNode_, name, fullName, signature) Ast.storeInDiffGraph(astForLambda.merge(typeDeclAst), diffGraph) - Ast(methodRefNode(lambdaExpression, codeString, fullname, methodNode_.astParentFullName)) + Ast(methodRefNode(lambdaExpression, codeString, fullName, registerType(fullName))) } protected def astForFunctionDeclarator(funcDecl: IASTFunctionDeclarator): Ast = { funcDecl.getName.resolveBinding() match { - case function: IFunction => - val returnType = typeForDeclSpecifier(funcDecl.getParent.asInstanceOf[IASTSimpleDeclaration].getDeclSpecifier) - val fullname = fullName(funcDecl) - val templateParams = templateParameters(funcDecl).getOrElse("") - val signature = - s"$returnType${parameterListSignature(funcDecl)}" - - if (seenFunctionFullnames.add(fullname)) { - val name = shortName(funcDecl) - val codeString = code(funcDecl.getParent) - val filename = fileName(funcDecl) - val methodNode_ = methodNode(funcDecl, name, codeString, fullname, Some(signature), filename) - - scope.pushNewScope(methodNode_) - - val parameterNodes = withIndex(parameters(funcDecl)) { (p, i) => - parameterNode(p, i) - } - setVariadic(parameterNodes, funcDecl) - - scope.popScope() - - val stubAst = - methodStubAst( - methodNode_, - parameterNodes.map(Ast(_)), - newMethodReturnNode(funcDecl, registerType(returnType)) - ) - val typeDeclAst = createFunctionTypeAndTypeDecl(funcDecl, methodNode_, name, fullname, signature) - stubAst.merge(typeDeclAst) - } else { - Ast() + case _: IFunction => + val MethodFullNameInfo(name, fullName, signature, returnType) = this.methodFullNameInfo(funcDecl) + val codeString = code(funcDecl.getParent) + val filename = fileName(funcDecl) + + val parameterNodeInfos = thisForCPPFunctions(funcDecl) ++ withIndex(parameters(funcDecl)) { (p, i) => + parameterNodeInfo(p, i) } + setVariadicParameterInfo(parameterNodeInfos, funcDecl) + + val (astParentType, astParentFullName) = methodDeclarationParentInfo() + + val methodInfo = CGlobal.MethodInfo( + name, + code = codeString, + fileName = filename, + returnType = registerType(returnType), + astParentType = astParentType, + astParentFullName = astParentFullName, + lineNumber = line(funcDecl), + columnNumber = column(funcDecl), + lineNumberEnd = lineEnd(funcDecl), + columnNumberEnd = columnEnd(funcDecl), + signature = signature, + offset(funcDecl), + parameter = parameterNodeInfos, + modifier = modifierFor(funcDecl).map(_.modifierType) + ) + registerMethodDeclaration(fullName, methodInfo) + Ast() + case cVariable: CVariable => + val name = shortName(funcDecl) + val tpe = cleanType(ASTTypeUtil.getType(cVariable.getType)) + val codeString = code(funcDecl.getParent) + val node = localNode(funcDecl, name, codeString, registerType(tpe)) + scope.addToScope(name, (node, tpe)) + Ast(node) case field: IField => // TODO create a member for the field // We get here a least for function pointer member declarations in classes like: @@ -180,72 +183,117 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th case typeDef: ITypedef => // TODO handle typeDecl for now we just ignore this. Ast() + case other => + notHandledYet(funcDecl) } } - private def isCppConstructor(funcDef: IASTFunctionDefinition): Boolean = { + private def modifierFromString(image: String): List[NewModifier] = { + image match { + case "static" => List(newModifierNode(ModifierTypes.STATIC)) + case _ => Nil + } + } + + private def modifierFor(funcDef: IASTFunctionDefinition): List[NewModifier] = { + val constructorModifier = if (isCppConstructor(funcDef)) { + List(newModifierNode(ModifierTypes.CONSTRUCTOR), newModifierNode(ModifierTypes.PUBLIC)) + } else Nil + val visibilityModifier = Try(modifierFromString(funcDef.getSyntax.getImage)).getOrElse(Nil) + constructorModifier ++ visibilityModifier + } + + private def modifierFor(funcDecl: IASTFunctionDeclarator): List[NewModifier] = { + Try(modifierFromString(funcDecl.getParent.getSyntax.getImage)).getOrElse(Nil) + } + + protected def isCppConstructor(funcDef: IASTFunctionDefinition): Boolean = { funcDef match { case cppFunc: CPPASTFunctionDefinition => cppFunc.getMemberInitializers.nonEmpty case _ => false } } + private def thisForCPPFunctions(func: IASTNode): Seq[CGlobal.ParameterInfo] = { + func match { + case cppFunc: ICPPASTFunctionDefinition => + val maybeOwner = Try(cppFunc.getDeclarator.getName.getBinding).toOption match { + case Some(o: ICPPBinding) if o.getOwner.isInstanceOf[CPPClassType] => + Some(o.getOwner.asInstanceOf[CPPClassType].getQualifiedName.mkString(".")) + case Some(o: ICPPBinding) if o.getOwner.isInstanceOf[CPPEnumeration] => + Some(o.getOwner.asInstanceOf[CPPEnumeration].getQualifiedName.mkString(".")) + case Some(o: ICPPBinding) if o.getOwner.isInstanceOf[CPPStructuredBindingComposite] => + Some(o.getOwner.asInstanceOf[CPPStructuredBindingComposite].getQualifiedName.mkString(".")) + case _ => None + } + maybeOwner.toSeq.map { owner => + new CGlobal.ParameterInfo( + "this", + "this", + 0, + false, + EvaluationStrategies.BY_VALUE, + line(cppFunc), + column(cppFunc), + registerType(owner) + ) + } + case _ => Seq.empty + } + } + protected def astForFunctionDefinition(funcDef: IASTFunctionDefinition): Ast = { - val filename = fileName(funcDef) - val returnType = if (isCppConstructor(funcDef)) { - typeFor(funcDef.asInstanceOf[CPPASTFunctionDefinition].getMemberInitializers.head.getInitializer) - } else typeForDeclSpecifier(funcDef.getDeclSpecifier) - val name = shortName(funcDef) - val fullname = fullName(funcDef) - val templateParams = templateParameters(funcDef).getOrElse("") - - val signature = - s"$returnType${parameterListSignature(funcDef)}" - seenFunctionFullnames.add(fullname) + val filename = fileName(funcDef) + val MethodFullNameInfo(name, fullName, signature, returnType) = this.methodFullNameInfo(funcDef) + registerMethodDefinition(fullName) val codeString = code(funcDef) - val methodNode_ = methodNode(funcDef, name, codeString, fullname, Some(signature), filename) + val methodNode_ = methodNode(funcDef, name, codeString, fullName, Some(signature), filename) methodAstParentStack.push(methodNode_) scope.pushNewScope(methodNode_) - val parameterNodes = withIndex(parameters(funcDef)) { (p, i) => + val implicitThisParam = thisForCPPFunctions(funcDef).map { thisParam => + val parameterNode = parameterInNode( + funcDef, + thisParam.name, + thisParam.code, + thisParam.index, + thisParam.isVariadic, + thisParam.evaluationStrategy, + thisParam.typeFullName + ) + scope.addToScope(thisParam.name, (parameterNode, thisParam.typeFullName)) + parameterNode + } + val parameterNodes = implicitThisParam ++ withIndex(parameters(funcDef)) { (p, i) => parameterNode(p, i) } setVariadic(parameterNodes, funcDef) - val modifiers = if (isCppConstructor(funcDef)) { - List(newModifierNode(ModifierTypes.CONSTRUCTOR), newModifierNode(ModifierTypes.PUBLIC)) - } else Nil - val astForMethod = methodAst( methodNode_, parameterNodes.map(Ast(_)), astForMethodBody(Option(funcDef.getBody)), - newMethodReturnNode(funcDef, registerType(returnType)), - modifiers = modifiers + methodReturnNode(funcDef, registerType(returnType)), + modifiers = modifierFor(funcDef) ) scope.popScope() methodAstParentStack.pop() - val typeDeclAst = createFunctionTypeAndTypeDecl(funcDef, methodNode_, name, fullname, signature) + val typeDeclAst = createFunctionTypeAndTypeDecl(funcDef, methodNode_, name, fullName, signature) astForMethod.merge(typeDeclAst) } - private def parameterNode(parameter: IASTNode, paramIndex: Int): NewMethodParameterIn = { + private def parameterNodeInfo(parameter: IASTNode, paramIndex: Int): CGlobal.ParameterInfo = { val (name, codeString, tpe, variadic) = parameter match { case p: CASTParameterDeclaration => - ( - ASTStringUtil.getSimpleName(p.getDeclarator.getName), - code(p), - cleanType(typeForDeclSpecifier(p.getDeclSpecifier)), - false - ) + (shortName(p.getDeclarator), code(p), cleanType(typeForDeclSpecifier(p.getDeclSpecifier)), false) case p: CPPASTParameterDeclaration => ( - ASTStringUtil.getSimpleName(p.getDeclarator.getName), + shortName(p.getDeclarator), code(p), cleanType(typeForDeclSpecifier(p.getDeclSpecifier)), p.getDeclarator.declaresParameterPack() @@ -262,25 +310,38 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th case other => (code(other), code(other), cleanType(typeForDeclSpecifier(other)), false) } + new CGlobal.ParameterInfo( + name, + codeString, + paramIndex, + variadic, + EvaluationStrategies.BY_VALUE, + lineNumber = line(parameter), + columnNumber = column(parameter), + typeFullName = registerType(tpe) + ) + } + private def parameterNode(parameter: IASTNode, paramIndex: Int): NewMethodParameterIn = { + val parameterInfo = parameterNodeInfo(parameter, paramIndex) val parameterNode = parameterInNode( parameter, - name, - codeString, - paramIndex, - variadic, - EvaluationStrategies.BY_VALUE, - registerType(tpe) + parameterInfo.name, + parameterInfo.code, + parameterInfo.index, + parameterInfo.isVariadic, + parameterInfo.evaluationStrategy, + parameterInfo.typeFullName ) - scope.addToScope(name, (parameterNode, tpe)) + scope.addToScope(parameterInfo.name, (parameterNode, parameterInfo.typeFullName)) parameterNode } private def astForMethodBody(body: Option[IASTStatement]): Ast = body match { case Some(b: IASTCompoundStatement) => astForBlockStatement(b) case Some(b) => astForNode(b) - case None => blockAst(NewBlock()) + case None => blockAst(NewBlock().typeFullName(Defines.Any)) } } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForPrimitivesCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForPrimitivesCreator.scala index b1c268e2cb7e..fe54a7ab3325 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForPrimitivesCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForPrimitivesCreator.scala @@ -1,35 +1,61 @@ package io.joern.c2cpg.astcreation -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.x2cpg.{Ast, ValidationMode} +import io.joern.x2cpg.Ast +import io.joern.x2cpg.ValidationMode +import io.joern.x2cpg.Defines as X2CpgDefines +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.NewMethod import io.shiftleft.codepropertygraph.generated.nodes.NewMethodRef +import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.internal.core.dom.parser.c.ICInternalBinding import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTQualifiedName import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPInternalBinding +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFunctionDeclarator +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTIdExpression +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPField +import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalMemberAccess import org.eclipse.cdt.internal.core.model.ASTStringUtil +import scala.util.Try + trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def astForComment(comment: IASTComment): Ast = Ast(newCommentNode(comment, code(comment), fileName(comment))) protected def astForLiteral(lit: IASTLiteralExpression): Ast = { - val tpe = cleanType(ASTTypeUtil.getType(lit.getExpressionType)) - Ast(literalNode(lit, code(lit), registerType(tpe))) + val codeString = code(lit) + val tpe = registerType(cleanType(safeGetType(lit.getExpressionType))) + if (codeString == "this") { + val thisIdentifier = identifierNode(lit, "this", "this", tpe) + scope.lookupVariable("this") match { + case Some((variable, _)) => Ast(thisIdentifier).withRefEdge(thisIdentifier, variable) + case None => Ast(identifierNode(lit, codeString, codeString, tpe)) + } + } else { + Ast(literalNode(lit, codeString, tpe)) + } } private def namesForBinding(binding: ICInternalBinding | ICPPInternalBinding): (Option[String], Option[String]) = { val definition = binding match { - // sadly, there is no common interface defining .getDefinition - case b: ICInternalBinding => b.getDefinition.asInstanceOf[IASTFunctionDeclarator] - case b: ICPPInternalBinding => b.getDefinition.asInstanceOf[IASTFunctionDeclarator] + // sadly, there is no common interface + case b: ICInternalBinding if b.getDefinition.isInstanceOf[IASTFunctionDeclarator] => + Some(b.getDefinition.asInstanceOf[IASTFunctionDeclarator]) + case b: ICPPInternalBinding if b.getDefinition.isInstanceOf[IASTFunctionDeclarator] => + Some(b.getDefinition.asInstanceOf[IASTFunctionDeclarator]) + case b: ICInternalBinding => b.getDeclarations.find(_.isInstanceOf[IASTFunctionDeclarator]) + case b: ICPPInternalBinding => b.getDeclarations.find(_.isInstanceOf[IASTFunctionDeclarator]) + case null => None } - val typeFullName = definition.getParent match { - case d: IASTFunctionDefinition => Some(typeForDeclSpecifier(d.getDeclSpecifier)) - case _ => None + val typeFullName = definition.map(_.getParent) match { + case Some(d: IASTFunctionDefinition) => Some(typeForDeclSpecifier(d.getDeclSpecifier)) + case Some(d: IASTSimpleDeclaration) => Some(typeForDeclSpecifier(d.getDeclSpecifier)) + case _ => None } - (Some(this.fullName(definition)), typeFullName) + (definition.map(fullName), typeFullName) } private def maybeMethodRefForIdentifier(ident: IASTNode): Option[NewMethodRef] = { @@ -39,58 +65,112 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t val (mayBeFullName, mayBeTypeFullName) = id.getName.getBinding match { case binding: ICInternalBinding if binding.getDefinition.isInstanceOf[IASTFunctionDeclarator] => namesForBinding(binding) + case binding: ICInternalBinding + if binding.getDeclarations != null && + binding.getDeclarations.exists(_.isInstanceOf[IASTFunctionDeclarator]) => + namesForBinding(binding) case binding: ICPPInternalBinding if binding.getDefinition.isInstanceOf[IASTFunctionDeclarator] => namesForBinding(binding) + case binding: ICPPInternalBinding + if binding.getDeclarations != null && + binding.getDeclarations.exists(_.isInstanceOf[CPPASTFunctionDeclarator]) => + namesForBinding(binding) case _ => (None, None) } for { fullName <- mayBeFullName typeFullName <- mayBeTypeFullName - } yield methodRefNode(ident, code(ident), fullName, typeFullName) + } yield methodRefNode(ident, code(ident), fullName, registerType(cleanType(typeFullName))) case _ => None } } + private def isInCurrentScope(owner: String): Boolean = { + methodAstParentStack.collectFirst { + case typeDecl: NewTypeDecl if typeDecl.fullName == owner => typeDecl + case method: NewMethod if method.fullName.startsWith(owner) => method + }.nonEmpty + } + + private def nameForIdentifier(ident: IASTNode): String = { + ident match { + case id: IASTIdExpression => ASTStringUtil.getSimpleName(id.getName) + case id: IASTName => + val name = ASTStringUtil.getSimpleName(id) + if (name.isEmpty) Try(id.resolveBinding().getName).getOrElse(uniqueName("name", "", "")._1) + else name + case _ => code(ident) + } + } + + private def syntheticThisAccess(ident: CPPASTIdExpression, identifierName: String): String | Ast = { + val tpe = ident.getName.getBinding match { + case f: CPPField => cleanType(f.getType.toString) + case _ => typeFor(ident) + } + Try(ident.getEvaluation).toOption match { + case Some(e: EvalMemberAccess) => + val tpe = registerType(typeFor(ident)) + val ownerType = registerType(cleanType(e.getOwnerType.toString)) + if (isInCurrentScope(ownerType)) { + scope.lookupVariable("this") match { + case Some((variable, _)) => + val op = Operators.indirectFieldAccess + val code = s"this->$identifierName" + val thisIdentifier = identifierNode(ident, "this", "this", tpe) + val member = fieldIdentifierNode(ident, identifierName, identifierName) + val ma = callNode(ident, code, op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) + callAst(ma, Seq(Ast(thisIdentifier).withRefEdge(thisIdentifier, variable), Ast(member))) + case None => tpe + } + } else tpe + case _ => tpe + } + } + + private def typeNameForIdentifier(ident: IASTNode, identifierName: String): String | Ast = { + val variableOption = scope.lookupVariable(identifierName) + variableOption match { + case Some((_, variableTypeName)) => variableTypeName + case None if ident.isInstanceOf[IASTName] && ident.asInstanceOf[IASTName].getBinding != null => + val id = ident.asInstanceOf[IASTName] + id.getBinding match { + case v: IVariable => + v.getType match { + case f: IFunctionType => f.getReturnType.toString + case other => other.toString + } + case other => other.getName + } + case None if ident.isInstanceOf[IASTName] => + typeFor(ident.getParent) + case None if ident.isInstanceOf[CPPASTIdExpression] => + syntheticThisAccess(ident.asInstanceOf[CPPASTIdExpression], identifierName) + case None => typeFor(ident) + } + } + protected def astForIdentifier(ident: IASTNode): Ast = { maybeMethodRefForIdentifier(ident) match { case Some(ref) => Ast(ref) case None => - val identifierName = ident match { - case id: IASTIdExpression => ASTStringUtil.getSimpleName(id.getName) - case id: IASTName if ASTStringUtil.getSimpleName(id).isEmpty && id.getBinding != null => id.getBinding.getName - case id: IASTName if ASTStringUtil.getSimpleName(id).isEmpty => uniqueName("name", "", "")._1 - case _ => code(ident) - } - val variableOption = scope.lookupVariable(identifierName) - val identifierTypeName = variableOption match { - case Some((_, variableTypeName)) => variableTypeName - case None if ident.isInstanceOf[IASTName] && ident.asInstanceOf[IASTName].getBinding != null => - val id = ident.asInstanceOf[IASTName] - id.getBinding match { - case v: IVariable => - v.getType match { - case f: IFunctionType => f.getReturnType.toString - case other => other.toString - } - case other => other.getName + val identifierName = nameForIdentifier(ident) + typeNameForIdentifier(ident, identifierName) match { + case identifierTypeName: String => + val node = identifierNode(ident, identifierName, code(ident), registerType(cleanType(identifierTypeName))) + scope.lookupVariable(identifierName) match { + case Some((variable, _)) => + Ast(node).withRefEdge(node, variable) + case None => Ast(node) } - case None if ident.isInstanceOf[IASTName] => - typeFor(ident.getParent) - case None => typeFor(ident) - } - - val node = identifierNode(ident, identifierName, code(ident), registerType(cleanType(identifierTypeName))) - variableOption match { - case Some((variable, _)) => - Ast(node).withRefEdge(node, variable) - case None => Ast(node) + case ast: Ast => ast } } } protected def astForFieldReference(fieldRef: IASTFieldReference): Ast = { val op = if (fieldRef.isPointerDereference) Operators.indirectFieldAccess else Operators.fieldAccess - val ma = callNode(fieldRef, code(fieldRef), op, op, DispatchTypes.STATIC_DISPATCH) + val ma = callNode(fieldRef, code(fieldRef), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val owner = astForExpression(fieldRef.getFieldOwner) val member = fieldIdentifierNode(fieldRef, fieldRef.getFieldName.toString, fieldRef.getFieldName.toString) callAst(ma, List(owner, Ast(member))) @@ -101,7 +181,7 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t protected def astForInitializerList(l: IASTInitializerList): Ast = { val op = Operators.arrayInitializer - val initCallNode = callNode(l, code(l), op, op, DispatchTypes.STATIC_DISPATCH) + val initCallNode = callNode(l, code(l), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val MAX_INITIALIZERS = 1000 val clauses = l.getClauses.slice(0, MAX_INITIALIZERS) @@ -111,7 +191,7 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t val ast = callAst(initCallNode, args) if (l.getClauses.length > MAX_INITIALIZERS) { val placeholder = - literalNode(l, "", Defines.anyTypeName).argumentIndex(MAX_INITIALIZERS) + literalNode(l, "", Defines.Any).argumentIndex(MAX_INITIALIZERS) ast.withChild(Ast(placeholder)).withArgEdge(initCallNode, placeholder) } else { ast @@ -120,7 +200,7 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t protected def astForQualifiedName(qualId: CPPASTQualifiedName): Ast = { val op = Operators.fieldAccess - val ma = callNode(qualId, code(qualId), op, op, DispatchTypes.STATIC_DISPATCH) + val ma = callNode(qualId, code(qualId), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) def fieldAccesses(names: List[IASTNode], argIndex: Int = -1): Ast = names match { case Nil => Ast() @@ -129,7 +209,7 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t case head :: tail => val codeString = s"${code(head)}::${tail.map(code).mkString("::")}" val callNode_ = - callNode(head, code(head), op, op, DispatchTypes.STATIC_DISPATCH) + callNode(head, code(head), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) .argumentIndex(argIndex) callNode_.code = codeString val arg1 = astForNode(head) @@ -142,7 +222,7 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { t val owner = if (qualifier != Ast()) { qualifier } else { - Ast(literalNode(qualId.getLastName, "", Defines.anyTypeName)) + Ast(literalNode(qualId.getLastName, "", Defines.Any)) } val member = fieldIdentifierNode( diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForStatementsCreator.scala index badeda55e61a..8db9c0f119cc 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForStatementsCreator.scala @@ -5,6 +5,8 @@ import io.shiftleft.codepropertygraph.generated.ControlStructureTypes import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.AstNodeNew import io.shiftleft.codepropertygraph.generated.nodes.ExpressionNew +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.Operators import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.* import org.eclipse.cdt.core.dom.ast.gnu.IGNUASTGotoStatement @@ -21,8 +23,8 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t protected def astForBlockStatement(blockStmt: IASTCompoundStatement, order: Int = -1): Ast = { val codeString = code(blockStmt) - val blockCode = if (codeString == "{}" || codeString.isEmpty) Defines.empty else codeString - val node = blockNode(blockStmt, blockCode, registerType(Defines.voidTypeName)) + val blockCode = if (codeString == "{}" || codeString.isEmpty) Defines.Empty else codeString + val node = blockNode(blockStmt, blockCode, registerType(Defines.Void)) .order(order) .argumentIndex(order) scope.pushNewScope(node) @@ -36,19 +38,34 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t blockAst(node, childAsts.toList) } + private def hasValidArrayModifier(arrayDecl: IASTArrayDeclarator): Boolean = + arrayDecl.getArrayModifiers.nonEmpty && arrayDecl.getArrayModifiers.forall(_.getConstantExpression != null) + private def astsForDeclarationStatement(decl: IASTDeclarationStatement): Seq[Ast] = decl.getDeclaration match { - case simplDecl: IASTSimpleDeclaration - if simplDecl.getDeclarators.headOption.exists(_.isInstanceOf[IASTFunctionDeclarator]) => - Seq(astForFunctionDeclarator(simplDecl.getDeclarators.head.asInstanceOf[IASTFunctionDeclarator])) - case simplDecl: IASTSimpleDeclaration => - val locals = - simplDecl.getDeclarators.zipWithIndex.toList.map { case (d, i) => astForDeclarator(simplDecl, d, i) } - val calls = - simplDecl.getDeclarators.filter(_.getInitializer != null).toList.map { d => - astForInitializer(d, d.getInitializer) + case simpleDecl: IASTSimpleDeclaration + if simpleDecl.getDeclarators.headOption.exists(_.isInstanceOf[IASTFunctionDeclarator]) => + Seq(astForFunctionDeclarator(simpleDecl.getDeclarators.head.asInstanceOf[IASTFunctionDeclarator])) + case simpleDecl: IASTSimpleDeclaration => + val locals = simpleDecl.getDeclarators.zipWithIndex.map { case (d, i) => astForDeclarator(simpleDecl, d, i) } + val arrayModCalls = simpleDecl.getDeclarators + .collect { case d: IASTArrayDeclarator if hasValidArrayModifier(d) => d } + .map { d => + val name = Operators.alloc + val tpe = registerType(typeFor(d)) + val codeString = code(d) + val allocCallNode = callNode(d, codeString, name, name, DispatchTypes.STATIC_DISPATCH, None, Some(tpe)) + val allocCallAst = callAst(allocCallNode, d.getArrayModifiers.toIndexedSeq.map(astForNode)) + val operatorName = Operators.assignment + val assignmentCallNode = + callNode(d, codeString, operatorName, operatorName, DispatchTypes.STATIC_DISPATCH, None, Some(tpe)) + val left = astForNode(d.getName) + callAst(assignmentCallNode, List(left, allocCallAst)) } - locals ++ calls + val initCalls = simpleDecl.getDeclarators.filter(_.getInitializer != null).map { d => + astForInitializer(d, d.getInitializer) + } + Seq.from(locals ++ arrayModCalls ++ initCalls) case s: ICPPASTStaticAssertDeclaration => Seq(astForStaticAssert(s)) case usingDeclaration: ICPPASTUsingDeclaration => handleUsingDeclaration(usingDeclaration) case alias: ICPPASTAliasDeclaration => Seq(astForAliasDeclaration(alias)) @@ -174,7 +191,8 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t // We only handle un-parsable macros here for now val isFromMacroExpansion = statement.getProblem.getNodeLocations.exists(_.isInstanceOf[IASTMacroExpansionLocation]) val asts = if (isFromMacroExpansion) { - new CdtParser(config).parse(statement.getRawSignature, Paths.get(statement.getContainingFilename)) match + new CdtParser(config, List.empty) + .parse(statement.getRawSignature, Paths.get(statement.getContainingFilename)) match case Some(node) => node.getDeclarations.toIndexedSeq.flatMap(astsForDeclaration) case None => Seq.empty } else { @@ -193,7 +211,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t private def astForConditionExpression(expression: IASTExpression, explicitArgumentIndex: Option[Int] = None): Ast = { val ast = expression match { case exprList: IASTExpressionList => - val compareAstBlock = blockNode(expression, Defines.empty, registerType(Defines.voidTypeName)) + val compareAstBlock = blockNode(expression, Defines.Empty, registerType(Defines.Void)) scope.pushNewScope(compareAstBlock) val compareBlockAstChildren = exprList.getExpressions.toList.map(nullSafeAst) setArgumentIndices(compareBlockAstChildren) @@ -217,7 +235,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t val code = s"for ($codeInit$codeCond;$codeIter)" val forNode = controlStructureNode(forStmt, ControlStructureTypes.FOR, code) - val initAstBlock = blockNode(forStmt, Defines.empty, registerType(Defines.voidTypeName)) + val initAstBlock = blockNode(forStmt, Defines.Empty, registerType(Defines.Void)) scope.pushNewScope(initAstBlock) val initAst = blockAst(initAstBlock, nullSafeAst(forStmt.getInitializerStatement, 1).toList) scope.popScope() @@ -262,7 +280,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t (c, compareAst) case s: CPPASTIfStatement if s.getConditionExpression == null => val c = s"if (${nullSafeCode(s.getConditionDeclaration)})" - val exprBlock = blockNode(s.getConditionDeclaration, Defines.empty, Defines.voidTypeName) + val exprBlock = blockNode(s.getConditionDeclaration, Defines.Empty, Defines.Void) scope.pushNewScope(exprBlock) val a = astsForDeclaration(s.getConditionDeclaration) setArgumentIndices(a) @@ -275,7 +293,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t val thenAst = ifStmt.getThenClause match { case block: IASTCompoundStatement => astForBlockStatement(block) case other if other != null => - val thenBlock = blockNode(other, Defines.empty, Defines.voidTypeName) + val thenBlock = blockNode(other, Defines.Empty, Defines.Void) scope.pushNewScope(thenBlock) val a = astsForStatement(other) setArgumentIndices(a) @@ -291,7 +309,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t Ast(elseNode).withChild(elseAst) case other if other != null => val elseNode = controlStructureNode(ifStmt.getElseClause, ControlStructureTypes.ELSE, "else") - val elseBlock = blockNode(other, Defines.empty, Defines.voidTypeName) + val elseBlock = blockNode(other, Defines.Empty, Defines.Void) scope.pushNewScope(elseBlock) val a = astsForStatement(other) setArgumentIndices(a) diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForTypesCreator.scala index 8bfbf0a2e05a..e20136dc70a3 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstForTypesCreator.scala @@ -3,12 +3,15 @@ package io.joern.c2cpg.astcreation import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.joern.x2cpg.{Ast, ValidationMode} +import io.joern.x2cpg.Defines as X2CpgDefines import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.* import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTAliasDeclaration import org.eclipse.cdt.internal.core.model.ASTStringUtil import io.joern.x2cpg.datastructures.Stack.* +import scala.util.Try + trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => private def parentIsClassDef(node: IASTNode): Boolean = Option(node.getParent) match { @@ -19,7 +22,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: private def isTypeDef(decl: IASTSimpleDeclaration): Boolean = code(decl).startsWith("typedef") - protected def templateParameters(e: IASTNode): Option[String] = { + private def templateParameters(e: IASTNode): Option[String] = { val templateDeclaration = e match { case _: IASTElaboratedTypeSpecifier | _: IASTFunctionDeclarator | _: IASTCompositeTypeSpecifier if e.getParent != null => @@ -34,11 +37,10 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: } private def astForNamespaceDefinition(namespaceDefinition: ICPPASTNamespaceDefinition): Ast = { - val (name, fullname) = - uniqueName("namespace", namespaceDefinition.getName.getLastName.toString, fullName(namespaceDefinition)) - val codeString = code(namespaceDefinition) + val TypeFullNameInfo(name, fullName) = typeFullNameInfo(namespaceDefinition) + val codeString = code(namespaceDefinition) val cpgNamespace = - newNamespaceBlockNode(namespaceDefinition, name, fullname, codeString, fileName(namespaceDefinition)) + newNamespaceBlockNode(namespaceDefinition, name, fullName, codeString, fileName(namespaceDefinition)) scope.pushNewScope(cpgNamespace) val childrenAsts = namespaceDefinition.getDeclarations.flatMap { decl => @@ -52,25 +54,36 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: } protected def astForNamespaceAlias(namespaceAlias: ICPPASTNamespaceAlias): Ast = { - val name = ASTStringUtil.getSimpleName(namespaceAlias.getAlias) - val fullname = fullName(namespaceAlias) - + val TypeFullNameInfo(name, fullName) = typeFullNameInfo(namespaceAlias) if (!isQualifiedName(name)) { - usingDeclarationMappings.put(name, fullname) + usingDeclarationMappings.put(name, fullName) } - val codeString = code(namespaceAlias) - val cpgNamespace = newNamespaceBlockNode(namespaceAlias, name, fullname, codeString, fileName(namespaceAlias)) + val cpgNamespace = newNamespaceBlockNode(namespaceAlias, name, fullName, codeString, fileName(namespaceAlias)) Ast(cpgNamespace) } protected def astForDeclarator(declaration: IASTSimpleDeclaration, declarator: IASTDeclarator, index: Int): Ast = { - val name = ASTStringUtil.getSimpleName(declarator.getName) + val name = shortName(declarator) declaration match { case d if isTypeDef(d) && shortName(d.getDeclSpecifier).nonEmpty => val filename = fileName(declaration) - val tpe = registerType(typeFor(declarator)) - Ast(typeDeclNode(declarator, name, registerType(name), filename, code(d), alias = Option(tpe))) + val typeDefName = if (name.isEmpty) { + Try(declarator.getName.resolveBinding()).toOption.map(b => registerType(b.getName)) + } else { + Option(registerType(name)) + } + val tpe = registerType(typeFor(declarator)) + Ast( + typeDeclNode( + declarator, + typeDefName.getOrElse(name), + typeDefName.getOrElse(name), + filename, + code(d), + alias = Option(tpe) + ) + ) case d if parentIsClassDef(d) => val tpe = declarator match { case _: IASTArrayDeclarator => registerType(typeFor(declarator)) @@ -99,19 +112,36 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: case i: IASTEqualsInitializer => val operatorName = Operators.assignment val callNode_ = - callNode(declarator, code(declarator), operatorName, operatorName, DispatchTypes.STATIC_DISPATCH) + callNode( + declarator, + code(declarator), + operatorName, + operatorName, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val left = astForNode(declarator.getName) val right = astForNode(i.getInitializerClause) callAst(callNode_, List(left, right)) case i: ICPPASTConstructorInitializer => - val name = ASTStringUtil.getSimpleName(declarator.getName) - val callNode_ = callNode(declarator, code(declarator), name, name, DispatchTypes.STATIC_DISPATCH) - val args = i.getArguments.toList.map(x => astForNode(x)) + val name = ASTStringUtil.getSimpleName(declarator.getName) + val callNode_ = + callNode(declarator, code(declarator), name, name, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) + val args = i.getArguments.toList.map(x => astForNode(x)) callAst(callNode_, args) case i: IASTInitializerList => val operatorName = Operators.assignment val callNode_ = - callNode(declarator, code(declarator), operatorName, operatorName, DispatchTypes.STATIC_DISPATCH) + callNode( + declarator, + code(declarator), + operatorName, + operatorName, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val left = astForNode(declarator.getName) val right = astForNode(i) callAst(callNode_, List(left, right)) @@ -151,7 +181,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: protected def astForASMDeclaration(asm: IASTASMDeclaration): Ast = Ast(unknownNode(asm, code(asm))) private def astForStructuredBindingDeclaration(decl: ICPPASTStructuredBindingDeclaration): Ast = { - val node = blockNode(decl, Defines.empty, Defines.voidTypeName) + val node = blockNode(decl, Defines.Empty, Defines.Void) scope.pushNewScope(node) val childAsts = decl.getNames.toList.map { name => astForNode(name) @@ -212,8 +242,9 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: case d: IASTDeclarator if d.getInitializer != null => astForInitializer(d, d.getInitializer) case arrayDecl: IASTArrayDeclarator => - val op = Operators.arrayInitializer - val initCallNode = callNode(arrayDecl, code(arrayDecl), op, op, DispatchTypes.STATIC_DISPATCH) + val op = Operators.arrayInitializer + val initCallNode = + callNode(arrayDecl, code(arrayDecl), op, op, DispatchTypes.STATIC_DISPATCH, None, Some(X2CpgDefines.Any)) val initArgs = arrayDecl.getArrayModifiers.toList.filter(m => m.getConstantExpression != null).map(astForNode) callAst(initCallNode, initArgs) @@ -235,25 +266,21 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astForDeclarator(typeSpecifier.getParent.asInstanceOf[IASTSimpleDeclaration], d, i) } - val lineNumber = line(typeSpecifier) - val columnNumber = column(typeSpecifier) - val fullname = registerType(cleanType(fullName(typeSpecifier))) - val name = ASTStringUtil.getSimpleName(typeSpecifier.getName) match { - case n if n.isEmpty => lastNameOfQualifiedName(fullname) - case other => other - } - val codeString = code(typeSpecifier) - val nameAlias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) - val nameWithTemplateParams = templateParameters(typeSpecifier).map(t => registerType(s"$fullname$t")) - val alias = (nameAlias.toList ++ nameWithTemplateParams.toList).headOption + val lineNumber = line(typeSpecifier) + val columnNumber = column(typeSpecifier) + val TypeFullNameInfo(name, fullName) = typeFullNameInfo(typeSpecifier) + val codeString = code(typeSpecifier) + val nameAlias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) + val nameWithTemplateParams = templateParameters(typeSpecifier).map(t => registerType(s"$fullName$t")) + val alias = (nameAlias.toList ++ nameWithTemplateParams.toList).headOption val typeDecl = typeSpecifier match { case cppClass: ICPPASTCompositeTypeSpecifier => val baseClassList = cppClass.getBaseSpecifiers.toSeq.map(s => registerType(s.getNameSpecifier.toString)) - typeDeclNode(typeSpecifier, name, fullname, filename, codeString, inherits = baseClassList, alias = alias) + typeDeclNode(typeSpecifier, name, fullName, filename, codeString, inherits = baseClassList, alias = alias) case _ => - typeDeclNode(typeSpecifier, name, fullname, filename, codeString, alias = alias) + typeDeclNode(typeSpecifier, name, fullName, filename, codeString, alias = alias) } methodAstParentStack.push(typeDecl) @@ -270,9 +297,9 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: } else { val init = staticInitMethodAst( calls, - s"$fullname:${io.joern.x2cpg.Defines.StaticInitMethodName}", + s"$fullName.${io.joern.x2cpg.Defines.StaticInitMethodName}", None, - Defines.anyTypeName, + Defines.Any, Some(filename), lineNumber, columnNumber @@ -289,16 +316,11 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: val declAsts = decls.zipWithIndex.map { case (d, i) => astForDeclarator(typeSpecifier.getParent.asInstanceOf[IASTSimpleDeclaration], d, i) } - - val name = ASTStringUtil.getSimpleName(typeSpecifier.getName) - val fullname = registerType(cleanType(fullName(typeSpecifier))) - val nameAlias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) - val nameWithTemplateParams = templateParameters(typeSpecifier).map(t => registerType(s"$fullname$t")) - val alias = (nameAlias.toList ++ nameWithTemplateParams.toList).headOption - - val typeDecl = - typeDeclNode(typeSpecifier, name, fullname, filename, code(typeSpecifier), alias = alias) - + val TypeFullNameInfo(name, fullName) = typeFullNameInfo(typeSpecifier) + val nameAlias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) + val nameWithTemplateParams = templateParameters(typeSpecifier).map(t => registerType(s"$fullName$t")) + val alias = (nameAlias.toList ++ nameWithTemplateParams.toList).headOption + val typeDecl = typeDeclNode(typeSpecifier, name, fullName, filename, code(typeSpecifier), alias = alias) Ast(typeDecl) +: declAsts } @@ -318,7 +340,15 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: if (enumerator.getValue != null) { val operatorName = Operators.assignment val callNode_ = - callNode(enumerator, code(enumerator), operatorName, operatorName, DispatchTypes.STATIC_DISPATCH) + callNode( + enumerator, + code(enumerator), + operatorName, + operatorName, + DispatchTypes.STATIC_DISPATCH, + None, + Some(X2CpgDefines.Any) + ) val left = astForNode(enumerator.getName) val right = astForNode(enumerator.getValue) val ast = callAst(callNode_, List(left, right)) @@ -334,15 +364,14 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astForDeclarator(typeSpecifier.getParent.asInstanceOf[IASTSimpleDeclaration], d, i) } - val lineNumber = line(typeSpecifier) - val columnNumber = column(typeSpecifier) - val (name, fullname) = - uniqueName("enum", ASTStringUtil.getSimpleName(typeSpecifier.getName), fullName(typeSpecifier)) - val alias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) + val lineNumber = line(typeSpecifier) + val columnNumber = column(typeSpecifier) + val TypeFullNameInfo(name, fullName) = typeFullNameInfo(typeSpecifier) + val alias = decls.headOption.map(d => registerType(shortName(d))).filter(_.nonEmpty) val (deAliasedName, deAliasedFullName, newAlias) = if (name.contains("anonymous_enum") && alias.isDefined) { - (alias.get, fullname.substring(0, fullname.indexOf("anonymous_enum")) + alias.get, None) - } else { (name, fullname, alias) } + (alias.get, fullName.substring(0, fullName.indexOf("anonymous_enum")) + alias.get, None) + } else { (name, fullName, alias) } val typeDecl = typeDeclNode( @@ -370,7 +399,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: calls, s"$deAliasedFullName:${io.joern.x2cpg.Defines.StaticInitMethodName}", None, - Defines.anyTypeName, + Defines.Any, Some(filename), lineNumber, columnNumber diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala index f3d7316835f2..5499f2d655a4 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstNodeBuilder.scala @@ -1,12 +1,12 @@ package io.joern.c2cpg.astcreation -import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode => newMethodReturnNode_} -import io.shiftleft.codepropertygraph.generated.nodes._ -import org.eclipse.cdt.core.dom.ast.{IASTLabelStatement, IASTNode} -import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement +import io.shiftleft.codepropertygraph.generated.nodes.* +import org.eclipse.cdt.core.dom.ast.IASTLabelStatement +import org.eclipse.cdt.core.dom.ast.IASTNode import org.eclipse.cdt.internal.core.model.ASTStringUtil trait AstNodeBuilder { this: AstCreator => + protected def newCommentNode(node: IASTNode, code: String, filename: String): NewComment = { NewComment().code(code).filename(filename).lineNumber(line(node)).columnNumber(column(node)) } @@ -14,7 +14,7 @@ trait AstNodeBuilder { this: AstCreator => protected def newNamespaceBlockNode( node: IASTNode, name: String, - fullname: String, + fullName: String, code: String, filename: String ): NewNamespaceBlock = { @@ -24,12 +24,7 @@ trait AstNodeBuilder { this: AstCreator => .columnNumber(column(node)) .filename(filename) .name(name) - .fullName(fullname) - } - - // TODO: We should get rid of this method as its being used at multiple places and use it from x2cpg/AstNodeBuilder "methodReturnNode" - protected def newMethodReturnNode(node: IASTNode, typeFullName: String): NewMethodReturn = { - newMethodReturnNode_(typeFullName, None, line(node), column(node)) + .fullName(fullName) } protected def newJumpTargetNode(node: IASTNode): NewJumpTarget = { diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/CGlobal.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/CGlobal.scala new file mode 100644 index 000000000000..bb417bd27a9a --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/CGlobal.scala @@ -0,0 +1,43 @@ +package io.joern.c2cpg.astcreation + +import io.joern.x2cpg.datastructures.Global +import java.util.concurrent.ConcurrentHashMap + +object CGlobal { + + final case class MethodInfo( + name: String, + code: String, + fileName: String, + returnType: String, + astParentType: String, + astParentFullName: String, + lineNumber: Option[Int], + columnNumber: Option[Int], + lineNumberEnd: Option[Int], + columnNumberEnd: Option[Int], + signature: String, + offset: Option[(Int, Int)], + parameter: Seq[ParameterInfo], + modifier: Seq[String] + ) + final class ParameterInfo( + val name: String, + var code: String, + val index: Int, + var isVariadic: Boolean, + val evaluationStrategy: String, + val lineNumber: Option[Int], + val columnNumber: Option[Int], + val typeFullName: String + ) + +} + +class CGlobal extends Global { + import io.joern.c2cpg.astcreation.CGlobal.MethodInfo + + val methodDeclarations: ConcurrentHashMap[String, MethodInfo] = new ConcurrentHashMap() + val methodDefinitions: ConcurrentHashMap[String, Boolean] = new ConcurrentHashMap() + +} diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/Defines.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/Defines.scala index f697eb70ca23..612200d00f18 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/Defines.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/Defines.scala @@ -1,10 +1,23 @@ package io.joern.c2cpg.astcreation object Defines { - val anyTypeName: String = "ANY" - val voidTypeName: String = "void" - val qualifiedNameSeparator: String = "::" - val empty = "" + val Any: String = "ANY" + val Void: String = "void" + val Function: String = "std.function" + val Array: String = "std.array" + val QualifiedNameSeparator: String = "::" + val Empty = "" - val operatorPointerCall = ".pointerCall" + val OperatorPointerCall = ".pointerCall" + val OperatorConstructorInitializer = ".constructorInitializer" + val OperatorTypeOf = ".typeOf" + val OperatorMax = ".max" + val OperatorMin = ".min" + val OperatorEllipses = ".op_ellipses" + val OperatorUnknown = ".unknown" + val OperatorCall = "()" + val OperatorExpressionList = ".expressionList" + val OperatorNew = ".new" + val OperatorThrow = ".throw" + val OperatorBracketedPrimary = ".bracketedPrimary" } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/FullNameProvider.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/FullNameProvider.scala new file mode 100644 index 000000000000..798fa0c8ccc6 --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/FullNameProvider.scala @@ -0,0 +1,391 @@ +package io.joern.c2cpg.astcreation + +import org.apache.commons.lang3.StringUtils +import org.eclipse.cdt.core.dom.ast.* +import org.eclipse.cdt.core.dom.ast.cpp.* +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTIdExpression +import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalBinding +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFunctionDeclarator +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTFunctionDefinition +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPFunction +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVariable +import org.eclipse.cdt.internal.core.model.ASTStringUtil +import io.joern.x2cpg.Defines as X2CpgDefines +import org.eclipse.cdt.internal.core.dom.parser.c.CASTFunctionDeclarator +import org.eclipse.cdt.internal.core.dom.parser.c.CVariable + +import scala.util.Try + +trait FullNameProvider { this: AstCreator => + + protected type MethodLike = IASTFunctionDeclarator | IASTFunctionDefinition | ICPPASTLambdaExpression + + protected type TypeLike = IASTEnumerationSpecifier | ICPPASTNamespaceDefinition | ICPPASTNamespaceAlias | + IASTCompositeTypeSpecifier | IASTElaboratedTypeSpecifier + + protected def fixQualifiedName(name: String): String = { + if (name.isEmpty) { name } + else { + val normalizedName = StringUtils.normalizeSpace(name) + normalizedName + .stripPrefix(Defines.QualifiedNameSeparator) + .replace(Defines.QualifiedNameSeparator, ".") + .stripPrefix(".") + } + } + + protected def isQualifiedName(name: String): Boolean = + name.startsWith(Defines.QualifiedNameSeparator) + + protected def lastNameOfQualifiedName(name: String): String = { + val normalizedName = StringUtils.normalizeSpace(replaceOperator(name)) + val cleanedName = if (normalizedName.contains("<") && normalizedName.contains(">")) { + name.substring(0, normalizedName.indexOf("<")) + } else { + normalizedName + } + cleanedName.split(Defines.QualifiedNameSeparator).lastOption.getOrElse(cleanedName) + } + + protected def methodFullNameInfo(methodLike: MethodLike): MethodFullNameInfo = { + val returnType_ = returnType(methodLike) + val signature_ = signature(returnType_, methodLike) + val name_ = shortName(methodLike) + val fullName_ = fullName(methodLike) + val sanitizedFullName = sanitizeMethodLikeFullName(name_, fullName_, signature_, methodLike) + MethodFullNameInfo(name_, sanitizedFullName, signature_, returnType_) + } + + protected def typeFullNameInfo(typeLike: TypeLike): TypeFullNameInfo = { + typeLike match { + case _: IASTElaboratedTypeSpecifier => + val name_ = shortName(typeLike) + val fullName_ = registerType(cleanType(fullName(typeLike))) + TypeFullNameInfo(name_, fullName_) + case e: IASTEnumerationSpecifier => + val name_ = shortName(e) + val fullName_ = fullName(e) + val (uniqueName_, uniqueNameFullName_) = uniqueName("enum", name_, fullName_) + TypeFullNameInfo(uniqueName_, uniqueNameFullName_) + case n: ICPPASTNamespaceDefinition => + val name_ = shortName(n) + val fullName_ = fullName(n) + val (uniqueName_, uniqueNameFullName_) = uniqueName("namespace", name_, fullName_) + TypeFullNameInfo(uniqueName_, uniqueNameFullName_) + case a: ICPPASTNamespaceAlias => + val name_ = shortName(a) + val fullName_ = fullName(a) + TypeFullNameInfo(name_, fullName_) + case s: IASTCompositeTypeSpecifier => + val fullName_ = registerType(cleanType(fullName(s))) + val name_ = shortName(s) match { + case n if n.isEmpty => lastNameOfQualifiedName(fullName_) + case other => other + } + TypeFullNameInfo(name_, fullName_) + } + } + + protected def shortName(node: IASTNode): String = { + val name = node match { + case s: IASTSimpleDeclSpecifier => s.getRawSignature + case d: IASTDeclarator => shortNameForIASTDeclarator(d) + case f: ICPPASTFunctionDefinition => shortNameForICPPASTFunctionDefinition(f) + case f: IASTFunctionDefinition => shortNameForIASTFunctionDefinition(f) + case u: IASTUnaryExpression => shortName(u.getOperand) + case c: IASTFunctionCallExpression => shortName(c.getFunctionNameExpression) + case d: CPPASTIdExpression => shortNameForCPPASTIdExpression(d) + case d: IASTIdExpression => shortNameForIASTIdExpression(d) + case a: ICPPASTNamespaceAlias => ASTStringUtil.getSimpleName(a.getAlias) + case n: ICPPASTNamespaceDefinition => ASTStringUtil.getSimpleName(n.getName) + case e: IASTEnumerationSpecifier => ASTStringUtil.getSimpleName(e.getName) + case c: IASTCompositeTypeSpecifier => ASTStringUtil.getSimpleName(c.getName) + case e: IASTElaboratedTypeSpecifier => ASTStringUtil.getSimpleName(e.getName) + case s: IASTNamedTypeSpecifier => ASTStringUtil.getSimpleName(s.getName) + case _: ICPPASTLambdaExpression => nextClosureName() + case other => + notHandledYet(other) + nextClosureName() + } + StringUtils.normalizeSpace(name) + } + + protected def fullName(node: IASTNode): String = { + fullNameFromBinding(node) match { + case Some(fullName) => + StringUtils.normalizeSpace(fullName) + case None => + val qualifiedName = node match { + case _: IASTTranslationUnit => "" + case alias: ICPPASTNamespaceAlias => fixQualifiedName(ASTStringUtil.getQualifiedName(alias.getMappingName)) + case namespace: ICPPASTNamespaceDefinition => fullNameForICPPASTNamespaceDefinition(namespace) + case compType: IASTCompositeTypeSpecifier => fullNameForIASTCompositeTypeSpecifier(compType) + case enumSpecifier: IASTEnumerationSpecifier => fullNameForIASTEnumerationSpecifier(enumSpecifier) + case f: IASTFunctionDeclarator => fullNameForIASTFunctionDeclarator(f) + case f: IASTFunctionDefinition => fullNameForIASTFunctionDefinition(f) + case e: IASTElaboratedTypeSpecifier => fullNameForIASTElaboratedTypeSpecifier(e) + case d: IASTIdExpression => ASTStringUtil.getSimpleName(d.getName) + case u: IASTUnaryExpression => code(u.getOperand) + case x: ICPPASTQualifiedName => fixQualifiedName(ASTStringUtil.getQualifiedName(x)) + case other if other != null && other.getParent != null => fullName(other.getParent) + case other if other != null => notHandledYet(other); "" + case null => "" + } + fixQualifiedName(qualifiedName).stripPrefix(".") + } + } + + private def isCPPFunction(methodLike: MethodLike): Boolean = { + methodLike.isInstanceOf[CPPASTFunctionDeclarator] || methodLike.isInstanceOf[CPPASTFunctionDefinition] + } + + private def sanitizeMethodLikeFullName( + name: String, + fullName: String, + signature: String, + methodLike: MethodLike + ): String = { + fullName match { + case f if methodLike.isInstanceOf[ICPPASTLambdaExpression] && (f.contains("[") || f.contains("{")) => + s"${X2CpgDefines.UnresolvedNamespace}.$name" + case f if methodLike.isInstanceOf[ICPPASTLambdaExpression] && f.isEmpty => + name + case f if methodLike.isInstanceOf[ICPPASTLambdaExpression] => + s"$f.$name" + case f if isCPPFunction(methodLike) && (f.isEmpty || f == s"${X2CpgDefines.UnresolvedNamespace}.") => + s"${X2CpgDefines.UnresolvedNamespace}.$name:$signature" + case f if isCPPFunction(methodLike) && f.contains("?") => + s"${StringUtils.normalizeSpace(f).takeWhile(_ != ':')}:$signature" + case f if f.isEmpty || f == s"${X2CpgDefines.UnresolvedNamespace}." => + s"${X2CpgDefines.UnresolvedNamespace}.$name" + case other if other.nonEmpty => other + case _ => s"${X2CpgDefines.UnresolvedNamespace}.$name" + } + } + + private def returnTypeForIASTFunctionDeclarator(declarator: IASTFunctionDeclarator): String = { + cleanType(typeForDeclSpecifier(declarator.getParent.asInstanceOf[IASTSimpleDeclaration].getDeclSpecifier)) + } + + private def returnTypeForIASTFunctionDefinition(definition: IASTFunctionDefinition): String = { + if (isCppConstructor(definition)) { + typeFor(definition.asInstanceOf[CPPASTFunctionDefinition].getMemberInitializers.head.getInitializer) + } else { + typeForDeclSpecifier(definition.getDeclSpecifier) + } + } + + private def returnTypeForICPPASTLambdaExpression(lambda: ICPPASTLambdaExpression): String = { + lambda.getDeclarator match { + case declarator: IASTDeclarator => + Option(declarator.getTrailingReturnType) + .map(id => typeForDeclSpecifier(id.getDeclSpecifier)) + .getOrElse(Defines.Any) + case null => Defines.Any + } + } + + private def returnType(methodLike: MethodLike): String = { + methodLike match { + case declarator: IASTFunctionDeclarator => returnTypeForIASTFunctionDeclarator(declarator) + case definition: IASTFunctionDefinition => returnTypeForIASTFunctionDefinition(definition) + case lambda: ICPPASTLambdaExpression => returnTypeForICPPASTLambdaExpression(lambda) + } + } + + private def parameterListSignature(func: IASTNode): String = { + val variadic = if (isVariadic(func)) "..." else "" + val elements = parameters(func).map { + case p: IASTParameterDeclaration => typeForDeclSpecifier(p.getDeclSpecifier) + case other => typeForDeclSpecifier(other) + } + s"(${elements.mkString(",")}$variadic)" + } + + private def signature(returnType: String, methodLike: MethodLike): String = { + StringUtils.normalizeSpace(s"$returnType${parameterListSignature(methodLike)}") + } + + private def shortNameForIASTDeclarator(declarator: IASTDeclarator): String = { + Try(declarator.getName.resolveBinding().getName).getOrElse { + if (ASTStringUtil.getSimpleName(declarator.getName).isEmpty && declarator.getNestedDeclarator != null) { + shortName(declarator.getNestedDeclarator) + } else { + ASTStringUtil.getSimpleName(declarator.getName) + } + } + } + + private def shortNameForICPPASTFunctionDefinition(definition: ICPPASTFunctionDefinition): String = { + if ( + ASTStringUtil.getSimpleName(definition.getDeclarator.getName).isEmpty + && definition.getDeclarator.getNestedDeclarator != null + ) { + shortName(definition.getDeclarator.getNestedDeclarator) + } else { + lastNameOfQualifiedName(ASTStringUtil.getSimpleName(definition.getDeclarator.getName)) + } + } + + private def shortNameForIASTFunctionDefinition(definition: IASTFunctionDefinition): String = { + if ( + ASTStringUtil.getSimpleName(definition.getDeclarator.getName).isEmpty + && definition.getDeclarator.getNestedDeclarator != null + ) { + shortName(definition.getDeclarator.getNestedDeclarator) + } else { + ASTStringUtil.getSimpleName(definition.getDeclarator.getName) + } + } + + private def shortNameForCPPASTIdExpression(d: CPPASTIdExpression): String = { + val name = safeGetEvaluation(d) match { + case Some(evalBinding: EvalBinding) => + evalBinding.getBinding match { + case f: CPPFunction if f.getDeclarations != null => + f.getDeclarations.headOption.map(n => ASTStringUtil.getSimpleName(n.getName)).getOrElse(f.getName) + case f: CPPFunction if f.getDefinition != null => ASTStringUtil.getSimpleName(f.getDefinition.getName) + case other => other.getName + } + case _ => ASTStringUtil.getSimpleName(d.getName) + } + lastNameOfQualifiedName(name) + } + + private def shortNameForIASTIdExpression(d: IASTIdExpression): String = { + lastNameOfQualifiedName(ASTStringUtil.getSimpleName(d.getName)) + } + + private def replaceOperator(name: String): String = { + name + .replace("operator class ", "") + .replace("operator enum ", "") + .replace("operator struct ", "") + .replace("operator ", "") + } + + private def fullNameFromBinding(node: IASTNode): Option[String] = { + node match { + case id: CPPASTIdExpression => + safeGetEvaluation(id) match { + case Some(evalBinding: EvalBinding) => + evalBinding.getBinding match { + case f: CPPFunction if f.getDeclarations != null => + Option(f.getDeclarations.headOption.map(n => s"${fullName(n)}").getOrElse(f.getName)) + case f: CPPFunction if f.getDefinition != null => + Option(s"${fullName(f.getDefinition)}") + case other => + Option(other.getName) + } + case _ => None + } + case declarator: CPPASTFunctionDeclarator => + declarator.getName.resolveBinding() match { + case function: ICPPFunction if declarator.getName.isInstanceOf[ICPPASTConversionName] => + val tpe = typeFor(declarator.getName.asInstanceOf[ICPPASTConversionName].getTypeId) + val fullNameNoSig = fixQualifiedName( + function.getQualifiedName.takeWhile(!_.startsWith("operator ")).mkString(".") + ) + val fn = if (function.isExternC) { + tpe + } else { + s"$fullNameNoSig.$tpe:${functionTypeToSignature(function.getType)}" + } + Option(fn) + case function: ICPPFunction => + val fullNameNoSig = fixQualifiedName(replaceOperator(function.getQualifiedName.mkString("."))) + val fn = if (function.isExternC) { + replaceOperator(function.getName) + } else { + s"$fullNameNoSig:${functionTypeToSignature(function.getType)}" + } + Option(fn) + case x @ (_: ICPPField | _: CPPVariable) => + val fullNameNoSig = fixQualifiedName(x.getQualifiedName.mkString(".")) + val fn = if (x.isExternC) { + x.getName + } else { + s"$fullNameNoSig:${cleanType(safeGetType(x.getType))}" + } + Option(fn) + case _: IProblemBinding => + val fullNameNoSig = replaceOperator(ASTStringUtil.getQualifiedName(declarator.getName)) + val fixedFullName = fixQualifiedName(fullNameNoSig) + if (fixedFullName.isEmpty) { + Option(s"${X2CpgDefines.UnresolvedNamespace}:${X2CpgDefines.UnresolvedSignature}") + } else { + Option(s"$fixedFullName:${X2CpgDefines.UnresolvedSignature}") + } + case _ => None + } + case declarator: CASTFunctionDeclarator => + declarator.getName.resolveBinding() match { + case cVariable: CVariable => Option(cVariable.getName) + case _ => Option(declarator.getName.toString) + } + case definition: ICPPASTFunctionDefinition => + Some(fullName(definition.getDeclarator)) + case namespace: ICPPASTNamespaceDefinition => + namespace.getName.resolveBinding() match { + case b: ICPPBinding => Option(b.getQualifiedName.mkString(".")) + case _ => None + } + case compType: IASTCompositeTypeSpecifier => + compType.getName.resolveBinding() match { + case b: ICPPBinding => Option(b.getQualifiedName.mkString(".")) + case _ => None + } + case enumSpecifier: IASTEnumerationSpecifier => + enumSpecifier.getName.resolveBinding() match { + case b: ICPPBinding => Option(b.getQualifiedName.mkString(".")) + case _ => None + } + case e: IASTElaboratedTypeSpecifier => + e.getName.resolveBinding() match { + case b: ICPPBinding => Option(b.getQualifiedName.mkString(".")) + case _ => None + } + case _ => None + } + } + + private def fullNameForICPPASTNamespaceDefinition(namespace: ICPPASTNamespaceDefinition): String = { + s"${fullName(namespace.getParent)}.${ASTStringUtil.getSimpleName(namespace.getName)}" + } + + private def fullNameForIASTCompositeTypeSpecifier(compType: IASTCompositeTypeSpecifier): String = { + if (ASTStringUtil.getSimpleName(compType.getName).nonEmpty) { + s"${fullName(compType.getParent)}.${ASTStringUtil.getSimpleName(compType.getName)}" + } else { + val name = compType.getParent match { + case decl: IASTSimpleDeclaration => + decl.getDeclarators.headOption + .map(n => ASTStringUtil.getSimpleName(n.getName)) + .getOrElse(uniqueName("composite_type", "", "")._1) + case _ => uniqueName("composite_type", "", "")._1 + } + s"${fullName(compType.getParent)}.$name" + } + } + + private def fullNameForIASTEnumerationSpecifier(enumSpecifier: IASTEnumerationSpecifier): String = { + s"${fullName(enumSpecifier.getParent)}.${ASTStringUtil.getSimpleName(enumSpecifier.getName)}" + } + + private def fullNameForIASTElaboratedTypeSpecifier(e: IASTElaboratedTypeSpecifier): String = { + s"${fullName(e.getParent)}.${ASTStringUtil.getSimpleName(e.getName)}" + } + + private def fullNameForIASTFunctionDeclarator(f: IASTFunctionDeclarator): String = { + Try(fixQualifiedName(ASTStringUtil.getQualifiedName(f.getName))).getOrElse(nextClosureName()) + } + + private def fullNameForIASTFunctionDefinition(f: IASTFunctionDefinition): String = { + Try(fixQualifiedName(ASTStringUtil.getQualifiedName(f.getDeclarator.getName))).getOrElse(nextClosureName()) + } + + protected final case class MethodFullNameInfo(name: String, fullName: String, signature: String, returnType: String) + + protected final case class TypeFullNameInfo(name: String, fullName: String) + +} diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala index 9001ddcd0855..61394858a887 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/MacroHandler.scala @@ -1,18 +1,20 @@ package io.joern.c2cpg.astcreation +import io.joern.x2cpg.Ast +import io.joern.x2cpg.AstEdge +import io.joern.x2cpg.ValidationMode import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.codepropertygraph.generated.nodes.{ - AstNodeNew, - ExpressionNew, - NewBlock, - NewCall, - NewFieldIdentifier, - NewNode -} -import io.joern.x2cpg.{Ast, AstEdge, ValidationMode} +import io.shiftleft.codepropertygraph.generated.nodes.AstNodeNew +import io.shiftleft.codepropertygraph.generated.nodes.ExpressionNew +import io.shiftleft.codepropertygraph.generated.nodes.NewBlock +import io.shiftleft.codepropertygraph.generated.nodes.NewCall +import io.shiftleft.codepropertygraph.generated.nodes.NewFieldIdentifier +import io.shiftleft.codepropertygraph.generated.nodes.NewNode import io.shiftleft.codepropertygraph.generated.nodes.NewLocal import org.apache.commons.lang3.StringUtils -import org.eclipse.cdt.core.dom.ast.{IASTMacroExpansionLocation, IASTNode, IASTPreprocessorMacroDefinition} +import org.eclipse.cdt.core.dom.ast.IASTMacroExpansionLocation +import org.eclipse.cdt.core.dom.ast.IASTNode +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition import org.eclipse.cdt.core.dom.ast.IASTBinaryExpression import org.eclipse.cdt.internal.core.model.ASTStringUtil @@ -45,18 +47,19 @@ trait MacroHandler(implicit withSchemaValidation: ValidationMode) { this: AstCre val macroCallAst = matchingMacro.map { case (mac, args) => createMacroCallAst(ast, node, mac, args) } macroCallAst match { case Some(callAst) => - val lostLocals = ast.refEdges.collect { case AstEdge(_, dst: NewLocal) => Ast(dst) }.toList - val newAst = ast.subTreeCopy(ast.root.get.asInstanceOf[AstNodeNew], argIndex = 1) + val newAst = ast.subTreeCopy(ast.root.get.asInstanceOf[AstNodeNew], argIndex = 1) // We need to wrap the copied AST as it may contain CPG nodes not being allowed // to be connected via AST edges under a CALL. E.g., LOCALs but only if its not already a BLOCK. val childAst = newAst.root match { - case Some(_: NewBlock) => - newAst - case _ => - val b = NewBlock().argumentIndex(1).typeFullName(registerType(Defines.voidTypeName)) - blockAst(b, List(newAst)) + case Some(_: NewBlock) => newAst + case _ => blockAst(blockNode(node), List(newAst)) } - callAst.withChildren(lostLocals).withChild(childAst) + val lostLocals = ast.edges.collect { + case AstEdge(_, dst: NewLocal) if !newAst.edges.exists(_.dst == dst) => Ast(dst) + }.distinct + val childrenAsts = lostLocals :+ childAst + setArgumentIndices(childrenAsts.toList) + callAst.withChildren(childrenAsts) case None => ast } } @@ -124,13 +127,14 @@ trait MacroHandler(implicit withSchemaValidation: ValidationMode) { this: AstCre val callName = StringUtils.normalizeSpace(name) val callFullName = StringUtils.normalizeSpace(fullName(macroDef, argAsts)) + val typeFullName = registerType(cleanType(typeFor(node))) val callNode = NewCall() .name(callName) .dispatchType(DispatchTypes.INLINED) .methodFullName(callFullName) .code(code) - .typeFullName(typeFor(node)) + .typeFullName(typeFullName) .lineNumber(line(node)) .columnNumber(column(node)) callAst(callNode, argAsts) diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala index 8c4f051070dd..1a8cfce01c62 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala @@ -2,6 +2,7 @@ package io.joern.c2cpg.parser import better.files.File import io.joern.c2cpg.Config +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser.CommandObject import io.shiftleft.utils.IOUtils import org.eclipse.cdt.core.dom.ast.gnu.c.GCCLanguage import org.eclipse.cdt.core.dom.ast.gnu.cpp.GPPLanguage @@ -30,20 +31,26 @@ object CdtParser { failure: Option[Throwable] = None ) - def readFileAsFileContent(path: Path): FileContent = { + def loadLinesAsFileContent(path: Path, lines: Array[Char]): InternalFileContent = { + FileContent.create(path.toString, true, lines).asInstanceOf[InternalFileContent] + } + + def readFileAsFileContent(path: Path): InternalFileContent = { val lines = IOUtils.readLinesInFile(path).mkString("\n").toArray - FileContent.create(path.toString, true, lines) + loadLinesAsFileContent(path, lines) } } -class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorStatementsLogger { +class CdtParser(config: Config, compilationDatabase: List[CommandObject]) + extends ParseProblemsLogger + with PreprocessorStatementsLogger { import io.joern.c2cpg.parser.CdtParser._ private val headerFileFinder = new HeaderFileFinder(config.inputPath) - private val parserConfig = ParserConfig.fromConfig(config) - private val definedSymbols = parserConfig.definedSymbols.asJava + private val parserConfig = ParserConfig.fromConfig(config, compilationDatabase) + private val definedSymbols = parserConfig.definedSymbols private val includePaths = parserConfig.userIncludePaths private val log = new DefaultLogService @@ -76,7 +83,12 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta val additionalIncludes = if (FileDefaults.isCPPFile(file.toString)) parserConfig.systemIncludePathsCPP else parserConfig.systemIncludePathsC - new ScannerInfo(definedSymbols, (includePaths ++ additionalIncludes).map(_.toString).toArray) + val fileSpecificDefines = parserConfig.definedSymbolsPerFile.getOrElse(file.toString, Map.empty) + val fileSpecificIncludes = parserConfig.includesPerFile.getOrElse(file.toString, List.empty) + new ScannerInfo( + (definedSymbols ++ fileSpecificDefines).asJava, + fileSpecificIncludes.toArray ++ (includePaths ++ additionalIncludes).map(_.toString).toArray + ) } private def parseInternal(code: String, inFile: File): IASTTranslationUnit = { @@ -97,8 +109,8 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta try { val fileContent = readFileAsFileContent(realPath.path) val fileContentProvider = new CustomFileContentProvider(headerFileFinder) - val lang = createParseLanguage(realPath.path, fileContent.asInstanceOf[InternalFileContent].toString) - val scannerInfo = createScannerInfo(realPath.path) + val lang = createParseLanguage(realPath.path, fileContent.toString) + val scannerInfo = createScannerInfo(realPath.path) val translationUnit = lang.getASTTranslationUnit(fileContent, scannerInfo, fileContentProvider, null, opts, log) val problems = CPPVisitor.getProblems(translationUnit) if (parserConfig.logProblems) logProblems(problems.toList) diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala index b22a21bc4c8d..e30846d86f68 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala @@ -1,14 +1,24 @@ package io.joern.c2cpg.parser +import io.joern.c2cpg.parser.CustomFileContentProvider.missingHeaderFiles +import io.shiftleft.utils.IOUtils import org.eclipse.cdt.core.index.IIndexFileLocation import org.eclipse.cdt.internal.core.parser.IMacroDictionary import org.eclipse.cdt.internal.core.parser.scanner.{InternalFileContent, InternalFileContentProvider} import org.slf4j.LoggerFactory import java.nio.file.Paths +import java.util.concurrent.ConcurrentHashMap + +object CustomFileContentProvider { + private val headerFileToLines: ConcurrentHashMap[String, Array[Char]] = new ConcurrentHashMap() + private val missingHeaderFiles: ConcurrentHashMap[String, Boolean] = new ConcurrentHashMap() +} class CustomFileContentProvider(headerFileFinder: HeaderFileFinder) extends InternalFileContentProvider { + import io.joern.c2cpg.parser.CustomFileContentProvider.headerFileToLines + private val logger = LoggerFactory.getLogger(classOf[CustomFileContentProvider]) private def loadContent(path: String): InternalFileContent = { @@ -19,11 +29,24 @@ class CustomFileContentProvider(headerFileFinder: HeaderFileFinder) extends Inte } maybeFullPath .map { foundPath => - logger.debug(s"Loading header file '$foundPath'") - CdtParser.readFileAsFileContent(Paths.get(foundPath)).asInstanceOf[InternalFileContent] + val p = Paths.get(foundPath) + val content = headerFileToLines.computeIfAbsent( + foundPath, + _ => { + logger.debug(s"Loading header file '$foundPath'") + IOUtils.readLinesInFile(p).mkString("\n").toArray + } + ) + CdtParser.loadLinesAsFileContent(p, content) } .getOrElse { - logger.debug(s"Cannot find header file for '$path'") + missingHeaderFiles.computeIfAbsent( + path, + _ => { + logger.debug(s"Cannot find header file for '$path'") + true + } + ) null } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/HeaderFileFinder.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/HeaderFileFinder.scala index dbc6f36a1be9..af72687a9315 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/HeaderFileFinder.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/HeaderFileFinder.scala @@ -1,6 +1,6 @@ package io.joern.c2cpg.parser -import better.files._ +import better.files.* import io.joern.x2cpg.SourceFiles import org.jline.utils.Levenshtein diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/JSONCompilationDatabaseParser.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/JSONCompilationDatabaseParser.scala new file mode 100644 index 000000000000..ede3e37f621a --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/JSONCompilationDatabaseParser.scala @@ -0,0 +1,97 @@ +package io.joern.c2cpg.parser + +import io.joern.x2cpg.SourceFiles +import io.shiftleft.utils.IOUtils +import org.slf4j.LoggerFactory +import ujson.Value + +import java.nio.file.Paths +import scala.util.Try + +object JSONCompilationDatabaseParser { + + private val logger = LoggerFactory.getLogger(getClass) + + /** {{{ + * 1) -D: Matches the -D flag, which is the key prefix for defining macros. + * 2) ([A-Za-z_][A-Za-z0-9_]+): Matches a valid macro name (which must start with a letter or underscore and can be followed by letters, numbers, or underscores). + * 3) (=(\\*".*"))?: Optionally matches = followed by either: + * a) A quoted string: Allows for strings in quotes. + * b) Any char sequence (.*") closed with a quote. + * }}} + */ + private val defineInCommandPattern = """-D([A-Za-z_][A-Za-z0-9_]+)(=(\\*".*"))?""".r + + /** {{{ + * 1) -I: Matches the -I flag, which indicates an include directory. + * 2) (\S+): Matches one or more non-whitespace characters, which represent the path of the directory. + * }}} + */ + private val includeInCommandPattern = """-I(\S+)""".r + + case class CommandObject(directory: String, arguments: List[String], command: List[String], file: String) { + + /** @return + * the file path (guaranteed to be absolute) + */ + def compiledFile(): String = SourceFiles.toAbsolutePath(file, directory) + + private def nameValuePairFromDefine(define: String): (String, String) = { + val s = define.stripPrefix("-D") + if (s.contains("=")) { + val split = s.split("=") + (split.head, split(1)) + } else { + (s, "") + } + } + + private def pathFromInclude(include: String): String = include.stripPrefix("-I") + + def includes(): List[String] = { + val includesFromArguments = arguments.filter(a => a.startsWith("-I")).map(pathFromInclude) + val includesFromCommand = command.flatMap { c => + val includes = includeInCommandPattern.findAllIn(c).toList + includes.map(pathFromInclude) + } + includesFromArguments ++ includesFromCommand + } + + def defines(): List[(String, String)] = { + val definesFromArguments = arguments.filter(a => a.startsWith("-D")).map(nameValuePairFromDefine) + val definesFromCommand = command.flatMap { c => + val defines = defineInCommandPattern.findAllIn(c).toList + defines.map(nameValuePairFromDefine) + } + definesFromArguments ++ definesFromCommand + } + } + + private def hasKey(node: Value, key: String): Boolean = Try(node(key)).isSuccess + + private def safeArguments(obj: Value): List[String] = { + if (hasKey(obj, "arguments")) obj("arguments").arrOpt.map(_.toList.map(_.str)).getOrElse(List.empty) + else List.empty + } + + private def safeCommand(obj: Value): List[String] = { + if (hasKey(obj, "command")) List(obj("command").str) + else List.empty + } + + def parse(compileCommandsJson: String): List[CommandObject] = { + try { + val jsonContent = IOUtils.readEntireFile(Paths.get(compileCommandsJson)) + val json = ujson.read(jsonContent) + val allCommandObjects = json.arr.toList + allCommandObjects.map { obj => + CommandObject(obj("directory").str, safeArguments(obj), safeCommand(obj), obj("file").str) + } + } catch { + case t: Throwable => + logger.warn(s"Could not parse '$compileCommandsJson'", t) + List.empty + } + } + +} diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/ParserConfig.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/ParserConfig.scala index 7bb82a6b751e..955430d1a5ee 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/ParserConfig.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/ParserConfig.scala @@ -1,6 +1,7 @@ package io.joern.c2cpg.parser import io.joern.c2cpg.Config +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser.CommandObject import io.joern.c2cpg.utils.IncludeAutoDiscovery import java.nio.file.{Path, Paths} @@ -8,21 +9,42 @@ import java.nio.file.{Path, Paths} object ParserConfig { def empty: ParserConfig = - ParserConfig(Set.empty, Set.empty, Set.empty, Map.empty, logProblems = false, logPreprocessor = false) + ParserConfig( + Set.empty, + Set.empty, + Set.empty, + Map.empty, + Map.empty, + Map.empty, + logProblems = false, + logPreprocessor = false + ) - def fromConfig(config: Config): ParserConfig = ParserConfig( - config.includePaths.map(Paths.get(_).toAbsolutePath), - IncludeAutoDiscovery.discoverIncludePathsC(config), - IncludeAutoDiscovery.discoverIncludePathsCPP(config), - config.defines.map { - case define if define.contains("=") => - val s = define.split("=") - s.head -> s(1) - case define => define -> "true" - }.toMap ++ DefaultDefines.DEFAULT_CALL_CONVENTIONS, - config.logProblems, - config.logPreprocessor - ) + def fromConfig(config: Config, compilationDatabase: List[CommandObject]): ParserConfig = { + val compilationDatabaseDefines = compilationDatabase.map { c => + c.compiledFile() -> c.defines().toMap + }.toMap + val includes = compilationDatabase.map { c => + c.compiledFile() -> c.includes() + }.toMap + ParserConfig( + config.includePaths.map(Paths.get(_).toAbsolutePath), + IncludeAutoDiscovery.discoverIncludePathsC(config), + IncludeAutoDiscovery.discoverIncludePathsCPP(config), + config.defines.map { define => + if (define.contains("=")) { + val split = define.split("=") + split.head -> split(1) + } else { + define -> "" + } + }.toMap ++ DefaultDefines.DEFAULT_CALL_CONVENTIONS, + compilationDatabaseDefines, + includes, + config.logProblems, + config.logPreprocessor + ) + } } @@ -31,6 +53,8 @@ case class ParserConfig( systemIncludePathsC: Set[Path], systemIncludePathsCPP: Set[Path], definedSymbols: Map[String, String], + definedSymbolsPerFile: Map[String, Map[String, String]], + includesPerFile: Map[String, List[String]], logProblems: Boolean, logPreprocessor: Boolean ) diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala index a140db208fe4..0325a19b0395 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala @@ -3,31 +3,44 @@ package io.joern.c2cpg.passes import io.joern.c2cpg.C2Cpg.DefaultIgnoredFolders import io.joern.c2cpg.Config import io.joern.c2cpg.astcreation.AstCreator -import io.joern.c2cpg.astcreation.Defines +import io.joern.c2cpg.astcreation.CGlobal import io.joern.c2cpg.parser.{CdtParser, FileDefaults} +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser.CommandObject import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.ForkJoinParallelCpgPass import io.joern.x2cpg.SourceFiles -import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.utils.Report import io.joern.x2cpg.utils.TimeUtils import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap +import org.slf4j.{Logger, LoggerFactory} + import scala.util.matching.Regex +import scala.util.{Failure, Success, Try} import scala.jdk.CollectionConverters.* class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) extends ForkJoinParallelCpgPass[String](cpg) { + private val logger: Logger = LoggerFactory.getLogger(classOf[AstCreationPass]) + + private val global = new CGlobal() private val file2OffsetTable: ConcurrentHashMap[String, Array[Int]] = new ConcurrentHashMap() - private val parser: CdtParser = new CdtParser(config) - private val global = new Global() + private val compilationDatabase: List[CommandObject] = + config.compilationDatabase.map(JSONCompilationDatabaseParser.parse).getOrElse(List.empty) - def typesSeen(): List[String] = global.usedTypes.keys().asScala.filterNot(_ == Defines.anyTypeName).toList + private val parser: CdtParser = new CdtParser(config, compilationDatabase) - override def generateParts(): Array[String] = { + def typesSeen(): List[String] = global.usedTypes.keys().asScala.toList + + def unhandledMethodDeclarations(): Map[String, CGlobal.MethodInfo] = { + global.methodDeclarations.asScala.toMap -- global.methodDefinitions.asScala.keys + } + + private def sourceFilesFromDirectory(): Array[String] = { val sourceFileExtensions = FileDefaults.SOURCE_FILE_EXTENSIONS ++ FileDefaults.HEADER_FILE_EXTENSIONS ++ Option.when(config.withPreprocessedFiles)(FileDefaults.PREPROCESSED_EXT).toList @@ -52,6 +65,29 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) } } + private def sourceFilesFromCompilationDatabase(compilationDatabaseFile: String): Array[String] = { + if (compilationDatabase.isEmpty) { + logger.warn(s"'$compilationDatabaseFile' contains no source files. CPG will be empty.") + } + SourceFiles + .filterFiles( + compilationDatabase.map(_.compiledFile()), + config.inputPath, + ignoredDefaultRegex = Option(DefaultIgnoredFolders), + ignoredFilesRegex = Option(config.ignoredFilesRegex), + ignoredFilesPath = Option(config.ignoredFiles) + ) + .toArray + } + + override def generateParts(): Array[String] = { + if (config.compilationDatabase.isEmpty) { + sourceFilesFromDirectory() + } else { + sourceFilesFromCompilationDatabase(config.compilationDatabase.get) + } + } + override def runOnPart(diffGraph: DiffGraphBuilder, filename: String): Unit = { val path = Paths.get(filename).toAbsolutePath val relPath = SourceFiles.toRelativePath(path.toString, config.inputPath) @@ -61,11 +97,17 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) parseResult match { case Some(translationUnit) => report.addReportInfo(relPath, fileLOC, parsed = true) - val localDiff = new AstCreator(relPath, global, config, translationUnit, file2OffsetTable)( - config.schemaValidation - ).createAst() - diffGraph.absorb(localDiff) - true + Try { + val localDiff = new AstCreator(relPath, global, config, translationUnit, file2OffsetTable)( + config.schemaValidation + ).createAst() + diffGraph.absorb(localDiff) + } match { + case Failure(exception) => + logger.warn(s"Failed to generate a CPG for: '$filename'", exception) + false + case Success(_) => true + } case None => report.addReportInfo(relPath, fileLOC) false diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/FunctionDeclNodePass.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/FunctionDeclNodePass.scala new file mode 100644 index 000000000000..ceba5ba84df0 --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/FunctionDeclNodePass.scala @@ -0,0 +1,174 @@ +package io.joern.c2cpg.passes + +import io.joern.c2cpg.astcreation.CGlobal +import io.joern.x2cpg.Ast +import io.joern.x2cpg.Defines +import io.joern.x2cpg.ValidationMode +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.NewBlock +import io.shiftleft.codepropertygraph.generated.nodes.NewMethod +import io.shiftleft.codepropertygraph.generated.nodes.NewMethodParameterIn +import io.shiftleft.codepropertygraph.generated.nodes.NewMethodReturn +import io.shiftleft.codepropertygraph.generated.EvaluationStrategies +import io.shiftleft.codepropertygraph.generated.nodes.NewBinding +import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.NodeTypes +import io.shiftleft.codepropertygraph.generated.nodes.NewModifier +import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* +import org.apache.commons.lang3.StringUtils + +import scala.collection.immutable.Map + +class FunctionDeclNodePass(cpg: Cpg, methodDeclarations: Map[String, CGlobal.MethodInfo])(implicit + withSchemaValidation: ValidationMode +) extends CpgPass(cpg) { + + private def methodNode(fullName: String, methodNodeInfo: CGlobal.MethodInfo): NewMethod = { + val node_ = + NewMethod() + .name(StringUtils.normalizeSpace(methodNodeInfo.name)) + .code(methodNodeInfo.code) + .fullName(StringUtils.normalizeSpace(fullName)) + .filename(methodNodeInfo.fileName) + .astParentType(methodNodeInfo.astParentType) + .astParentFullName(methodNodeInfo.astParentFullName) + .isExternal(false) + .lineNumber(methodNodeInfo.lineNumber) + .columnNumber(methodNodeInfo.columnNumber) + .lineNumberEnd(methodNodeInfo.lineNumberEnd) + .columnNumberEnd(methodNodeInfo.columnNumberEnd) + .signature(StringUtils.normalizeSpace(methodNodeInfo.signature)) + methodNodeInfo.offset.foreach { case (offset, offsetEnd) => + node_.offset(offset).offsetEnd(offsetEnd) + } + node_ + } + + private def parameterInNode(parameterNodeInfo: CGlobal.ParameterInfo): NewMethodParameterIn = { + NewMethodParameterIn() + .name(parameterNodeInfo.name) + .code(parameterNodeInfo.code) + .index(parameterNodeInfo.index) + .order(parameterNodeInfo.index) + .isVariadic(parameterNodeInfo.isVariadic) + .evaluationStrategy(parameterNodeInfo.evaluationStrategy) + .lineNumber(parameterNodeInfo.lineNumber) + .columnNumber(parameterNodeInfo.columnNumber) + .typeFullName(parameterNodeInfo.typeFullName) + } + + private def methodReturnNode(typeFullName: String, line: Option[Int], column: Option[Int]): NewMethodReturn = + NewMethodReturn() + .typeFullName(typeFullName) + .code("RET") + .evaluationStrategy(EvaluationStrategies.BY_VALUE) + .lineNumber(line) + .columnNumber(column) + + private def typeDeclNode( + name: String, + fullName: String, + filename: String, + code: String, + astParentType: String, + astParentFullName: String, + line: Option[Int], + column: Option[Int], + offset: Option[(Int, Int)] + ): NewTypeDecl = { + val node_ = NewTypeDecl() + .name(name) + .fullName(fullName) + .code(code) + .isExternal(false) + .filename(filename) + .astParentType(astParentType) + .astParentFullName(astParentFullName) + .lineNumber(line) + .columnNumber(column) + offset.foreach { case (offset, offsetEnd) => + node_.offset(offset).offsetEnd(offsetEnd) + } + node_ + } + + private def methodStubAst( + method: NewMethod, + parameters: Seq[Ast], + methodReturn: NewMethodReturn, + modifier: Seq[Ast] + ): Ast = + Ast(method) + .withChildren(parameters) + .withChild(Ast(NewBlock().typeFullName(Defines.Any))) + .withChildren(modifier) + .withChild(Ast(methodReturn)) + + private def createFunctionTypeAndTypeDecl( + methodInfo: CGlobal.MethodInfo, + method: NewMethod, + methodName: String, + methodFullName: String, + signature: String, + dstGraph: DiffGraphBuilder + ): Ast = { + val normalizedName = StringUtils.normalizeSpace(methodName) + val normalizedFullName = StringUtils.normalizeSpace(methodFullName) + + if (methodInfo.astParentType == NodeTypes.TYPE_DECL) { + val parentTypeDecl = cpg.typeDecl.nameExact(methodInfo.astParentFullName).headOption + parentTypeDecl + .map { typeDecl => + val functionBinding = + NewBinding().name(normalizedName).methodFullName(normalizedFullName).signature(signature) + dstGraph.addEdge(typeDecl, functionBinding, EdgeTypes.BINDS) + Ast(functionBinding).withRefEdge(functionBinding, method) + } + .getOrElse(Ast()) + } else { + val typeDecl = typeDeclNode( + normalizedName, + normalizedFullName, + method.filename, + normalizedName, + methodInfo.astParentType, + methodInfo.astParentFullName, + methodInfo.lineNumber, + methodInfo.columnNumber, + methodInfo.offset + ) + Ast.storeInDiffGraph(Ast(typeDecl), dstGraph) + method.astParentFullName = typeDecl.fullName + method.astParentType = typeDecl.label + val functionBinding = NewBinding().name(normalizedName).methodFullName(normalizedFullName).signature(signature) + Ast(functionBinding).withBindsEdge(typeDecl, functionBinding).withRefEdge(functionBinding, method) + } + } + + override def run(dstGraph: DiffGraphBuilder): Unit = { + methodDeclarations.foreach { case (fullName, methodNodeInfo) => + val methodNode_ = methodNode(fullName, methodNodeInfo) + val parameterNodes = methodNodeInfo.parameter.map(p => Ast(parameterInNode(p))) + val stubAst = + methodStubAst( + methodNode_, + parameterNodes, + methodReturnNode(methodNodeInfo.returnType, methodNodeInfo.lineNumber, methodNodeInfo.columnNumber), + methodNodeInfo.modifier.map(m => Ast(NewModifier().modifierType(m))) + ) + val typeDeclAst = createFunctionTypeAndTypeDecl( + methodNodeInfo, + methodNode_, + methodNodeInfo.name, + fullName, + methodNodeInfo.signature, + dstGraph + ) + val ast = stubAst.merge(typeDeclAst) + Ast.storeInDiffGraph(ast, dstGraph) + } + } + +} diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/PreprocessorPass.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/PreprocessorPass.scala index 3a884d7a9257..a8e435413799 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/PreprocessorPass.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/PreprocessorPass.scala @@ -3,12 +3,15 @@ package io.joern.c2cpg.passes import io.joern.c2cpg.C2Cpg.DefaultIgnoredFolders import io.joern.c2cpg.Config import io.joern.c2cpg.parser.{CdtParser, FileDefaults} +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser.CommandObject import io.joern.x2cpg.SourceFiles import org.eclipse.cdt.core.dom.ast.{ - IASTPreprocessorIfStatement, IASTPreprocessorIfdefStatement, + IASTPreprocessorIfStatement, IASTPreprocessorStatement } +import org.slf4j.LoggerFactory import java.nio.file.Paths import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable @@ -16,9 +19,14 @@ import scala.collection.parallel.immutable.ParIterable class PreprocessorPass(config: Config) { - private val parser = new CdtParser(config) + private val logger = LoggerFactory.getLogger(classOf[PreprocessorPass]) + + private val compilationDatabase: List[CommandObject] = + config.compilationDatabase.map(JSONCompilationDatabaseParser.parse).getOrElse(List.empty) - def run(): ParIterable[String] = + private val parser = new CdtParser(config, compilationDatabase) + + private def sourceFilesFromDirectory(): ParIterable[String] = { SourceFiles .determine( config.inputPath, @@ -29,6 +37,32 @@ class PreprocessorPass(config: Config) { ) .par .flatMap(runOnPart) + } + + private def sourceFilesFromCompilationDatabase(compilationDatabaseFile: String): ParIterable[String] = { + if (compilationDatabase.isEmpty) { + logger.warn(s"'$compilationDatabaseFile' contains no source files.") + } + SourceFiles + .filterFiles( + compilationDatabase.map(_.compiledFile()), + config.inputPath, + ignoredDefaultRegex = Option(DefaultIgnoredFolders), + ignoredFilesRegex = Option(config.ignoredFilesRegex), + ignoredFilesPath = Option(config.ignoredFiles) + ) + .par + .flatMap(runOnPart) + } + + def run(): ParIterable[String] = { + if (config.compilationDatabase.isEmpty) { + sourceFilesFromDirectory() + } else { + sourceFilesFromCompilationDatabase(config.compilationDatabase.get) + } + + } private def preprocessorStatement2String(stmt: IASTPreprocessorStatement): Option[String] = stmt match { case s: IASTPreprocessorIfStatement => diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/TypeDeclNodePass.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/TypeDeclNodePass.scala index f8437a16004b..f4dc1ead9293 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/TypeDeclNodePass.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/TypeDeclNodePass.scala @@ -34,8 +34,8 @@ class TypeDeclNodePass(cpg: Cpg)(implicit withSchemaValidation: ValidationMode) .lineNumber(1) .astParentType(NodeTypes.NAMESPACE_BLOCK) .astParentFullName(fullName) - val blockNode = NewBlock().typeFullName(Defines.anyTypeName) - val methodReturn = newMethodReturnNode(Defines.anyTypeName, line = None, column = None) + val blockNode = NewBlock().typeFullName(Defines.Any) + val methodReturn = newMethodReturnNode(Defines.Any, line = None, column = None) Ast(includesFile).withChild( Ast(namespaceBlock) .withChild(Ast(fakeGlobalIncludesMethod).withChild(Ast(blockNode)).withChild(Ast(methodReturn))) diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala index c213b6e8fe71..aa47f4c8d423 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala @@ -1,33 +1,24 @@ package io.joern.c2cpg.utils -import java.util.concurrent.ConcurrentLinkedQueue -import scala.sys.process.{Process, ProcessLogger} import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters.* -object ExternalCommand extends io.joern.x2cpg.utils.ExternalCommand { +object ExternalCommand { - override def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - result match { - case Success(0) => + import io.joern.x2cpg.utils.ExternalCommand.ExternalCommandResult + + private val IsWin = scala.util.Properties.isWin + + def run(command: Seq[String], cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { + io.joern.x2cpg.utils.ExternalCommand.run(command, cwd, mergeStdErrInStdOut = true, extraEnv) match { + case ExternalCommandResult(0, stdOut, _) => Success(stdOut) - case Success(1) if IsWin && IncludeAutoDiscovery.gccAvailable() => + case ExternalCommandResult(1, stdOut, _) if IsWin && IncludeAutoDiscovery.gccAvailable() => // the command to query the system header file locations within a Windows // environment always returns Success(1) for whatever reason... Success(stdOut) - case _ => + case ExternalCommandResult(_, stdOut, _) => Failure(new RuntimeException(stdOut.mkString(System.lineSeparator()))) } } - override def run(command: String, cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { - val stdOutOutput = new ConcurrentLinkedQueue[String] - val processLogger = ProcessLogger(stdOutOutput.add, stdOutOutput.add) - val process = shellPrefix match { - case Nil => Process(command, new java.io.File(cwd), extraEnv.toList*) - case _ => Process(shellPrefix :+ command, new java.io.File(cwd), extraEnv.toList*) - } - handleRunResult(Try(process.!(processLogger)), stdOutOutput.asScala.toSeq, Nil) - } - } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala index 131434e29564..286f67d94cf4 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala @@ -13,13 +13,15 @@ object IncludeAutoDiscovery { private val IS_WIN = scala.util.Properties.isWin - val GCC_VERSION_COMMAND = "gcc --version" + val GCC_VERSION_COMMAND = Seq("gcc", "--version") private val CPP_INCLUDE_COMMAND = - if (IS_WIN) "gcc -xc++ -E -v . -o nul" else "gcc -xc++ -E -v /dev/null -o /dev/null" + if (IS_WIN) Seq("gcc", "-xc++", "-E", "-v", ".", "-o", "nul") + else Seq("gcc", "-xc++", "-E", "-v", "/dev/null", "-o", "/dev/null") private val C_INCLUDE_COMMAND = - if (IS_WIN) "gcc -xc -E -v . -o nul" else "gcc -xc -E -v /dev/null -o /dev/null" + if (IS_WIN) Seq("gcc", "-xc", "-E", "-v", ".", "-o", "nul") + else Seq("gcc", "-xc", "-E", "-v", "/dev/null", "-o", "/dev/null") // Only check once private var isGccAvailable: Option[Boolean] = None @@ -57,7 +59,7 @@ object IncludeAutoDiscovery { output.slice(startIndex, endIndex).map(p => Paths.get(p.trim).toRealPath()).toSet } - private def discoverPaths(command: String): Set[Path] = ExternalCommand.run(command, ".") match { + private def discoverPaths(command: Seq[String]): Set[Path] = ExternalCommand.run(command, ".") match { case Success(output) => extractPaths(output) case Failure(exception) => logger.warn(s"Unable to discover system include paths. Running '$command' failed.", exception) diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala index a050ffde48cd..644bf471a117 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/DataFlowTests.scala @@ -6,8 +6,7 @@ import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Identifier, Literal} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Table.AvailableWidthProvider -import overflowdb.traversal.toNodeTraversal +import flatgraph.help.Table.AvailableWidthProvider class DataFlowTests extends DataFlowCodeToCpgSuite { @@ -1082,7 +1081,7 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { cpg .call("bar") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } } @@ -1989,4 +1988,137 @@ class DataFlowTestsWithCallDepth extends DataFlowCodeToCpgSuite { ) } } + + "DataFlowTest73" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x%=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x%=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x%=2", 4), ("call1(x%=2)", 4))) + } + + "the literal in x%=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x%=2", 4), ("call2(x)", 5))) + } + + } + + "DataFlowTest74" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x^=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x^=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x^=2", 4), ("call1(x^=2)", 4))) + } + + "the literal in x^=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x^=2", 4), ("call2(x)", 5))) + } + } + + "DataFlowTest75" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x|=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x|=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x|=2", 4), ("call1(x|=2)", 4))) + } + + "the literal in x|=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x|=2", 4), ("call2(x)", 5))) + } + } + + "DataFlowTest76" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x&=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x&=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x&=2", 4), ("call1(x&=2)", 4))) + } + + "the literal in x&=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x&=2", 4), ("call2(x)", 5))) + } + } + + "DataFlowTest77" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x<<=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x<<=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x<<=2", 4), ("call1(x<<=2)", 4))) + } + + "the literal in x<<=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x<<=2", 4), ("call2(x)", 5))) + } + } + + "DataFlowTest78" should { + val cpg = code(""" + |int main(void) { + | int x = 5; + | call1(x>>=2); + | call2(x); + |} + |""".stripMargin) + + "the literal in x>>=2 should taint the outer expression" in { + val source = cpg.literal("2") + val sink = cpg.call("call1") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x>>=2", 4), ("call1(x>>=2)", 4))) + } + + "the literal in x>>=2 should taint the next occurrence of x" in { + val source = cpg.literal("2") + val sink = cpg.call("call2") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List(List(("x>>=2", 4), ("call2(x)", 5))) + } + } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/ReachingDefTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/ReachingDefTests.scala index c1c41699ed98..c8c0771dc094 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/ReachingDefTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/dataflow/ReachingDefTests.scala @@ -3,7 +3,7 @@ package io.joern.c2cpg.dataflow import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite import io.joern.dataflowengineoss.passes.reachingdef.ReachingDefFlowGraph import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ReachingDefTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/C2CpgHTTPServerTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/C2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..b39faa919658 --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/C2CpgHTTPServerTests.scala @@ -0,0 +1,83 @@ +package io.joern.c2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.util.Failure +import scala.util.Success +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable + +class C2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("c2cpgTestsHttpTest") + val file = dir / "main.c" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |int main$indexStr(int argc, char *argv[]) { + | print("Hello World!"); + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.c2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.c2cpg.Main.stop() + } + + "Using c2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("c2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("c2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain(s"main$index") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + } + } + +} diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromFileTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromFileTests.scala index e05a794841d5..fcd7e87c1502 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromFileTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromFileTests.scala @@ -3,7 +3,7 @@ package io.joern.c2cpg.io import better.files.File import io.joern.c2cpg.testfixtures.C2CpgSuite import io.shiftleft.semanticcpg.codedumper.CodeDumper -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/ExcludeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/ExcludeTests.scala index 4d750ebda36e..09c8dad7fe05 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/ExcludeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/ExcludeTests.scala @@ -4,7 +4,7 @@ import better.files.File import io.joern.c2cpg.Config import io.joern.c2cpg.C2Cpg import io.joern.x2cpg.X2Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks @@ -62,6 +62,28 @@ class ExcludeTests extends AnyWordSpec with Matchers with TableDrivenPropertyChe ) } + "Using case sensitive excludes" should { + "exclude the given files correctly" in { + if (scala.util.Properties.isWin) { + // both are written uppercase and are ignored nevertheless + testWithArguments(Seq("Folder", "Index.c"), "", Set("a.c", "foo.bar/d.c")) + } + if (scala.util.Properties.isMac) { + // Folder written uppercase and it is not ignored while Index.c is. + // This might be an issue within Files.isSameFile but we take it for now. + testWithArguments(Seq("Folder", "Index.c"), "", Set("a.c", "folder/b.c", "folder/c.c", "foo.bar/d.c")) + } + if (scala.util.Properties.isLinux) { + // both are written uppercase and are not ignored + testWithArguments( + Seq("Folder", "Index.c"), + "", + Set("a.c", "folder/b.c", "folder/c.c", "foo.bar/d.c", "index.c") + ) + } + } + } + "Using different excludes via program arguments" should { val testInput = Table( diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/FileHandlingTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/FileHandlingTests.scala new file mode 100644 index 000000000000..42950f8835f5 --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/FileHandlingTests.scala @@ -0,0 +1,75 @@ +package io.joern.c2cpg.io + +import better.files.File +import io.joern.c2cpg.parser.FileDefaults +import io.joern.c2cpg.testfixtures.CDefaultTestCpg +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.x2cpg.testfixtures.Code2CpgFixture +import io.shiftleft.semanticcpg.language.* + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +object FileHandlingTests { + private val brokenLinkedFile: String = "broken.c" + private val cyclicLinkedFile: String = "loop.c" +} + +class FileHandlingTests + extends Code2CpgFixture(() => + new CDefaultTestCpg(FileDefaults.C_EXT) { + override def codeFilePreProcessing(codeFile: Path): Unit = { + if (codeFile.toString.endsWith(FileHandlingTests.brokenLinkedFile)) { + File(codeFile).delete().symbolicLinkTo(File("does/not/exist.c")) + } + if (codeFile.toString.endsWith(FileHandlingTests.cyclicLinkedFile)) { + val dir = File(codeFile).delete().parent + val folderA = Paths.get(dir.toString(), "FolderA") + val folderB = Paths.get(dir.toString(), "FolderB") + val symlinkAtoB = folderA.resolve("LinkToB") + val symlinkBtoA = folderB.resolve("LinkToA") + Files.createDirectory(folderA) + Files.createDirectory(folderB) + Files.createSymbolicLink(symlinkAtoB, folderB) + Files.createSymbolicLink(symlinkBtoA, folderA) + } + } + } + .withOssDataflow(false) + .withSemantics(DefaultSemantics()) + .withPostProcessingPasses(false) + ) { + + "File handling 1" should { + val cpg = code( + """ + |int a() {} + |""".stripMargin, + "a.c" + ).moreCode("", FileHandlingTests.brokenLinkedFile) + + "not crash on broken symlinks" in { + val fileNames = cpg.file.name.l + fileNames should contain("a.c").and(not contain FileHandlingTests.brokenLinkedFile) + } + + } + + "File handling 2" should { + val cpg = code( + """ + |int a() {} + |""".stripMargin, + "a.c" + ).moreCode("", FileHandlingTests.cyclicLinkedFile) + + "not crash on cyclic symlinks" in { + val fileNames = cpg.file.name.l + fileNames should contain("a.c") + } + + } + +} diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/JSONCompilationDatabaseParserTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/JSONCompilationDatabaseParserTests.scala new file mode 100644 index 000000000000..a57c4612d8c9 --- /dev/null +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/JSONCompilationDatabaseParserTests.scala @@ -0,0 +1,114 @@ +package io.joern.c2cpg.io + +import better.files.File +import io.joern.c2cpg.parser.JSONCompilationDatabaseParser +import io.joern.c2cpg.C2Cpg +import io.joern.c2cpg.Config +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.types.structure.FileTraversal +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.nio.file.Paths + +class JSONCompilationDatabaseParserTests extends AnyWordSpec with Matchers { + + "Parsing a simple compile_commands.json" should { + "generate a proper list of CommandObjects" in { + val content = + """ + |[ + | { "directory": "/home/user/llvm/build", + | "arguments": ["/usr/bin/clang++", "-I/usr/include", "-I./include", "-DSOMEDEFA=With spaces, quotes and \\-es.", "-c", "-o", "file.o", "file.cc"], + | "file": "file.cc" }, + | { "directory": "/home/user/llvm/build", + | "command": "/usr/bin/clang++ -I/home/user/project/includes -DSOMEDEFB=\"With spaces, quotes and \\-es.\" -DSOMEDEFC -c -o file.o file.cc", + | "file": "file2.cc" } + |]""".stripMargin + + File.usingTemporaryFile("compile_commands.json") { commandJsonFile => + commandJsonFile.writeText(content) + + val commandObjects = JSONCompilationDatabaseParser.parse(commandJsonFile.pathAsString) + commandObjects.map(_.compiledFile()) shouldBe List( + Paths.get("/home/user/llvm/build/file.cc").toString, + Paths.get("/home/user/llvm/build/file2.cc").toString + ) + commandObjects.flatMap(_.defines()) shouldBe List( + ("SOMEDEFA", "With spaces, quotes and \\-es."), + ("SOMEDEFB", "\"With spaces, quotes and \\-es.\""), + ("SOMEDEFC", "") + ) + commandObjects.flatMap(_.includes()) shouldBe List("/usr/include", "./include", "/home/user/project/includes") + } + } + } + + private def newProjectUnderTest(): File = { + val dir = File.newTemporaryDirectory("c2cpgJSONCompilationDatabaseParserTests") + + val mainText = + """ + |int main(int argc, char *argv[]) { + | print("Hello World!"); + |} + |#ifdef SOMEDEFA + |void foo() {} + |#endif + |#ifdef SOMEDEFC + |void bar() {} + |#endif + |""".stripMargin + + val fileA = dir / "fileA.c" + fileA.createIfNotExists(createParents = true) + fileA.writeText(mainText) + fileA.deleteOnExit() + val fileB = dir / "fileB.c" + fileB.createIfNotExists(createParents = true) + fileB.writeText(mainText) + fileB.deleteOnExit() + val fileC = dir / "fileC.c" + fileC.createIfNotExists(createParents = true) + fileC.writeText(mainText) + fileC.deleteOnExit() + + val compilerCommands = dir / "compile_commands.json" + compilerCommands.createIfNotExists(createParents = true) + val content = s""" + |[ + | { "directory": "${dir.pathAsString}", + | "arguments": ["/usr/bin/clang++", "-Irelative", "-DSOMEDEFA=With spaces, quotes and \\-es.", "-c", "-o", "fileA.o", "fileA.cc"], + | "file": "fileA.c" }, + | { "directory": ".", + | "arguments": ["/usr/bin/clang++", "-Irelative", "-DSOMEDEFB=With spaces, quotes and \\-es.", "-c", "-o", "fileB.o", "fileB.cc"], + | "file": "${fileB.pathAsString}" } + |]""".stripMargin.replace("\\", "\\\\") // escape for tests under Windows + compilerCommands.writeText(content) + compilerCommands.deleteOnExit() + + dir.deleteOnExit() + } + + "Using a simple compile_commands.json" should { + "respect the files listed" in { + val cpgOutFile = File.newTemporaryFile("c2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val config = Config() + .withInputPath(input) + .withOutputPath(output) + .withCompilationDatabase((File(input) / "compile_commands.json").pathAsString) + val c2cpg = new C2Cpg() + val cpg = c2cpg.createCpg(config).get + cpg.file.nameNot(FileTraversal.UNKNOWN, "").name.sorted.l should contain theSameElementsAs List( + "fileA.c", + "fileB.c" + // fileC.c is ignored because it is not listed in the compile_commands.json + ) + cpg.method.nameNot("").name.sorted.l shouldBe List("foo", "main", "main") + } + } +} diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotAstGeneratorTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotAstGeneratorTests.scala index 76a1bc882684..32e598393484 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotAstGeneratorTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotAstGeneratorTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.io.dotgenerator import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DotAstGeneratorTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCdgGeneratorTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCdgGeneratorTests.scala index 08778bd8ec6e..550e5a2349dd 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCdgGeneratorTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCdgGeneratorTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.io.dotgenerator import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DotCdgGeneratorTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCfgGeneratorTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCfgGeneratorTests.scala index 4cd62011ce76..a281be9e1b01 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCfgGeneratorTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotCfgGeneratorTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.io.dotgenerator import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DotCfgGeneratorTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotDdgGeneratorTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotDdgGeneratorTests.scala index 846ca622d3d7..f043b7fbe7f5 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotDdgGeneratorTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/dotgenerator/DotDdgGeneratorTests.scala @@ -1,8 +1,8 @@ package io.joern.c2cpg.io.dotgenerator import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* class DotDdgGeneratorTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala index 56192a053e2f..94a80e22521e 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/macros/MacroHandlingTests.scala @@ -2,13 +2,13 @@ package io.joern.c2cpg.macros import io.joern.c2cpg.testfixtures.C2CpgSuite import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Block import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MacroHandlingTests extends C2CpgSuite { @@ -231,7 +231,7 @@ class MacroHandlingTests extends C2CpgSuite { """.stripMargin) "should not result in malformed CFGs when expanding a nested macro with block" in { - cpg.all.collectAll[Block].l.count(b => b.cfgOut.size > 1) shouldBe 0 + cpg.all.collectAll[Block].l.count(b => b._cfgOut.size > 1) shouldBe 0 } } @@ -298,6 +298,52 @@ class MacroHandlingTests extends C2CpgSuite { typeNumCall.columnNumber shouldBe Some(11) } } + + "MacroHandlingTests10" should { + + "have ast parents" in { + val cpg = code(""" + |#define FFSWAP(type,a,b) do{type SWAP_tmp=b; b=a; a=SWAP_tmp;}while(0) + |struct elem_to_channel { + | uint64_t av_position; + | uint8_t syn_ele; + | uint8_t elem_id; + | uint8_t aac_position; + |}; + |int main () { + | struct elem_to_channel e2c_vec[4 * 1] = { { 0 } }; + | int i = 1; + | FFSWAP(struct elem_to_channel, e2c_vec[i - 1], e2c_vec[i]); + |} + |""".stripMargin) + cpg.local.count(l => l._astIn.isEmpty) shouldBe 0 + cpg.local.count(l => l._astIn.size == 1) shouldBe 4 + cpg.local.count(l => l._astIn.size > 1) shouldBe 0 + } + + "only have locals with exactly one ast parent" in { + val cpg = code( + """ + |#define deleteReset(ptr) do { delete ptr; ptr = nullptr; } while(0) + |void func(void) { + | int *foo = new int; + | int *bar = new int; + | int *baz = new int; + | deleteReset(foo); + | deleteReset(bar); + | deleteReset(baz); + |} + |""".stripMargin, + "foo.cc" + ) + val List(foo) = cpg.local.nameExact("foo").l + foo._astIn.size shouldBe 1 + val List(bar) = cpg.local.nameExact("bar").l + bar._astIn.size shouldBe 1 + val List(baz) = cpg.local.nameExact("baz").l + baz._astIn.size shouldBe 1 + } + } } class CfgMacroTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala index 2068e17210ff..9996ac6b6995 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/MetaDataPassTests.scala @@ -2,14 +2,12 @@ package io.joern.c2cpg.passes import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import io.joern.x2cpg.passes.frontend.MetaDataPass import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import scala.jdk.CollectionConverters._ - class MetaDataPassTests extends AnyWordSpec with Matchers { "MetaDataPass" should { @@ -17,11 +15,11 @@ class MetaDataPassTests extends AnyWordSpec with Matchers { new MetaDataPass(cpg, Languages.C, "").createAndApply() "create exactly two nodes" in { - cpg.graph.V.asScala.size shouldBe 2 + cpg.graph.allNodes.size shouldBe 2 } "create no edges" in { - cpg.graph.E.asScala.size shouldBe 0 + cpg.graph.allNodes.outE.size shouldBe 0 } "create a metadata node with correct language" in { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala index 1fa6e7de118c..5f0bbdf7b68e 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/AstCreationPassTests.scala @@ -12,7 +12,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal -import overflowdb.traversal.toNodeTraversal class AstCreationPassTests extends AstC2CpgSuite { @@ -112,8 +111,8 @@ class AstCreationPassTests extends AstC2CpgSuite { val lambda1FullName = "0" val lambda2FullName = "1" - cpg.local.name("x").order.l shouldBe List(1) - cpg.local.name("y").order.l shouldBe List(3) + cpg.local.nameExact("x").order.l shouldBe List(1) + cpg.local.nameExact("y").order.l shouldBe List(3) inside(cpg.assignment.l) { case List(assignment1, assignment2) => assignment1.order shouldBe 2 @@ -176,7 +175,7 @@ class AstCreationPassTests extends AstC2CpgSuite { val lambdaFullName = s"Foo.$lambdaName" val signature = s"int(int,int)" - cpg.member.name("x").order.l shouldBe List(1) + cpg.member.nameExact("x").order.l shouldBe List(1) inside(cpg.assignment.l) { case List(assignment1) => inside(assignment1.astMinusRoot.isMethodRef.l) { case List(ref) => @@ -219,7 +218,7 @@ class AstCreationPassTests extends AstC2CpgSuite { val lambdaFullName = s"A.B.Foo.$lambdaName" val signature = s"int(int,int)" - cpg.member.name("x").order.l shouldBe List(1) + cpg.member.nameExact("x").order.l shouldBe List(1) inside(cpg.assignment.l) { case List(assignment1) => inside(assignment1.astMinusRoot.isMethodRef.l) { case List(ref) => @@ -265,9 +264,9 @@ class AstCreationPassTests extends AstC2CpgSuite { val lambda2Name = "1" val signature2 = s"int(int)" - cpg.local.name("x").order.l shouldBe List(1) - cpg.local.name("foo1").order.l shouldBe List(3) - cpg.local.name("foo2").order.l shouldBe List(5) + cpg.local.nameExact("x").order.l shouldBe List(1) + cpg.local.nameExact("foo1").order.l shouldBe List(3) + cpg.local.nameExact("foo2").order.l shouldBe List(5) inside(cpg.assignment.l) { case List(assignment1, assignment2, assignment3) => assignment1.order shouldBe 2 @@ -338,7 +337,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "be correct for empty method" in { val cpg = code("void method(int x) { }") - inside(cpg.method.name("method").astChildren.l) { + inside(cpg.method.nameExact("method").astChildren.l) { case List(param: MethodParameterIn, _: Block, ret: MethodReturn) => ret.typeFullName shouldBe "void" param.typeFullName shouldBe "int" @@ -354,7 +353,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.typeFullName shouldBe "a_struct_type*" param.name shouldBe "a_struct" param.code shouldBe "a_struct_type *a_struct" @@ -369,7 +368,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.code shouldBe "struct date *date" param.typeFullName shouldBe "date*" param.name shouldBe "date" @@ -384,7 +383,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.typeFullName shouldBe "int[]" param.name shouldBe "x" } @@ -398,7 +397,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.typeFullName shouldBe "int[]" param.name shouldBe "" } @@ -412,7 +411,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.typeFullName shouldBe "a_struct_type[]" param.name shouldBe "a_struct" } @@ -426,7 +425,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | free(x); |} |""".stripMargin) - inside(cpg.method.name("method").parameter.l) { case List(param: MethodParameterIn) => + inside(cpg.method.nameExact("method").parameter.l) { case List(param: MethodParameterIn) => param.typeFullName shouldBe "a_struct_type[]*" param.name shouldBe "a_struct" } @@ -438,7 +437,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | int local = 1; |} |""".stripMargin) - inside(cpg.method.name("method").block.astChildren.l) { case List(local: Local, call: Call) => + inside(cpg.method.nameExact("method").block.astChildren.l) { case List(local: Local, call: Call) => local.name shouldBe "local" local.typeFullName shouldBe "int" local.order shouldBe 1 @@ -467,7 +466,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "test.cpp" ) - inside(cpg.method.name("method").block.astChildren.l) { case List(_, call1: Call, _, call2: Call) => + inside(cpg.method.nameExact("method").block.astChildren.l) { case List(_, call1: Call, _, call2: Call) => call1.name shouldBe Operators.assignment inside(call2.astChildren.l) { case List(identifier: Identifier, call: Call) => identifier.name shouldBe "is_std_array_v" @@ -492,7 +491,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |void method(int x) { | int local = x; |}""".stripMargin) - cpg.local.name("local").order.l shouldBe List(1) + cpg.local.nameExact("local").order.l shouldBe List(1) inside(cpg.method("method").block.astChildren.assignment.source.l) { case List(identifier: Identifier) => identifier.code shouldBe "x" identifier.typeFullName shouldBe "int" @@ -501,6 +500,20 @@ class AstCreationPassTests extends AstC2CpgSuite { } } + "be correct for decl assignment with references" in { + val cpg = code( + """ + |int addrOfLocalRef(struct x **foo) { + | struct x &bar = **foo; + | *foo = &bar; + |}""".stripMargin, + "foo.cc" + ) + val List(barLocal) = cpg.method.nameExact("addrOfLocalRef").local.l + barLocal.name shouldBe "bar" + barLocal.code shouldBe "struct x& bar" + } + "be correct for decl assignment of multiple locals" in { val cpg = code(""" |void method(int x, int y) { @@ -544,7 +557,7 @@ class AstCreationPassTests extends AstC2CpgSuite { val localZ = cpg.local.order(3) localZ.name.l shouldBe List("z") - inside(cpg.method.name("method").ast.isCall.name(Operators.assignment).cast[OpNodes.Assignment].l) { + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.assignment).cast[OpNodes.Assignment].l) { case List(assignment) => assignment.target.code shouldBe "x" assignment.source.start.isCall.name.l shouldBe List(Operators.addition) @@ -566,7 +579,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | } |} """.stripMargin) - inside(cpg.method.name("method").block.astChildren.l) { case List(local: Local, innerBlock: Block) => + inside(cpg.method.nameExact("method").block.astChildren.l) { case List(local: Local, innerBlock: Block) => local.name shouldBe "x" local.order shouldBe 1 inside(innerBlock.astChildren.l) { case List(localInBlock: Local) => @@ -584,7 +597,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | } |} """.stripMargin) - inside(cpg.method.name("method").block.astChildren.isControlStructure.l) { + inside(cpg.method.nameExact("method").block.astChildren.isControlStructure.l) { case List(controlStruct: ControlStructure) => controlStruct.code shouldBe "while (x < 1)" controlStruct.controlStructureType shouldBe ControlStructureTypes.WHILE @@ -606,7 +619,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | } |} """.stripMargin) - inside(cpg.method.name("method").controlStructure.l) { case List(controlStruct: ControlStructure) => + inside(cpg.method.nameExact("method").controlStructure.l) { case List(controlStruct: ControlStructure) => controlStruct.code shouldBe "if (x > 0)" controlStruct.controlStructureType shouldBe ControlStructureTypes.IF inside(controlStruct.condition.l) { case List(cndNode) => @@ -628,7 +641,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | } |} """.stripMargin) - inside(cpg.method.name("method").controlStructure.l) { case List(ifStmt, elseStmt) => + inside(cpg.method.nameExact("method").controlStructure.l) { case List(ifStmt, elseStmt) => ifStmt.controlStructureType shouldBe ControlStructureTypes.IF ifStmt.code shouldBe "if (x > 0)" elseStmt.controlStructureType shouldBe ControlStructureTypes.ELSE @@ -653,7 +666,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | int x = (true ? vlc_dccp_CreateFD : vlc_datagram_CreateFD)(fd); | } """.stripMargin) - inside(cpg.method.name("method").ast.isCall.name(Operators.conditional).l) { case List(call) => + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.conditional).l) { case List(call) => call.code shouldBe "true ? vlc_dccp_CreateFD : vlc_datagram_CreateFD" } } @@ -668,7 +681,7 @@ class AstCreationPassTests extends AstC2CpgSuite { // `cpg.method.call` will not work at this stage // either because there are no CONTAINS edges - inside(cpg.method.name("method").ast.isCall.name(Operators.conditional).l) { case List(call) => + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.conditional).l) { case List(call) => call.code shouldBe "(foo == 1) ? bar : 0" inside(call.argument.l) { case List(condition, trueBranch, falseBranch) => condition.argumentIndex shouldBe 1 @@ -691,7 +704,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |}""".stripMargin, "file.cpp" ) - inside(cpg.method.name("method").controlStructure.l) { case List(forStmt) => + inside(cpg.method.nameExact("method").controlStructure.l) { case List(forStmt) => forStmt.controlStructureType shouldBe ControlStructureTypes.FOR inside(forStmt.astChildren.order(1).l) { case List(ident: Identifier) => ident.code shouldBe "list" @@ -717,7 +730,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "test.cpp" ) - inside(cpg.method.name("method").controlStructure.l) { case List(forStmt) => + inside(cpg.method.nameExact("method").controlStructure.l) { case List(forStmt) => forStmt.controlStructureType shouldBe ControlStructureTypes.FOR inside(forStmt.astChildren.order(1).l) { case List(ident) => ident.code shouldBe "foo" @@ -741,7 +754,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | } |} """.stripMargin) - inside(cpg.method.name("method").controlStructure.l) { case List(forStmt) => + inside(cpg.method.nameExact("method").controlStructure.l) { case List(forStmt) => forStmt.controlStructureType shouldBe ControlStructureTypes.FOR childContainsAssignments(forStmt, 1, List("x = 0", "y = 0")) @@ -768,10 +781,10 @@ class AstCreationPassTests extends AstC2CpgSuite { |} """.stripMargin) cpg.method - .name("method") + .nameExact("method") .ast .isCall - .name(Operators.preIncrement) + .nameExact(Operators.preIncrement) .argument(1) .code .l shouldBe List("x") @@ -819,10 +832,10 @@ class AstCreationPassTests extends AstC2CpgSuite { |} """.stripMargin) cpg.method - .name("method") + .nameExact("method") .ast .isCall - .name("foo") + .nameExact("foo") .argument(1) .code .l shouldBe List("x") @@ -835,7 +848,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | foo(x); |} """.stripMargin) - inside(cpg.method.name("method").ast.isCall.l) { case List(call: Call) => + inside(cpg.method.nameExact("method").ast.isCall.l) { case List(call: Call) => call.code shouldBe "foo(x)" call.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH val rec = call.receiver.l @@ -850,7 +863,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | x.a; |} """.stripMargin) - inside(cpg.method.name("method").ast.isCall.name(Operators.fieldAccess).l) { case List(call) => + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.fieldAccess).l) { case List(call) => val arg1 = call.argument(1) val arg2 = call.argument(2) arg1.isIdentifier shouldBe true @@ -869,7 +882,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | x->a; |} """.stripMargin) - inside(cpg.method.name("method").ast.isCall.name(Operators.indirectFieldAccess).l) { case List(call) => + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.indirectFieldAccess).l) { case List(call) => val arg1 = call.argument(1) val arg2 = call.argument(2) arg1.isIdentifier shouldBe true @@ -888,7 +901,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return (x->a)(1, 2); |} """.stripMargin) - inside(cpg.method.name("method").ast.isCall.name(Operators.indirectFieldAccess).l) { case List(call) => + inside(cpg.method.nameExact("method").ast.isCall.nameExact(Operators.indirectFieldAccess).l) { case List(call) => val arg1 = call.argument(1) val arg2 = call.argument(2) arg1.isIdentifier shouldBe true @@ -909,9 +922,9 @@ class AstCreationPassTests extends AstC2CpgSuite { | return (*strLenFunc)("123"); |} """.stripMargin) - inside(cpg.method.name("main").ast.isCall.codeExact("(*strLenFunc)(\"123\")").l) { case List(call) => - call.name shouldBe Defines.operatorPointerCall - call.methodFullName shouldBe Defines.operatorPointerCall + inside(cpg.method.nameExact("main").ast.isCall.codeExact("(*strLenFunc)(\"123\")").l) { case List(call) => + call.name shouldBe Defines.OperatorPointerCall + call.methodFullName shouldBe Defines.OperatorPointerCall } } @@ -923,13 +936,13 @@ class AstCreationPassTests extends AstC2CpgSuite { |} """.stripMargin) cpg.method - .name("method") + .nameExact("method") .ast .isCall - .name(Operators.sizeOf) + .nameExact(Operators.sizeOf) .argument(1) .isIdentifier - .name("a") + .nameExact("a") .argumentIndex(1) .size shouldBe 1 } @@ -942,13 +955,13 @@ class AstCreationPassTests extends AstC2CpgSuite { |} """.stripMargin) cpg.method - .name("method") + .nameExact("method") .ast .isCall - .name(Operators.sizeOf) + .nameExact(Operators.sizeOf) .argument(1) .isIdentifier - .name("a") + .nameExact("a") .argumentIndex(1) .size shouldBe 1 } @@ -962,13 +975,13 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) cpg.method - .name("method") + .nameExact("method") .ast .isCall - .name(Operators.sizeOf) + .nameExact(Operators.sizeOf) .argument(1) .isIdentifier - .name("int") + .nameExact("int") .argumentIndex(1) .size shouldBe 1 } @@ -981,7 +994,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | void method() { | }; """.stripMargin) - cpg.method.name("method").size shouldBe 1 + cpg.method.nameExact("method").size shouldBe 1 } "be correct for empty named struct" in { @@ -989,14 +1002,14 @@ class AstCreationPassTests extends AstC2CpgSuite { | struct foo { | }; """.stripMargin) - cpg.typeDecl.name("foo").size shouldBe 1 + cpg.typeDecl.nameExact("foo").size shouldBe 1 } "be correct for struct decl" in { val cpg = code(""" | struct foo; """.stripMargin) - cpg.typeDecl.name("foo").size shouldBe 1 + cpg.typeDecl.nameExact("foo").size shouldBe 1 } "be correct for named struct with single field" in { @@ -1006,10 +1019,10 @@ class AstCreationPassTests extends AstC2CpgSuite { | }; """.stripMargin) cpg.typeDecl - .name("foo") + .nameExact("foo") .member .code("x") - .name("x") + .nameExact("x") .typeFullName("int") .size shouldBe 1 } @@ -1022,7 +1035,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | int z; | }; """.stripMargin) - cpg.typeDecl.name("foo").member.code.toSetMutable shouldBe Set("x", "y", "z") + cpg.typeDecl.nameExact("foo").member.code.toSetMutable shouldBe Set("x", "y", "z") } "be correct for named struct with nested struct" in { @@ -1037,12 +1050,12 @@ class AstCreationPassTests extends AstC2CpgSuite { | }; | }; """.stripMargin) - inside(cpg.typeDecl.name("foo").l) { case List(fooStruct: TypeDecl) => - fooStruct.member.name("x").size shouldBe 1 + inside(cpg.typeDecl.nameExact("foo").l) { case List(fooStruct: TypeDecl) => + fooStruct.member.nameExact("x").size shouldBe 1 inside(fooStruct.astChildren.isTypeDecl.l) { case List(barStruct: TypeDecl) => - barStruct.member.name("y").size shouldBe 1 + barStruct.member.nameExact("y").size shouldBe 1 inside(barStruct.astChildren.isTypeDecl.l) { case List(foo2Struct: TypeDecl) => - foo2Struct.member.name("z").size shouldBe 1 + foo2Struct.member.nameExact("z").size shouldBe 1 } } } @@ -1053,7 +1066,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |typedef struct foo { |} abc; """.stripMargin) - cpg.typeDecl.name("foo").aliasTypeFullName("abc").size shouldBe 1 + cpg.typeDecl.nameExact("foo").aliasTypeFullName("abc").size shouldBe 1 } "be correct for struct with local" in { @@ -1067,7 +1080,7 @@ class AstCreationPassTests extends AstC2CpgSuite { x.name shouldBe "x" x.typeFullName shouldBe "int" } - cpg.typeDecl.name("B").size shouldBe 1 + cpg.typeDecl.nameExact("B").size shouldBe 1 inside(cpg.local.l) { case List(localA, localB) => localA.name shouldBe "a" localA.typeFullName shouldBe "A" @@ -1104,10 +1117,10 @@ class AstCreationPassTests extends AstC2CpgSuite { | i = 0; |} """.stripMargin) - val List(localMyOtherFs) = cpg.method("main").local.name("my_other_fs").l + val List(localMyOtherFs) = cpg.method("main").local.nameExact("my_other_fs").l localMyOtherFs.order shouldBe 2 localMyOtherFs.referencingIdentifiers.name.l shouldBe List("my_other_fs") - val List(localMyFs) = cpg.local.name("my_fs").l + val List(localMyFs) = cpg.local.nameExact("my_fs").l localMyFs.order shouldBe 4 localMyFs.referencingIdentifiers.name.l shouldBe List("my_fs") cpg.typeDecl.nameNot(NamespaceTraversal.globalNamespaceName).fullName.l.distinct shouldBe List("filesystem") @@ -1118,7 +1131,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |typedef enum foo { |} abc; """.stripMargin) - cpg.typeDecl.name("foo").aliasTypeFullName("abc").size shouldBe 1 + cpg.typeDecl.nameExact("foo").aliasTypeFullName("abc").size shouldBe 1 } "be correct for classes with friends" in { @@ -1150,7 +1163,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) cpg.typeDecl - .name("Derived") + .nameExact("Derived") .count(_.inheritsFromTypeFullName == List("Base")) shouldBe 1 } @@ -1161,7 +1174,7 @@ class AstCreationPassTests extends AstC2CpgSuite { """.stripMargin, "file.cpp" ) - inside(cpg.call.name(Operators.cast).l) { case List(call: Call) => + inside(cpg.call.nameExact(Operators.cast).l) { case List(call: Call) => call.argument(2).code shouldBe "{ 1 }" call.argument(1).code shouldBe "int" } @@ -1300,7 +1313,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) cpg.typeDecl - .name("Y") + .nameExact("Y") .l .size shouldBe 1 } @@ -1319,7 +1332,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) cpg.method - .name("f") + .nameExact("f") .l .size shouldBe 1 } @@ -1352,10 +1365,10 @@ class AstCreationPassTests extends AstC2CpgSuite { |} |""".stripMargin) cpg.method - .name("foo") + .nameExact("foo") .ast .isCall - .name("bar") + .nameExact("bar") .argument .code("x") .size shouldBe 1 @@ -1368,16 +1381,14 @@ class AstCreationPassTests extends AstC2CpgSuite { |} |""".stripMargin) // TODO no step class defined for `Return` nodes - cpg.method.name("d").ast.isReturn.astChildren.order(1).isCall.code.l shouldBe List("x * 2") + cpg.method.nameExact("d").ast.isReturn.astChildren.order(1).isCall.code.l shouldBe List("x * 2") cpg.method - .name("d") + .nameExact("d") .ast .isReturn - .outE(EdgeTypes.ARGUMENT) + .out(EdgeTypes.ARGUMENT) .head - .inNode() - .get - .asInstanceOf[CallDb] + .asInstanceOf[Call] .code shouldBe "x * 2" } @@ -1387,7 +1398,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return x * 2; |} |""".stripMargin) - cpg.call.name(Operators.multiplication).code.l shouldBe List("x * 2") + cpg.call.nameExact(Operators.multiplication).code.l shouldBe List("x * 2") } "be correct for unary method calls" in { @@ -1396,7 +1407,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return !b; |} |""".stripMargin) - cpg.call.name(Operators.logicalNot).argument(1).code.l shouldBe List("b") + cpg.call.nameExact(Operators.logicalNot).argument(1).code.l shouldBe List("b") } "be correct for unary expr" in { @@ -1407,7 +1418,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return end ? (int)(end - str) : max; | } |""".stripMargin) - inside(cpg.call.name(Operators.cast).astChildren.l) { case List(tpe: Unknown, call: Call) => + inside(cpg.call.nameExact(Operators.cast).astChildren.l) { case List(tpe: Unknown, call: Call) => call.code shouldBe "end - str" call.argumentIndex shouldBe 2 tpe.code shouldBe "int" @@ -1423,8 +1434,8 @@ class AstCreationPassTests extends AstC2CpgSuite { | return pos; |} |""".stripMargin) - cpg.call.name(Operators.postIncrement).argument(1).code("x").size shouldBe 1 - cpg.call.name(Operators.postDecrement).argument(1).code("x").size shouldBe 1 + cpg.call.nameExact(Operators.postIncrement).argument(1).code("x").size shouldBe 1 + cpg.call.nameExact(Operators.postDecrement).argument(1).code("x").size shouldBe 1 } "be correct for conditional expressions containing calls" in { @@ -1433,7 +1444,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return x > 0 ? x : -x; |} |""".stripMargin) - cpg.call.name(Operators.conditional).argument.code.l shouldBe List("x > 0", "x", "-x") + cpg.call.nameExact(Operators.conditional).argument.code.l shouldBe List("x > 0", "x", "-x") } "be correct for sizeof expressions" in { @@ -1442,7 +1453,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return sizeof(int); |} |""".stripMargin) - inside(cpg.call.name(Operators.sizeOf).argument(1).l) { case List(i: Identifier) => + inside(cpg.call.nameExact(Operators.sizeOf).argument(1).l) { case List(i: Identifier) => i.code shouldBe "int" i.name shouldBe "int" } @@ -1459,7 +1470,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return x[0]; |} |""".stripMargin) - cpg.call.name(Operators.indirectIndexAccess).argument.code.l shouldBe List("x", "0") + cpg.call.nameExact(Operators.indirectIndexAccess).argument.code.l shouldBe List("x", "0") } "be correct for type casts" in { @@ -1468,7 +1479,7 @@ class AstCreationPassTests extends AstC2CpgSuite { | return (int) x; |} |""".stripMargin) - cpg.call.name(Operators.cast).argument.code.l shouldBe List("int", "x") + cpg.call.nameExact(Operators.cast).argument.code.l shouldBe List("int", "x") } "be correct for 'new' array" in { @@ -1482,7 +1493,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) // TODO: ".new" is not part of Operators - cpg.call.name(".new").code("new int\\[n\\]").argument.code("int").size shouldBe 1 + cpg.call.nameExact(".new").code("new int\\[n\\]").argument.code("int").size shouldBe 1 } "be correct for 'new' with explicit identifier" in { @@ -1496,7 +1507,7 @@ class AstCreationPassTests extends AstC2CpgSuite { "file.cpp" ) // TODO: ".new" is not part of Operators - val List(newCall) = cpg.call.name(".new").l + val List(newCall) = cpg.call.nameExact(".new").l val List(string, hi, buf) = newCall.argument.l string.argumentIndex shouldBe 1 string.code shouldBe "string" @@ -1510,17 +1521,61 @@ class AstCreationPassTests extends AstC2CpgSuite { "be correct for array size" in { val cpg = code(""" |int main() { - | char buf[256]; - | printf("%s", buf); + | char bufA[256]; + | char bufB[1+2]; |} |""".stripMargin) - inside(cpg.local.l) { case List(buf: Local) => - buf.typeFullName shouldBe "char[256]" - buf.name shouldBe "buf" - buf.code shouldBe "char[256] buf" + inside(cpg.call.nameExact(Operators.assignment).l) { case List(bufCallAAssign: Call, bufCallBAssign: Call) => + val List(bufAId, bufCallA) = bufCallAAssign.argument.l + bufAId.code shouldBe "bufA" + val List(bufBId, bufCallB) = bufCallBAssign.argument.l + bufBId.code shouldBe "bufB" + + inside(cpg.call.nameExact(Operators.alloc).l) { case List(bufCallAAlloc: Call, bufCallBAlloc: Call) => + bufCallAAlloc shouldBe bufCallA + bufCallBAlloc shouldBe bufCallB + + bufCallAAlloc.code shouldBe "bufA[256]" + bufCallAAlloc.typeFullName shouldBe "char[256]" + val List(argA) = bufCallAAlloc.argument.isLiteral.l + argA.code shouldBe "256" + + bufCallBAlloc.code shouldBe "bufB[1+2]" + bufCallBAlloc.typeFullName shouldBe "char[1+2]" + val List(argB) = bufCallBAlloc.argument.isCall.l + argB.name shouldBe Operators.addition + argB.code shouldBe "1+2" + val List(one, two) = argB.argument.isLiteral.l + one.code shouldBe "1" + two.code shouldBe "2" + } + } + + inside(cpg.local.l) { case List(bufA: Local, bufB: Local) => + bufA.typeFullName shouldBe "char[256]" + bufA.name shouldBe "bufA" + bufA.code shouldBe "char[256] bufA" + + bufB.typeFullName shouldBe "char[1+2]" + bufB.name shouldBe "bufB" + bufB.code shouldBe "char[1+2] bufB" } } + "be correct for empty array init" in { + val cpg = code(""" + |void other(void) { + | int i = 0; + | char str[] = "abc"; + | printf("%d %s", i, str); + |} + |""".stripMargin) + val List(str1, str2) = cpg.identifier.nameExact("str").l + str1.typeFullName shouldBe "char[]" + str2.typeFullName shouldBe "char[]" + cpg.call.nameExact(Operators.alloc) shouldBe empty + } + "be correct for array init" in { val cpg = code(""" |int x[] = {0, 1, 2, 3}; @@ -1658,6 +1713,27 @@ class AstCreationPassTests extends AstC2CpgSuite { } } + "be correct for method refs from function pointers" in { + val cpg = code(""" + |uid_t getuid(void); + |void someFunction() {} + |void checkFunctionPointerComparison() { + | if (getuid == 0 || someFunction == 0) {} + |} + |""".stripMargin) + val List(methodA) = cpg.method.fullNameExact("getuid").l + val List(methodB) = cpg.method.fullNameExact("someFunction").l + val List(methodC) = cpg.method.fullNameExact("checkFunctionPointerComparison").l + inside(cpg.call.nameExact(Operators.equals).l) { case List(callA: Call, callB: Call) => + val getuidRef = callA.argument(1).asInstanceOf[MethodRef] + getuidRef.methodFullName shouldBe methodA.fullName + getuidRef.typeFullName shouldBe methodA.methodReturn.typeFullName + val someFunctionRef = callB.argument(1).asInstanceOf[MethodRef] + someFunctionRef.methodFullName shouldBe methodB.fullName + someFunctionRef.typeFullName shouldBe methodB.methodReturn.typeFullName + } + } + "be correct for locals for array init" in { val cpg = code(""" |bool x[2] = { TRUE, FALSE }; @@ -1738,7 +1814,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(".new").codeExact("new Foo(n, 42)").argument.code("Foo").size shouldBe 1 + cpg.call.nameExact(".new").codeExact("new Foo(n, 42)").argument.code("Foo").size shouldBe 1 } "be correct for simple 'delete'" in { @@ -1750,7 +1826,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.delete).code("delete n").argument.code("n").size shouldBe 1 + cpg.call.nameExact(Operators.delete).code("delete n").argument.code("n").size shouldBe 1 } "be correct for array 'delete'" in { @@ -1762,7 +1838,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.delete).codeExact("delete[] n").argument.code("n").size shouldBe 1 + cpg.call.nameExact(Operators.delete).codeExact("delete[] n").argument.code("n").size shouldBe 1 } "be correct for const_cast" in { @@ -1775,7 +1851,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.cast).codeExact("const_cast(n)").argument.code.l shouldBe List("int", "n") + cpg.call.nameExact(Operators.cast).codeExact("const_cast(n)").argument.code.l shouldBe List("int", "n") } "be correct for static_cast" in { @@ -1788,7 +1864,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.cast).codeExact("static_cast(n)").argument.code.l shouldBe List("int", "n") + cpg.call.nameExact(Operators.cast).codeExact("static_cast(n)").argument.code.l shouldBe List("int", "n") } "be correct for dynamic_cast" in { @@ -1801,7 +1877,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.cast).codeExact("dynamic_cast(n)").argument.code.l shouldBe List("int", "n") + cpg.call.nameExact(Operators.cast).codeExact("dynamic_cast(n)").argument.code.l shouldBe List("int", "n") } "be correct for reinterpret_cast" in { @@ -1814,7 +1890,7 @@ class AstCreationPassTests extends AstC2CpgSuite { |""".stripMargin, "file.cpp" ) - cpg.call.name(Operators.cast).codeExact("reinterpret_cast(n)").argument.code.l shouldBe List("int", "n") + cpg.call.nameExact(Operators.cast).codeExact("reinterpret_cast(n)").argument.code.l shouldBe List("int", "n") } "be correct for designated initializers in plain C" in { @@ -1823,14 +1899,14 @@ class AstCreationPassTests extends AstC2CpgSuite { | int a[3] = { [1] = 5, [2] = 10, [3 ... 9] = 15 }; |}; """.stripMargin) - inside(cpg.assignment.head.astChildren.l) { case List(ident: Identifier, call: Call) => + inside(cpg.assignment.l(1).astChildren.l) { case List(ident: Identifier, call: Call) => ident.typeFullName shouldBe "int[3]" ident.order shouldBe 1 call.code shouldBe "{ [1] = 5, [2] = 10, [3 ... 9] = 15 }" call.order shouldBe 2 call.name shouldBe Operators.arrayInitializer call.methodFullName shouldBe Operators.arrayInitializer - val children = call.astMinusRoot.isCall.name(Operators.assignment).l + val children = call.astMinusRoot.isCall.nameExact(Operators.assignment).l val args = call.argument.astChildren.l inside(children) { case List(call1, call2, call3) => call1.code shouldBe "[1] = 5" @@ -1863,14 +1939,14 @@ class AstCreationPassTests extends AstC2CpgSuite { """.stripMargin, "test.cpp" ) - inside(cpg.assignment.head.astChildren.l) { case List(ident: Identifier, call: Call) => + inside(cpg.assignment.l(1).astChildren.l) { case List(ident: Identifier, call: Call) => ident.typeFullName shouldBe "int[3]" ident.order shouldBe 1 call.code shouldBe "{ [1] = 5, [2] = 10, [3 ... 9] = 15 }" call.order shouldBe 2 call.name shouldBe Operators.arrayInitializer call.methodFullName shouldBe Operators.arrayInitializer - val children = call.astMinusRoot.isCall.name(Operators.assignment).l + val children = call.astMinusRoot.isCall.nameExact(Operators.assignment).l val args = call.argument.astChildren.l inside(children) { case List(call1, call2, call3) => call1.code shouldBe "[1] = 5" @@ -2047,8 +2123,8 @@ class AstCreationPassTests extends AstC2CpgSuite { | x = 1; | } """.stripMargin) - cpg.method.name("method").lineNumber.l shouldBe List(6) - cpg.method.name("method").block.assignment.lineNumber.l shouldBe List(8) + cpg.method.nameExact("method").lineNumber.l shouldBe List(6) + cpg.method.nameExact("method").block.assignment.lineNumber.l shouldBe List(8) } // for https://github.com/ShiftLeftSecurity/codepropertygraph/issues/1321 @@ -2123,7 +2199,7 @@ class AstCreationPassTests extends AstC2CpgSuite { val cpg = code("class Foo { char (*(*x())[5])() }", "test.cpp") val List(method) = cpg.method.nameNot("").l method.name shouldBe "x" - method.fullName shouldBe "Foo.x:char (* (*)[5])()()" + method.fullName shouldBe "Foo.x:char(*(*)[5])()()" method.code shouldBe "char (*(*x())[5])()" method.signature shouldBe "char()" } @@ -2135,10 +2211,10 @@ class AstCreationPassTests extends AstC2CpgSuite { | char *x; |} |""".stripMargin) - cpg.member.name("z").typeFullName.head shouldBe "char*" - cpg.parameter.name("y").typeFullName.head shouldBe "char*" - cpg.local.name("x").typeFullName.head shouldBe "char*" - cpg.method.name("a").methodReturn.typeFullName.head shouldBe "char*" + cpg.member.nameExact("z").typeFullName.head shouldBe "char*" + cpg.parameter.nameExact("y").typeFullName.head shouldBe "char*" + cpg.local.nameExact("x").typeFullName.head shouldBe "char*" + cpg.method.nameExact("a").methodReturn.typeFullName.head shouldBe "char*" } "be consistent with array types" in { @@ -2148,9 +2224,9 @@ class AstCreationPassTests extends AstC2CpgSuite { | char x[1]; |} |""".stripMargin) - cpg.member.name("z").typeFullName.head shouldBe "char[1]" - cpg.parameter.name("y").typeFullName.head shouldBe "char[1]" - cpg.local.name("x").typeFullName.head shouldBe "char[1]" + cpg.member.nameExact("z").typeFullName.head shouldBe "char[1]" + cpg.parameter.nameExact("y").typeFullName.head shouldBe "char[1]" + cpg.local.nameExact("x").typeFullName.head shouldBe "char[1]" } "be consistent with long number types" in { @@ -2164,7 +2240,9 @@ class AstCreationPassTests extends AstC2CpgSuite { val List(bufLocal) = cpg.local.nameExact("buf").l bufLocal.typeFullName shouldBe "char[0x111111111111111]" bufLocal.code shouldBe "char[0x111111111111111] buf" - cpg.literal.code.l shouldBe List("0x111111111111111") + val List(bufAllocCall) = cpg.call.nameExact(Operators.alloc).l + bufAllocCall.code shouldBe "buf[BUFSIZE]" + bufAllocCall.argument.ast.isLiteral.code.l shouldBe List("0x111111111111111") } } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallConventionsTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallConventionsTests.scala index 1adaddce886d..41f1ee51627c 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallConventionsTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallConventionsTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.AstC2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class CallConventionsTests extends AstC2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala index a7efec6f1df8..b928b1b3d38d 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/CallTests.scala @@ -9,8 +9,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.Literal import io.shiftleft.semanticcpg.language.NoResolve import io.shiftleft.semanticcpg.language.* -import java.nio.file.{Files, Path} - class CallTests extends C2CpgSuite { implicit val resolver: NoResolve.type = NoResolve @@ -112,7 +110,7 @@ class CallTests extends C2CpgSuite { "have the correct callIn" in { val List(m) = cpg.method.nameNot("").where(_.ast.isReturn.code(".*nullptr.*")).l val List(c) = cpg.call.codeExact("b->GetObj()").l - c.callee.head shouldBe m + c.callee.l should contain(m) val List(callIn) = m.callIn.l callIn.code shouldBe "b->GetObj()" } @@ -341,9 +339,9 @@ class CallTests extends C2CpgSuite { "test.cpp" ) - val List(call) = cpg.call.nameExact(Defines.operatorPointerCall).l + val List(call) = cpg.call.nameExact(Defines.OperatorPointerCall).l call.signature shouldBe "" - call.methodFullName shouldBe Defines.operatorPointerCall + call.methodFullName shouldBe Defines.OperatorPointerCall call.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH call.typeFullName shouldBe "void" @@ -408,9 +406,9 @@ class CallTests extends C2CpgSuite { "test.c" ) - val List(call) = cpg.call.nameExact(Defines.operatorPointerCall).l + val List(call) = cpg.call.nameExact(Defines.OperatorPointerCall).l call.signature shouldBe "" - call.methodFullName shouldBe Defines.operatorPointerCall + call.methodFullName shouldBe Defines.OperatorPointerCall call.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH call.typeFullName shouldBe "void" @@ -562,9 +560,9 @@ class CallTests extends C2CpgSuite { "test.c" ) - val List(call) = cpg.call.nameExact(Defines.operatorPointerCall).l + val List(call) = cpg.call.nameExact(Defines.OperatorPointerCall).l call.signature shouldBe "" - call.methodFullName shouldBe Defines.operatorPointerCall + call.methodFullName shouldBe Defines.OperatorPointerCall call.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH call.typeFullName shouldBe X2CpgDefines.Any @@ -610,9 +608,9 @@ class CallTests extends C2CpgSuite { "test.c" ) - val List(call) = cpg.call.nameExact(Defines.operatorPointerCall).l + val List(call) = cpg.call.nameExact(Defines.OperatorPointerCall).l call.signature shouldBe "" - call.methodFullName shouldBe Defines.operatorPointerCall + call.methodFullName shouldBe Defines.OperatorPointerCall call.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH call.typeFullName shouldBe X2CpgDefines.Any diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ControlStructureTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ControlStructureTests.scala index b918bb66bca4..5313ff41f2d4 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ControlStructureTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ControlStructureTests.scala @@ -3,7 +3,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.parser.FileDefaults import io.joern.c2cpg.testfixtures.C2CpgSuite import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ControlStructureTests extends C2CpgSuite(FileDefaults.CPP_EXT) { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/DependencyTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/DependencyTests.scala index 5961a22bd2f1..ff9882703a7d 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/DependencyTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/DependencyTests.scala @@ -2,7 +2,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.AstC2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DependencyTests extends AstC2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/FileTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/FileTests.scala index 83da69ec968d..244250740568 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/FileTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/FileTests.scala @@ -65,6 +65,51 @@ class FileTests extends C2CpgSuite { } } + "File test for single file with a header include" should { + + val cpg = code( + """ + |#include "fetch.h" + |#include "cache.h" + |const char *write_ref = NULL; + |void pull_say(const char *fmt, const char *hex { + | if (get_verbosely) { fprintf(stderr, fmt, hex); } + |} + |""".stripMargin, + "fetch.c" + ) + + "contain the correct file nodes" in { + cpg.file.name.sorted.l shouldBe List("", "", "fetch.c") + } + + } + + "File test for single file with a header include that actually exists" should { + + val cpg = code( + """ + |#include "fetch.h" + |#include "cache.h" + |const char *write_ref = NULL; + |void pull_say(const char *fmt, const char *hex { + | if (get_verbosely) { fprintf(stderr, fmt, hex); } + |} + |""".stripMargin, + "fetch.c" + ).moreCode( + """ + |extern const char *write_ref; + |""".stripMargin, + "fetch.h" + ) + + "contain the correct file nodes" in { + cpg.file.name.sorted.l shouldBe List("", "", "fetch.c", "fetch.h") + } + + } + "File test for multiple source files and preprocessed files" should { val cpg = code("int foo() {}", "main.c") diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/HeaderAstCreationPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/HeaderAstCreationPassTests.scala index 53a624e68242..e09854f53372 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/HeaderAstCreationPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/HeaderAstCreationPassTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class HeaderAstCreationPassTests extends C2CpgSuite { @@ -37,20 +37,17 @@ class HeaderAstCreationPassTests extends C2CpgSuite { "de-duplicate content correctly" in { inside(cpg.method.nameNot(NamespaceTraversal.globalNamespaceName).sortBy(_.fullName)) { - case Seq(bar, foo, m1, m2, printf) => + case Seq(bar, foo, m, printf) => // note that we don't see bar twice even so it is contained // in main.h and included in main.c and we do scan both bar.fullName shouldBe "bar" bar.filename shouldBe "main.h" foo.fullName shouldBe "foo" foo.filename shouldBe "other.h" - // main is include twice. First time for the header file, - // second time for the actual implementation in the source file - // We do not de-duplicate this as line/column numbers differ - m1.fullName shouldBe "main" - m1.filename shouldBe "main.c" - m2.fullName shouldBe "main" - m2.filename shouldBe "main.h" + // main is also deduplicated. It is defined within the header file, + // and has an actual implementation in the source file + m.fullName shouldBe "main" + m.filename shouldBe "main.c" printf.fullName shouldBe "printf" } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MemberTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MemberTests.scala index 82b91be23f27..6117f4ae71e0 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MemberTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MemberTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MemberTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MetaDataTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MetaDataTests.scala index 0724f67523f9..c3c91d81c19b 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MetaDataTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MetaDataTests.scala @@ -6,7 +6,7 @@ import io.joern.x2cpg.layers.CallGraph import io.joern.x2cpg.layers.ControlFlow import io.joern.x2cpg.layers.TypeRelations import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodParameterTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodParameterTests.scala index 4c4535132524..d3c12d8c82c2 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodParameterTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodParameterTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodParameterTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodReturnTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodReturnTests.scala index 64ab131c7323..ca33f31a7595 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodReturnTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodReturnTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class MethodReturnTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodTests.scala index 194649699a10..c431f087becc 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/MethodTests.scala @@ -3,7 +3,9 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite import io.shiftleft.codepropertygraph.generated.EvaluationStrategies import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.Identifier +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class MethodTests extends C2CpgSuite { @@ -92,7 +94,7 @@ class MethodTests extends C2CpgSuite { data.index shouldBe 1 data.name shouldBe "data" data.code shouldBe "int &data" - data.typeFullName shouldBe "int" + data.typeFullName shouldBe "int&" data.isVariadic shouldBe false } } @@ -270,8 +272,53 @@ class MethodTests extends C2CpgSuite { } + "Static modifier for methods" should { + "be correct" in { + val cpg = code( + """ + |static void staticCMethodDecl(); + |static void staticCMethodDef() {} + |""".stripMargin, + "test.c" + ).moreCode( + """ + |class A { + | static void staticCPPMethodDecl(); + | static void staticCPPMethodDef() {} + |}; + |""".stripMargin, + "test.cpp" + ) + val List(staticCMethodDecl) = cpg.method.nameExact("staticCMethodDecl").isStatic.l + val List(staticCMethodDef) = cpg.method.nameExact("staticCMethodDef").isStatic.l + val List(staticCPPMethodDecl) = cpg.method.nameExact("staticCPPMethodDecl").isStatic.l + val List(staticCPPMethodDef) = cpg.method.nameExact("staticCPPMethodDef").isStatic.l + staticCMethodDecl.fullName shouldBe "staticCMethodDecl" + staticCMethodDef.fullName shouldBe "staticCMethodDef" + staticCPPMethodDecl.fullName shouldBe "A.staticCPPMethodDecl:void()" + staticCPPMethodDef.fullName shouldBe "A.staticCPPMethodDef:void()" + } + } + + "Name for method parameter in parentheses" should { + "be correct" in { + val cpg = code(""" + |int foo(int * (a)) { + | int (x) = a; + | return 2 * *a; + |} + |""".stripMargin) + val List(paramA) = cpg.method("foo").parameter.l + paramA.code shouldBe "int * (a)" + paramA.typeFullName shouldBe "int*" + paramA.name shouldBe "a" + cpg.identifier.nameExact("x").size shouldBe 1 + cpg.method("foo").local.nameExact("x").size shouldBe 1 + } + } + "Method name, signature and full name tests" should { - "be correct for plain method C" in { + "be correct for plain C method" in { val cpg = code( """ |int method(int); @@ -283,6 +330,21 @@ class MethodTests extends C2CpgSuite { method.fullName shouldBe "method" } + "be correct for C function pointer" in { + val cpg = code( + """ + |int (*foo)(int, int) = { 0 }; + |int (*bar[])(int, int) = { 0 }; + |""".stripMargin, + "test.c" + ) + val List(foo, bar) = cpg.local.l + foo.name shouldBe "foo" + foo.typeFullName shouldBe "int(*)(int,int)" + bar.name shouldBe "bar" + bar.typeFullName shouldBe "int(*[])(int,int)" + } + "be correct for plain method CPP" in { val cpg = code( """ @@ -328,5 +390,45 @@ class MethodTests extends C2CpgSuite { method.signature shouldBe "int(int)" method.fullName shouldBe "NNN.CCC.method:int(int)" } + + "be correct for class method with implicit member access" in { + val cpg = code( + """ + |class A { + | int var; + | void meth(); + |}; + |namespace Foo { + | void A::meth() { + | assert(this->var == var); + | } + |}""".stripMargin, + "test.cpp" + ) + val List(implicitThisParam) = cpg.method.name("meth").parameter.l + implicitThisParam.name shouldBe "this" + implicitThisParam.typeFullName shouldBe "A" + val List(trueVarAccess) = cpg.call.name(Operators.equals).argument.argumentIndex(1).isCall.l + trueVarAccess.code shouldBe "this->var" + trueVarAccess.name shouldBe Operators.indirectFieldAccess + val List(trueThisId, trueVarFieldIdent) = trueVarAccess.argument.l + trueThisId.code shouldBe "this" + trueThisId.isIdentifier shouldBe true + trueThisId.asInstanceOf[Identifier].typeFullName shouldBe "A*" + trueThisId._refOut.l shouldBe List(implicitThisParam) + trueVarFieldIdent.code shouldBe "var" + trueVarFieldIdent.isFieldIdentifier shouldBe true + + val List(varAccess) = cpg.call.name(Operators.equals).argument.argumentIndex(2).isCall.l + varAccess.code shouldBe "this->var" + varAccess.name shouldBe Operators.indirectFieldAccess + val List(thisId, varFieldIdent) = varAccess.argument.l + thisId.code shouldBe "this" + thisId.isIdentifier shouldBe true + thisId.asInstanceOf[Identifier].typeFullName shouldBe "A*" + thisId._refOut.l shouldBe List(implicitThisParam) + varFieldIdent.code shouldBe "var" + varFieldIdent.isFieldIdentifier shouldBe true + } } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/NamespaceBlockTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/NamespaceBlockTests.scala index 200c7bed77ce..335c187314d6 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/NamespaceBlockTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/NamespaceBlockTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ProgramStructureTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ProgramStructureTests.scala index cda8183dcc8a..bef636625dac 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ProgramStructureTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/ast/ProgramStructureTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.passes.ast import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class ProgramStructureTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/CfgCreationPassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/CfgCreationPassTests.scala index 6ade9688aa0e..4a26d09b77a6 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/CfgCreationPassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/CfgCreationPassTests.scala @@ -14,319 +14,341 @@ class CfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg) { "Cfg" should { "contain an entry and exit node at least" in { implicit val cpg: Cpg = code("") - succOf("func") shouldBe expected(("RET", AlwaysEdge)) - succOf("RET") shouldBe expected() + succOf("func") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("RET") should contain theSameElementsAs expected() } "be correct for decl statement with assignment" in { implicit val cpg: Cpg = code("int x = 1;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x = 1", AlwaysEdge)) - succOf("x = 1") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x = 1", AlwaysEdge)) + succOf("x = 1") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested expression" in { implicit val cpg: Cpg = code("x = y + 1;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y + 1", AlwaysEdge)) - succOf("y + 1") shouldBe expected(("x = y + 1", AlwaysEdge)) - succOf("x = y + 1") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y + 1", AlwaysEdge)) + succOf("y + 1") should contain theSameElementsAs expected(("x = y + 1", AlwaysEdge)) + succOf("x = y + 1") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for return statement" in { implicit val cpg: Cpg = code("return x;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("return x;", AlwaysEdge)) - succOf("return x;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("return x;", AlwaysEdge)) + succOf("return x;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for consecutive return statements" in { implicit val cpg: Cpg = code("return x; return y;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("return x;", AlwaysEdge)) - succOf("y") shouldBe expected(("return y;", AlwaysEdge)) - succOf("return x;") shouldBe expected(("RET", AlwaysEdge)) - succOf("return y;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("return x;", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("return y;", AlwaysEdge)) + succOf("return x;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("return y;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for void return statement" in { implicit val cpg: Cpg = code("return;") - succOf("func") shouldBe expected(("return;", AlwaysEdge)) - succOf("return;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("return;", AlwaysEdge)) + succOf("return;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for call expression" in { implicit val cpg: Cpg = code("foo(a + 1, b);") - succOf("func") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("a + 1", AlwaysEdge)) - succOf("a + 1") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("foo(a + 1, b)", AlwaysEdge)) - succOf("foo(a + 1, b)") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("a + 1", AlwaysEdge)) + succOf("a + 1") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("foo(a + 1, b)", AlwaysEdge)) + succOf("foo(a + 1, b)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for unary expression '+'" in { implicit val cpg: Cpg = code("+x;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("+x", AlwaysEdge)) - succOf("+x") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("+x", AlwaysEdge)) + succOf("+x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for unary expression '++'" in { implicit val cpg: Cpg = code("++x;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("++x", AlwaysEdge)) - succOf("++x") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("++x", AlwaysEdge)) + succOf("++x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for conditional expression" in { implicit val cpg: Cpg = code("x ? y : z;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("z", FalseEdge)) - succOf("y") shouldBe expected(("x ? y : z", AlwaysEdge)) - succOf("z") shouldBe expected(("x ? y : z", AlwaysEdge)) - succOf("x ? y : z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("z", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("x ? y : z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("x ? y : z", AlwaysEdge)) + succOf("x ? y : z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for conditional expression with empty then" in { implicit val cpg: Cpg = code("x ? : z;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("x ? : z", TrueEdge), ("z", FalseEdge)) - succOf("z") shouldBe expected(("x ? : z", AlwaysEdge)) - succOf("x ? : z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("x ? : z", TrueEdge), ("z", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("x ? : z", AlwaysEdge)) + succOf("x ? : z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for short-circuit AND expression" in { implicit val cpg: Cpg = code("int z = x && y;") - succOf("func") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("x && y", FalseEdge)) - succOf("y") shouldBe expected(("x && y", AlwaysEdge)) - succOf("x && y") shouldBe expected(("z = x && y", AlwaysEdge)) - succOf("z = x && y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("x && y", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("x && y", AlwaysEdge)) + succOf("x && y") should contain theSameElementsAs expected(("z = x && y", AlwaysEdge)) + succOf("z = x && y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for short-circuit OR expression" in { implicit val cpg: Cpg = code("x || y;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", FalseEdge), ("x || y", TrueEdge)) - succOf("y") shouldBe expected(("x || y", AlwaysEdge)) - succOf("x || y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", FalseEdge), ("x || y", TrueEdge)) + succOf("y") should contain theSameElementsAs expected(("x || y", AlwaysEdge)) + succOf("x || y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } "Cfg for while-loop" should { "be correct" in { implicit val cpg: Cpg = code("while (x < 1) { y = 2; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y = 2", AlwaysEdge)) - succOf("y = 2") shouldBe expected(("x", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y = 2", AlwaysEdge)) + succOf("y = 2") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct with break" in { implicit val cpg: Cpg = code("while (x < 1) { break; y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct with continue" in { implicit val cpg: Cpg = code("while (x < 1) { continue; y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) - succOf("continue;") shouldBe expected(("x", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("continue;") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct with nested while-loop" in { implicit val cpg: Cpg = code("while (x) { while (y) { z; }}") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("z", TrueEdge), ("x", FalseEdge)) - succOf("z") shouldBe expected(("y", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("z", TrueEdge), ("x", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("y", AlwaysEdge)) } } "Cfg for do-while-loop" should { "be correct" in { implicit val cpg: Cpg = code("do { y = 2; } while (x < 1);") - succOf("func") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y = 2", AlwaysEdge)) - succOf("y = 2") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y = 2", AlwaysEdge)) + succOf("y = 2") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) } "be correct with break" in { implicit val cpg: Cpg = code("do { break; y; } while (x < 1);") - succOf("func") shouldBe expected(("break;", AlwaysEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("break;", AlwaysEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) } "be correct with continue" in { implicit val cpg: Cpg = code("do { continue; y; } while (x < 1);") - succOf("func") shouldBe expected(("continue;", AlwaysEdge)) - succOf("continue;") shouldBe expected(("x", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("continue;", AlwaysEdge)) + succOf("continue;") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) } "be correct with nested do-while-loop" in { implicit val cpg: Cpg = code("do { do { x; } while (y); } while (z);") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("x", TrueEdge), ("z", FalseEdge)) - succOf("z") shouldBe expected(("x", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", TrueEdge), ("z", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("x", TrueEdge), ("RET", FalseEdge)) } "be correct for do-while-loop with empty body" in { implicit val cpg: Cpg = code("do { } while(x > 1);") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("1") shouldBe expected(("x > 1", AlwaysEdge)) - succOf("x > 1") shouldBe expected(("x", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x > 1", AlwaysEdge)) + succOf("x > 1") should contain theSameElementsAs expected(("x", TrueEdge), ("RET", FalseEdge)) + } + + "be correct with multiple macro calls" in { + implicit val cpg: Cpg = code( + """ + |#define deleteReset(ptr) do { delete ptr; ptr = nullptr; } while(0) + |void func(void) { + | int *foo = new int; + | int *bar = new int; + | int *baz = new int; + | deleteReset(foo); + | deleteReset(bar); + | deleteReset(baz); + |} + |""".stripMargin, + "foo.cc" + ) + succOf("deleteReset(foo)") should contain theSameElementsAs expected(("foo", 2, AlwaysEdge), ("bar", AlwaysEdge)) + succOf("foo", 2) should contain theSameElementsAs expected(("delete foo", AlwaysEdge)) + succOf("deleteReset(bar)") should contain theSameElementsAs expected(("bar", 2, AlwaysEdge), ("baz", AlwaysEdge)) + succOf("bar", 2) should contain theSameElementsAs expected(("delete bar", AlwaysEdge)) + succOf("deleteReset(baz)") should contain theSameElementsAs expected(("baz", 2, AlwaysEdge), ("RET", AlwaysEdge)) + succOf("baz", 2) should contain theSameElementsAs expected(("delete baz", AlwaysEdge)) } - } "Cfg for for-loop" should { "be correct" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { a = 3; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("x = 0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("a", TrueEdge), ("RET", FalseEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("x = 0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("a", TrueEdge), ("RET", FalseEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct with break" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { break; a = 3; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct with continue" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { continue; a = 3; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("x = 0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) - succOf("continue;") shouldBe expected(("z", AlwaysEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("x = 0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("continue;") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct with nested for-loop" in { implicit val cpg: Cpg = code("for (x; y; z) { for (a; b; c) { u; } }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("a", TrueEdge), ("RET", FalseEdge)) - succOf("z") shouldBe expected(("y", AlwaysEdge)) - succOf("a") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("u", TrueEdge), ("z", FalseEdge)) - succOf("c") shouldBe expected(("b", AlwaysEdge)) - succOf("u") shouldBe expected(("c", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("a", TrueEdge), ("RET", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("u", TrueEdge), ("z", FalseEdge)) + succOf("c") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("u") should contain theSameElementsAs expected(("c", AlwaysEdge)) } "be correct with empty condition" in { implicit val cpg: Cpg = code("for (;;) { a = 1; }") - succOf("func") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("a = 1", AlwaysEdge)) - succOf("a = 1") shouldBe expected(("a", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("a = 1", AlwaysEdge)) + succOf("a = 1") should contain theSameElementsAs expected(("a", AlwaysEdge)) } "be correct with empty condition with break" in { implicit val cpg: Cpg = code("for (;;) { break; }") - succOf("func") shouldBe expected(("break;", AlwaysEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("break;", AlwaysEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with empty condition with continue" in { implicit val cpg: Cpg = code("for (;;) { continue ; }") - succOf("func") shouldBe expected(("continue ;", AlwaysEdge)) - succOf("continue ;") shouldBe expected(("continue ;", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("continue ;", AlwaysEdge)) + succOf("continue ;") should contain theSameElementsAs expected(("continue ;", AlwaysEdge)) } "be correct with empty condition with nested empty for-loop" in { implicit val cpg: Cpg = code("for (;;) { for (;;) { x; } }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("x", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct with empty condition with empty block" in { implicit val cpg: Cpg = code("for (;;) ;") - succOf("func") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct when empty for-loop is skipped" in { implicit val cpg: Cpg = code("for (;;) {}; return;") - succOf("func") shouldBe expected(("return;", AlwaysEdge)) - succOf("return;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("return;", AlwaysEdge)) + succOf("return;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with function call condition with empty block" in { implicit val cpg: Cpg = code("for (; x(1);) ;") - succOf("func") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x(1)", AlwaysEdge)) - succOf("x(1)") shouldBe expected(("1", TrueEdge), ("RET", FalseEdge)) + succOf("func") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x(1)", AlwaysEdge)) + succOf("x(1)") should contain theSameElementsAs expected(("1", TrueEdge), ("RET", FalseEdge)) } } "Cfg for goto" should { "be correct for single label" in { implicit val cpg: Cpg = code("x; goto l1; y; l1: ;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("goto l1;", AlwaysEdge)) - succOf("goto l1;") shouldBe expected(("l1: ;", AlwaysEdge)) - succOf("l1: ;") shouldBe expected(("RET", AlwaysEdge)) - succOf("y") shouldBe expected(("l1: ;", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("goto l1;", AlwaysEdge)) + succOf("goto l1;") should contain theSameElementsAs expected(("l1: ;", AlwaysEdge)) + succOf("l1: ;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("l1: ;", AlwaysEdge)) } "be correct for GNU goto labels as values" in { @@ -336,40 +358,40 @@ class CfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg) { |otherCall(); |foo: someCall(); |""".stripMargin) - succOf("func") shouldBe expected(("ptr", AlwaysEdge)) - succOf("ptr") shouldBe expected(("foo", AlwaysEdge)) - succOf("ptr", 1) shouldBe expected(("*ptr", AlwaysEdge)) - succOf("foo") shouldBe expected(("&&foo", AlwaysEdge)) - succOf("*ptr = &&foo") shouldBe expected(("goto *;", AlwaysEdge)) - succOf("goto *;") shouldBe expected(("foo: someCall();", AlwaysEdge)) - succOf("foo: someCall();") shouldBe expected(("someCall()", AlwaysEdge)) - succOf("otherCall()") shouldBe expected(("foo: someCall();", AlwaysEdge)) - succOf("someCall()") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("ptr", AlwaysEdge)) + succOf("ptr") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("ptr", 1) should contain theSameElementsAs expected(("*ptr", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("&&foo", AlwaysEdge)) + succOf("*ptr = &&foo") should contain theSameElementsAs expected(("goto *;", AlwaysEdge)) + succOf("goto *;") should contain theSameElementsAs expected(("foo: someCall();", AlwaysEdge)) + succOf("foo: someCall();") should contain theSameElementsAs expected(("someCall()", AlwaysEdge)) + succOf("otherCall()") should contain theSameElementsAs expected(("foo: someCall();", AlwaysEdge)) + succOf("someCall()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for multiple labels" in { implicit val cpg: Cpg = code("x; goto l1; l2: y; l1: ;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("goto l1;", AlwaysEdge)) - succOf("goto l1;") shouldBe expected(("l1: ;", AlwaysEdge)) - succOf("y") shouldBe expected(("l1: ;", AlwaysEdge)) - succOf("l1: ;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("goto l1;", AlwaysEdge)) + succOf("goto l1;") should contain theSameElementsAs expected(("l1: ;", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("l1: ;", AlwaysEdge)) + succOf("l1: ;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for multiple labels on same spot" in { implicit val cpg: Cpg = code("x; goto l2; y; l1: ;l2: ;") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("goto l2;", AlwaysEdge)) - succOf("goto l2;") shouldBe expected(("l2: ;", AlwaysEdge)) - succOf("y") shouldBe expected(("l1: ;", AlwaysEdge)) - succOf("l1: ;") shouldBe expected(("l2: ;", AlwaysEdge)) - succOf("l2: ;") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("goto l2;", AlwaysEdge)) + succOf("goto l2;") should contain theSameElementsAs expected(("l2: ;", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("l1: ;", AlwaysEdge)) + succOf("l1: ;") should contain theSameElementsAs expected(("l2: ;", AlwaysEdge)) + succOf("l2: ;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "work correctly with if block" in { implicit val cpg: Cpg = code("if(foo) goto end; if(bar) { f(x); } end: ;") - succOf("func") shouldBe expected(("foo", AlwaysEdge)) - succOf("goto end;") shouldBe expected(("end: ;", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("goto end;") should contain theSameElementsAs expected(("end: ;", AlwaysEdge)) } } @@ -377,85 +399,93 @@ class CfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg) { "Cfg for switch" should { "be correct with one case" in { implicit val cpg: Cpg = code("switch (x) { case 1: y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("case 1:", CaseEdge), ("RET", CaseEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with multiple cases" in { implicit val cpg: Cpg = code("switch (x) { case 1: y; case 2: z;}") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("case 2:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("case 2:", AlwaysEdge)) - succOf("case 2:") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected( + ("case 1:", CaseEdge), + ("case 2:", CaseEdge), + ("RET", CaseEdge) + ) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("case 2:", AlwaysEdge)) + succOf("case 2:") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with multiple cases on same spot" in { implicit val cpg: Cpg = code("switch (x) { case 1: case 2: y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("case 2:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("case 2:", AlwaysEdge)) - succOf("case 2:") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected( + ("case 1:", CaseEdge), + ("case 2:", CaseEdge), + ("RET", CaseEdge) + ) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("case 2:", AlwaysEdge)) + succOf("case 2:") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with multiple cases and multiple cases on same spot" in { implicit val cpg: Cpg = code("switch (x) { case 1: case 2: y; case 3: z;}") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected( + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected( ("case 1:", CaseEdge), ("case 2:", CaseEdge), ("case 3:", CaseEdge), ("RET", CaseEdge) ) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("case 2:", AlwaysEdge)) - succOf("case 2:") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("case 3:", AlwaysEdge)) - succOf("case 3:") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("case 2:", AlwaysEdge)) + succOf("case 2:") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("case 3:", AlwaysEdge)) + succOf("case 3:") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with default case" in { implicit val cpg: Cpg = code("switch (x) { default: y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("default:", CaseEdge)) - succOf("default:") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("default:", CaseEdge)) + succOf("default:") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for case and default combined" in { implicit val cpg: Cpg = code("switch (x) { case 1: y; break; default: z;}") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("default:", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("break;", AlwaysEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("default:") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("case 1:", CaseEdge), ("default:", CaseEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", AlwaysEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("default:") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested switch" in { implicit val cpg: Cpg = code("switch (x) { case 1: switch(y) { default: z; } }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("RET", AlwaysEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("default:", CaseEdge)) - succOf("default:") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("case 1:", CaseEdge), ("RET", AlwaysEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("default:", CaseEdge)) + succOf("default:") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch containing continue statement" in { @@ -467,42 +497,62 @@ class CfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg) { | } |} |""".stripMargin) - succOf("continue;") shouldBe expected(("i", AlwaysEdge)) + succOf("continue;") should contain theSameElementsAs expected(("i", AlwaysEdge)) } } "Cfg for if" should { "be correct" in { implicit val cpg: Cpg = code("if (x) { y; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with else block" in { implicit val cpg: Cpg = code("if (x) { y; } else { z; }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("z", FalseEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("z", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with nested if" in { implicit val cpg: Cpg = code("if (x) { if (y) { z; } }") - succOf("func") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("z", TrueEdge), ("RET", FalseEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("z", TrueEdge), ("RET", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct with else if chain" in { implicit val cpg: Cpg = code("if (a) { b; } else if (c) { d;} else { e; }") - succOf("func") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("b", TrueEdge), ("c", FalseEdge)) - succOf("b") shouldBe expected(("RET", AlwaysEdge)) - succOf("c") shouldBe expected(("d", TrueEdge), ("e", FalseEdge)) - succOf("d") shouldBe expected(("RET", AlwaysEdge)) - succOf("e") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", TrueEdge), ("c", FalseEdge)) + succOf("b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("c") should contain theSameElementsAs expected(("d", TrueEdge), ("e", FalseEdge)) + succOf("d") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("e") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + } + + "be correct for empty 'then' block" in { + implicit val cpg: Cpg = code("if (cond()) {} else { foo(); }") + succOf("func") should contain theSameElementsAs expected(("cond()", AlwaysEdge)) + succOf("cond()") should contain theSameElementsAs expected(("RET", TrueEdge), ("foo()", FalseEdge)) + succOf("foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + } + + "be correct for empty 'else' block" in { + implicit val cpg: Cpg = code("if (cond()) {foo();} else {}") + succOf("func") should contain theSameElementsAs expected(("cond()", AlwaysEdge)) + succOf("cond()") should contain theSameElementsAs expected(("RET", FalseEdge), ("foo()", TrueEdge)) + succOf("foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + } + + "be correct for empty 'then' and 'else' block" in { + implicit val cpg: Cpg = code("if (cond()) {} else {}") + succOf("func") should contain theSameElementsAs expected(("cond()", AlwaysEdge)) + succOf("cond()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } } @@ -516,9 +566,9 @@ class CppCfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg(FileD "be correct for try with a single catch" in { implicit val cpg: Cpg = code("try { a; } catch (int x) { b; }") - succOf("func") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("b", AlwaysEdge), ("RET", AlwaysEdge)) - succOf("b") shouldBe expected(("RET", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", AlwaysEdge), ("RET", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for try with multiple catches" in { @@ -533,13 +583,41 @@ class CppCfgCreationPassTests extends CfgTestFixture(() => new CCfgTestCpg(FileD | d; |} |""".stripMargin) - succOf("func") shouldBe expected(("a", AlwaysEdge)) + succOf("func") should contain theSameElementsAs expected(("a", AlwaysEdge)) // Try should have an edge to all catches and return - succOf("a") shouldBe expected(("b", AlwaysEdge), ("c", AlwaysEdge), ("d", AlwaysEdge), ("RET", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected( + ("b", AlwaysEdge), + ("c", AlwaysEdge), + ("d", AlwaysEdge), + ("RET", AlwaysEdge) + ) // But catches should only have edges to return - succOf("b") shouldBe expected(("RET", AlwaysEdge)) - succOf("c") shouldBe expected(("RET", AlwaysEdge)) - succOf("d") shouldBe expected(("RET", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("c") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("d") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + } + + "be correct for throw statement" in { + implicit val cpg: Cpg = code(""" + |throw foo(); + |bar(); + |""".stripMargin) + succOf("func") should contain theSameElementsAs expected(("foo()", AlwaysEdge)) + succOf("foo()") should contain theSameElementsAs expected(("throw foo()", AlwaysEdge)) + succOf("throw foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("bar()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + } + + "be correct for throw statement in if-else" in { + implicit val cpg: Cpg = code(""" + |if (true) throw foo(); + |else bar(); + |""".stripMargin) + succOf("func") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("foo()", TrueEdge), ("bar()", FalseEdge)) + succOf("foo()") should contain theSameElementsAs expected(("throw foo()", AlwaysEdge)) + succOf("throw foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("bar()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/MethodCfgLayoutTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/MethodCfgLayoutTests.scala index d2c5042faa88..27b285307716 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/MethodCfgLayoutTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/cfg/MethodCfgLayoutTests.scala @@ -1,9 +1,9 @@ package io.joern.c2cpg.passes.cfg import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodCfgLayoutTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/ClassTypeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/ClassTypeTests.scala index c746c034f705..c7ac3a2eb5ed 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/ClassTypeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/ClassTypeTests.scala @@ -2,7 +2,7 @@ package io.joern.c2cpg.passes.types import io.joern.c2cpg.parser.FileDefaults import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class ClassTypeTests extends C2CpgSuite(FileDefaults.CPP_EXT) { @@ -145,6 +145,7 @@ class ClassTypeTests extends C2CpgSuite(FileDefaults.CPP_EXT) { |public: | void foo1() { | b.foo2(); + | B x = b; | } |}; | @@ -156,6 +157,7 @@ class ClassTypeTests extends C2CpgSuite(FileDefaults.CPP_EXT) { val List(call) = cpg.call("foo2").l call.methodFullName shouldBe "B.foo2:void()" + cpg.fieldIdentifier.canonicalNameExact("b").inCall.code.l shouldBe List("this->b", "this->b") } } @@ -170,10 +172,55 @@ class ClassTypeTests extends C2CpgSuite(FileDefaults.CPP_EXT) { | ): Bar::Foo(a, b) {} |}""".stripMargin) val List(constructor) = cpg.typeDecl.nameExact("FooT").method.isConstructor.l - constructor.signature shouldBe "Bar.Foo(std.string,Bar.SomeClass)" - val List(p1, p2) = constructor.parameter.l - p1.typ.fullName shouldBe "std.string" - p2.typ.fullName shouldBe "Bar.SomeClass" + constructor.signature shouldBe "Bar.Foo(std.string&,Bar.SomeClass&)" + val List(thisP, p1, p2) = constructor.parameter.l + thisP.name shouldBe "this" + thisP.typeFullName shouldBe "FooT" + thisP.index shouldBe 0 + p1.typ.fullName shouldBe "std.string&" + p1.index shouldBe 1 + p2.typ.fullName shouldBe "Bar.SomeClass&" + p2.index shouldBe 2 + } + } + + "handling C++ operator definitions" should { + "generate correct fullnames in classes" in { + val cpg = code(""" + |class Foo { + | public: + | void operator delete (void *d) { free(d); } + | bool operator == (const Foo &lhs, const Foo &rhs) { return false; } + | Foo &Foo::operator + (const Foo &lhs, const Foo &rhs) { return null; } + | Foo &Foo::operator() (const Foo &a) { return null; } + | Foo &Foo::operator[] (int index) { return null; } + |} + |Foo &Foo::operator + (const Foo &lhs, const Foo &rhs) + |""".stripMargin) + val List(del, eq, plus, apply, idx) = cpg.typeDecl.nameExact("Foo").method.l + del.name shouldBe "delete" + del.fullName shouldBe "Foo.delete:void(void*)" + eq.name shouldBe "==" + eq.fullName shouldBe "Foo.==:bool(Foo &,Foo &)" + plus.name shouldBe "+" + plus.fullName shouldBe "Foo.+:Foo &(Foo &,Foo &)" + apply.name shouldBe "()" + apply.fullName shouldBe "Foo.():Foo &(Foo &)" + idx.name shouldBe "[]" + idx.fullName shouldBe "Foo.[]:Foo &(int)" + } + + "generate correct fullnames in classes with conversions" in { + val cpg = code(""" + |class Foo { + | enum Kind { A, B, C } kind; + | public: + | operator Kind() const { return kind; } + |}; + |""".stripMargin) + val List(k) = cpg.typeDecl.nameExact("Foo").method.l + k.name shouldBe "Kind" + k.fullName shouldBe "Foo.Kind:Foo.Kind()" } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/EnumTypeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/EnumTypeTests.scala index 27dd1b780dd2..48d4cafc9034 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/EnumTypeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/EnumTypeTests.scala @@ -6,7 +6,7 @@ import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.codepropertygraph.generated.nodes.FieldIdentifier import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class EnumTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/NamespaceTypeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/NamespaceTypeTests.scala index a58938f089aa..e219fb585fca 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/NamespaceTypeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/NamespaceTypeTests.scala @@ -6,7 +6,7 @@ import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.codepropertygraph.generated.nodes.FieldIdentifier import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class NamespaceTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { @@ -77,12 +77,10 @@ class NamespaceTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { | // enclosing namespaces are the global namespace, Q, and Q::V |{ return 0; } |""".stripMargin) - inside(cpg.method.nameNot("").fullName.l) { case List(m1, f1, f2, h, m2) => - m1 shouldBe "Q.V.C.m:int()" - f1 shouldBe "Q.V.f:int()" - f2 shouldBe "Q.V.f:int()" + inside(cpg.method.nameNot("").fullName.l) { case List(f, m, h) => + f shouldBe "Q.V.f:int()" + m shouldBe "Q.V.C.m:int()" h shouldBe "h:void()" - m2 shouldBe "Q.V.C.m:int()" } inside(cpg.namespaceBlock.nameNot("").l) { case List(q, v) => @@ -162,10 +160,10 @@ class NamespaceTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { namespaceX.fullName shouldBe "X" } - inside(cpg.method.internal.nameNot("").fullName.l) { case List(f, g, h) => + inside(cpg.method.internal.nameNot("").fullName.l) { case List(h, f, g) => + h shouldBe "h:void()" f shouldBe "f:void()" g shouldBe "A.g:void()" - h shouldBe "h:void()" } inside(cpg.call.filterNot(_.name == Operators.fieldAccess).l) { case List(f, g) => @@ -201,7 +199,7 @@ class NamespaceTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { a2.fullName shouldBe "A" } - inside(cpg.method.internal.nameNot("").l) { case List(f1, f2, foo, bar) => + inside(cpg.method.internal.nameNot("").l) { case List(foo, bar, f1, f2) => f1.fullName shouldBe "A.f:void(int)" f1.signature shouldBe "void(int)" f2.fullName shouldBe "A.f:void(char)" @@ -377,9 +375,7 @@ class NamespaceTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { "FinalClasses.C22", "FinalClasses.C23", "IntermediateClasses.B1", - "IntermediateClasses.B1*", - "IntermediateClasses.B2", - "IntermediateClasses.B2*" + "IntermediateClasses.B2" ) } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/StructTypeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/StructTypeTests.scala index c6cfba93a1d4..8b4b9b76c92f 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/StructTypeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/StructTypeTests.scala @@ -2,7 +2,7 @@ package io.joern.c2cpg.passes.types import io.joern.c2cpg.testfixtures.C2CpgSuite import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class StructTypeTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TemplateTypeTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TemplateTypeTests.scala index 59720a9a3042..050c9603ea52 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TemplateTypeTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TemplateTypeTests.scala @@ -2,7 +2,7 @@ package io.joern.c2cpg.passes.types import io.joern.c2cpg.parser.FileDefaults import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class TemplateTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { @@ -72,10 +72,10 @@ class TemplateTypeTests extends C2CpgSuite(fileSuffix = FileDefaults.CPP_EXT) { |""".stripMargin) inside(cpg.method.nameNot("").internal.l) { case List(x, y) => x.name shouldBe "x" - x.fullName shouldBe "x:void(#0,#1)" + x.fullName shouldBe "x:void(ANY,ANY)" x.signature shouldBe "void(T,U)" y.name shouldBe "y" - y.fullName shouldBe "y:void(#0,#1)" + y.fullName shouldBe "y:void(ANY,ANY)" y.signature shouldBe "void(T,U)" } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TypeNodePassTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TypeNodePassTests.scala index 01a1ffa70919..9e48c9ffdc65 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TypeNodePassTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/passes/types/TypeNodePassTests.scala @@ -1,9 +1,9 @@ package io.joern.c2cpg.passes.types import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TypeNodePassTests extends C2CpgSuite { @@ -16,8 +16,20 @@ class TypeNodePassTests extends C2CpgSuite { |""".stripMargin) val List(foo) = cpg.typeDecl.nameExact("foo").l val List(bar) = cpg.typeDecl.nameExact("bar").l - foo.aliasTypeFullName shouldBe Option("char") - bar.aliasTypeFullName shouldBe Option("char") + foo.aliasTypeFullName shouldBe Option("char*") + bar.aliasTypeFullName shouldBe Option("char**") + } + + "be correct for reference to type" in { + val cpg = code( + """ + |typedef const char (&TwoChars)[2]; + |""".stripMargin, + "twochars.cpp" + ) + val List(bar) = cpg.typeDecl.nameExact("TwoChars").l + bar.fullName shouldBe "TwoChars" + bar.aliasTypeFullName shouldBe Option("char(&)[2]") } "be correct for static decl assignment" in { @@ -126,12 +138,11 @@ class TypeNodePassTests extends C2CpgSuite { |} |""".stripMargin) inside(cpg.call("free").argument(1).l) { case List(arg) => - arg.evalType.l shouldBe List("test") + arg.evalType.l shouldBe List("test*") arg.code shouldBe "ptr" inside(arg.typ.referencedTypeDecl.l) { case List(tpe) => - tpe.fullName shouldBe "test" - tpe.name shouldBe "test" - tpe.code should startWith("struct test") + tpe.fullName shouldBe "test*" + tpe.name shouldBe "test*" } inside(cpg.local.l) { case List(ptr) => ptr.name shouldBe "ptr" @@ -139,9 +150,8 @@ class TypeNodePassTests extends C2CpgSuite { ptr.code shouldBe "struct test* ptr" } inside(cpg.local.typ.referencedTypeDecl.l) { case List(tpe) => - tpe.name shouldBe "test" - tpe.fullName shouldBe "test" - tpe.code should startWith("struct test") + tpe.name shouldBe "test*" + tpe.fullName shouldBe "test*" } } } @@ -169,7 +179,7 @@ class TypeNodePassTests extends C2CpgSuite { |} |""".stripMargin) inside(cpg.local.typ.referencedTypeDecl.l) { case List(tpe) => - tpe.fullName shouldBe "Foo" + tpe.fullName shouldBe "Foo*" } } @@ -194,6 +204,42 @@ class TypeNodePassTests extends C2CpgSuite { } } } + + "be correct for volatile types" in { + val cpg = code(""" + |void func(void) { + | static volatile int **ipp; + | static int *ip; + | static volatile int i = 0; + | + | ipp = &ip; + | ipp = (int**) &ip; + | *ipp = &i; + | if (*ip != 0) {} + |}""".stripMargin) + cpg.identifier.nameExact("ipp").typeFullName.distinct.l shouldBe List("volatile int**") + cpg.identifier.nameExact("ip").typeFullName.distinct.l shouldBe List("int*") + cpg.identifier.nameExact("i").typeFullName.distinct.l shouldBe List("volatile int") + cpg.local.nameExact("ipp").typeFullName.l shouldBe List("volatile int**") + cpg.local.nameExact("ip").typeFullName.l shouldBe List("int*") + cpg.local.nameExact("i").typeFullName.l shouldBe List("volatile int") + } + + "be correct for referenced types from locals" in { + val cpg = code(""" + |struct flex { + | int a; + | char b[]; + |}; + |void foo() { + | struct flex *ptr = malloc(sizeof(struct flex)); + | struct flex value = {0}; + |}""".stripMargin) + val List(value) = cpg.typeDecl.fullNameExact("flex").referencingType.fullNameExact("flex").localOfType.l + value.name shouldBe "value" + value.typeFullName shouldBe "flex" + value.code shouldBe "struct flex value" + } } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/AstQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/AstQueryTests.scala index b77feba015d4..f3b9675ec390 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/AstQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/AstQueryTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.querying import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AstQueryTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/CfgQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/CfgQueryTests.scala index f66694c2ceb2..8c31b9ce4a69 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/CfgQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/CfgQueryTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.querying import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CfgQueryTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/DdgCfgQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/DdgCfgQueryTests.scala index befa06bf7d00..f2e558ed8dd9 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/DdgCfgQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/DdgCfgQueryTests.scala @@ -2,8 +2,8 @@ package io.joern.c2cpg.querying import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* class DdgCfgQueryTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocalQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocalQueryTests.scala index d858744b122e..ae0acd31eb25 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocalQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocalQueryTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.querying import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** Language primitives for navigating local variables */ diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocationQueryTests.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocationQueryTests.scala index a4bc22e264c4..7ac71e73f9fe 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocationQueryTests.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/querying/LocationQueryTests.scala @@ -1,7 +1,7 @@ package io.joern.c2cpg.querying import io.joern.c2cpg.testfixtures.C2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocationQueryTests extends C2CpgSuite { diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/AstC2CpgFrontend.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/AstC2CpgFrontend.scala index bc837309b80f..71d522a20ae0 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/AstC2CpgFrontend.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/AstC2CpgFrontend.scala @@ -3,7 +3,9 @@ package io.joern.c2cpg.testfixtures import better.files.File import io.joern.c2cpg.Config import io.joern.c2cpg.passes.AstCreationPass +import io.joern.c2cpg.passes.FunctionDeclNodePass import io.joern.x2cpg.testfixtures.LanguageFrontend +import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.X2Cpg.newEmptyCpg import io.shiftleft.codepropertygraph.generated.Cpg @@ -19,6 +21,8 @@ trait AstC2CpgFrontend extends LanguageFrontend { .withOutputPath(pathAsString) val astCreationPass = new AstCreationPass(cpg, config) astCreationPass.createAndApply() + new FunctionDeclNodePass(cpg, astCreationPass.unhandledMethodDeclarations())(ValidationMode.Enabled) + .createAndApply() cpg } } diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/C2CpgSuite.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/C2CpgSuite.scala index 95cb9238cfcc..e54a5acd067e 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/C2CpgSuite.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/C2CpgSuite.scala @@ -1,17 +1,18 @@ package io.joern.c2cpg.testfixtures import io.joern.c2cpg.parser.FileDefaults -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.x2cpg.testfixtures.Code2CpgFixture class C2CpgSuite( fileSuffix: String = FileDefaults.C_EXT, withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), withPostProcessing: Boolean = false ) extends Code2CpgFixture(() => new CDefaultTestCpg(fileSuffix) .withOssDataflow(withOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/DataFlowCodeToCpgSuite.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/DataFlowCodeToCpgSuite.scala index 16211eb85508..7e3498ff8d24 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/DataFlowCodeToCpgSuite.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/testfixtures/DataFlowCodeToCpgSuite.scala @@ -27,7 +27,7 @@ class DataFlowCodeToCpgSuite extends Code2CpgFixture(() => new DataFlowTestCpg() protected implicit val context: EngineContext = EngineContext() protected def flowToResultPairs(path: Path): List[(String, Integer)] = - path.resultPairs().collect { case (firstElement: String, secondElement: Option[Integer]) => + path.resultPairs().collect { case (firstElement: String, secondElement) => (firstElement, secondElement.getOrElse(-1)) } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala index 2333179302cb..273366a8d6a4 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala @@ -5,6 +5,7 @@ import io.joern.x2cpg.astgen.AstGenConfig import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} import io.joern.x2cpg.utils.Environment import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import org.slf4j.LoggerFactory import scopt.OParser @@ -40,16 +41,21 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) with FrontendHTTPServer[Config, CSharpSrc2Cpg] { private val logger = LoggerFactory.getLogger(getClass) + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, csharpsrc2cpg: CSharpSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - if (Environment.pathExists(absPath)) { - csharpsrc2cpg.run(config.withInputPath(absPath)) - } else { - logger.warn(s"Given path '$absPath' does not exist, skipping") + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + if (Environment.pathExists(absPath)) { + csharpsrc2cpg.run(config.withInputPath(absPath)) + } else { + logger.warn(s"Given path '$absPath' does not exist, skipping") + } } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala index 90291c57fcb6..bd420286d8bb 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala @@ -8,9 +8,8 @@ import io.joern.x2cpg.astgen.{AstGenNodeBuilder, ParserResult} import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode} import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewFile, NewTypeDecl} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.Value import java.math.BigInteger diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala index a070f8c8b539..7e284e250531 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala @@ -3,14 +3,15 @@ package io.joern.csharpsrc2cpg.astcreation import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.csharpsrc2cpg.parser.{DotNetJsonAst, DotNetNodeInfo, ParserKeys} import io.joern.csharpsrc2cpg.{CSharpDefines, Constants, astcreation} +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.{Ast, Defines, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} -import io.shiftleft.passes.IntervalKeyPool import ujson.Value import scala.annotation.tailrec import scala.util.{Failure, Success, Try} + trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => private val anonymousTypeKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) @@ -83,7 +84,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As case x: NewMethodParameterIn => identifierNode(dotNetNode.orNull, x.name, x.code, x.typeFullName, x.dynamicTypeHintFullName) case x => - logger.warn(s"Unhandled declaration type '${x.label()}' for ${x.name}") + logger.warn(s"Unhandled declaration type '${x.label}' for ${x.name}") identifierNode(dotNetNode.orNull, x.name, x.name, Defines.Any) } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala index cf2de044a993..681300ab1fcc 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForStatementsCreator.scala @@ -6,10 +6,14 @@ import io.joern.csharpsrc2cpg.parser.ParserKeys import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.x2cpg.Ast import io.joern.x2cpg.ValidationMode -import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.codepropertygraph.generated.nodes.NewControlStructure -import io.shiftleft.codepropertygraph.generated.nodes.NewIdentifier +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{ + NewControlStructure, + NewFieldIdentifier, + NewIdentifier, + NewLiteral, + NewLocal +} import scala.:: import scala.util.Try @@ -166,22 +170,100 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } private def astForForEachStatement(forEachStmt: DotNetNodeInfo): Seq[Ast] = { - val forEachNode = controlStructureNode(forEachStmt, ControlStructureTypes.FOR, forEachStmt.code) - val iterableAst = astForNode(forEachStmt.json(ParserKeys.Expression)) - val forEachBlockAst = astForBlock(createDotNetNodeInfo(forEachStmt.json(ParserKeys.Statement))) - - val identifierValue = forEachStmt.json(ParserKeys.Identifier)(ParserKeys.Value).str - val _identifierNode = + val int32Tfn = BuiltinTypes.DotNetTypeMap(BuiltinTypes.Int) + val forEachNode = controlStructureNode(forEachStmt, ControlStructureTypes.FOR, forEachStmt.code) + // Create the collection AST + def newCollectionAst = astForNode(forEachStmt.json(ParserKeys.Expression)) + val collectionNode = createDotNetNodeInfo(forEachStmt.json(ParserKeys.Expression)) + val collectionCode = code(collectionNode) + // Create the iterator variable + val iterName = forEachStmt.json(ParserKeys.Identifier)(ParserKeys.Value).str + val iterNode = forEachStmt.json(ParserKeys.Type) + val iterNodeTfn = nodeTypeFullName(createDotNetNodeInfo(iterNode)) + val iterIdentifier = identifierNode( - node = createDotNetNodeInfo(forEachStmt.json(ParserKeys.Type)), - name = identifierValue, - code = identifierValue, - typeFullName = nodeTypeFullName(createDotNetNodeInfo(forEachStmt.json(ParserKeys.Type))) + node = createDotNetNodeInfo(iterNode), + name = iterName, + code = iterName, + typeFullName = iterNodeTfn + ) + val iterVarLocal = NewLocal().name(iterName).code(iterName).typeFullName(iterNodeTfn) + scope.addToScope(iterName, iterVarLocal) + // Create a de-sugared `idx` variable, i.e., var _idx_ = 0 + val idxName = "_idx_" + val idxLocal = NewLocal().name(idxName).code(idxName).typeFullName(int32Tfn) + val idxIdenAtAssign = identifierNode(node = collectionNode, name = idxName, code = idxName, typeFullName = int32Tfn) + val idxAssignment = + callNode(forEachStmt, s"$idxName = 0", Operators.assignment, Operators.assignment, DispatchTypes.STATIC_DISPATCH) + val idxAssigmentArgs = + List(Ast(idxIdenAtAssign), Ast(NewLiteral().code("0").typeFullName(BuiltinTypes.DotNetTypeMap(BuiltinTypes.Int)))) + val idxAssignmentAst = callAst(idxAssignment, idxAssigmentArgs) + // Create condition based on `idx` variable, i.e., _idx_ < $collection.Count + val idxIdAtCond = idxIdenAtAssign.copy + val collectCountAccess = callNode( + forEachStmt, + s"$collectionCode.Count", + Operators.fieldAccess, + Operators.fieldAccess, + DispatchTypes.STATIC_DISPATCH + ) + val fieldAccessAst = + callAst(collectCountAccess, newCollectionAst :+ Ast(NewFieldIdentifier().canonicalName("Count").code("Count"))) + val idxLt = + callNode( + forEachStmt, + s"$idxName < $collectionCode.Count", + Operators.lessThan, + Operators.lessThan, + DispatchTypes.STATIC_DISPATCH ) + val idxLtArgs = + List(Ast(idxIdAtCond), fieldAccessAst) + val ltCallCond = callAst(idxLt, idxLtArgs) + // Create the assignment from $element = $collection[_idx_++] + val idxIdAtCollAccess = idxIdenAtAssign.copy + val collectIdxAccess = callNode( + forEachStmt, + s"$collectionCode[$idxName++]", + Operators.indexAccess, + Operators.indexAccess, + DispatchTypes.STATIC_DISPATCH + ) + val postIncrAst = callAst( + callNode( + forEachStmt, + s"$idxName++", + Operators.postIncrement, + Operators.postIncrement, + DispatchTypes.STATIC_DISPATCH + ), + Ast(idxIdAtCollAccess) :: Nil + ) + val indexAccessAst = callAst(collectIdxAccess, newCollectionAst :+ postIncrAst) + val iteratorAssignmentNode = + callNode( + forEachStmt, + s"$iterName = $collectionCode[$idxName++]", + Operators.assignment, + Operators.assignment, + DispatchTypes.STATIC_DISPATCH + ) + val iteratorAssignmentArgs = List(Ast(iterIdentifier), indexAccessAst) + val iteratorAssignmentAst = callAst(iteratorAssignmentNode, iteratorAssignmentArgs) - val iteratorVarAst = Ast(_identifierNode) + val forEachBlockAst = astForBlock(createDotNetNodeInfo(forEachStmt.json(ParserKeys.Statement))) - Seq(Ast(forEachNode).withChild(iteratorVarAst).withChildren(iterableAst).withChild(forEachBlockAst)) + forAst( + forNode = forEachNode, + locals = Ast(idxLocal) + .withRefEdge(idxIdenAtAssign, idxLocal) + .withRefEdge(idxIdAtCond, idxLocal) + .withRefEdge(idxIdAtCollAccess, idxLocal) :: Ast(iterVarLocal).withRefEdge(iterIdentifier, iterVarLocal) :: Nil, + conditionAsts = ltCallCond :: Nil, + initAsts = idxAssignmentAst :: Nil, + updateAsts = iteratorAssignmentAst :: Nil, + bodyAst = forEachBlockAst + ) :: Nil } private def astForElseStatement(elseParserNode: DotNetNodeInfo): Ast = { diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala index 9b1ba7c73646..9915b71c51d9 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstSummaryVisitor.scala @@ -1,5 +1,6 @@ package io.joern.csharpsrc2cpg.astcreation +import flatgraph.DiffGraphApplier.applyDiff import io.joern.csharpsrc2cpg.Constants import io.joern.csharpsrc2cpg.datastructures.{ CSharpField, @@ -12,9 +13,8 @@ import io.joern.csharpsrc2cpg.datastructures.{ import io.joern.csharpsrc2cpg.parser.ParserKeys import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder, EdgeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} import io.shiftleft.semanticcpg.language.* -import overflowdb.{BatchedUpdate, Config} import scala.collection.mutable import scala.util.Using @@ -29,11 +29,11 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A this.parseLevel = AstParseLevel.SIGNATURES val fileNode = NewFile().name(relativeFileName) val compilationUnit = createDotNetNodeInfo(parserResult.json(ParserKeys.AstRoot)) - Using.resource(Cpg.withConfig(Config.withoutOverflow())) { cpg => + Using.resource(Cpg.empty) { cpg => // Build and store compilation unit AST val ast = Ast(fileNode).withChildren(astForCompilationUnit(compilationUnit)) Ast.storeInDiffGraph(ast, diffGraph) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) + applyDiff(cpg.graph, diffGraph) // Simulate AST Linker for global namespace val globalNode = NewNamespaceBlock().fullName(Constants.Global).name(Constants.Global) @@ -41,7 +41,7 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A cpg.typeDecl .where(_.astParentFullNameExact(Constants.Global)) .foreach(globalDiffGraph.addEdge(globalNode, _, EdgeTypes.AST)) - BatchedUpdate.applyDiff(cpg.graph, globalDiffGraph) + applyDiff(cpg.graph, globalDiffGraph) // Summarize findings summarize(cpg) diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala index 6ff1c0715021..ca6598bea083 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala @@ -25,7 +25,7 @@ type NamespaceToTypeMap = mutable.Map[String, mutable.Set[CSharpType]] * @see * [[CSharpProgramSummary.jsonToInitialMapping]] for generating initial mappings. */ -case class CSharpProgramSummary(val namespaceToType: NamespaceToTypeMap, val imports: Set[String]) +case class CSharpProgramSummary(namespaceToType: NamespaceToTypeMap, imports: Set[String]) extends ProgramSummary[CSharpType, CSharpMethod, CSharpField] { def findGlobalTypes: Set[CSharpType] = namespaceToType.getOrElse(Constants.Global, Set.empty).toSet diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala index e5ac1153b08c..8a86765291f3 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala @@ -63,8 +63,8 @@ class DotNetAstGenRunner(config: Config) extends AstGenRunnerBase(config) { override def runAstGenNative(in: String, out: File, exclude: String, include: String)(implicit metaData: AstGenProgramMetaData ): Try[Seq[String]] = { - val excludeCommand = if (exclude.isEmpty) "" else s"-e \"$exclude\"" - ExternalCommand.run(s"$astGenCommand -o ${out.toString()} -i \"$in\" $excludeCommand", ".") + val excludeCommand = if (exclude.isEmpty) Seq.empty else Seq("-e", exclude) + ExternalCommand.run(Seq(astGenCommand, "-o", out.toString(), "-i", in) ++ excludeCommand, ".").toTry } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..639704906c97 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala @@ -0,0 +1,78 @@ +package io.joern.csharpsrc2cpg.io + +import better.files.File +import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class CSharp2CpgHTTPServerTests extends CSharpCode2CpgFixture with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("csharp2cpgTestsHttpTest") + val file = dir / "main.cs" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(basicBoilerplate(s"Console.WriteLine($indexStr);")) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.csharpsrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.csharpsrc2cpg.Main.stop() + } + + "Using csharp2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("Main") + cpg.call.code.l shouldBe List("Console.WriteLine()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("Main") + cpg.call.code.l shouldBe List(s"Console.WriteLine($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala index 1dc9312deaad..bd00ad8a77bc 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala @@ -1,11 +1,11 @@ package io.joern.csharpsrc2cpg.io import better.files.File -import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary +import io.joern.csharpsrc2cpg.CSharpSrc2Cpg +import io.joern.csharpsrc2cpg.Config import io.joern.csharpsrc2cpg.passes.AstCreationPass import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture import io.joern.csharpsrc2cpg.utils.DotNetAstGenRunner -import io.joern.csharpsrc2cpg.{CSharpSrc2Cpg, Config} import io.joern.x2cpg.X2Cpg.newEmptyCpg import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.generated.Cpg diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/LoopsTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/LoopsTests.scala index eecace4df9f1..d932655a6771 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/LoopsTests.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/LoopsTests.scala @@ -1,10 +1,11 @@ package io.joern.csharpsrc2cpg.querying.ast import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Local} import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} import io.shiftleft.semanticcpg.language.* -class LoopsTests extends CSharpCode2CpgFixture { +class LoopsTests extends CSharpCode2CpgFixture(withDataFlow = true) { "AST Creation for loops" should { "be correct for foreach statement" in { val cpg = code(basicBoilerplate(""" @@ -19,22 +20,28 @@ class LoopsTests extends CSharpCode2CpgFixture { case forEachNode :: Nil => forEachNode.controlStructureType shouldBe ControlStructureTypes.FOR - inside(forEachNode.astChildren.isIdentifier.l) { - case iteratorNode :: iterableNode :: Nil => - iteratorNode.code shouldBe "element" - iteratorNode.typeFullName shouldBe "System.Int32" + inside(forEachNode.astChildren.l) { + case (idxLocal: Local) :: (elementLocal: Local) :: (initAssign: Call) :: (cond: Call) :: (update: Call) :: (forBlock: Block) :: Nil => + idxLocal.name shouldBe "_idx_" + idxLocal.typeFullName shouldBe "System.Int32" - iterableNode.code shouldBe "fibNumbers" - // TODO: List will be fully qualified once the System types are known - iterableNode.typeFullName shouldBe "List" - case _ => fail("No node for iterable found in `foreach` statement") - } + elementLocal.name shouldBe "element" + elementLocal.typeFullName shouldBe "System.Int32" + + initAssign.code shouldBe "_idx_ = 0" + initAssign.name shouldBe Operators.assignment + initAssign.methodFullName shouldBe Operators.assignment + + cond.code shouldBe "_idx_ < fibNumbers.Count" + cond.name shouldBe Operators.lessThan + cond.methodFullName shouldBe Operators.lessThan + + update.code shouldBe "element = fibNumbers[_idx_++]" + update.name shouldBe Operators.assignment + update.methodFullName shouldBe Operators.assignment - inside(forEachNode.astChildren.isBlock.l) { - case blockNode :: Nil => val List(writeCall) = cpg.call.nameExact("Write").l - writeCall.astParent shouldBe blockNode - case _ => fail("Correct blockNode as child not found for `foreach` statement") + writeCall.astParent shouldBe forBlock } case _ => fail("No control structure node found for `foreach`.") diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/dataflow/ControlStructureDataflowTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/dataflow/ControlStructureDataflowTests.scala index 9f086f405db0..8d2c5d2e2a66 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/dataflow/ControlStructureDataflowTests.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/dataflow/ControlStructureDataflowTests.scala @@ -38,7 +38,7 @@ class ControlStructureDataflowTests extends CSharpCode2CpgFixture(withDataFlow = "find a path from element to Write and from i to assignment through a foreach loop" in { val elementSrc = cpg.identifier.nameExact("element").l val writeSink = cpg.call.nameExact("Write").l - writeSink.reachableBy(elementSrc).size shouldBe 1 + writeSink.reachableBy(elementSrc).size shouldBe 2 val assignmentSrc = cpg.identifier.nameExact("i").lineNumber(10).l val newI = cpg.identifier.nameExact("newI").l diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/testfixtures/CSharpCode2CpgFixture.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/testfixtures/CSharpCode2CpgFixture.scala index 9b31f150451f..a410d8a0ff06 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/testfixtures/CSharpCode2CpgFixture.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/testfixtures/CSharpCode2CpgFixture.scala @@ -1,8 +1,9 @@ package io.joern.csharpsrc2cpg.testfixtures import io.joern.csharpsrc2cpg.{CSharpSrc2Cpg, Config} +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} import io.joern.x2cpg.{ValidationMode, X2Cpg} @@ -16,14 +17,14 @@ import java.io.File class CSharpCode2CpgFixture( withPostProcessing: Boolean = false, withDataFlow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty + semantics: Semantics = DefaultSemantics() ) extends Code2CpgFixture(() => new DefaultTestCpgWithCSharp() .withOssDataflow(withDataFlow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) - with SemanticCpgTestFixture(extraFlows) + with SemanticCpgTestFixture(semantics) with Inside { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Ghidra2Cpg.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Ghidra2Cpg.scala index 751ec5c511fe..02c66985fa8a 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Ghidra2Cpg.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Ghidra2Cpg.scala @@ -13,7 +13,7 @@ import ghidra.program.model.listing.Program import ghidra.program.util.{DefinedDataIterator, GhidraProgramUtilities} import ghidra.util.exception.InvalidInputException import ghidra.util.task.TaskMonitor -import io.joern.ghidra2cpg.passes._ +import io.joern.ghidra2cpg.passes.* import io.joern.ghidra2cpg.passes.arm.ArmFunctionPass import io.joern.ghidra2cpg.passes.mips.{LoHiPass, MipsFunctionPass} import io.joern.ghidra2cpg.passes.x86.{ReturnEdgesPass, X86FunctionPass} @@ -26,7 +26,7 @@ import utilities.util.FileUtilities import java.io.File import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.util.Try class Ghidra2Cpg extends X2CpgFrontend[Config] { @@ -149,9 +149,6 @@ class Ghidra2Cpg extends X2CpgFrontend[Config] { new LiteralPass(cpg, flatProgramAPI).createAndApply() } - private class HeadlessProjectConnection(projectManager: HeadlessGhidraProjectManager, connection: GhidraURLConnection) - extends DefaultProject(projectManager, connection) {} - private class HeadlessGhidraProjectManager extends DefaultProjectManager {} } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Main.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Main.scala index 3c6ee6139f2f..189f3f8fd034 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Main.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/Main.scala @@ -1,6 +1,6 @@ package io.joern.ghidra2cpg -import io.joern.ghidra2cpg.Frontend._ +import io.joern.ghidra2cpg.Frontend.* import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} import scopt.OParser diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala index 4af29997e1d5..c32880c96990 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala @@ -6,17 +6,17 @@ import ghidra.program.model.lang.Register import ghidra.program.model.listing.{CodeUnitFormat, CodeUnitFormatOptions, Function, Instruction, Program} import ghidra.program.model.pcode.{HighFunction, HighSymbol} import ghidra.program.model.scalar.Scalar -import io.joern.ghidra2cpg._ -import io.joern.ghidra2cpg.processors._ +import io.joern.ghidra2cpg.* +import io.joern.ghidra2cpg.processors.* import io.joern.ghidra2cpg.utils.Decompiler -import io.joern.ghidra2cpg.utils.Utils._ +import io.joern.ghidra2cpg.utils.Utils.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{CfgNodeNew, NewBlock, NewMethod} import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} import io.shiftleft.passes.ForkJoinParallelCpgPass import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions abstract class FunctionPass( @@ -60,7 +60,7 @@ abstract class FunctionPass( override def generateParts(): Array[Function] = functions.toArray - implicit def intToIntegerOption(intOption: Option[Int]): Option[Integer] = intOption.map(intValue => { + implicit def intToIntegerOption(intOption: Option[Int]): Option[Int] = intOption.map(intValue => { val integerValue = intValue integerValue }) diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/JumpPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/JumpPass.scala index dfbd5fa2beca..7a58ae23608e 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/JumpPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/JumpPass.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import scala.util.Try diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/LiteralPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/LiteralPass.scala index 97e27d06c0b9..97f37248b57a 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/LiteralPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/LiteralPass.scala @@ -6,7 +6,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.passes.ForkJoinParallelCpgPass -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions class LiteralPass(cpg: Cpg, flatProgramAPI: FlatProgramAPI) extends ForkJoinParallelCpgPass[String](cpg) { diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala index 7905e95c11df..de8fca302cdf 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/PCodePass.scala @@ -2,15 +2,14 @@ package io.joern.ghidra2cpg.passes import ghidra.program.model.listing.{Function, Program} import ghidra.program.util.DefinedDataIterator -import io.joern.ghidra2cpg._ -import io.joern.ghidra2cpg.utils.Utils._ +import io.joern.ghidra2cpg.* +import io.joern.ghidra2cpg.utils.Utils.* import io.joern.ghidra2cpg.utils.{Decompiler, PCodeMapper} -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{NewBlock, NewMethod} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, nodes} import io.shiftleft.passes.ForkJoinParallelCpgPass -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions class PCodePass(currentProgram: Program, fileName: String, functions: List[Function], cpg: Cpg, decompiler: Decompiler) diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala index c0ebcf7d26df..60963b18126e 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/arm/ArmFunctionPass.scala @@ -5,9 +5,8 @@ import io.joern.ghidra2cpg.utils.Decompiler import io.joern.ghidra2cpg.passes.FunctionPass import io.joern.ghidra2cpg.processors.ArmProcessor import io.joern.ghidra2cpg.utils.Utils.{checkIfExternal, createMethodNode, createReturnNode} -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewBlock -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, nodes} class ArmFunctionPass( currentProgram: Program, diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala index 2b592a15e403..70f7bbe485dc 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/LoHiPass.scala @@ -1,10 +1,10 @@ package io.joern.ghidra2cpg.passes.mips import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LoHiPass(cpg: Cpg) extends ForkJoinParallelCpgPass[(Call, Call)](cpg) { override def generateParts(): Array[(Call, Call)] = { @@ -23,6 +23,9 @@ class LoHiPass(cpg: Cpg) extends ForkJoinParallelCpgPass[(Call, Call)](cpg) { }.toArray override def runOnPart(diffGraph: DiffGraphBuilder, pair: (Call, Call)): Unit = { - diffGraph.addEdge(pair._1, pair._2, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, pair._1.code) + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = pair._1.code + diffGraph.addEdge(pair._1, pair._2, EdgeTypes.REACHING_DEF, variableProperty) } } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala index 6e5a238b570d..71222e896100 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsFunctionPass.scala @@ -2,12 +2,12 @@ package io.joern.ghidra2cpg.passes.mips import ghidra.program.model.address.GenericAddress import ghidra.program.model.lang.Register import ghidra.program.model.listing.{Function, Instruction, Program} -import ghidra.program.model.pcode.PcodeOp._ +import ghidra.program.model.pcode.PcodeOp.* import ghidra.program.model.pcode.{HighFunction, PcodeOp, PcodeOpAST, Varnode} import ghidra.program.model.scalar.Scalar import io.joern.ghidra2cpg.passes.FunctionPass import io.joern.ghidra2cpg.processors.MipsProcessor -import io.joern.ghidra2cpg.utils.Utils._ +import io.joern.ghidra2cpg.utils.Utils.* import io.joern.ghidra2cpg.Types import io.joern.ghidra2cpg.utils.Decompiler import io.shiftleft.codepropertygraph.generated.Cpg @@ -15,7 +15,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{CfgNodeNew, NewBlock} import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} import org.slf4j.LoggerFactory -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions class MipsFunctionPass( diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala index 03f63d30fb6a..468d6a3e82f5 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/mips/MipsReturnEdgesPass.scala @@ -3,7 +3,7 @@ package io.joern.ghidra2cpg.passes.mips import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} class MipsReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { @@ -17,7 +17,10 @@ class MipsReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { // the first .cfgNext is skipping a _nop instruction after the call val to = from.cfgNext.cfgNext.isCall.argument.code("v(0|1)").headOption if (to.nonEmpty) { - diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, from.code) + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = from.code + diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, variableProperty) } } } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala index 956c55cb58e9..0669e45b7a16 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/ReturnEdgesPass.scala @@ -3,7 +3,7 @@ package io.joern.ghidra2cpg.passes.x86 import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory class ReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { @@ -15,7 +15,11 @@ class ReturnEdgesPass(cpg: Cpg) extends CpgPass(cpg) { cpg.call.nameNot(".*").foreach { from => // We expect RAX/EAX as return val to = from.cfgNext.isCall.argument.code("(R|E)AX").headOption - if (to.nonEmpty) diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, PropertyNames.VARIABLE, from.code) + + // in flatgraph an edge may have zero or one properties and they're not named... + // in this case we know that we're dealing with ReachingDef edges which has the `variable` property + val variableProperty = from.code + if (to.nonEmpty) diffGraph.addEdge(from, to.get, EdgeTypes.REACHING_DEF, variableProperty) } } diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/X86FunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/X86FunctionPass.scala index c7e5ae1c0134..aff0452836d8 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/X86FunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/x86/X86FunctionPass.scala @@ -4,7 +4,7 @@ import ghidra.program.model.listing.{Function, Program} import io.joern.ghidra2cpg.utils.Decompiler import io.joern.ghidra2cpg.passes.FunctionPass import io.joern.ghidra2cpg.processors.X86Processor -import io.joern.ghidra2cpg.utils.Utils._ +import io.joern.ghidra2cpg.utils.Utils.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} import io.shiftleft.codepropertygraph.generated.nodes.{NewBlock, NewMethod} diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/PCodeMapper.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/PCodeMapper.scala index ccd8fa695d6a..d24f15f3b5db 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/PCodeMapper.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/PCodeMapper.scala @@ -2,17 +2,17 @@ package io.joern.ghidra2cpg.utils import ghidra.app.util.template.TemplateSimplifier import ghidra.program.model.listing.{CodeUnitFormat, CodeUnitFormatOptions, Function, Instruction} -import ghidra.program.model.pcode.PcodeOp._ +import ghidra.program.model.pcode.PcodeOp.* import ghidra.program.model.pcode.{HighFunction, PcodeOp, PcodeOpAST, Varnode} import io.joern.ghidra2cpg.Types //import io.joern.ghidra2cpg.utils.Utils.{createCallNode, createIdentifier, createLiteral} -import io.joern.ghidra2cpg.utils.Utils._ +import io.joern.ghidra2cpg.utils.Utils.* import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.CfgNodeNew import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions class State(argumentIndex: Int) { var argument: Int = argumentIndex diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Utils.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Utils.scala index 5185c815db89..1d9128ba886c 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Utils.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Utils.scala @@ -2,11 +2,11 @@ package io.joern.ghidra2cpg.utils import ghidra.program.model.listing.{Function, Instruction, Program} import io.joern.ghidra2cpg.Types -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.proto.cpg.Cpg.DispatchTypes -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.language.implicitConversions object Utils { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala index f8b6abfbdf82..a586f3206acd 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/DataFlowBinToCpgSuite.scala @@ -9,12 +9,13 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.dotextension.ImageViewer import io.shiftleft.semanticcpg.layers.* +import scala.compiletime.uninitialized import scala.sys.process.Process import scala.util.Try class DataFlowBinToCpgSuite extends GhidraBinToCpgSuite { - implicit var context: EngineContext = scala.compiletime.uninitialized + implicit var context: EngineContext = uninitialized override def beforeAll(): Unit = { super.beforeAll() @@ -33,7 +34,7 @@ class DataFlowBinToCpgSuite extends GhidraBinToCpgSuite { new OssDataFlow(options).run(context) } - protected implicit def int2IntegerOption(x: Int): Option[Integer] = + protected implicit def int2IntegerOption(x: Int): Option[Int] = Some(x) protected def getMemberOfType(cpg: Cpg, typeName: String, memberName: String): Iterator[Member] = diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala index 07546bea0205..21d619dcb701 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala @@ -6,8 +6,8 @@ import io.joern.x2cpg.testfixtures.LanguageFrontend import io.shiftleft.utils.ProjectRoot import org.apache.commons.io.FileUtils import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* import java.nio.file.Files diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/CallArgumentsTest.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/CallArgumentsTest.scala index f885b944657c..9fc3cf0d22ec 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/CallArgumentsTest.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/CallArgumentsTest.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.mips import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallArgumentsTest extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowTests.scala index 99f43aace6f6..dc6babe19dae 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowTests.scala @@ -1,13 +1,13 @@ package io.joern.ghidra2cpg.querying.mips -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, _} -import io.shiftleft.semanticcpg.layers._ +import io.shiftleft.semanticcpg.layers.* class DataFlowTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowThroughLoHiRegistersTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowThroughLoHiRegistersTests.scala index e0c1b017f17f..f42b3b965165 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowThroughLoHiRegistersTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/mips/DataFlowThroughLoHiRegistersTests.scala @@ -1,14 +1,14 @@ package io.joern.ghidra2cpg.querying.mips -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.{Parser, Semantics} +import io.joern.dataflowengineoss.semanticsloader.{FullNameSemanticsParser, FullNameSemantics, Semantics} import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.layers._ +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.layers.* class DataFlowThroughLoHiRegistersTests extends GhidraBinToCpgSuite { override def passes(cpg: Cpg): Unit = { @@ -37,7 +37,7 @@ class DataFlowThroughLoHiRegistersTests extends GhidraBinToCpgSuite { |".incBy" 1->1 2->1 3->1 4->1 |".rotateRight" 2->1 |""".stripMargin - implicit val semantics: Semantics = Semantics.fromList(new Parser().parse(customSemantics)) + implicit val semantics: Semantics = FullNameSemantics.fromList(new FullNameSemanticsParser().parse(customSemantics)) implicit val context: EngineContext = EngineContext(semantics) "should find flows through `div*` instructions" in { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/CFGTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/CFGTests.scala index 5b9e420a5059..15f43cc1807b 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/CFGTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/CFGTests.scala @@ -2,7 +2,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CFGTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/DataFlowTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/DataFlowTests.scala index c0e439760c4f..de5a4ea2f40b 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/DataFlowTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/DataFlowTests.scala @@ -2,14 +2,14 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite import io.shiftleft.codepropertygraph.generated.Cpg -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext import io.joern.dataflowengineoss.semanticsloader.Semantics import io.joern.dataflowengineoss.DefaultSemantics import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.layers._ +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.layers.* class DataFlowTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/FileTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/FileTests.scala index 2d77c4520a6c..645b906d5873 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/FileTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/FileTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LiteralNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LiteralNodeTests.scala index 2b10f2019861..f7bab3771104 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LiteralNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LiteralNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LiteralNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LocalNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LocalNodeTests.scala index 009cf922c0fa..1a54bd8a3bc4 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LocalNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/LocalNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocalNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MetaDataNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MetaDataNodeTests.scala index f8b5143c4252..9f141e9ff412 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MetaDataNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MetaDataNodeTests.scala @@ -2,7 +2,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MethodNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MethodNodeTests.scala index c4c04ce1c827..85f99d1afa53 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MethodNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/MethodNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/NamespaceBlockTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/NamespaceBlockTests.scala index f9341692c73a..abe9dcd5f9db 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/NamespaceBlockTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/NamespaceBlockTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.{FileTraversal, NamespaceTraversal} class NamespaceBlockTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ParameterNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ParameterNodeTests.scala index 9f2c03f4b1fb..c37d5ac9c4d5 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ParameterNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ParameterNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ParameterNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/RefNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/RefNodeTests.scala index b3b317abc4f1..2db2de746f84 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/RefNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/RefNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class RefNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ReturnNodeTests.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ReturnNodeTests.scala index 795b879e4f38..dd564ce01662 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ReturnNodeTests.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/querying/x86/ReturnNodeTests.scala @@ -1,7 +1,7 @@ package io.joern.ghidra2cpg.querying.x86 import io.joern.ghidra2cpg.fixtures.GhidraBinToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ReturnNodeTests extends GhidraBinToCpgSuite { diff --git a/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc b/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc index 81cab22add9e..2cd441ae3888 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc +++ b/joern-cli/frontends/ghidra2cpg/src/test/testbinaries/coverage/testscript.sc @@ -2,7 +2,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.joern.dataflowengineoss.language._ import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment -import overflowdb.traversal._ +import flatgraph.traversal._ @main def main(testBinary: String) = { importCode.ghidra(testBinary) diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt index b91c175eaa13..61a15734e6be 100644 --- a/joern-cli/frontends/gosrc2cpg/build.sbt +++ b/joern-cli/frontends/gosrc2cpg/build.sbt @@ -37,7 +37,7 @@ lazy val GoAstgenMac = "goastgen-macos" lazy val GoAstgenMacArm = "goastgen-macos-arm64" lazy val goAstGenDlUrl = settingKey[String]("goastgen download url") -goAstGenDlUrl := s"https://github.com/Privado-Inc/goastgen/releases/download/v${goAstGenVersion.value}/" +goAstGenDlUrl := s"https://github.com/joernio/goastgen/releases/download/v${goAstGenVersion.value}/" def hasCompatibleAstGenVersion(goAstGenVersion: String): Boolean = { Try("goastgen -version".!!).toOption.map(_.strip()) match { diff --git a/joern-cli/frontends/gosrc2cpg/src/main/resources/application.conf b/joern-cli/frontends/gosrc2cpg/src/main/resources/application.conf index 2d296c7eb816..cd729e0c5ef1 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/resources/application.conf +++ b/joern-cli/frontends/gosrc2cpg/src/main/resources/application.conf @@ -1,3 +1,3 @@ gosrc2cpg { - goastgen_version: "0.17.0" + goastgen_version: "0.1.0" } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala index efbcaa30af5e..9fa86ed511d4 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/GoSrc2Cpg.scala @@ -23,30 +23,33 @@ class GoSrc2Cpg(goGlobalOption: Option[GoGlobal] = Option(GoGlobal())) extends X def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => File.usingTemporaryDirectory("gosrc2cpgOut") { tmpDir => - goGlobalOption - .orElse(Option(GoGlobal())) - .foreach(goGlobal => { - MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() - val astGenResult = new AstGenRunner(config).execute(tmpDir).asInstanceOf[GoAstGenRunnerResult] - goMod = Some( - GoModHelper( - Some(config), - astGenResult.parsedModFile - .flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) + MetaDataPass(cpg, Languages.GOLANG, config.inputPath).createAndApply() + val astGenResults = new AstGenRunner(config).executeForGo(tmpDir) + astGenResults.foreach(astGenResult => { + goGlobalOption + .orElse(Option(GoGlobal())) + .foreach(goGlobal => { + goMod = Some( + GoModHelper( + Some(astGenResult.modulePath), + astGenResult.parsedModFile + .flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) + ) ) - ) - goGlobal.mainModule = goMod.flatMap(modHelper => modHelper.getModMetaData().map(mod => mod.module.name)) - InitialMainSrcPass(cpg, astGenResult.parsedFiles, config, goMod.get, goGlobal, tmpDir).createAndApply() - if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then - PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() - if (config.fetchDependencies) { - goGlobal.processingDependencies = true - DownloadDependenciesPass(cpg, goMod.get, goGlobal, config).process() - goGlobal.processingDependencies = false - } - AstCreationPass(cpg, astGenResult.parsedFiles, config, goMod.get, goGlobal, tmpDir, report).createAndApply() - report.print() - }) + goGlobal.mainModule = goMod.flatMap(modHelper => modHelper.getModMetaData().map(mod => mod.module.name)) + InitialMainSrcPass(cpg, astGenResult.parsedFiles, config, goMod.get, goGlobal, tmpDir).createAndApply() + if goGlobal.pkgLevelVarAndConstantAstMap.size() > 0 then + PackageCtorCreationPass(cpg, config, goGlobal).createAndApply() + if (config.fetchDependencies) { + goGlobal.processingDependencies = true + DownloadDependenciesPass(cpg, goMod.get, goGlobal, config).process() + goGlobal.processingDependencies = false + } + AstCreationPass(cpg, astGenResult.parsedFiles, config, goMod.get, goGlobal, tmpDir, report) + .createAndApply() + report.print() + }) + }) } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala index 5112f2ccf523..4b3e9a15b0b6 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.gosrc2cpg import io.joern.gosrc2cpg.Frontend.* import io.joern.x2cpg.astgen.AstGenConfig import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -42,10 +43,15 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) with FrontendHTTPServer[Config, GoSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() def run(config: Config, gosrc2cpg: GoSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - gosrc2cpg.run(config.withInputPath(absPath)) + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + gosrc2cpg.run(config.withInputPath(absPath)) + } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala index 3fc1aa1d21c8..1d0a90cbdd2a 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala @@ -12,7 +12,7 @@ import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.NewNode import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.Value import java.nio.file.Paths diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala index d7c2f7b6a4e5..31b8d03a1732 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForPackageConstructorCreator.scala @@ -7,7 +7,7 @@ import io.joern.x2cpg.astgen.AstGenNodeBuilder import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.NodeTypes import org.apache.commons.lang3.StringUtils -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.Value import scala.collection.immutable.Set diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/InitialMainSrcProcessor.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/InitialMainSrcProcessor.scala index 5893116d563a..d56020f92998 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/InitialMainSrcProcessor.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/InitialMainSrcProcessor.scala @@ -5,7 +5,7 @@ import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import ujson.{Arr, Obj, Value} import java.io.File diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala index 619aa1834231..df2d810c3f6f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala @@ -1,6 +1,5 @@ package io.joern.gosrc2cpg.model -import io.joern.gosrc2cpg.Config import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern import upickle.default.* @@ -9,11 +8,10 @@ import java.util.Set import java.util.concurrent.ConcurrentSkipListSet import scala.util.control.Breaks.* -class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { +class GoModHelper(modulePath: Option[String] = None, meta: Option[GoMod] = None) { def getModMetaData(): Option[GoMod] = meta def getNameSpace(compilationUnitFilePath: String, pkg: String): String = { - if (meta.isEmpty || compilationUnitFilePath == null || compilationUnitFilePath.isEmpty) { // When there no go.mod file, we don't have the information about the module prefix // In this case we will use package name as a namespace @@ -29,7 +27,7 @@ class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { // 1. if there is go file inside /first/second/test.go (package main) => '/first/second/main' // 2. /test.go (package main) => 'main' - val remainingpath = compilationUnitFilePath.stripPrefix(config.get.inputPath) + val remainingpath = compilationUnitFilePath.stripPrefix(modulePath.get) val pathTokens = remainingpath.split(fileSeparateorPattern) val tokens = pathTokens.dropRight(1).filterNot(x => x == null || x.trim.isEmpty) :+ pkg return tokens.mkString("/") @@ -39,7 +37,7 @@ class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { // go.mod (module jorn.io/trial) and /foo.go (package foo) => jorn.io/trial>foo // go.mod (module jorn.io/trial) and /first/foo.go (package first) => jorn.io/trial/first // go.mod (module jorn.io/trial) and /first/foo.go (package bar) => jorn.io/trial/first - val remainingpath = compilationUnitFilePath.stripPrefix(config.get.inputPath) + val remainingpath = compilationUnitFilePath.stripPrefix(modulePath.get) val pathTokens = remainingpath.split(fileSeparateorPattern) // prefixing module name i.e. jorn.io/trial val tokens = meta.get.module.name +: pathTokens.dropRight(1).filterNot(x => x == null || x.trim.isEmpty) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala index 00b5d2558202..d3234b5bc1db 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala @@ -27,13 +27,13 @@ class DownloadDependenciesPass(cpg: Cpg, parentGoMod: GoModHelper, goGlobal: GoG parentGoMod .getModMetaData() .foreach(mod => { - ExternalCommand.run("go mod init joern.io/temp", projDir) match { + ExternalCommand.run(Seq("go", "mod", "init", "joern.io/temp"), projDir).toTry match { case Success(_) => mod.dependencies .filter(dep => dep.beingUsed) .map(dependency => { - val cmd = s"go get ${dependency.dependencyStr()}" - val results = ExternalCommand.run(cmd, projDir) + val cmd = Seq("go", "get", dependency.dependencyStr()) + val results = ExternalCommand.run(cmd, projDir).toTry results match { case Success(_) => print(". ") @@ -83,10 +83,11 @@ class DownloadDependenciesPass(cpg: Cpg, parentGoMod: GoModHelper, goGlobal: GoG .withIgnoredFilesRegex(config.ignoredFilesRegex.toString()) .withIgnoredFiles(config.ignoredFiles.toList) val astGenResult = new AstGenRunner(depConfig, dependency.getIncludePackagesList()) - .execute(astLocation) - .asInstanceOf[GoAstGenRunnerResult] + .executeForGo(astLocation) + .headOption + .getOrElse(GoAstGenRunnerResult()) val goMod = new GoModHelper( - Some(depConfig), + Some(dependencyLocation), astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) ) DependencySrcProcessorPass(cpg, astGenResult.parsedFiles, depConfig, goMod, goGlobal, astLocation) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala index 93f06aeea3f7..48fa174edbeb 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala @@ -10,12 +10,16 @@ import io.joern.x2cpg.utils.Environment.OperatingSystemType.OperatingSystemType import io.joern.x2cpg.utils.{Environment, ExternalCommand} import org.slf4j.LoggerFactory +import java.nio.file.Paths +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters.* import scala.util.matching.Regex import scala.util.{Failure, Success, Try} object AstGenRunner { private val logger = LoggerFactory.getLogger(getClass) case class GoAstGenRunnerResult( + modulePath: String = "", parsedModFile: Option[String] = None, parsedFiles: List[String] = List.empty, skippedFiles: List[String] = List.empty @@ -71,16 +75,18 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen override def runAstGenNative(in: String, out: File, exclude: String, include: String)(implicit metaData: AstGenProgramMetaData ): Try[Seq[String]] = { - val excludeCommand = if (exclude.isEmpty) "" else s"-exclude \"$exclude\"" - val includeCommand = if (include.isEmpty) "" else s"-include-packages \"$include\"" - ExternalCommand.run(s"$astGenCommand $excludeCommand $includeCommand -out ${out.toString()} $in", ".") + val excludeCommand = if (exclude.isEmpty) Seq.empty else Seq("-exclude", exclude) + val includeCommand = if (include.isEmpty) Seq.empty else Seq("-include-packages", include) + ExternalCommand + .run((astGenCommand +: excludeCommand) ++ includeCommand ++ Seq("-out", out.toString(), in), ".") + .toTry } - override def execute(out: File): AstGenRunnerResult = { + def executeForGo(out: File): List[GoAstGenRunnerResult] = { implicit val metaData: AstGenProgramMetaData = config.astGenMetaData val in = File(config.inputPath) logger.info(s"Running goastgen in '$config.inputPath' ...") - runAstGenNative(config.inputPath, out, config.ignoredFilesRegex.toString(), includeFileRegex.toString()) match { + runAstGenNative(config.inputPath, out, config.ignoredFilesRegex.toString(), includeFileRegex) match { case Success(result) => val srcFiles = SourceFiles.determine( out.toString(), @@ -91,11 +97,108 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen val parsedModFile = filterModFile(srcFiles, out) val parsed = filterFiles(srcFiles, out) val skipped = skippedFiles(in, result.toList) - GoAstGenRunnerResult(parsedModFile.headOption, parsed, skipped) + segregateByModule(config.inputPath, out.toString, parsedModFile, parsed, skipped) case Failure(f) => logger.error("\t- running astgen failed!", f) - GoAstGenRunnerResult() + List() } } + /** Segregate all parsed files including go.mod files under separate modules. This will also segregate modules defined + * inside another module + */ + private def segregateByModule( + inputPath: String, + outPath: String, + parsedModFiles: List[String], + parsedFiles: List[String], + skippedFiles: List[String] + ): List[GoAstGenRunnerResult] = { + val moduleMeta: ModuleMeta = + ModuleMeta(inputPath, outPath, None, ListBuffer[String](), ListBuffer[String](), ListBuffer[ModuleMeta]()) + if (parsedModFiles.nonEmpty) { + parsedModFiles + .sortBy(_.split(UtilityConstants.fileSeparateorPattern).length) + .foreach(modFile => { + moduleMeta.addModFile(modFile, inputPath, outPath) + }) + parsedFiles.foreach(moduleMeta.addParsedFile) + skippedFiles.foreach(moduleMeta.addSkippedFile) + moduleMeta.getOnlyChildren + } else { + parsedFiles.foreach(moduleMeta.addParsedFile) + skippedFiles.foreach(moduleMeta.addSkippedFile) + moduleMeta.getAllChildren + } + } + + private def getParentFolder(path: String): String = { + val parent = Paths.get(path).getParent + if (parent != null) parent.toString else "" + } + + case class ModuleMeta( + modulePath: String, + outputModulePath: String, + modFilePath: Option[String], + parsedFiles: ListBuffer[String], + skippedFiles: ListBuffer[String], + childModules: ListBuffer[ModuleMeta] + ) { + def addModFile(modFile: String, inputPath: String, outPath: String): Unit = { + childModules.collectFirst { + case childMod if modFile.startsWith(childMod.outputModulePath) => + childMod.addModFile(modFile, inputPath, outPath) + } match { + case None => + val outmodpath = getParentFolder(modFile) + childModules.addOne( + ModuleMeta( + outmodpath.replace(outPath, inputPath), + outmodpath, + Some(modFile), + ListBuffer[String](), + ListBuffer[String](), + ListBuffer[ModuleMeta]() + ) + ) + case _ => + } + } + + def addParsedFile(parsedFile: String): Unit = { + childModules.collectFirst { + case childMod if parsedFile.startsWith(childMod.outputModulePath) => + childMod.addParsedFile(parsedFile) + } match { + case None => parsedFiles.addOne(parsedFile) + case _ => + } + } + + def addSkippedFile(skippedFile: String): Unit = { + childModules.collectFirst { + case childMod if skippedFile.startsWith(childMod.outputModulePath) => + childMod.addSkippedFile(skippedFile) + } match { + case None => skippedFiles.addOne(skippedFile) + case _ => + } + } + + def getOnlyChildren: List[GoAstGenRunnerResult] = { + childModules.flatMap(_.getAllChildren).toList + } + + def getAllChildren: List[GoAstGenRunnerResult] = { + getOnlyChildren ++ List( + GoAstGenRunnerResult( + modulePath = modulePath, + parsedModFile = modFilePath, + parsedFiles = parsedFiles.toList, + skippedFiles = skippedFiles.toList + ) + ) + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ConditionalsDataflowTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ConditionalsDataflowTests.scala index a0f4fbac6484..6dd2003ef65a 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ConditionalsDataflowTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/ConditionalsDataflowTests.scala @@ -1,8 +1,8 @@ package io.joern.go2cpg.dataflow import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite class ConditionalsDataflowTests extends GoCodeToCpgSuite(withOssDataflow = true) { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/LoopsDataflowTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/LoopsDataflowTests.scala index ce889f319522..f69b72680ea0 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/LoopsDataflowTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/LoopsDataflowTests.scala @@ -1,8 +1,8 @@ package io.joern.go2cpg.dataflow import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite class LoopsDataflowTests extends GoCodeToCpgSuite(withOssDataflow = true) { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/SwitchDataflowTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/SwitchDataflowTests.scala index ad4d2a22445a..9eb77b9272ba 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/SwitchDataflowTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/SwitchDataflowTests.scala @@ -1,8 +1,8 @@ package io.joern.go2cpg.dataflow import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite class SwitchDataflowTests extends GoCodeToCpgSuite(withOssDataflow = true) { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/TypeDeclConstructorDataflowTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/TypeDeclConstructorDataflowTests.scala index f2829f37f35e..5f79899ff2fe 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/TypeDeclConstructorDataflowTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/dataflow/TypeDeclConstructorDataflowTests.scala @@ -1,8 +1,8 @@ package io.joern.go2cpg.dataflow import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* class TypeDeclConstructorDataflowTests extends GoCodeToCpgSuite(withOssDataflow = true) { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..d9e1ac60a2ac --- /dev/null +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala @@ -0,0 +1,84 @@ +package io.joern.go2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class GoSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("gosrc2cpgTestsHttpTest") + val file = dir / "main.go" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |package main + |func main$indexStr() { + | print("Hello World!") + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.gosrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.gosrc2cpg.Main.stop() + } + + "Using gosrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain(s"main$index") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + } + } + +} diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/model/GoModTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/model/GoModTest.scala index 960916e97852..6691ab69e1db 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/model/GoModTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/model/GoModTest.scala @@ -17,13 +17,12 @@ class GoModTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { namespace shouldBe "main" } "invalid compilation file unit with main pkg" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) @@ -37,128 +36,121 @@ class GoModTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { } "with .mod file and main pkg 1 use case" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "first" / "second" / "test.go" pathAsString, "main") + goMod.getNameSpace(File(inputPath) / "first" / "second" / "test.go" pathAsString, "main") namespace shouldBe "first/second/main" } "with .mod file and main pkg 2 use case" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + JFile.separator + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "first" / "second" / "test.go" pathAsString, "main") + goMod.getNameSpace(File(inputPath) / "first" / "second" / "test.go" pathAsString, "main") namespace shouldBe "first/second/main" } "with .mod file and main pkg 3 use case" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "test.go" pathAsString, "main") + goMod.getNameSpace(File(inputPath) / "test.go" pathAsString, "main") namespace shouldBe "main" } "with .mod file and pkg other than main matching with folder" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "test.go" pathAsString, "trial") + goMod.getNameSpace(File(inputPath) / "test.go" pathAsString, "trial") namespace shouldBe "joern.io/trial" } "with .mod file, pkg other than main, one level child folder, and package matching with last folder" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "first" / "test.go" pathAsString, "first") + goMod.getNameSpace(File(inputPath) / "first" / "test.go" pathAsString, "first") namespace shouldBe "joern.io/trial/first" } "with .mod file and pkg other than main and not matching with folder" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "test.go" pathAsString, "foo") + goMod.getNameSpace(File(inputPath) / "test.go" pathAsString, "foo") namespace shouldBe "joern.io/trial" } "with .mod file, pkg other than main, one level child folder, and package not matching with last folder" in { - val config = Config() - config.inputPath = File.currentWorkingDirectory.toString() + val inputPath = File.currentWorkingDirectory.toString() + JFile.separator val goMod = new GoModHelper( - Some(config), + Some(inputPath), Some( GoMod( - fileFullPath = File(config.inputPath) / "go.mod" pathAsString, + fileFullPath = File(inputPath) / "go.mod" pathAsString, module = GoModModule("joern.io/trial"), dependencies = List[GoModDependency]() ) ) ) val namespace = - goMod.getNameSpace(File(config.inputPath) / "first" / "test.go" pathAsString, "bar") + goMod.getNameSpace(File(inputPath) / "first" / "test.go" pathAsString, "bar") namespace shouldBe "joern.io/trial/first" } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ConditionalsTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ConditionalsTests.scala index 776c8c5a5142..36183ae1d152 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ConditionalsTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ConditionalsTests.scala @@ -4,8 +4,8 @@ import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.nodes.Call import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import scala.collection.immutable.List diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DeclarationsTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DeclarationsTests.scala index 0d4deb9b345f..827b7812364e 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DeclarationsTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DeclarationsTests.scala @@ -1,8 +1,8 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala index 4453314d54ba..4cc107c8ad32 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala @@ -225,9 +225,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in method full name to return type map" in { // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -236,7 +236,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in struct member to type map" in { // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } @@ -298,9 +298,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in method full name to return type map" ignore { // This should only contain the `main` method return type mapping as main source code is not invoking any of the dependency method. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -310,7 +310,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { "not create any entry in struct member to type map" ignore { // This should be empty as neither main code has defined any struct type nor we are accessing the third party struct type. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } @@ -397,9 +397,9 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { // TODO: While doing the implementation we need update this test // Lambda expression return types are also getting recorded under this map goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.methodMetaMap.size() shouldBe 1 - val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().toList + val List(mainfullname) = metadata.methodMetaMap.keys().asIterator().asScala.toList mainfullname shouldBe "main" val Array(returnType) = metadata.methodMetaMap.values().toArray returnType shouldBe MethodCacheMetaData(Defines.voidTypeName, "main.main()") @@ -412,7 +412,7 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { // 2. Struct Type is being passed as parameter or returned as value of method that is being used. // 3. A method of Struct Type being used. goGlobal.nameSpaceMetaDataMap.size() shouldBe 1 - val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().toArray + val Array(metadata) = goGlobal.nameSpaceMetaDataMap.values().iterator().asScala.toArray metadata.structTypeMembers.size() shouldBe 0 } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ExpressionsTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ExpressionsTests.scala index aff9c3dda63f..50f14e27db6b 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ExpressionsTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ExpressionsTests.scala @@ -3,8 +3,8 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes class ExpressionsTests extends GoCodeToCpgSuite { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/FileTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/FileTests.scala index f338ba6084aa..357fa250f93f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/FileTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/FileTests.scala @@ -1,7 +1,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import java.io.File diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ImportTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ImportTests.scala index dbacf6940a4a..571d6632a569 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ImportTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/ImportTests.scala @@ -1,7 +1,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ImportTests extends GoCodeToCpgSuite { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MetaDataTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MetaDataTests.scala index 83662d57b493..98370d93b790 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MetaDataTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MetaDataTests.scala @@ -3,7 +3,7 @@ package io.joern.go2cpg.passes.ast import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} import io.shiftleft.codepropertygraph.generated.Languages import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataTests extends GoCodeToCpgSuite { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala index ceb574e867ed..56972e8865a6 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MethodCallTests.scala @@ -7,9 +7,8 @@ import io.shiftleft.codepropertygraph.generated.edges.Ref import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.{jIteratortoTraversal, toNodeTraversal} - import java.io.File + class MethodCallTests extends GoCodeToCpgSuite(withOssDataflow = true) { "Simple method call use case" should { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MultiModuleTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MultiModuleTests.scala new file mode 100644 index 000000000000..d3d117a00c4d --- /dev/null +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/MultiModuleTests.scala @@ -0,0 +1,318 @@ +package io.joern.go2cpg.passes.ast + +import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite +import io.shiftleft.semanticcpg.language.* + +import java.io.File +import scala.collection.immutable.List + +class MultiModuleTests extends GoCodeToCpgSuite { + "Module defined under another directory" should { + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + |""".stripMargin, + Seq("module1", "go.mod").mkString(File.separator) + ).moreCode( + """ + |package fpkg + |type Sample struct { + | Name string + |} + |func Woo(a int) int{ + | return 0 + |} + |""".stripMargin, + Seq("module1", "lib", "lib.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/sample/lib" + |func main() { + | var a = fpkg.Woo(10) + | var b = fpkg.Sample{name: "Pandurang"} + | var c = b.Name + | var d fpkg.Sample + |} + |""".stripMargin, + Seq("module1", "main.go").mkString(File.separator) + ) + + "Check METHOD Node" in { + cpg.method("Woo").size shouldBe 1 + val List(x) = cpg.method("Woo").l + x.fullName shouldBe "joern.io/sample/lib.Woo" + x.signature shouldBe "joern.io/sample/lib.Woo(int)int" + } + + "Check CALL Node" in { + val List(x) = cpg.call("Woo").l + x.methodFullName shouldBe "joern.io/sample/lib.Woo" + x.typeFullName shouldBe "int" + } + + "Traversal from call to callee method node" in { + val List(x) = cpg.call("Woo").callee.l + x.fullName shouldBe "joern.io/sample/lib.Woo" + x.isExternal shouldBe false + } + + "Check TypeDecl Node" in { + val List(x) = cpg.typeDecl("Sample").l + x.fullName shouldBe "joern.io/sample/lib.Sample" + } + + "Check LOCAL Nodes" in { + val List(a, b, c, d) = cpg.local.l + a.typeFullName shouldBe "int" + b.typeFullName shouldBe "joern.io/sample/lib.Sample" + c.typeFullName shouldBe "string" + d.typeFullName shouldBe "joern.io/sample/lib.Sample" + } + } + + "Multiple modules defined under one directory" should { + val cpg = code( + """ + |module joern.io/module1 + |go 1.18 + |""".stripMargin, + Seq("module1", "go.mod").mkString(File.separator) + ).moreCode( + """ + |package pkg + |type ModoneSample struct { + | Name string + |} + |func ModoneWoo(a int) int{ + | return 0 + |} + |""".stripMargin, + Seq("module1", "pkg", "lib.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/module1/pkg" + |func main() { + | var a = pkg.ModoneWoo(10) + | var b = pkg.ModoneSample{name: "Pandurang"} + | var c = b.Name + | var d pkg.ModoneSample + |} + |""".stripMargin, + Seq("module1", "main.go").mkString(File.separator) + ).moreCode( + """ + |module joern.io/module2 + |go 1.18 + |""".stripMargin, + Seq("module2", "go.mod").mkString(File.separator) + ).moreCode( + """ + |package pkg + |type ModtwoSample struct { + | Name string + |} + |func ModtwoWoo(a int) int{ + | return 0 + |} + |""".stripMargin, + Seq("module2", "pkg", "lib.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/module2/pkg" + |func main() { + | var a = pkg.ModtwoWoo(10) + | var b = pkg.ModtwoSample{name: "Pandurang"} + | var c = b.Name + | var d pkg.ModtwoSample + |} + |""".stripMargin, + Seq("module2", "main.go").mkString(File.separator) + ) + "Check METHOD Node module 1" in { + cpg.method("ModoneWoo").size shouldBe 1 + val List(x) = cpg.method("ModoneWoo").l + x.fullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.signature shouldBe "joern.io/module1/pkg.ModoneWoo(int)int" + } + + "Check METHOD Node module 2" in { + cpg.method("ModtwoWoo").size shouldBe 1 + val List(x) = cpg.method("ModtwoWoo").l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.signature shouldBe "joern.io/module2/pkg.ModtwoWoo(int)int" + } + + "Check CALL Node module 1" in { + val List(x) = cpg.call("ModoneWoo").l + x.methodFullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.typeFullName shouldBe "int" + } + + "Check CALL Node module 2" in { + val List(x) = cpg.call("ModtwoWoo").l + x.methodFullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.typeFullName shouldBe "int" + } + + "Traversal from call to callee method node module 1" in { + val List(x) = cpg.call("ModoneWoo").callee.l + x.fullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.isExternal shouldBe false + } + + "Traversal from call to callee method node module 2" in { + val List(x) = cpg.call("ModtwoWoo").callee.l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.isExternal shouldBe false + } + + "Check TypeDecl Node module 1" in { + val List(x) = cpg.typeDecl("ModoneSample").l + x.fullName shouldBe "joern.io/module1/pkg.ModoneSample" + } + + "Check TypeDecl Node module 2" in { + val List(x) = cpg.typeDecl("ModtwoSample").l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoSample" + } + + "Check LOCAL Nodes Module 1 and 2" in { + val List(a, b, c, d, e, f, g, h) = cpg.local.l + a.typeFullName shouldBe "int" + b.typeFullName shouldBe "joern.io/module1/pkg.ModoneSample" + c.typeFullName shouldBe "string" + d.typeFullName shouldBe "joern.io/module1/pkg.ModoneSample" + + e.typeFullName shouldBe "int" + f.typeFullName shouldBe "joern.io/module2/pkg.ModtwoSample" + g.typeFullName shouldBe "string" + h.typeFullName shouldBe "joern.io/module2/pkg.ModtwoSample" + } + } + + "Multiple modules defined one inside another" should { + val cpg = code( + """ + |module joern.io/module1 + |go 1.18 + |""".stripMargin, + Seq("module1", "go.mod").mkString(File.separator) + ).moreCode( + """ + |package pkg + |type ModoneSample struct { + | Name string + |} + |func ModoneWoo(a int) int{ + | return 0 + |} + |""".stripMargin, + Seq("module1", "pkg", "lib.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/module1/pkg" + |func main() { + | var a = pkg.ModoneWoo(10) + | var b = pkg.ModoneSample{name: "Pandurang"} + | var c = b.Name + | var d pkg.ModoneSample + |} + |""".stripMargin, + Seq("module1", "main.go").mkString(File.separator) + ).moreCode( + """ + |module joern.io/module2 + |go 1.18 + |""".stripMargin, + Seq("module1", "stage", "src", "module2", "go.mod").mkString(File.separator) + ).moreCode( + """ + |package pkg + |type ModtwoSample struct { + | Name string + |} + |func ModtwoWoo(a int) int{ + | return 0 + |} + |""".stripMargin, + Seq("module1", "stage", "src", "module2", "pkg", "lib.go").mkString(File.separator) + ).moreCode( + """ + |package main + |import "joern.io/module2/pkg" + |func main() { + | var a = pkg.ModtwoWoo(10) + | var b = pkg.ModtwoSample{name: "Pandurang"} + | var c = b.Name + | var d pkg.ModtwoSample + |} + |""".stripMargin, + Seq("module1", "stage", "src", "module2", "main.go").mkString(File.separator) + ) + "Check METHOD Node module 1" in { + cpg.method("ModoneWoo").size shouldBe 1 + val List(x) = cpg.method("ModoneWoo").l + x.fullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.signature shouldBe "joern.io/module1/pkg.ModoneWoo(int)int" + } + + "Check METHOD Node module 2" in { + cpg.method("ModtwoWoo").size shouldBe 1 + val List(x) = cpg.method("ModtwoWoo").l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.signature shouldBe "joern.io/module2/pkg.ModtwoWoo(int)int" + } + + "Check CALL Node module 1" in { + val List(x) = cpg.call("ModoneWoo").l + x.methodFullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.typeFullName shouldBe "int" + } + + "Check CALL Node module 2" in { + val List(x) = cpg.call("ModtwoWoo").l + x.methodFullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.typeFullName shouldBe "int" + } + + "Traversal from call to callee method node module 1" in { + val List(x) = cpg.call("ModoneWoo").callee.l + x.fullName shouldBe "joern.io/module1/pkg.ModoneWoo" + x.isExternal shouldBe false + } + + "Traversal from call to callee method node module 2" in { + val List(x) = cpg.call("ModtwoWoo").callee.l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoWoo" + x.isExternal shouldBe false + } + + "Check TypeDecl Node module 1" in { + val List(x) = cpg.typeDecl("ModoneSample").l + x.fullName shouldBe "joern.io/module1/pkg.ModoneSample" + } + + "Check TypeDecl Node module 2" in { + val List(x) = cpg.typeDecl("ModtwoSample").l + x.fullName shouldBe "joern.io/module2/pkg.ModtwoSample" + } + + "Check LOCAL Nodes Module 1 and 2" in { + val List(a, b, c, d, e, f, g, h) = cpg.local.l + a.typeFullName shouldBe "int" + b.typeFullName shouldBe "joern.io/module2/pkg.ModtwoSample" + c.typeFullName shouldBe "string" + d.typeFullName shouldBe "joern.io/module2/pkg.ModtwoSample" + + e.typeFullName shouldBe "int" + f.typeFullName shouldBe "joern.io/module1/pkg.ModoneSample" + g.typeFullName shouldBe "string" + h.typeFullName shouldBe "joern.io/module1/pkg.ModoneSample" + } + } +} diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/NamespaceBlockTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/NamespaceBlockTests.scala index b9013d929b36..e860d36a142f 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/NamespaceBlockTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/NamespaceBlockTests.scala @@ -1,7 +1,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/OperatorsTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/OperatorsTests.scala index 21238c36cb69..5b56a56d070c 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/OperatorsTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/OperatorsTests.scala @@ -3,7 +3,7 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class OperatorsTests extends GoCodeToCpgSuite { diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala index 878af00ed211..34d2bc145379 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala @@ -4,8 +4,8 @@ import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.nodes.Call import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import scala.collection.immutable.List import io.joern.gosrc2cpg.astcreation.Defines diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala index 2e7601ac3979..e5be2e191a4b 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/testfixtures/GoCodeToCpgSuite.scala @@ -1,7 +1,8 @@ package io.joern.go2cpg.testfixtures import better.files.File -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} import io.joern.gosrc2cpg.datastructures.GoGlobal import io.joern.gosrc2cpg.model.GoModHelper @@ -49,11 +50,11 @@ class DefaultTestCpgWithGo(val fileSuffix: String) extends DefaultTestCpg with S class GoCodeToCpgSuite( fileSuffix: String = ".go", withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty + semantics: Semantics = DefaultSemantics() ) extends Code2CpgFixture(() => - new DefaultTestCpgWithGo(fileSuffix).withOssDataflow(withOssDataflow).withExtraFlows(extraFlows) + new DefaultTestCpgWithGo(fileSuffix).withOssDataflow(withOssDataflow).withSemantics(semantics) ) - with SemanticCpgTestFixture(extraFlows) + with SemanticCpgTestFixture(semantics) with Inside { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala index e89fc8d7e883..c7f3068523b2 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala @@ -2,11 +2,16 @@ package io.joern.javasrc2cpg import io.joern.javasrc2cpg.Frontend.* import io.joern.javasrc2cpg.jpastprinter.JavaParserAstPrinter +import io.joern.x2cpg.X2CpgConfig +import io.joern.x2cpg.X2CpgMain import io.joern.x2cpg.frontendspecific.javasrc2cpg -import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} -import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} +import io.joern.x2cpg.passes.frontend.TypeRecoveryParserConfig +import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser +import java.util.concurrent.ExecutorService + /** Command line configuration parameters */ final case class Config( @@ -133,7 +138,9 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) with FrontendHTTPServer[Config, JavaSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() override def main(args: Array[String]): Unit = { // TODO: This is a hack to allow users to use the "--show-env" option without having @@ -146,14 +153,14 @@ object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) { } def run(config: Config, javasrc2Cpg: JavaSrc2Cpg): Unit = { - if (config.showEnv) { - JavaSrc2Cpg.showEnv() - } else if (config.dumpJavaparserAsts) { - JavaParserAstPrinter.printJpAsts(config) - } else { - javasrc2Cpg.run(config) + config match { + case c if c.serverMode => startup() + case c if c.showEnv => JavaSrc2Cpg.showEnv() + case c if c.dumpJavaparserAsts => JavaParserAstPrinter.printJpAsts(c) + case _ => javasrc2Cpg.run(config) } } def getCmdLineParser = cmdLineParser + } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala index 3d661070e83c..c7b5e5a46c59 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/AstCreator.scala @@ -51,7 +51,7 @@ import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewClosureBinding, NewFile, NewImport, NewNamespaceBlock} import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import java.util.concurrent.ConcurrentHashMap import scala.collection.mutable @@ -135,10 +135,10 @@ class AstCreator( .removeOption(new DefaultConfigurationOption(ConfigOption.PRINT_COMMENTS)) .removeOption(new DefaultConfigurationOption(ConfigOption.PRINT_JAVADOC)) - protected def line(node: Node): Option[Int] = node.getBegin.map(x => x.line).toScala - protected def column(node: Node): Option[Int] = node.getBegin.map(x => x.column).toScala - protected def lineEnd(node: Node): Option[Int] = node.getEnd.map(x => x.line).toScala - protected def columnEnd(node: Node): Option[Int] = node.getEnd.map(x => x.column).toScala + protected def line(node: Node): Option[Int] = node.getBegin.map(_.line).toScala + protected def column(node: Node): Option[Int] = node.getBegin.map(_.column).toScala + protected def lineEnd(node: Node): Option[Int] = node.getEnd.map(_.line).toScala + protected def columnEnd(node: Node): Option[Int] = node.getEnd.map(_.column).toScala protected def code(node: Node): String = node.toString(codePrinterOptions) private val lineOffsetTable = OffsetUtils.getLineOffsetTable(fileContent) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala index 9352e254b8f8..5fecec488de1 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/declarations/AstForMethodsCreator.scala @@ -40,6 +40,7 @@ import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.EdgeTypes import com.github.javaparser.ast.Node +import com.github.javaparser.ast.`type`.ClassOrInterfaceType import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserParameterDeclaration import io.joern.javasrc2cpg.astcreation.declarations.AstForMethodsCreator.PartialConstructorDeclaration import io.joern.javasrc2cpg.util.{NameConstants, Util} @@ -59,11 +60,9 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator => val returnTypeFullName = expectedReturnType .flatMap(typeInfoCalc.fullName) .orElse(simpleMethodReturnType.flatMap(scope.lookupType(_))) - .orElse( - tryWithSafeStackOverflow(methodDeclaration.getType.asClassOrInterfaceType).toOption.flatMap(t => - scope.lookupType(t.getNameAsString) - ) - ) + .orElse(tryWithSafeStackOverflow(methodDeclaration.getType).toOption.collect { case t: ClassOrInterfaceType => + scope.lookupType(t.getNameAsString) + }.flatten) .orElse(typeParameters.find(typeParam => simpleMethodReturnType.contains(typeParam.name)).map(_.typeFullName)) scope.pushMethodScope( @@ -464,8 +463,8 @@ private[declarations] trait AstForMethodsCreator { this: AstCreator => } private def constructorReturnNode(constructorDeclaration: ConstructorDeclaration): NewMethodReturn = { - val line = constructorDeclaration.getEnd.map(x => x.line).toScala - val column = constructorDeclaration.getEnd.map(x => x.column).toScala + val line = constructorDeclaration.getEnd.map(_.line).toScala + val column = constructorDeclaration.getEnd.map(_.column).toScala newMethodReturnNode(TypeConstants.Void, None, line, column) } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala index f38c1d958779..98e756c4a52d 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForLambdasCreator.scala @@ -17,13 +17,12 @@ import io.joern.javasrc2cpg.util.BindingTable.createBindingTable import io.joern.javasrc2cpg.util.Util.{composeMethodFullName, composeMethodLikeSignature, composeUnresolvedSignature} import io.joern.javasrc2cpg.util.{BindingTable, BindingTableAdapterForLambdas, LambdaBindingInfo, NameConstants} import io.joern.x2cpg.utils.AstPropertiesUtil.* -import io.joern.x2cpg.utils.NodeBuilders +import io.joern.x2cpg.utils.{IntervalKeyPool, NodeBuilders} import io.joern.x2cpg.utils.NodeBuilders.* import io.joern.x2cpg.{Ast, Defines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn.PropertyDefaults as ParameterDefaults import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, ModifierTypes} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala index 7982e905e43e..867636fa1e43 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/expressions/AstForSimpleExpressionsCreator.scala @@ -59,7 +59,11 @@ trait AstForSimpleExpressionsCreator { this: AstCreator => } private[expressions] def astForArrayCreationExpr(expr: ArrayCreationExpr, expectedType: ExpectedType): Ast = { - val maybeInitializerAst = expr.getInitializer.toScala.map(astForArrayInitializerExpr(_, expectedType)) + val elementType = tryWithSafeStackOverflow(expr.getElementType.resolve()).map(elementType => + ExpectedType(typeInfoCalc.fullName(elementType).map(_ ++ "[]"), Option(elementType)) + ) + val maybeInitializerAst = + expr.getInitializer.toScala.map(astForArrayInitializerExpr(_, elementType.getOrElse(expectedType))) maybeInitializerAst.flatMap(_.root) match { case Some(initializerRoot: NewCall) => initializerRoot.code(expr.toString) @@ -84,11 +88,12 @@ trait AstForSimpleExpressionsCreator { this: AstCreator => } private[expressions] def astForArrayInitializerExpr(expr: ArrayInitializerExpr, expectedType: ExpectedType): Ast = { - val typeFullName = - expressionReturnTypeFullName(expr) - .orElse(expectedType.fullName) - .map(typeInfoCalc.registerType) - .getOrElse(TypeConstants.Any) + // In the expression `new int[] { 1, 2 }`, the ArrayInitializerExpr is only the `{ 1, 2 }` part and does not have + // a type itself. We need to use the expected type from the parent expr here. + val typeFullName = expectedType.fullName + .map(typeInfoCalc.registerType) + .getOrElse(TypeConstants.Any) + val callNode = newOperatorCallNode( Operators.arrayInitializer, code = expr.toString, diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala index 0370d6ccc52e..5938bff74273 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/astcreation/statements/AstForForLoopsCreator.scala @@ -6,6 +6,7 @@ import io.joern.javasrc2cpg.astcreation.{AstCreator, ExpectedType} import io.joern.javasrc2cpg.scope.NodeTypeInfo import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants import io.joern.x2cpg.Ast +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.{newCallNode, newFieldIdentifierNode, newIdentifierNode, newOperatorCallNode} import io.shiftleft.codepropertygraph.generated.nodes.Call.PropertyDefaults import io.shiftleft.codepropertygraph.generated.nodes.{ @@ -18,7 +19,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ NewNode } import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} -import io.shiftleft.passes.IntervalKeyPool import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TokenParser.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TokenParser.scala index 2984520ca54c..e053b780a2f0 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TokenParser.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TokenParser.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.jartypereader.descriptorparser import io.joern.javasrc2cpg.jartypereader.model.PrimitiveType -import io.joern.javasrc2cpg.jartypereader.model.Model.TypeConstants._ +import io.joern.javasrc2cpg.jartypereader.model.Model.TypeConstants.* import org.slf4j.LoggerFactory import scala.util.parsing.combinator.RegexParsers diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TypeParser.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TypeParser.scala index d1f735573013..6694c27abf3a 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TypeParser.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jartypereader/descriptorparser/TypeParser.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.jartypereader.descriptorparser import io.joern.javasrc2cpg.jartypereader.model.Bound.{BoundAbove, BoundBelow} -import io.joern.javasrc2cpg.jartypereader.model._ +import io.joern.javasrc2cpg.jartypereader.model.* import org.slf4j.LoggerFactory trait TypeParser extends TokenParser { diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala index 34cd3c42241c..a92e51fe2c42 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/TypeInferencePass.scala @@ -3,11 +3,10 @@ package io.joern.javasrc2cpg.passes import com.github.javaparser.symbolsolver.cache.GuavaCache import com.google.common.cache.CacheBuilder import io.joern.x2cpg.Defines -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.ModifierTypes +import io.shiftleft.codepropertygraph.generated.{Cpg, ModifierTypes, Properties} import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import scala.jdk.OptionConverters.RichOptional @@ -55,11 +54,8 @@ class TypeInferencePass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { val callArgs = if (skipCallThis) call.argument.toList.tail else call.argument.toList val hasDifferingArg = method.parameter.zip(callArgs).exists { case (parameter, argument) => - val maybeArgumentType = Option(argument.property(PropertyNames.TypeFullName)) - .map(_.toString()) - .getOrElse(TypeConstants.Any) - - val argMatches = maybeArgumentType == TypeConstants.Any || maybeArgumentType == parameter.typeFullName + val maybeArgumentType = argument.propertyOption(Properties.TypeFullName).getOrElse(TypeConstants.Any) + val argMatches = maybeArgumentType == TypeConstants.Any || maybeArgumentType == parameter.typeFullName !argMatches } @@ -80,10 +76,8 @@ class TypeInferencePass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { } private def getReplacementMethod(call: Call): Option[Method] = { - val argTypes = - call.argument.flatMap(arg => Option(arg.property(PropertyNames.TypeFullName)).map(_.toString)).mkString(":") - val callKey = - s"${call.methodFullName}:$argTypes" + val argTypes = call.argument.property(Properties.TypeFullName).mkString(":") + val callKey = s"${call.methodFullName}:$argTypes" cache.get(callKey).toScala.getOrElse { val callNameParts = getNameParts(call.name, call.methodFullName) resolvedMethodIndex.get(call.name).flatMap { candidateMethods => diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala index 04f5df7ea105..005cf188c7f3 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/JavaScopeElement.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewMethodParameterIn import io.shiftleft.codepropertygraph.generated.nodes.NewLocal import io.shiftleft.codepropertygraph.generated.nodes.NewMember import io.joern.javasrc2cpg.util.{BindingTable, BindingTableEntry, NameConstants} -import io.shiftleft.passes.IntervalKeyPool +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.Ast trait JavaScopeElement { diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/Scope.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/Scope.scala index b0025ca254be..5a98bdb7ecbf 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/Scope.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/scope/Scope.scala @@ -74,10 +74,10 @@ class Scope { def popNamespaceScope(): NamespaceScope = popScope[NamespaceScope]() - private def popScope[ScopeType <: JavaScopeElement](): ScopeType = { + private def popScope[ScopeType0 <: JavaScopeElement](): ScopeType0 = { val scope = scopeStack.head scopeStack = scopeStack.tail - scope.asInstanceOf[ScopeType] + scope.asInstanceOf[ScopeType0] } def addTopLevelType(name: String, typeFullName: String): Unit = { diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/JmodClassPath.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/JmodClassPath.scala index e234475e2508..a9891ff5aabf 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/JmodClassPath.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/JmodClassPath.scala @@ -1,10 +1,10 @@ package io.joern.javasrc2cpg.typesolvers import better.files.File -import io.joern.javasrc2cpg.typesolvers.JmodClassPath._ +import io.joern.javasrc2cpg.typesolvers.JmodClassPath.* import javassist.ClassPath -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.util.Try import java.io.InputStream import java.net.URL diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/TypeSizeReducer.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/TypeSizeReducer.scala index 11fa1fdd082f..5e1e0e250267 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/TypeSizeReducer.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/typesolvers/TypeSizeReducer.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.typesolvers import com.github.javaparser.ast.body.TypeDeclaration import com.github.javaparser.ast.stmt.BlockStmt -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* object TypeSizeReducer { def simplifyType(typeDeclaration: TypeDeclaration[?]): Unit = { diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/BindingTable.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/BindingTable.scala index 27d8c2634036..1e4c3f190960 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/BindingTable.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/BindingTable.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.util import scala.collection.mutable import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.joern.x2cpg.utils.NodeBuilders.newBindingNode -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.EdgeTypes case class BindingTableEntry(name: String, signature: String, implementingMethodFullName: String) diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala index 1d1ab11b932a..69b1354d07be 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala @@ -1,14 +1,14 @@ package io.joern.javasrc2cpg.util import better.files.File -import io.joern.x2cpg.utils.ExternalCommand import io.joern.javasrc2cpg.util.Delombok.DelombokMode.* +import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory -import java.nio.file.{Path, Paths} -import scala.collection.mutable -import scala.util.matching.Regex -import scala.util.{Failure, Success, Try} +import java.nio.file.Path +import scala.util.Failure +import scala.util.Success +import scala.util.Try object Delombok { @@ -53,8 +53,17 @@ object Delombok { System.getProperty("java.class.path") } val command = - s"$javaPath -cp $classPathArg lombok.launch.Main delombok ${inputPath.toAbsolutePath.toString} -d ${outputDir.canonicalPath}" - logger.debug(s"Executing delombok with command $command") + Seq( + javaPath, + "-cp", + classPathArg, + "lombok.launch.Main", + "delombok", + inputPath.toAbsolutePath.toString, + "-d", + outputDir.canonicalPath + ) + logger.debug(s"Executing delombok with command ${command.mkString(" ")}") command } @@ -72,6 +81,7 @@ object Delombok { Try(delombokTempDir.createChild(relativeOutputPath, asDirectory = true)).flatMap { packageOutputDir => ExternalCommand .run(delombokToTempDirCommand(inputDir, packageOutputDir, analysisJavaHome), ".") + .toTry .map(_ => delombokTempDir.path.toAbsolutePath.toString) } } diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala index 8bc1e05e920d..9d844758da35 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Util.scala @@ -2,15 +2,12 @@ package io.joern.javasrc2cpg.util import com.github.javaparser.resolution.declarations.ResolvedReferenceTypeDeclaration import com.github.javaparser.resolution.types.ResolvedReferenceType -import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants -import io.joern.x2cpg.{Ast, Defines} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, PropertyNames} -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewFieldIdentifier, NewMember} +import io.joern.x2cpg.Defines import org.slf4j.LoggerFactory import scala.collection.mutable import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* object Util { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/JavaSrc2CpgTestContext.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/JavaSrc2CpgTestContext.scala deleted file mode 100644 index afc3a72a0da0..000000000000 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/JavaSrc2CpgTestContext.scala +++ /dev/null @@ -1,61 +0,0 @@ -package io.joern.javasrc2cpg - -import io.shiftleft.codepropertygraph.generated.Cpg -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic -import io.joern.x2cpg.X2Cpg.writeCodeToFile -import io.shiftleft.semanticcpg.layers.LayerCreatorContext - -class JavaSrc2CpgTestContext { - private var code: String = "" - private var buildResult = Option.empty[Cpg] - private var _extraFlows = List.empty[FlowSemantic] - - def buildCpg(runDataflow: Boolean, inferenceJarPaths: Set[String]): Cpg = { - if (buildResult.isEmpty) { - val javaSrc2Cpg = JavaSrc2Cpg() - val config = Config(inferenceJarPaths = inferenceJarPaths) - .withInputPath(writeCodeToFile(code, "javasrc2cpgTest", ".java").getAbsolutePath) - .withOutputPath("") - .withCacheJdkTypeSolver(true) - val cpg = javaSrc2Cpg.createCpgWithOverlays(config) - if (runDataflow) { - val context = new LayerCreatorContext(cpg.get) - val options = new OssDataFlowOptions(extraFlows = _extraFlows) - new OssDataFlow(options).run(context) - } - buildResult = Some(cpg.get) - } - buildResult.get - } - - private def withSource(code: String): JavaSrc2CpgTestContext = { - this.code = code - this - } - - private def withExtraFlows(value: List[FlowSemantic] = List.empty): this.type = { - this._extraFlows = value - this - } - -} - -object JavaSrc2CpgTestContext { - def buildCpg(code: String, inferenceJarPaths: Set[String] = Set.empty): Cpg = { - new JavaSrc2CpgTestContext() - .withSource(code) - .buildCpg(runDataflow = false, inferenceJarPaths = inferenceJarPaths) - } - - def buildCpgWithDataflow( - code: String, - inferenceJarPaths: Set[String] = Set.empty, - extraFlows: List[FlowSemantic] = List.empty - ): Cpg = { - new JavaSrc2CpgTestContext() - .withSource(code) - .withExtraFlows(extraFlows) - .buildCpg(runDataflow = true, inferenceJarPaths = inferenceJarPaths) - } -} diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..773fce3a89c7 --- /dev/null +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala @@ -0,0 +1,85 @@ +package io.joern.javasrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class JavaSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("javasrc2cpgTestsHttpTest") + val file = dir / "Main.java" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |class HelloWorld { + | public static void main(String[] args) { + | System.out.println($indexStr); + | } + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.javasrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.javasrc2cpg.Main.stop() + } + + "Using javasrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("System.out.println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"System.out.println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala index 2891f13b3eaa..d48a1a873f6e 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala @@ -1,14 +1,13 @@ package io.joern.javasrc2cpg.passes import better.files.File +import flatgraph.misc.TestUtils.* import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.x2cpg.passes.frontend.JavaConfigFileCreationPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewMetaData import io.shiftleft.semanticcpg.language.* import io.shiftleft.utils.ProjectRoot -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder import java.nio.file.Paths @@ -18,8 +17,8 @@ class ConfigFileCreationPassTests extends JavaSrcCode2CpgFixture { ProjectRoot.relativise("joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests") "it should find the correct config files" in { - val cpg = new Cpg() - BatchedUpdate.applyDiff(cpg.graph, Cpg.newDiffGraphBuilder.addNode(NewMetaData().root(testConfigDir)).build()) + val cpg = Cpg.from(_.addNode(NewMetaData().root(testConfigDir))) + val foundFiles = new JavaConfigFileCreationPass(cpg).generateParts().map(_.canonicalPath) val absoluteConfigDir = File(testConfigDir).canonicalPath foundFiles should contain theSameElementsAs Array( diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArithmeticOperationsTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArithmeticOperationsTests.scala index e804e57eee6f..6e33a3755eef 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArithmeticOperationsTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArithmeticOperationsTests.scala @@ -5,7 +5,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Identifier import io.shiftleft.semanticcpg.language.toNodeTypeStarters -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ArithmeticOperationsTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArrayTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArrayTests.scala index e1f247b52fd8..1c44df3ef426 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArrayTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ArrayTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ArrayTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BindingTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BindingTests.scala index e7904883cec5..153b99d7e4a1 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BindingTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BindingTests.scala @@ -1,6 +1,6 @@ package io.joern.javasrc2cpg.querying -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture class BindingTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BooleanOperationsTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BooleanOperationsTests.scala index 7ec4357a2891..76e69e381f6c 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BooleanOperationsTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/BooleanOperationsTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class BooleanOperationsTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallGraphTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallGraphTests.scala index c54121618cf8..1c680d2d689b 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallGraphTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallGraphTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallGraphTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala index 3072ad5574d3..9e62224b211f 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CallTests.scala @@ -6,9 +6,7 @@ import io.shiftleft.codepropertygraph.generated.edges.Ref import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal, MethodParameterIn} import io.shiftleft.semanticcpg.language.NoResolve -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.jIteratortoTraversal -import overflowdb.traversal.toNodeTraversal +import io.shiftleft.semanticcpg.language.* class NewCallTests extends JavaSrcCode2CpgFixture { "calls to imported methods" when { @@ -284,7 +282,7 @@ class NewCallTests extends JavaSrcCode2CpgFixture { cpg.method.name("test").call.name("foo").argument(0).outE.collectAll[Ref].l match { case List(ref) => - ref.inNode match { + ref.dst match { case param: MethodParameterIn => param.name shouldBe "this" param.index shouldBe 0 @@ -309,7 +307,7 @@ class NewCallTests extends JavaSrcCode2CpgFixture { cpg.method.name("test").call.name("foo").argument(0).outE.collectAll[Ref].l match { case List(ref) => - ref.inNode match { + ref.dst match { case param: MethodParameterIn => param.name shouldBe "this" param.index shouldBe 0 diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CfgTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CfgTests.scala index 5bcf4d6c7fec..9b7e75ec1bd0 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CfgTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/CfgTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CfgTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ClassLoaderTypeTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ClassLoaderTypeTests.scala index 1178f9113a77..ceb60c86b9a5 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ClassLoaderTypeTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ClassLoaderTypeTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.Config import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.utils.ExternalCommand class ClassLoaderTypeTests extends JavaSrcCode2CpgFixture { @@ -48,7 +48,6 @@ class ClassLoaderTypeTests extends JavaSrcCode2CpgFixture { } "be resolved by the system classloader (java 17)" in { - println(System.getProperty("java.version")) val cpg = code(testCode) cpg.call.name("getIconHeight").methodFullName.head.startsWith(" val List(_: Local, assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked - assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() assign.name shouldBe Operators.assignment val alloc = assign.argument(2).asInstanceOf[Call] alloc.name shouldBe ".alloc" alloc.code shouldBe "new Bar(4, 2)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.methodFullName shouldBe ".alloc" alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" init.code shouldBe "new Bar(4, 2)" @@ -303,20 +303,20 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { case List(method) => val List(assign: Call, init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked - assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() assign.name shouldBe Operators.assignment val alloc = assign.argument(2).asInstanceOf[Call] alloc.name shouldBe ".alloc" alloc.code shouldBe "new Bar(4, 2)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.methodFullName shouldBe ".alloc" alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int,int)" + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" init.code shouldBe "new Bar(4, 2)" @@ -362,16 +362,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { alloc.order shouldBe 2 alloc.argumentIndex shouldBe 2 alloc.code shouldBe "new Bar(42)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" init.signature shouldBe "void(int)" init.code shouldBe "new Bar(42)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.argument.size shouldBe 2 val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked @@ -411,16 +411,16 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { alloc.order shouldBe 2 alloc.argumentIndex shouldBe 2 alloc.code shouldBe "new Bar(42)" - alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() alloc.typeFullName shouldBe "Bar" alloc.argument.size shouldBe 0 init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.callOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" + init._methodViaCallOut.head.fullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" init.signature shouldBe "void(int)" init.code shouldBe "new Bar(42)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.argument.size shouldBe 2 val List(obj: Identifier, initArg1: Literal) = init.argument.l: @unchecked @@ -447,7 +447,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Bar.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int)" @@ -475,7 +475,7 @@ class ConstructorInvocationTests extends JavaSrcCode2CpgFixture { val List(init: Call) = method.astChildren.isBlock.astChildren.l: @unchecked init.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName init.methodFullName shouldBe s"Foo.${io.joern.x2cpg.Defines.ConstructorMethodName}:void(int)" - init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString + init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.name() init.typeFullName shouldBe "void" init.signature shouldBe "void(int)" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala index e1fbe59ce842..c26cd0355f5b 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ControlStructureTests.scala @@ -13,10 +13,9 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ Local, Return } -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.toNodeTraversal +import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class NewControlStructureTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala index 84e8ed5e3df6..5130397d3462 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/EnumTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.Literal -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class EnumTests extends JavaSrcCode2CpgFixture { val cpg = code(""" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/FileTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/FileTests.scala index bd64f35330a9..728ffbdb36f1 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/FileTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/FileTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import org.scalatest.Ignore diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala index 609e8789244e..8620f7318b31 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/GenericsTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class GenericsTests extends JavaSrcCode2CpgFixture { "unresolved generic type declarations" should { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ImportTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ImportTests.scala index 8e01a0910bd9..aa26b95d1eab 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ImportTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ImportTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ImportTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/InferenceJarTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/InferenceJarTests.scala index 1d6b65f6dc97..8e77f6af02fe 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/InferenceJarTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/InferenceJarTests.scala @@ -1,16 +1,14 @@ package io.joern.javasrc2cpg.querying -import io.joern.javasrc2cpg.JavaSrc2CpgTestContext -import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants +import io.joern.javasrc2cpg.JavaSrc2Cpg.DefaultConfig +import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.x2cpg.Defines -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.utils.ProjectRoot -import org.scalatest.freespec.AnyFreeSpec -import org.scalatest.matchers.should.Matchers -class InferenceJarTests extends AnyFreeSpec with Matchers { +class InferenceJarTests extends JavaSrcCode2CpgFixture { - private val code: String = + private val _code: String = """ |class Test { | public void test1() { @@ -19,18 +17,18 @@ class InferenceJarTests extends AnyFreeSpec with Matchers { |} |""".stripMargin - "CPG for code where inference jar for dependencies is provided" - { + "CPG for code where inference jar for dependencies is provided" should { val inferenceJarPath = ProjectRoot.relativise("joern-cli/frontends/javasrc2cpg/src/test/resources/Deps.jar") - lazy val cpg = JavaSrc2CpgTestContext.buildCpg(code, inferenceJarPaths = Set(inferenceJarPath)) + lazy val cpg = code(_code).withConfig(DefaultConfig.withInferenceJarPaths(Set(inferenceJarPath))) - "it should resolve the type for Deps" in { + "resolve the type for Deps" in { val call = cpg.method.name("test1").call.name("foo").head call.methodFullName shouldBe "Deps.foo:int()" call.typeFullName shouldBe "int" call.signature shouldBe "int()" } - "it should create stubs for elements used in Deps" in { + "create stubs for elements used in Deps" in { cpg.typeDecl.name("Deps").size shouldBe 1 val depsTypeDecl = cpg.typeDecl.name("Deps").head depsTypeDecl.fullName shouldBe "Deps" @@ -43,10 +41,10 @@ class InferenceJarTests extends AnyFreeSpec with Matchers { } } - "CPG for code where inference jar for dependencies is not provided" - { - lazy val cpg = JavaSrc2CpgTestContext.buildCpg(code) + "CPG for code where inference jar for dependencies is not provided" should { + lazy val cpg = code(_code) - "it should fail to resolve the type for Deps" in { + "fail to resolve the type for Deps" in { val call = cpg.method.name("test1").call.name("foo").head call.methodFullName shouldBe s"${Defines.UnresolvedNamespace}.foo:${Defines.UnresolvedSignature}(0)" call.signature shouldBe s"${Defines.UnresolvedSignature}(0)" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LiteralTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LiteralTests.scala index ebb4c50a1907..4b95846be327 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LiteralTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LiteralTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import com.github.javaparser.ast.expr.LiteralExpr import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LiteralTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LocalTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LocalTests.scala index 6c2024b75004..7ba4a1df0eae 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LocalTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LocalTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.Local -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocalTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LombokTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LombokTests.scala index c8697f6fdef9..ca15ffbfe948 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LombokTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LombokTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants import io.joern.x2cpg.Defines -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.javasrc2cpg.Config class LombokTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala index 7ee3291c7607..8e707b219888 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MemberTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, ModifierTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal, Member} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NewMemberTests extends JavaSrcCode2CpgFixture { "locals shadowing members" should { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MetaDataTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MetaDataTests.scala index ea2f134d41fc..2a307d8ef928 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MetaDataTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MetaDataTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodParameterTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodParameterTests.scala index b0149ece816c..5462b381f3b9 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodParameterTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodParameterTests.scala @@ -2,7 +2,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.EvaluationStrategies -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodParameterTests2 extends JavaSrcCode2CpgFixture { "non generic method" should { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodReturnTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodReturnTests.scala index 67f350880208..ba6c505cc449 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodReturnTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodReturnTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Ignore class MethodReturnTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodTests.scala index 093544e797b5..03213b5d2467 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/MethodTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/NamespaceBlockTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/NamespaceBlockTests.scala index 77fca2f6ef15..b5d45219eb0d 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/NamespaceBlockTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/NamespaceBlockTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NamespaceBlockTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ScopeTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ScopeTests.scala index 34944d78812c..92ce9102f707 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ScopeTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/ScopeTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ScopeTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SpecialOperatorTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SpecialOperatorTests.scala index 9dbf0dfc7380..99a6dd5520d0 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SpecialOperatorTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SpecialOperatorTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, TypeRef} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SpecialOperatorTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SynchronizedTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SynchronizedTests.scala index 66eeb14c0d67..48b215d300c8 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SynchronizedTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/SynchronizedTests.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ Modifier, Return } -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SynchronizedTests extends JavaSrcCode2CpgFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeDeclTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeDeclTests.scala index 3d4857c67aca..8e287021afb9 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeDeclTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeDeclTests.scala @@ -3,7 +3,7 @@ package io.joern.javasrc2cpg.querying import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.shiftleft.codepropertygraph.generated.ModifierTypes import io.shiftleft.codepropertygraph.generated.nodes.Return -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeInferenceTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeInferenceTests.scala index 5f2562cbd982..ce3a116dd730 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeInferenceTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeInferenceTests.scala @@ -4,7 +4,7 @@ import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeTests.scala index 7ef189482af9..1e37c6ed4139 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/TypeTests.scala @@ -5,7 +5,7 @@ import io.joern.javasrc2cpg.typesolvers.TypeInfoCalculator.TypeConstants import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier} import io.shiftleft.proto.cpg.Cpg.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NewTypeTests extends JavaSrcCode2CpgFixture { "processing wildcard types should not crash (smoke test)" when { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ArrayTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ArrayTests.scala index 4dc784776b8d..e146937fda67 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ArrayTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ArrayTests.scala @@ -1,9 +1,9 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class ArrayTests extends JavaDataflowFixture { behavior of "Dataflow through arrays" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/FunctionCallTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/FunctionCallTests.scala index bb3b4b15a230..53a5570b7d6b 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/FunctionCallTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/FunctionCallTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.{JavaDataflowFixture, JavaSrcCode2CpgFixture} -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* class NewFunctionCallTests extends JavaSrcCode2CpgFixture(withOssDataflow = true) { "Dataflow through function calls" should { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/IfTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/IfTests.scala index 0a85145fd59f..f5384cab3c8c 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/IfTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/IfTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* class IfTests extends JavaDataflowFixture { behavior of "Dataflow through IF structures" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LambdaTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LambdaTests.scala index 2ae5354391e6..c18f615eedcc 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LambdaTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LambdaTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LambdaTests extends JavaSrcCode2CpgFixture(withOssDataflow = true) { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LoopTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LoopTests.scala index 556a263e0588..d664633ca5ef 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LoopTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/LoopTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* class LoopTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MemberTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MemberTests.scala index 2d104271385c..76a9cc9d6c2f 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MemberTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MemberTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.{JavaDataflowFixture, JavaSrcCode2CpgFixture} -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* class MemberTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MethodReturnTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MethodReturnTests.scala index 1f39c507cbef..e1db74388037 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MethodReturnTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/MethodReturnTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* class MethodReturnTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ObjectTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ObjectTests.scala index babcb1667967..0e261e9e7446 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ObjectTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ObjectTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* class ObjectTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/OperatorTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/OperatorTests.scala index 6a463d4e3819..b549f8ce6e50 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/OperatorTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/OperatorTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* class OperatorTests extends JavaDataflowFixture { behavior of "Dataflow through operators" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ReturnTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ReturnTests.scala index 6125def15d45..d2fd5859aec4 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ReturnTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/ReturnTests.scala @@ -1,8 +1,8 @@ package io.joern.javasrc2cpg.querying.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ReturnTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SemanticTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SemanticTests.scala index 8401b458b68d..e1dd18ac1d74 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SemanticTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SemanticTests.scala @@ -1,22 +1,24 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.{EngineContext, EngineConfig} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.semanticsloader.FlowSemantic import io.joern.x2cpg.Defines class SemanticTests - extends JavaDataflowFixture(extraFlows = - List( - FlowSemantic.from("Test.sanitize:java.lang.String(java.lang.String)", List((0, 0), (1, 1))), - FlowSemantic.from(s"ext.Library.killParam:${Defines.UnresolvedSignature}(1)", List.empty), - FlowSemantic.from("^ext\\.Library\\.taintNone:.*", List((0, 0), (1, 1)), regex = true), - FlowSemantic.from("^ext\\.Library\\.taint1to2:.*", List((1, 2)), regex = true) + extends JavaDataflowFixture(semantics = + DefaultSemantics().plus( + List( + FlowSemantic.from("Test.sanitize:java.lang.String(java.lang.String)", List((0, 0), (1, 1))), + FlowSemantic.from(s"ext.Library.killParam:${Defines.UnresolvedSignature}(1)", List.empty), + FlowSemantic.from("^ext\\.Library\\.taintNone:.*", List((0, 0), (1, 1)), regex = true), + FlowSemantic.from("^ext\\.Library\\.taint1to2:.*", List((1, 2)), regex = true) + ) ) ) { behavior of "Dataflow through custom semantics" diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SwitchTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SwitchTests.scala index 89131994d336..ba82845d22f1 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SwitchTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/SwitchTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* class SwitchTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/TryTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/TryTests.scala index 9f205a153558..8939c875e2a3 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/TryTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/dataflow/TryTests.scala @@ -1,7 +1,7 @@ package io.joern.javasrc2cpg.querying.dataflow import io.joern.javasrc2cpg.testfixtures.JavaDataflowFixture -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* class TryTests extends JavaDataflowFixture { diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaDataflowFixture.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaDataflowFixture.scala index ec6881f697e3..60c43e6555fd 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaDataflowFixture.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaDataflowFixture.scala @@ -1,22 +1,22 @@ package io.joern.javasrc2cpg.testfixtures +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic -import io.joern.javasrc2cpg.JavaSrc2CpgTestContext +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Literal} import io.shiftleft.semanticcpg.language.* import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class JavaDataflowFixture(extraFlows: List[FlowSemantic] = List.empty) extends AnyFlatSpec with Matchers { +class JavaDataflowFixture(semantics: Semantics = DefaultSemantics()) extends AnyFlatSpec with Matchers { implicit val resolver: ICallResolver = NoResolve implicit lazy val engineContext: EngineContext = EngineContext() val code: String = "" - lazy val cpg: Cpg = JavaSrc2CpgTestContext.buildCpgWithDataflow(code, extraFlows = extraFlows) + lazy val cpg: Cpg = JavaSrcTestCpg().withOssDataflow().withSemantics(semantics).moreCode(code) def getConstSourceSink( methodName: String, diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala index 717ce7bc3d95..f7dcf63602f3 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala @@ -1,13 +1,12 @@ package io.joern.javasrc2cpg.testfixtures -import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} -import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.frontendspecific.javasrc2cpg import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg} +import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Literal} import io.shiftleft.semanticcpg.language.* @@ -42,12 +41,12 @@ class JavaSrcTestCpg(enableTypeRecovery: Boolean = false) class JavaSrcCode2CpgFixture( withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), enableTypeRecovery: Boolean = false ) extends Code2CpgFixture(() => - new JavaSrcTestCpg(enableTypeRecovery).withOssDataflow(withOssDataflow).withExtraFlows(extraFlows) + new JavaSrcTestCpg(enableTypeRecovery).withOssDataflow(withOssDataflow).withSemantics(semantics) ) - with SemanticCpgTestFixture(extraFlows) { + with SemanticCpgTestFixture(semantics) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Main.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Main.scala index cc383563ea52..20421f73f158 100644 --- a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Main.scala +++ b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Main.scala @@ -1,9 +1,12 @@ package io.joern.jimple2cpg -import io.joern.jimple2cpg.Frontend._ +import io.joern.jimple2cpg.Frontend.* import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser +import java.util.concurrent.ExecutorService + /** Command line configuration parameters */ final case class Config( @@ -74,8 +77,14 @@ private object Frontend { /** Entry point for command line CPG creator */ -object Main extends X2CpgMain(cmdLineParser, new Jimple2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new Jimple2Cpg()) with FrontendHTTPServer[Config, Jimple2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + + override protected val executor: ExecutorService = FrontendHTTPServer.singleThreadExecutor() + def run(config: Config, jimple2Cpg: Jimple2Cpg): Unit = { - jimple2Cpg.run(config) + if (config.serverMode) { startup() } + else { jimple2Cpg.run(config) } } } diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/AstCreator.scala index ca9600e2cba9..6f248ffa494b 100644 --- a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/AstCreator.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* import org.objectweb.asm.Type import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import soot.jimple.* import soot.tagkit.* import soot.{Unit as SUnit, Local as _, *} diff --git a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala index a2af7244d5f3..e9286134bce8 100644 --- a/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala +++ b/joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/astcreation/declarations/AstForDeclarationsCreator.scala @@ -11,6 +11,7 @@ import soot.tagkit.* import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters.CollectionHasAsScala + trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) extends AstForTypeDeclsCreator with AstForMethodsCreator { this: AstCreator => diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/io/Jimple2CpgHTTPServerTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/io/Jimple2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..89d8b78aa59d --- /dev/null +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/io/Jimple2CpgHTTPServerTests.scala @@ -0,0 +1,88 @@ +package io.joern.jimple2cpg.io + +import better.files.File +import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture +import io.joern.jimple2cpg.testfixtures.JimpleCodeToCpgFixture +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class Jimple2CpgHTTPServerTests extends JimpleCode2CpgFixture with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("jimple2cpgTestsHttpTest") + val file = dir / "main.java" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |class Foo { + | static void main$indexStr(int argc, char argv) { + | System.out.println("Hello World!"); + | } + |} + |""".stripMargin) + JimpleCodeToCpgFixture.compileJava(dir.path, List(file.toJava)) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.jimple2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.jimple2cpg.Main.stop() + } + + "Using jimple2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("jimple2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("""$stack2.println("Hello World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("jimple2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain(s"main$index") + cpg.call.code.l should contain("""$stack2.println("Hello World!")""") + } + } + } + } + +} diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/AnnotationTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/AnnotationTests.scala index 11bb1da6a974..dd4e5c7f83fe 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/AnnotationTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/AnnotationTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, AnnotationLiteral, ArrayInitializer} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AnnotationTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ArrayTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ArrayTests.scala index a7ccd178c297..cd97abf470bf 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ArrayTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ArrayTests.scala @@ -4,7 +4,7 @@ import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Failed class ArrayTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CfgTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CfgTests.scala index 3dee0d91e31e..57325ce9c37f 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CfgTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CfgTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CfgTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CodeDumperTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CodeDumperTests.scala index 918b20fe503d..08d772ddac75 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CodeDumperTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/CodeDumperTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.Config import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CodeDumperTests extends JimpleCode2CpgFixture { private val config = Config().withDisableFileContent(false) diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ConstructorInvocationTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ConstructorInvocationTests.scala index 38ddf83d54c1..9bca4df2876c 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ConstructorInvocationTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ConstructorInvocationTests.scala @@ -3,9 +3,9 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.proto.cpg.Cpg.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** These tests are based off of those found in javasrc2cpg but modified to fit to Jimple's 3-address code rule and flat * AST. diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/EnumTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/EnumTests.scala index a1204ba6a8e0..2c5deae1521f 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/EnumTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/EnumTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.Literal -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class EnumTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FieldAccessTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FieldAccessTests.scala index 00835ff01fff..233c5a3a65e9 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FieldAccessTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FieldAccessTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class FieldAccessTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FileTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FileTests.scala index 39cf829b92c3..b0878237e449 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FileTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/FileTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.{File => JFile} diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/IfGotoTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/IfGotoTests.scala index aa5cb73bcd2e..20cc5fe5c232 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/IfGotoTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/IfGotoTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Unknown} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class IfGotoTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ImplementsInterfaceTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ImplementsInterfaceTests.scala index 0238a83d4225..8a1e81d2fbd0 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ImplementsInterfaceTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ImplementsInterfaceTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/InterfaceTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/InterfaceTests.scala index 4426dea1cb6d..e457ecf7108d 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/InterfaceTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/InterfaceTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/LocalTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/LocalTests.scala index dc9f97880ff4..8762d9b2f9ee 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/LocalTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/LocalTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Local -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Ignore class LocalTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MemberTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MemberTests.scala index 903dfeda8669..b86d2a728c6d 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MemberTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MemberTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Ignore class MemberTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala index 650fd5eca576..1830f176ce88 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MetaDataTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataTests extends JimpleCode2CpgFixture { @@ -19,8 +19,8 @@ class MetaDataTests extends JimpleCode2CpgFixture { "should not have any incoming or outgoing edges" in { cpg.metaData.size shouldBe 1 - cpg.metaData.in().l shouldBe List() - cpg.metaData.out().l shouldBe List() + cpg.metaData.in.l shouldBe List() + cpg.metaData.out.l shouldBe List() } } diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodParameterTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodParameterTests.scala index 005df7e16195..953ee19f0e25 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodParameterTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodParameterTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.EvaluationStrategies -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodParameterTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodReturnTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodReturnTests.scala index 676c2e40ff1a..90394c197098 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodReturnTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodReturnTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodReturnTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodTests.scala index a6f3cbbbede7..fa5284f4b101 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/MethodTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/NamespaceBlockTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/NamespaceBlockTests.scala index 4af8a2ae9956..b2b389bdd073 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/NamespaceBlockTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/NamespaceBlockTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NamespaceBlockTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ReflectionTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ReflectionTests.scala index d509ea4e5857..5b50d987c98c 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ReflectionTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/ReflectionTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** Right now reflection is mostly unsupported. This should be extended in later when it is. */ diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SpecialOperatorTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SpecialOperatorTests.scala index 7797f83427ed..3cb0f35d077e 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SpecialOperatorTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SpecialOperatorTests.scala @@ -4,7 +4,7 @@ import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, TypeRef} import io.shiftleft.proto.cpg.Cpg.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SpecialOperatorTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SwitchTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SwitchTests.scala index 41896808acaf..2214335eb27b 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SwitchTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SwitchTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.JumpTarget -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SwitchTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SynchronizedTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SynchronizedTests.scala index fad58de51c90..ce4481faaee6 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SynchronizedTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/SynchronizedTests.scala @@ -2,8 +2,8 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class SynchronizedTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeDeclTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeDeclTests.scala index 57d46e0f2853..78bf627e2ac4 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeDeclTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeDeclTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeTests.scala index 3dbc9ca13a40..8108fcbef372 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/TypeTests.scala @@ -2,7 +2,7 @@ package io.joern.jimple2cpg.querying import io.joern.jimple2cpg.testfixtures.JimpleCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TypeTests extends JimpleCode2CpgFixture { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/ArrayTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/ArrayTests.scala index b36a9e75e763..a375017251e0 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/ArrayTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/ArrayTests.scala @@ -1,6 +1,6 @@ package io.joern.jimple2cpg.querying.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.jimple2cpg.testfixtures.{JimpleDataFlowCodeToCpgSuite, JimpleDataflowTestCpg} class ArrayTests extends JimpleDataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/FunctionCallTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/FunctionCallTests.scala index 63a49bb0cfaf..942d79905634 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/FunctionCallTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/FunctionCallTests.scala @@ -3,7 +3,7 @@ package io.joern.jimple2cpg.querying.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.jimple2cpg.testfixtures.JimpleDataFlowCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.Operators class FunctionCallTests extends JimpleDataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SemanticTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SemanticTests.scala index 2cf966788c4d..dbf0956dbdae 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SemanticTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SemanticTests.scala @@ -1,15 +1,18 @@ package io.joern.jimple2cpg.querying.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.language.* import io.joern.jimple2cpg.testfixtures.{JimpleDataFlowCodeToCpgSuite, JimpleDataflowTestCpg} import io.joern.dataflowengineoss.semanticsloader.FlowSemantic import io.joern.x2cpg.Defines class SemanticTests - extends JimpleDataFlowCodeToCpgSuite(extraFlows = - List( - FlowSemantic.from("Test.sanitize:java.lang.String(java.lang.String)", List((0, 0), (1, 1))), - FlowSemantic.from("java.nio.file.Paths.get:.*\\(java.lang.String,.*\\)", List.empty, regex = true) + extends JimpleDataFlowCodeToCpgSuite(semantics = + DefaultSemantics().plus( + List( + FlowSemantic.from("Test.sanitize:java.lang.String(java.lang.String)", List((0, 0), (1, 1))), + FlowSemantic.from("java.nio.file.Paths.get:.*\\(java.lang.String,.*\\)", List.empty, regex = true) + ) ) ) { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SwitchTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SwitchTests.scala index 5949eda27a45..2611292db34a 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SwitchTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/querying/dataflow/SwitchTests.scala @@ -1,6 +1,6 @@ package io.joern.jimple2cpg.querying.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.jimple2cpg.testfixtures.JimpleDataFlowCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.Cpg diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleCodeToCpgFixture.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleCodeToCpgFixture.scala index 3d754650d482..ebdca165c635 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleCodeToCpgFixture.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleCodeToCpgFixture.scala @@ -1,6 +1,7 @@ package io.joern.jimple2cpg.testfixtures -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} import io.joern.jimple2cpg.{Config, Jimple2Cpg} import io.joern.x2cpg.X2Cpg @@ -23,9 +24,9 @@ trait Jimple2CpgFrontend extends LanguageFrontend { } } -class JimpleCode2CpgFixture(withOssDataflow: Boolean = false, extraFlows: List[FlowSemantic] = List.empty) - extends Code2CpgFixture(() => new JimpleTestCpg().withOssDataflow(withOssDataflow).withExtraFlows(extraFlows)) - with SemanticCpgTestFixture(extraFlows) {} +class JimpleCode2CpgFixture(withOssDataflow: Boolean = false, semantics: Semantics = DefaultSemantics()) + extends Code2CpgFixture(() => new JimpleTestCpg().withOssDataflow(withOssDataflow).withSemantics(semantics)) + with SemanticCpgTestFixture(semantics) {} class JimpleTestCpg extends DefaultTestCpg with Jimple2CpgFrontend with SemanticTestCpg { diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleDataflowCodeToCpgSuite.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleDataflowCodeToCpgSuite.scala index 8fa238410617..c2baab59cc80 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleDataflowCodeToCpgSuite.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/testfixtures/JimpleDataflowCodeToCpgSuite.scala @@ -1,15 +1,16 @@ package io.joern.jimple2cpg.testfixtures +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.x2cpg.testfixtures.Code2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext -class JimpleDataflowTestCpg(val extraFlows: List[FlowSemantic] = List.empty) extends JimpleTestCpg { +class JimpleDataflowTestCpg(val semantics: Semantics = DefaultSemantics()) extends JimpleTestCpg { implicit val resolver: ICallResolver = NoResolve implicit lazy val engineContext: EngineContext = EngineContext() @@ -17,14 +18,14 @@ class JimpleDataflowTestCpg(val extraFlows: List[FlowSemantic] = List.empty) ext override def applyPasses(): Unit = { super.applyPasses() val context = new LayerCreatorContext(this) - val options = new OssDataFlowOptions(extraFlows = extraFlows) + val options = new OssDataFlowOptions(semantics = semantics) new OssDataFlow(options).run(context) } } -class JimpleDataFlowCodeToCpgSuite(val extraFlows: List[FlowSemantic] = List.empty) - extends Code2CpgFixture(() => new JimpleDataflowTestCpg(extraFlows)) { +class JimpleDataFlowCodeToCpgSuite(val semantics: Semantics = DefaultSemantics()) + extends Code2CpgFixture(() => new JimpleDataflowTestCpg(semantics)) { implicit var context: EngineContext = EngineContext() diff --git a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala index b36deb4b9189..5f641af9a7b7 100644 --- a/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala +++ b/joern-cli/frontends/jimple2cpg/src/test/scala/io/joern/jimple2cpg/unpacking/JarUnpackingTests.scala @@ -12,14 +12,15 @@ import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec import java.nio.file.{Files, Path, Paths} +import scala.compiletime.uninitialized import scala.util.{Failure, Success, Try} class JarUnpackingTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { - var recurseCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var noRecurseCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var depthsCpgs: Map[String, Cpg] = scala.compiletime.uninitialized - var slippyCpg: Cpg = scala.compiletime.uninitialized + var recurseCpgs: Map[String, Cpg] = uninitialized + var noRecurseCpgs: Map[String, Cpg] = uninitialized + var depthsCpgs: Map[String, Cpg] = uninitialized + var slippyCpg: Cpg = uninitialized override protected def beforeAll(): Unit = { super.beforeAll() diff --git a/joern-cli/frontends/jssrc2cpg/src/main/resources/application.conf b/joern-cli/frontends/jssrc2cpg/src/main/resources/application.conf index caa918ef495a..0dc11de48300 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/resources/application.conf +++ b/joern-cli/frontends/jssrc2cpg/src/main/resources/application.conf @@ -1,3 +1,3 @@ jssrc2cpg { - astgen_version: "3.14.0" + astgen_version: "3.16.0" } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala index 22b3ce8120a4..258db2764f8c 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala @@ -18,11 +18,10 @@ import scala.util.Try class JsSrc2Cpg extends X2CpgFrontend[Config] { - private val report: Report = new Report() - def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => File.usingTemporaryDirectory("jssrc2cpgOut") { tmpDir => + val report = new Report() val astGenResult = new AstGenRunner(config).execute(tmpDir) val hash = HashUtil.sha256(astGenResult.parsedFiles.map { case (_, file) => File(file).path }) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/Main.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/Main.scala index 06b270dbc514..4582b898636e 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/Main.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/Main.scala @@ -4,6 +4,7 @@ import io.joern.jssrc2cpg.Frontend.* import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} import io.joern.x2cpg.utils.Environment import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -34,14 +35,19 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new JsSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new JsSrc2Cpg()) with FrontendHTTPServer[Config, JsSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() def run(config: Config, jssrc2cpg: JsSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - if (Environment.pathExists(absPath)) { - jssrc2cpg.run(config.withInputPath(absPath)) - } else { - System.exit(1) + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + if (Environment.pathExists(absPath)) { + jssrc2cpg.run(config.withInputPath(absPath)) + } else { + System.exit(1) + } } } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala index dd6b95bb45c5..a2c81ee3c6ea 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala @@ -1,24 +1,31 @@ package io.joern.jssrc2cpg.astcreation import io.joern.jssrc2cpg.Config -import io.joern.jssrc2cpg.datastructures.{MethodScope, Scope} +import io.joern.jssrc2cpg.datastructures.MethodScope +import io.joern.jssrc2cpg.datastructures.Scope import io.joern.jssrc2cpg.parser.BabelAst.* import io.joern.jssrc2cpg.parser.BabelJsonParser.ParseResult import io.joern.jssrc2cpg.parser.BabelNodeInfo +import io.joern.x2cpg.Ast +import io.joern.x2cpg.AstCreatorBase +import io.joern.x2cpg.ValidationMode +import io.joern.x2cpg.AstNodeBuilder as X2CpgAstNodeBuilder +import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines -import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode, newModifierNode} -import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} -import io.joern.x2cpg.datastructures.Global -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes, NodeTypes} +import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode +import io.joern.x2cpg.utils.NodeBuilders.newModifierNode +import io.shiftleft.codepropertygraph.generated.EvaluationStrategies +import io.shiftleft.codepropertygraph.generated.ModifierTypes +import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewBlock import io.shiftleft.codepropertygraph.generated.nodes.NewFile -import io.shiftleft.codepropertygraph.generated.nodes.NewMethod import io.shiftleft.codepropertygraph.generated.nodes.NewNode import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.shiftleft.codepropertygraph.generated.nodes.NewTypeRef -import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder +import org.slf4j.Logger +import org.slf4j.LoggerFactory import ujson.Value import scala.collection.mutable @@ -73,28 +80,13 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse } private def createProgramMethod(): Ast = { - val path = parserResult.filename - val astNodeInfo = createBabelNodeInfo(parserResult.json("ast")) - val lineNumber = astNodeInfo.lineNumber - val columnNumber = astNodeInfo.columnNumber - val lineNumberEnd = astNodeInfo.lineNumberEnd - val columnNumberEnd = astNodeInfo.columnNumberEnd - val name = Defines.Program - val fullName = s"$path:$name" + val path = parserResult.filename + val astNodeInfo = createBabelNodeInfo(parserResult.json("ast")) + val name = Defines.Program + val fullName = s"$path:$name" val programMethod = - NewMethod() - .order(1) - .name(name) - .code(name) - .fullName(fullName) - .filename(path) - .lineNumber(lineNumber) - .lineNumberEnd(lineNumberEnd) - .columnNumber(columnNumber) - .columnNumberEnd(columnNumberEnd) - .astParentType(NodeTypes.TYPE_DECL) - .astParentFullName(fullName) + methodNode(astNodeInfo, name, name, fullName, None, path, Some(NodeTypes.TYPE_DECL), Some(fullName)).order(1) val functionTypeAndTypeDeclAst = createFunctionTypeAndTypeDeclAst(astNodeInfo, programMethod, methodAstParentStack.head, name, fullName, path) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala index e4c16a4c1892..d5108eaa24c1 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala @@ -4,12 +4,12 @@ import io.joern.jssrc2cpg.datastructures.* import io.joern.jssrc2cpg.parser.BabelAst.* import io.joern.jssrc2cpg.parser.BabelNodeInfo import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode} import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies} import io.shiftleft.codepropertygraph.generated.nodes.File.PropertyDefaults -import io.shiftleft.passes.IntervalKeyPool import ujson.Value import scala.collection.{mutable, SortedMap} diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForExpressionsCreator.scala index 01fe0605917d..851c1657e641 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -21,9 +21,8 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case MemberExpression => code(callee.json("property")) case _ => callee.code } - val callNode = - createStaticCallNode(callExpr.code, callName, fullName, callee.lineNumber, callee.columnNumber) - val argAsts = astForNodes(callExpr.json("arguments").arr.toList) + val callNode = createStaticCallNode(callExpr.code, callName, fullName, callee.lineNumber, callee.columnNumber) + val argAsts = astForNodes(callExpr.json("arguments").arr.toList) callAst(callNode, argAsts) } @@ -114,9 +113,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { diffGraph.addEdge(localAstParentStack.head, localTmpAllocNode, EdgeTypes.AST) scope.addVariableReference(tmpAllocName, tmpAllocNode1) - val allocCallNode = - callNode(newExpr, ".alloc", Operators.alloc, DispatchTypes.STATIC_DISPATCH) - + val allocCallNode = callNode(newExpr, ".alloc", Operators.alloc, DispatchTypes.STATIC_DISPATCH) val assignmentTmpAllocCallNode = createAssignmentCallAst( tmpAllocNode1, @@ -126,12 +123,9 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { newExpr.columnNumber ) - val tmpAllocNode2 = identifierNode(newExpr, tmpAllocName) - - val receiverNode = astForNodeWithFunctionReference(callee) - - val callAst = handleCallNodeArgs(newExpr, receiverNode, tmpAllocNode2, Defines.OperatorsNew) - + val tmpAllocNode2 = identifierNode(newExpr, tmpAllocName) + val receiverNode = astForNodeWithFunctionReference(callee) + val callAst = handleCallNodeArgs(newExpr, receiverNode, tmpAllocNode2, Defines.OperatorsNew) val tmpAllocReturnNode = Ast(identifierNode(newExpr, tmpAllocName)) scope.popScope() @@ -217,31 +211,21 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { astForBinaryExpression(logicalExpr) protected def astForTSNonNullExpression(nonNullExpr: BabelNodeInfo): Ast = { - val op = Operators.notNullAssert - val callNode_ = - callNode(nonNullExpr, nonNullExpr.code, op, DispatchTypes.STATIC_DISPATCH) - val argAsts = List(astForNodeWithFunctionReference(nonNullExpr.json("expression"))) + val op = Operators.notNullAssert + val callNode_ = callNode(nonNullExpr, nonNullExpr.code, op, DispatchTypes.STATIC_DISPATCH) + val argAsts = List(astForNodeWithFunctionReference(nonNullExpr.json("expression"))) callAst(callNode_, argAsts) } protected def astForCastExpression(castExpr: BabelNodeInfo): Ast = { - val op = Operators.cast - val lhsNode = castExpr.json("typeAnnotation") - val rhsAst = astForNodeWithFunctionReference(castExpr.json("expression")) - typeFor(castExpr) match { - case tpe if GlobalBuiltins.builtins.contains(tpe) || Defines.isBuiltinType(tpe) => - val lhsAst = Ast(literalNode(castExpr, code(lhsNode), Option(tpe))) - val node = - callNode(castExpr, castExpr.code, op, DispatchTypes.STATIC_DISPATCH).dynamicTypeHintFullName(Seq(tpe)) - val argAsts = List(lhsAst, rhsAst) - callAst(node, argAsts) - case t => - val possibleTypes = Seq(t) - val lhsAst = Ast(literalNode(castExpr, code(lhsNode), None).possibleTypes(possibleTypes)) - val node = callNode(castExpr, castExpr.code, op, DispatchTypes.STATIC_DISPATCH).possibleTypes(possibleTypes) - val argAsts = List(lhsAst, rhsAst) - callAst(node, argAsts) - } + val op = Operators.cast + val lhsNode = castExpr.json("typeAnnotation") + val rhsAst = astForNodeWithFunctionReference(castExpr.json("expression")) + val possibleTypes = Seq(typeFor(castExpr)) + val lhsAst = Ast(literalNode(castExpr, code(lhsNode), None).possibleTypes(possibleTypes)) + val node = callNode(castExpr, castExpr.code, op, DispatchTypes.STATIC_DISPATCH).possibleTypes(possibleTypes) + val argAsts = List(lhsAst, rhsAst) + callAst(node, argAsts) } protected def astForBinaryExpression(binExpr: BabelNodeInfo): Ast = { @@ -340,8 +324,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForAwaitExpression(awaitExpr: BabelNodeInfo): Ast = { - val node = - callNode(awaitExpr, awaitExpr.code, ".await", DispatchTypes.STATIC_DISPATCH) + val node = callNode(awaitExpr, awaitExpr.code, ".await", DispatchTypes.STATIC_DISPATCH) val argAsts = List(astForNodeWithFunctionReference(awaitExpr.json("argument"))) callAst(node, argAsts) } @@ -417,12 +400,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } def astForTemplateExpression(templateExpr: BabelNodeInfo): Ast = { - val argumentAst = astForNodeWithFunctionReference(templateExpr.json("quasi")) - val callName = code(templateExpr.json("tag")) - val callCode = s"$callName(${codeOf(argumentAst.nodes.head)})" - val templateExprCall = - callNode(templateExpr, callCode, callName, DispatchTypes.STATIC_DISPATCH) - val argAsts = List(argumentAst) + val argumentAst = astForNodeWithFunctionReference(templateExpr.json("quasi")) + val callName = code(templateExpr.json("tag")) + val callCode = s"$callName(${codeOf(argumentAst.nodes.head)})" + val templateExprCall = callNode(templateExpr, callCode, callName, DispatchTypes.STATIC_DISPATCH) + val argAsts = List(argumentAst) callAst(templateExprCall, argAsts) } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForFunctionsCreator.scala index bcaea8374b4c..c357f76de027 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -5,7 +5,7 @@ import io.joern.jssrc2cpg.parser.BabelAst.* import io.joern.jssrc2cpg.parser.BabelNodeInfo import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines -import io.joern.x2cpg.utils.NodeBuilders.{newBindingNode, newModifierNode} +import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.{Identifier as _, *} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, EvaluationStrategies, ModifierTypes} @@ -325,10 +325,13 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } protected def astForTSDeclareFunction(func: BabelNodeInfo): Ast = { - val functionNode = createMethodDefinitionNode(func) - val bindingNode = newBindingNode("", "", "") - diffGraph.addEdge(getParentTypeDecl, bindingNode, EdgeTypes.BINDS) - diffGraph.addEdge(bindingNode, functionNode, EdgeTypes.REF) + val functionNode = createMethodDefinitionNode(func) + val tpe = typeFor(func) + val possibleTypes = Seq(tpe) + val typeFullName = if (Defines.isBuiltinType(tpe)) tpe else Defines.Any + val memberNode_ = memberNode(func, functionNode.name, func.code, typeFullName, Seq(functionNode.fullName)) + .possibleTypes(possibleTypes) + diffGraph.addEdge(getParentTypeDecl, memberNode_, EdgeTypes.AST) addModifier(functionNode, func.json) Ast(functionNode) } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala index ccb700cf3f0d..c20227c61512 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstForTypesCreator.scala @@ -6,9 +6,8 @@ import io.joern.jssrc2cpg.parser.BabelNodeInfo import io.joern.x2cpg.{Ast, ValidationMode} import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines -import io.joern.x2cpg.utils.NodeBuilders.newBindingNode import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators, PropertyNames} import ujson.Value import scala.util.Try @@ -30,7 +29,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: } else nameTpe val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString val aliasTypeDeclNode = typeDeclNode(alias, aliasName, aliasFullName, parserResult.filename, alias.code, astParentType, astParentFullName) @@ -94,17 +93,19 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: forElem: BabelNodeInfo, methodBlockContent: List[Ast] = List.empty ): MethodAst = { + val fakeStartEnd = + s""" + | "start": ${start(forElem.json).getOrElse(-1)}, + | "end": ${end(forElem.json).getOrElse(-1)} + |""".stripMargin + val fakeConstructorCode = s"""{ | "type": "ClassMethod", + | $fakeStartEnd, | "key": { | "type": "Identifier", | "name": "constructor", - | "loc": { - | "start": { - | "line": ${forElem.lineNumber.getOrElse(-1)}, - | "column": ${forElem.columnNumber.getOrElse(-1)} - | } - | } + | $fakeStartEnd | }, | "kind": "constructor", | "id": null, @@ -180,18 +181,12 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: val typeFullName = if (Defines.isBuiltinType(tpe)) tpe else Defines.Any val memberNode_ = nodeInfo.node match { case TSDeclareMethod | TSDeclareFunction => - val function = createMethodDefinitionNode(nodeInfo) - val bindingNode = newBindingNode("", "", "") - diffGraph.addEdge(typeDeclNode, bindingNode, EdgeTypes.BINDS) - diffGraph.addEdge(bindingNode, function, EdgeTypes.REF) + val function = createMethodDefinitionNode(nodeInfo) addModifier(function, nodeInfo.json) memberNode(nodeInfo, function.name, nodeInfo.code, typeFullName, Seq(function.fullName)) .possibleTypes(possibleTypes) case ClassMethod | ClassPrivateMethod => - val function = createMethodAstAndNode(nodeInfo).methodNode - val bindingNode = newBindingNode("", "", "") - diffGraph.addEdge(typeDeclNode, bindingNode, EdgeTypes.BINDS) - diffGraph.addEdge(bindingNode, function, EdgeTypes.REF) + val function = createMethodAstAndNode(nodeInfo).methodNode addModifier(function, nodeInfo.json) memberNode(nodeInfo, function.name, nodeInfo.code, typeFullName, Seq(function.fullName)) .possibleTypes(possibleTypes) @@ -237,7 +232,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString val typeDeclNode_ = typeDeclNode( tsEnum, @@ -319,7 +314,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString val superClass = Try(createBabelNodeInfo(clazz.json("superClass")).code).toOption.toSeq val implements = Try(clazz.json("implements").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten @@ -474,7 +469,7 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: registerType(typeFullName) val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.properties(PropertyNames.FULL_NAME).toString val extendz = Try(tsInterface.json("extends").arr.map(createBabelNodeInfo(_).code)).toOption.toSeq.flatten @@ -500,9 +495,9 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: val constructorNode = interfaceConstructor(typeName, tsInterface) diffGraph.addEdge(constructorNode, NewModifier().modifierType(ModifierTypes.CONSTRUCTOR), EdgeTypes.AST) - val constructorBindingNode = newBindingNode("", "", "") - diffGraph.addEdge(typeDeclNode_, constructorBindingNode, EdgeTypes.BINDS) - diffGraph.addEdge(constructorBindingNode, constructorNode, EdgeTypes.REF) + val memberNode_ = + memberNode(tsInterface, constructorNode.name, constructorNode.code, typeFullName, Seq(constructorNode.fullName)) + diffGraph.addEdge(typeDeclNode_, memberNode_, EdgeTypes.AST) val interfaceBodyElements = classMembers(tsInterface, withConstructor = false) @@ -514,9 +509,6 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: val memberNodes = nodeInfo.node match { case TSCallSignatureDeclaration | TSMethodSignature => val functionNode = createMethodDefinitionNode(nodeInfo) - val bindingNode = newBindingNode("", "", "") - diffGraph.addEdge(typeDeclNode_, bindingNode, EdgeTypes.BINDS) - diffGraph.addEdge(bindingNode, functionNode, EdgeTypes.REF) addModifier(functionNode, nodeInfo.json) Seq( memberNode(nodeInfo, functionNode.name, nodeInfo.code, typeFullName, Seq(functionNode.fullName)) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala index 2ef960276b6a..2c6ccdb5dc6c 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstNodeBuilder.scala @@ -6,8 +6,7 @@ import io.joern.x2cpg.{Ast, ValidationMode} import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, PropertyNames} trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def createMethodReturnNode(func: BabelNodeInfo): NewMethodReturn = { @@ -251,7 +250,7 @@ trait AstNodeBuilder(implicit withSchemaValidation: ValidationMode) { this: AstC registerType(methodFullName) val astParentType = parentNode.label - val astParentFullName = parentNode.properties("FULL_NAME").toString + val astParentFullName = parentNode.properties(PropertyNames.FULL_NAME).toString val functionTypeDeclNode = typeDeclNode( node, diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/TypeHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/TypeHelper.scala index 744fde0d8a05..726106fa22ad 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/TypeHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/TypeHelper.scala @@ -1,6 +1,6 @@ package io.joern.jssrc2cpg.astcreation -import io.joern.jssrc2cpg.parser.BabelAst._ +import io.joern.jssrc2cpg.parser.BabelAst.* import io.joern.jssrc2cpg.parser.BabelNodeInfo import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/parser/BabelAst.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/parser/BabelAst.scala index 2513121ab491..cae2a0ef5705 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/parser/BabelAst.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/parser/BabelAst.scala @@ -206,6 +206,7 @@ object BabelAst { object TSIndexSignature extends BabelNode object TSIndexedAccessType extends TSType object TSInferType extends TSType + object TSInstantiationExpression extends Expression object TSInterfaceBody extends BabelNode object TSInterfaceDeclaration extends BabelNode object TSIntersectionType extends TSType diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala index f9664be8c0c4..ad57e246e711 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala @@ -15,7 +15,7 @@ import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Paths import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: Config, report: Report = new Report())( implicit withSchemaValidation: ValidationMode diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/ImportsPass.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/ImportsPass.scala index a3aa62569e84..2ce72c78ef45 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/ImportsPass.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/ImportsPass.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.passes.frontend.XImportsPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment /** This pass creates `IMPORT` nodes by looking for calls to `require`. `IMPORT` nodes are linked to existing dependency diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala index b7eb8ea08bea..032b34655a4f 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeNodePass.scala @@ -4,14 +4,13 @@ import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.joern.x2cpg.passes.frontend.TypeNodePass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.* -import io.shiftleft.passes.KeyPool import scala.collection.mutable object JavaScriptTypeNodePass { - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) { + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) { override def fullToShortName(typeName: String): String = { typeName match { diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala index b99b9c2c6bf6..be065b36625f 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala @@ -126,7 +126,7 @@ object AstGenRunner { val astGenCommand = path.getOrElse("astgen") val localPath = path.flatMap(File(_).parentOption.map(_.pathAsString)).getOrElse(".") val debugMsgPath = path.getOrElse("PATH") - ExternalCommand.run(s"$astGenCommand --version", localPath).toOption.map(_.mkString.strip()) match { + ExternalCommand.run(Seq(astGenCommand, "--version"), localPath).successOption.map(_.mkString.strip()) match { case Some(installedVersion) if installedVersion != "unknown" && Try(VersionHelper.compare(installedVersion, astGenVersion)).toOption.getOrElse(-1) >= 0 => @@ -175,7 +175,7 @@ class AstGenRunner(config: Config) { import io.joern.jssrc2cpg.utils.AstGenRunner._ - private val executableArgs = if (!config.tsTypes) " --no-tsTypes" else "" + private val executableArgs = if (!config.tsTypes) Seq("--no-tsTypes") else Seq.empty private def skippedFiles(astGenOut: List[String]): List[String] = { val skipped = astGenOut.collect { @@ -297,7 +297,11 @@ class AstGenRunner(config: Config) { } val result = - ExternalCommand.run(s"$astGenCommand$executableArgs -t ts -o $out", out.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand.run( + (astGenCommand +: executableArgs) ++ Seq("-t", "ts", "-o", out.toString), + out.toString(), + extraEnv = NODE_OPTIONS + ) val jsons = SourceFiles.determine(out.toString(), Set(".json")) jsons.foreach { jsonPath => @@ -312,24 +316,49 @@ class AstGenRunner(config: Config) { } tmpJsFiles.foreach(_.delete()) - result + result.toTry } private def ejsFiles(in: File, out: File): Try[Seq[String]] = { - val files = SourceFiles.determine(in.pathAsString, Set(".ejs")) + val files = + SourceFiles.determine( + in.pathAsString, + Set(".ejs"), + ignoredDefaultRegex = Some(AstGenDefaultIgnoreRegex), + ignoredFilesRegex = Some(config.ignoredFilesRegex), + ignoredFilesPath = Some(config.ignoredFiles) + ) if (files.nonEmpty) processEjsFiles(in, out, files) else Success(Seq.empty) } private def vueFiles(in: File, out: File): Try[Seq[String]] = { - val files = SourceFiles.determine(in.pathAsString, Set(".vue")) + val files = SourceFiles.determine( + in.pathAsString, + Set(".vue"), + ignoredDefaultRegex = Some(AstGenDefaultIgnoreRegex), + ignoredFilesRegex = Some(config.ignoredFilesRegex), + ignoredFilesPath = Some(config.ignoredFiles) + ) if (files.nonEmpty) - ExternalCommand.run(s"$astGenCommand$executableArgs -t vue -o $out", in.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand + .run( + (astGenCommand +: executableArgs) ++ Seq("-t", "vue", "-o", out.toString), + in.toString(), + extraEnv = NODE_OPTIONS + ) + .toTry else Success(Seq.empty) } private def jsFiles(in: File, out: File): Try[Seq[String]] = - ExternalCommand.run(s"$astGenCommand$executableArgs -t ts -o $out", in.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand + .run( + (astGenCommand +: executableArgs) ++ Seq("-t", "ts", "-o", out.toString), + in.toString(), + extraEnv = NODE_OPTIONS + ) + .toTry private def runAstGenNative(in: File, out: File): Try[Seq[String]] = for { ejsResult <- ejsFiles(in, out) diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala index 24fde502b8a4..6696b70e20de 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTests.scala @@ -1,11 +1,11 @@ package io.joern.jssrc2cpg.dataflow -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.CfgNode -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DataflowTests extends DataFlowCodeToCpgSuite { @@ -461,7 +461,7 @@ class DataflowTests extends DataFlowCodeToCpgSuite { cpg.call .code("bar.*") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } "Flow from outer params to inner params" in { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTests.scala index 6a5fc2274375..739b893f0b59 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTests.scala @@ -40,18 +40,22 @@ class CodeDumperFromContentTests extends JsSrc2CpgSuite { | var x = foo(param1); |}""".stripMargin - val cpg = code( - s""" + val fullCode = s""" |// A comment |$myFuncContent - |""".stripMargin, - "index.js" - ).withConfig(Config().withDisableFileContent(false)) + |""".stripMargin + + val cpg = code(fullCode, "index.js").withConfig(Config().withDisableFileContent(false)) "allow one to dump a method node's source code from `Method.content`" in { val List(content) = cpg.method.nameExact("my_func").content.l content shouldBe myFuncContent } + + "allow one to dump the :program method node's source code from `Method.content`" in { + val List(content) = cpg.method.nameExact(":program").content.l + content shouldBe fullCode + } } "code from typedecl content" should { @@ -73,6 +77,11 @@ class CodeDumperFromContentTests extends JsSrc2CpgSuite { val List(content) = cpg.typeDecl.nameExact("Foo").content.l content shouldBe myClassContent } + + "allow one to dump the method node's source code from `Method.content`" in { + val List(content) = cpg.method.nameExact("").content.l + content shouldBe myClassContent + } } "content with UTF8 characters" should { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTests.scala index 9c27a157a302..ede0ff1342a9 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromFileTests.scala @@ -3,7 +3,7 @@ package io.joern.jssrc2cpg.io import better.files.File import io.joern.jssrc2cpg.testfixtures.JsSrc2CpgSuite import io.shiftleft.semanticcpg.codedumper.CodeDumper -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/JsSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/JsSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..02559d45de34 --- /dev/null +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/JsSrc2CpgHTTPServerTests.scala @@ -0,0 +1,83 @@ +package io.joern.jssrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.util.Failure +import scala.util.Success +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable + +class JsSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("jssrc2cpgTestsHttpTest") + val file = dir / "main.js" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |function main$indexStr() { + | console.log("Hello World!"); + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.jssrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.jssrc2cpg.Main.stop() + } + + "Using jssrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("jssrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("""console.log("Hello World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("jssrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain(s"main$index") + cpg.call.code.l should contain("""console.log("Hello World!")""") + } + } + } + } + +} diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/CallLinkerPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/CallLinkerPassTests.scala index 0e31d1703470..1f6d34a4f8ef 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/CallLinkerPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/CallLinkerPassTests.scala @@ -3,7 +3,7 @@ package io.joern.jssrc2cpg.passes import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallLinkerPassTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConfigPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConfigPassTests.scala index 87a0e66dad5e..3d2f165406e6 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConfigPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConfigPassTests.scala @@ -4,7 +4,7 @@ import better.files.File import io.joern.jssrc2cpg.Config import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConstClosurePassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConstClosurePassTests.scala index 6abed767948d..f1acaba369fa 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConstClosurePassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ConstClosurePassTests.scala @@ -1,7 +1,7 @@ package io.joern.jssrc2cpg.passes import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ConstClosurePassTests extends DataFlowCodeToCpgSuite { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/DomPassTestsHelper.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/DomPassTestsHelper.scala index dc9ac7cc858c..98433e4d23ef 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/DomPassTestsHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/DomPassTestsHelper.scala @@ -3,7 +3,7 @@ package io.joern.jssrc2cpg.passes import io.shiftleft.codepropertygraph.generated.nodes.Expression import io.shiftleft.codepropertygraph.generated.nodes.TemplateDom import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.apache.commons.lang3.StringUtils trait DomPassTestsHelper { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/InheritanceFullNamePassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/InheritanceFullNamePassTests.scala index cfccc1632b34..f580bf8cb688 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/InheritanceFullNamePassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/InheritanceFullNamePassTests.scala @@ -1,7 +1,7 @@ package io.joern.jssrc2cpg.passes import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File import scala.annotation.nowarn diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala index 399723cdc93c..bba23114b188 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/JsMetaDataPassTests.scala @@ -18,11 +18,11 @@ class JsMetaDataPassTests extends AnyWordSpec with Matchers with Inside { new JavaScriptMetaDataPass(cpg, "somehash", "").createAndApply() "create exactly 1 node" in { - cpg.graph.V.asScala.size shouldBe 1 + cpg.graph.allNodes.size shouldBe 1 } "create no edges" in { - cpg.graph.E.asScala.size shouldBe 0 + cpg.graph.allNodes.outE.size shouldBe 0 } "create a metadata node with correct language" in { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/RequirePassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/RequirePassTests.scala index 672a73ca0f65..b354a9942b94 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/RequirePassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/RequirePassTests.scala @@ -1,8 +1,8 @@ package io.joern.jssrc2cpg.passes -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.language.* import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/DependencyAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/DependencyAstCreationPassTests.scala index daa810ae9689..a581f7f76162 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/DependencyAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/DependencyAstCreationPassTests.scala @@ -3,7 +3,7 @@ package io.joern.jssrc2cpg.passes.ast import io.joern.jssrc2cpg.testfixtures.AstJsSrc2CpgSuite import io.joern.x2cpg.layers.Base import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DependencyAstCreationPassTests extends AstJsSrc2CpgSuite { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala index be5e156a9e4b..7653f019c727 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/MixedAstCreationPassTests.scala @@ -4,8 +4,8 @@ import io.joern.jssrc2cpg.testfixtures.AstJsSrc2CpgSuite import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.EvaluationStrategies import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.{ClosureBinding, MethodParameterIn} +import io.shiftleft.semanticcpg.language.* class MixedAstCreationPassTests extends AstJsSrc2CpgSuite { @@ -285,7 +285,7 @@ class MixedAstCreationPassTests extends AstJsSrc2CpgSuite { val List(fooLocalY) = fooBlock.astChildren.isLocal.nameExact("y").l val List(barRef) = fooBlock.astChildren.isCall.astChildren.isMethodRef.l - val List(closureBindForY, closureBindForX) = barRef.captureOut.l + val List(closureBindForY, closureBindForX) = barRef.captureOut.cast[ClosureBinding].l closureBindForX.closureOriginalName shouldBe Option("x") closureBindForX.closureBindingId shouldBe Option("Test0.js::program:foo:bar:x") diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala index f1bab7a456f8..cd85aac864df 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/SimpleAstCreationPassTests.scala @@ -859,7 +859,7 @@ class SimpleAstCreationPassTests extends AstJsSrc2CpgSuite { val List(typeDecl) = cpg.typeDecl.nameExact("method").l typeDecl.fullName should endWith("Test0.js::program:method") - val List(binding) = typeDecl.bindsOut.l + val List(binding) = typeDecl.bindsOut.cast[Binding].l binding.name shouldBe "" binding.signature shouldBe "" diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsAstCreationPassTests.scala index 71144ce1141b..33c2d5209640 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsAstCreationPassTests.scala @@ -108,8 +108,7 @@ class TsAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { arg.typeFullName shouldBe Defines.String arg.code shouldBe "arg: string" arg.index shouldBe 1 - val List(parentTypeDecl) = cpg.typeDecl.name(":program").l - parentTypeDecl.bindsOut.flatMap(_.refOut).l should contain(func) + cpg.method("foo").bindingTypeDecl.fullName.l shouldBe List("Test0.ts::program:foo") } "have correct structure for type assertion" in { diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala index fed85c602569..43d24af28886 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsClassesAstCreationPassTests.scala @@ -191,7 +191,10 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { greeter.fullName shouldBe "Test0.ts::program:Greeter" greeter.filename shouldBe "Test0.ts" greeter.file.name.head shouldBe "Test0.ts" - inside(cpg.typeDecl("Greeter").member.l) { case List(greeting, name, propName, foo, anon, toString) => + inside(cpg.typeDecl("Greeter").member.l) { case List(init, greeting, name, propName, foo, anon, toString) => + init.name shouldBe "" + init.typeFullName shouldBe "Test0.ts::program:Greeter" + init.dynamicTypeHintFullName shouldBe List("Test0.ts::program:Greeter:") greeting.name shouldBe "greeting" greeting.code shouldBe "greeting: string;" name.name shouldBe "name" @@ -339,7 +342,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { val List(credentialsParam) = cpg.parameter.nameExact("credentials").l credentialsParam.typeFullName shouldBe "Test0.ts::program:Test:run:0" // should not produce dangling nodes that are meant to be inside procedures - cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0 + cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0 cpg.identifier.count(_.refsTo.size > 1) shouldBe 0 cpg.identifier.whereNot(_.refsTo).size shouldBe 0 // should not produce assignment calls directly under typedecls @@ -359,7 +362,7 @@ class TsClassesAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { val List(credentialsParam) = cpg.parameter.nameExact("param1_0").l credentialsParam.typeFullName shouldBe "Test0.ts::program:apiCall:0" // should not produce dangling nodes that are meant to be inside procedures - cpg.all.collectAll[CfgNode].whereNot(_._astIn).size shouldBe 0 + cpg.all.collectAll[CfgNode].whereNot(_.astParent).size shouldBe 0 cpg.identifier.count(_.refsTo.size > 1) shouldBe 0 cpg.identifier.whereNot(_.refsTo).size shouldBe 0 // should not produce assignment calls directly under typedecls diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsDecoratorAstCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsDecoratorAstCreationPassTests.scala index c15f2dc879ae..38bd7b036a5c 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsDecoratorAstCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/ast/TsDecoratorAstCreationPassTests.scala @@ -1,8 +1,7 @@ package io.joern.jssrc2cpg.passes.ast import io.joern.jssrc2cpg.testfixtures.AstJsSrc2CpgSuite -import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TsDecoratorAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { @@ -324,308 +323,6 @@ class TsDecoratorAstCreationPassTests extends AstJsSrc2CpgSuite(".ts") { annotationD.parameterAssign.l shouldBe empty } } - - "create methods for const exports" in { - val cpg = code("export const getApiA = (req: Request) => { const user = req.user as UserDocument; }") - cpg.method.name.sorted.l shouldBe List(":program", "0") - cpg.assignment.code.l shouldBe List( - "const user = req.user as UserDocument", - "const getApiA = (req: Request) => { const user = req.user as UserDocument; }", - "exports.getApiA = getApiA" - ) - inside(cpg.method.name("0").l) { case List(anon) => - anon.fullName shouldBe "Test0.ts::program:0" - anon.ast.isIdentifier.name.l shouldBe List("user", "req") - } - } - - "have correct structure for import assignments" in { - val cpg = code(""" - |import fs = require('fs'); - |import models = require('../models/index'); - |""".stripMargin) - cpg.assignment.code.l shouldBe List("var fs = require(\"fs\")", "var models = require(\"../models/index\")") - cpg.local.code.l shouldBe List("fs", "models") - val List(fsDep, modelsDep) = cpg.dependency.l - fsDep.name shouldBe "fs" - fsDep.dependencyGroupId shouldBe Option("fs") - modelsDep.name shouldBe "models" - modelsDep.dependencyGroupId shouldBe Option("../models/index") - - val List(fs, models) = cpg.imports.l - fs.code shouldBe "import fs = require('fs')" - fs.importedEntity shouldBe Option("fs") - fs.importedAs shouldBe Option("fs") - models.code shouldBe "import models = require('../models/index')" - models.importedEntity shouldBe Option("../models/index") - models.importedAs shouldBe Option("models") - } - - "have correct structure for declared functions" in { - val cpg = code("declare function foo(arg: string): string") - val List(func) = cpg.method("foo").l - func.code shouldBe "declare function foo(arg: string): string" - func.name shouldBe "foo" - func.fullName shouldBe "Test0.ts::program:foo" - val List(_, arg) = cpg.method("foo").parameter.l - arg.name shouldBe "arg" - arg.typeFullName shouldBe Defines.String - arg.code shouldBe "arg: string" - arg.index shouldBe 1 - val List(parentTypeDecl) = cpg.typeDecl.name(":program").l - parentTypeDecl.bindsOut.flatMap(_.refOut).l should contain(func) - } - - } - - "AST generation for TS enums" should { - - "have correct structure for simple enum" in { - val cpg = code(""" - |enum Direction { - | Up = 1, - | Down, - | Left, - | Right, - |} - |""".stripMargin) - inside(cpg.typeDecl("Direction").l) { case List(direction) => - direction.name shouldBe "Direction" - direction.code shouldBe "enum Direction" - direction.fullName shouldBe "Test0.ts::program:Direction" - direction.filename shouldBe "Test0.ts" - direction.file.name.head shouldBe "Test0.ts" - inside(direction.method.name(io.joern.x2cpg.Defines.StaticInitMethodName).l) { case List(init) => - init.block.astChildren.isCall.code.head shouldBe "Up = 1" - } - inside(cpg.typeDecl("Direction").member.l) { case List(up, down, left, right) => - up.name shouldBe "Up" - up.code shouldBe "Up = 1" - down.name shouldBe "Down" - down.code shouldBe "Down" - left.name shouldBe "Left" - left.code shouldBe "Left" - right.name shouldBe "Right" - right.code shouldBe "Right" - } - } - } - - } - - "AST generation for TS classes" should { - - "have correct structure for simple classes" in { - val cpg = code(""" - |class Greeter { - | greeting: string; - | greet() { - | return "Hello, " + this.greeting; - | } - |} - |""".stripMargin) - inside(cpg.typeDecl("Greeter").l) { case List(greeter) => - greeter.name shouldBe "Greeter" - greeter.code shouldBe "class Greeter" - greeter.fullName shouldBe "Test0.ts::program:Greeter" - greeter.filename shouldBe "Test0.ts" - greeter.file.name.head shouldBe "Test0.ts" - val constructor = greeter.method.nameExact(io.joern.x2cpg.Defines.ConstructorMethodName).head - greeter.method.isConstructor.head shouldBe constructor - constructor.fullName shouldBe s"Test0.ts::program:Greeter:${io.joern.x2cpg.Defines.ConstructorMethodName}" - inside(cpg.typeDecl("Greeter").member.l) { case List(greeting, greet) => - greeting.name shouldBe "greeting" - greeting.code shouldBe "greeting: string;" - greet.name shouldBe "greet" - greet.dynamicTypeHintFullName shouldBe Seq("Test0.ts::program:Greeter:greet") - } - } - } - - "have correct structure for declared classes with empty constructor" in { - val cpg = code(""" - |declare class Greeter { - | greeting: string; - | constructor(arg: string); - |} - |""".stripMargin) - inside(cpg.typeDecl("Greeter").l) { case List(greeter) => - greeter.name shouldBe "Greeter" - greeter.code shouldBe "class Greeter" - greeter.fullName shouldBe "Test0.ts::program:Greeter" - greeter.filename shouldBe "Test0.ts" - greeter.file.name.head shouldBe "Test0.ts" - val constructor = greeter.method.nameExact(io.joern.x2cpg.Defines.ConstructorMethodName).head - constructor.fullName shouldBe s"Test0.ts::program:Greeter:${io.joern.x2cpg.Defines.ConstructorMethodName}" - greeter.method.isConstructor.head shouldBe constructor - inside(cpg.typeDecl("Greeter").member.l) { case List(greeting) => - greeting.name shouldBe "greeting" - greeting.code shouldBe "greeting: string;" - } - } - } - - "have correct modifier" in { - val cpg = code(""" - |abstract class Greeter { - | static a: string; - | private b: string; - | public c: string; - | protected d: string; - | #e: string; // also private - |} - |""".stripMargin) - inside(cpg.typeDecl.name("Greeter.*").l) { case List(greeter) => - greeter.name shouldBe "Greeter" - cpg.typeDecl.isAbstract.head shouldBe greeter - greeter.member.isStatic.head shouldBe greeter.member.name("a").head - greeter.member.isPrivate.l shouldBe greeter.member.name("b", "e").l - greeter.member.isPublic.head shouldBe greeter.member.name("c").head - greeter.member.isProtected.head shouldBe greeter.member.name("d").head - } - } - - "have correct structure for empty interfaces" in { - val cpg = code(""" - |interface A {}; - |interface B {}; - |""".stripMargin) - cpg.method.fullName.sorted.l shouldBe List( - "Test0.ts::program", - s"Test0.ts::program:A:${io.joern.x2cpg.Defines.ConstructorMethodName}", - s"Test0.ts::program:B:${io.joern.x2cpg.Defines.ConstructorMethodName}" - ) - } - - "have correct structure for simple interfaces" in { - val cpg = code(""" - |interface Greeter { - | greeting: string; - | name?: string; - | [propName: string]: any; - | "foo": string; - | (source: string, subString: string): boolean; - |} - |""".stripMargin) - inside(cpg.typeDecl("Greeter").l) { case List(greeter) => - greeter.name shouldBe "Greeter" - greeter.code shouldBe "interface Greeter" - greeter.fullName shouldBe "Test0.ts::program:Greeter" - greeter.filename shouldBe "Test0.ts" - greeter.file.name.head shouldBe "Test0.ts" - inside(cpg.typeDecl("Greeter").member.l) { case List(greeting, name, propName, foo, anon) => - greeting.name shouldBe "greeting" - greeting.code shouldBe "greeting: string;" - name.name shouldBe "name" - name.code shouldBe "name?: string;" - propName.name shouldBe "propName" - propName.code shouldBe "[propName: string]: any;" - foo.name shouldBe "foo" - foo.code shouldBe "\"foo\": string;" - anon.name shouldBe "0" - anon.dynamicTypeHintFullName shouldBe Seq("Test0.ts::program:Greeter:0") - anon.code shouldBe "(source: string, subString: string): boolean;" - } - inside(cpg.typeDecl("Greeter").method.l) { case List(constructor, anon) => - constructor.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName - constructor.fullName shouldBe s"Test0.ts::program:Greeter:${io.joern.x2cpg.Defines.ConstructorMethodName}" - constructor.code shouldBe "new: Greeter" - greeter.method.isConstructor.head shouldBe constructor - anon.name shouldBe "0" - anon.fullName shouldBe "Test0.ts::program:Greeter:0" - anon.code shouldBe "(source: string, subString: string): boolean;" - anon.parameter.name.l shouldBe List("this", "source", "subString") - anon.parameter.code.l shouldBe List("this", "source: string", "subString: string") - } - } - } - - "have correct structure for interface constructor" in { - val cpg = code(""" - |interface Greeter { - | new (param: string) : Greeter - |} - |""".stripMargin) - inside(cpg.typeDecl("Greeter").l) { case List(greeter) => - greeter.name shouldBe "Greeter" - greeter.code shouldBe "interface Greeter" - greeter.fullName shouldBe "Test0.ts::program:Greeter" - greeter.filename shouldBe "Test0.ts" - greeter.file.name.head shouldBe "Test0.ts" - inside(cpg.typeDecl("Greeter").method.l) { case List(constructor) => - constructor.name shouldBe io.joern.x2cpg.Defines.ConstructorMethodName - constructor.fullName shouldBe s"Test0.ts::program:Greeter:${io.joern.x2cpg.Defines.ConstructorMethodName}" - constructor.code shouldBe "new (param: string) : Greeter" - constructor.parameter.name.l shouldBe List("this", "param") - constructor.parameter.code.l shouldBe List("this", "param: string") - greeter.method.isConstructor.head shouldBe constructor - } - } - } - - "have correct structure for simple namespace" in { - val cpg = code(""" - |namespace A { - | class Foo {}; - |} - |""".stripMargin) - inside(cpg.namespaceBlock("A").l) { case List(namespaceA) => - namespaceA.code should startWith("namespace A") - namespaceA.fullName shouldBe "Test0.ts::program:A" - namespaceA.typeDecl.name("Foo").head.fullName shouldBe "Test0.ts::program:A:Foo" - } - } - - "have correct structure for nested namespaces" in { - val cpg = code(""" - |namespace A { - | namespace B { - | namespace C { - | class Foo {}; - | } - | } - |} - |""".stripMargin) - inside(cpg.namespaceBlock("A").l) { case List(namespaceA) => - namespaceA.code should startWith("namespace A") - namespaceA.fullName shouldBe "Test0.ts::program:A" - namespaceA.astChildren.astChildren.isNamespaceBlock.name("B").head shouldBe cpg.namespaceBlock("B").head - } - inside(cpg.namespaceBlock("B").l) { case List(namespaceB) => - namespaceB.code should startWith("namespace B") - namespaceB.fullName shouldBe "Test0.ts::program:A:B" - namespaceB.astChildren.astChildren.isNamespaceBlock.name("C").head shouldBe cpg.namespaceBlock("C").head - } - inside(cpg.namespaceBlock("C").l) { case List(namespaceC) => - namespaceC.code should startWith("namespace C") - namespaceC.fullName shouldBe "Test0.ts::program:A:B:C" - namespaceC.typeDecl.name("Foo").head.fullName shouldBe "Test0.ts::program:A:B:C:Foo" - } - } - - "have correct structure for nested namespaces with path" in { - val cpg = code(""" - |namespace A.B.C { - | class Foo {}; - |} - |""".stripMargin) - inside(cpg.namespaceBlock("A").l) { case List(namespaceA) => - namespaceA.code should startWith("namespace A") - namespaceA.fullName shouldBe "Test0.ts::program:A" - namespaceA.astChildren.isNamespaceBlock.name("B").head shouldBe cpg.namespaceBlock("B").head - } - inside(cpg.namespaceBlock("B").l) { case List(b) => - b.code should startWith("B.C") - b.fullName shouldBe "Test0.ts::program:A:B" - b.astChildren.isNamespaceBlock.name("C").head shouldBe cpg.namespaceBlock("C").head - } - inside(cpg.namespaceBlock("C").l) { case List(c) => - c.code should startWith("C") - c.fullName shouldBe "Test0.ts::program:A:B:C" - c.typeDecl.name("Foo").head.fullName shouldBe "Test0.ts::program:A:B:C:Foo" - } - } - } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/DependencyCfgCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/DependencyCfgCreationPassTests.scala index c83d24acb0b4..7e914d662a6f 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/DependencyCfgCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/DependencyCfgCreationPassTests.scala @@ -10,16 +10,16 @@ class DependencyCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTe "CFG generation for global builtins" should { "be correct for JSON.parse" in { implicit val cpg: Cpg = code("""JSON.parse("foo");""") - succOf(":program") shouldBe expected((""""foo"""", AlwaysEdge)) - succOf(""""foo"""") shouldBe expected(("""JSON.parse("foo")""", AlwaysEdge)) - succOf("""JSON.parse("foo")""") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected((""""foo"""", AlwaysEdge)) + succOf(""""foo"""") should contain theSameElementsAs expected(("""JSON.parse("foo")""", AlwaysEdge)) + succOf("""JSON.parse("foo")""") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "have correct structure for JSON.stringify" in { implicit val cpg: Cpg = code("""JSON.stringify(foo);""") - succOf(":program") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("JSON.stringify(foo)", AlwaysEdge)) - succOf("JSON.stringify(foo)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("JSON.stringify(foo)", AlwaysEdge)) + succOf("JSON.stringify(foo)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/JsClassesCfgCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/JsClassesCfgCreationPassTests.scala index 8301cf8a4afc..f827e03dbaab 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/JsClassesCfgCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/JsClassesCfgCreationPassTests.scala @@ -11,61 +11,65 @@ class JsClassesCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTes "CFG generation for constructor" should { "be correct for simple new" in { implicit val cpg: Cpg = code("new MyClass()") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected((".alloc", AlwaysEdge)) - succOf(".alloc") shouldBe expected(("_tmp_0 = .alloc", AlwaysEdge)) - succOf("_tmp_0 = .alloc") shouldBe expected(("MyClass", AlwaysEdge)) - succOf("MyClass") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("new MyClass()", AlwaysEdge)) - succOf("new MyClass()", NodeTypes.CALL) shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("new MyClass()", AlwaysEdge)) - succOf("new MyClass()") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected((".alloc", AlwaysEdge)) + succOf(".alloc") should contain theSameElementsAs expected(("_tmp_0 = .alloc", AlwaysEdge)) + succOf("_tmp_0 = .alloc") should contain theSameElementsAs expected(("MyClass", AlwaysEdge)) + succOf("MyClass") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("new MyClass()", AlwaysEdge)) + succOf("new MyClass()", NodeTypes.CALL) should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("new MyClass()", AlwaysEdge)) + succOf("new MyClass()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for simple new with arguments" in { implicit val cpg: Cpg = code("new MyClass(arg1, arg2)") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected((".alloc", AlwaysEdge)) - succOf(".alloc") shouldBe expected(("_tmp_0 = .alloc", AlwaysEdge)) - succOf("_tmp_0 = .alloc") shouldBe expected(("MyClass", AlwaysEdge)) - succOf("MyClass") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("arg1", AlwaysEdge)) - succOf("arg1") shouldBe expected(("arg2", AlwaysEdge)) - succOf("arg2") shouldBe expected(("new MyClass(arg1, arg2)", AlwaysEdge)) - succOf("new MyClass(arg1, arg2)", NodeTypes.CALL) shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("new MyClass(arg1, arg2)", AlwaysEdge)) - succOf("new MyClass(arg1, arg2)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected((".alloc", AlwaysEdge)) + succOf(".alloc") should contain theSameElementsAs expected(("_tmp_0 = .alloc", AlwaysEdge)) + succOf("_tmp_0 = .alloc") should contain theSameElementsAs expected(("MyClass", AlwaysEdge)) + succOf("MyClass") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("arg1", AlwaysEdge)) + succOf("arg1") should contain theSameElementsAs expected(("arg2", AlwaysEdge)) + succOf("arg2") should contain theSameElementsAs expected(("new MyClass(arg1, arg2)", AlwaysEdge)) + succOf("new MyClass(arg1, arg2)", NodeTypes.CALL) should contain theSameElementsAs expected( + ("_tmp_0", 2, AlwaysEdge) + ) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("new MyClass(arg1, arg2)", AlwaysEdge)) + succOf("new MyClass(arg1, arg2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for new with access path" in { implicit val cpg: Cpg = code("new foo.bar.MyClass()") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected((".alloc", AlwaysEdge)) - succOf(".alloc") shouldBe expected(("_tmp_0 = .alloc", AlwaysEdge)) - succOf("_tmp_0 = .alloc") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("bar", AlwaysEdge)) - succOf("bar") shouldBe expected(("foo.bar", AlwaysEdge)) - succOf("foo.bar") shouldBe expected(("MyClass", AlwaysEdge)) - succOf("MyClass") shouldBe expected(("foo.bar.MyClass", AlwaysEdge)) - succOf("foo.bar.MyClass") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("new foo.bar.MyClass()", AlwaysEdge)) - succOf("new foo.bar.MyClass()", NodeTypes.CALL) shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("new foo.bar.MyClass()", AlwaysEdge)) - succOf("new foo.bar.MyClass()") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected((".alloc", AlwaysEdge)) + succOf(".alloc") should contain theSameElementsAs expected(("_tmp_0 = .alloc", AlwaysEdge)) + succOf("_tmp_0 = .alloc") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("bar", AlwaysEdge)) + succOf("bar") should contain theSameElementsAs expected(("foo.bar", AlwaysEdge)) + succOf("foo.bar") should contain theSameElementsAs expected(("MyClass", AlwaysEdge)) + succOf("MyClass") should contain theSameElementsAs expected(("foo.bar.MyClass", AlwaysEdge)) + succOf("foo.bar.MyClass") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("new foo.bar.MyClass()", AlwaysEdge)) + succOf("new foo.bar.MyClass()", NodeTypes.CALL) should contain theSameElementsAs expected( + ("_tmp_0", 2, AlwaysEdge) + ) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("new foo.bar.MyClass()", AlwaysEdge)) + succOf("new foo.bar.MyClass()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be structure for throw new exceptions" in { implicit val cpg: Cpg = code("function foo() { throw new Foo() }") - succOf("foo", NodeTypes.METHOD) shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected((".alloc", AlwaysEdge)) - succOf(".alloc") shouldBe expected(("_tmp_0 = .alloc", AlwaysEdge)) - succOf("_tmp_0 = .alloc") shouldBe expected(("Foo", AlwaysEdge)) - succOf("Foo") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("new Foo()", AlwaysEdge)) - succOf("new Foo()", NodeTypes.CALL) shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("new Foo()", AlwaysEdge)) - succOf("new Foo()") shouldBe expected(("throw new Foo()", AlwaysEdge)) - succOf("throw new Foo()") shouldBe expected(("RET", AlwaysEdge)) + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected((".alloc", AlwaysEdge)) + succOf(".alloc") should contain theSameElementsAs expected(("_tmp_0 = .alloc", AlwaysEdge)) + succOf("_tmp_0 = .alloc") should contain theSameElementsAs expected(("Foo", AlwaysEdge)) + succOf("Foo") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("new Foo()", AlwaysEdge)) + succOf("new Foo()", NodeTypes.CALL) should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("new Foo()", AlwaysEdge)) + succOf("new Foo()") should contain theSameElementsAs expected(("throw new Foo()", AlwaysEdge)) + succOf("throw new Foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } @@ -78,10 +82,10 @@ class JsClassesCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTes | } |} |""".stripMargin) - succOf("foo", NodeTypes.METHOD) shouldBe expected(("bar", AlwaysEdge)) - succOf("bar") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("bar()", AlwaysEdge)) - succOf("bar()") shouldBe expected(("RET", 2, AlwaysEdge)) + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("bar", AlwaysEdge)) + succOf("bar") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("bar()", AlwaysEdge)) + succOf("bar()") should contain theSameElementsAs expected(("RET", 2, AlwaysEdge)) } "be correct for methods in class type decls with assignment" in { @@ -92,17 +96,17 @@ class JsClassesCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTes | } |} |""".stripMargin) - succOf(":program") shouldBe expected(("a", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("a", AlwaysEdge)) // call to constructor of ClassA - succOf("a") shouldBe expected(("class ClassA", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("class ClassA", AlwaysEdge)) } "be correct for outer method of anonymous class declaration" in { implicit val cpg: Cpg = code("var a = class {}") - succOf(":program") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("class 0", AlwaysEdge)) - succOf("class 0") shouldBe expected(("var a = class {}", AlwaysEdge)) - succOf("var a = class {}") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("class 0", AlwaysEdge)) + succOf("class 0") should contain theSameElementsAs expected(("var a = class {}", AlwaysEdge)) + succOf("var a = class {}") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/MixedCfgCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/MixedCfgCreationPassTests.scala index bef8d7d9f81c..f3c619eed31c 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/MixedCfgCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/MixedCfgCreationPassTests.scala @@ -7,164 +7,172 @@ import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.TrueEdge import io.joern.x2cpg.testfixtures.CfgTestFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MixedCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCpg()) { "CFG generation for destructing assignment" should { "be correct for object destruction assignment with declaration" in { implicit val cpg: Cpg = code("var {a, b} = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("_tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a") shouldBe expected(("a = _tmp_0.a", AlwaysEdge)) - - succOf("a = _tmp_0.a") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("_tmp_0.b", AlwaysEdge)) - succOf("_tmp_0.b") shouldBe expected(("b = _tmp_0.b", AlwaysEdge)) - succOf("b = _tmp_0.b") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("var {a, b} = x", AlwaysEdge)) - succOf("var {a, b} = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("_tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a") should contain theSameElementsAs expected(("a = _tmp_0.a", AlwaysEdge)) + + succOf("a = _tmp_0.a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("_tmp_0.b", AlwaysEdge)) + succOf("_tmp_0.b") should contain theSameElementsAs expected(("b = _tmp_0.b", AlwaysEdge)) + succOf("b = _tmp_0.b") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var {a, b} = x", AlwaysEdge)) + succOf("var {a, b} = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for object destruction assignment with declaration and ternary init" in { implicit val cpg: Cpg = code("const { a, b } = test() ? foo() : bar()") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("test", AlwaysEdge)) - succOf("test") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("test()", AlwaysEdge)) - succOf("test()") shouldBe expected(("foo", TrueEdge), ("bar", FalseEdge)) - succOf("foo") shouldBe expected(("this", 1, AlwaysEdge)) - succOf("this", 2) shouldBe expected(("foo()", AlwaysEdge)) - succOf("bar()") shouldBe expected(("test() ? foo() : bar()", AlwaysEdge)) - succOf("foo()") shouldBe expected(("test() ? foo() : bar()", AlwaysEdge)) - succOf("test() ? foo() : bar()") shouldBe expected(("_tmp_0 = test() ? foo() : bar()", AlwaysEdge)) - succOf("_tmp_0 = test() ? foo() : bar()") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("_tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a") shouldBe expected(("a = _tmp_0.a", AlwaysEdge)) - succOf("a = _tmp_0.a") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("_tmp_0.b", AlwaysEdge)) - succOf("_tmp_0.b") shouldBe expected(("b = _tmp_0.b", AlwaysEdge)) - succOf("b = _tmp_0.b") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("const { a, b } = test() ? foo() : bar()", AlwaysEdge)) - succOf("const { a, b } = test() ? foo() : bar()") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("test", AlwaysEdge)) + succOf("test") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("test()", AlwaysEdge)) + succOf("test()") should contain theSameElementsAs expected(("foo", TrueEdge), ("bar", FalseEdge)) + succOf("foo") should contain theSameElementsAs expected(("this", 1, AlwaysEdge)) + succOf("this", 2) should contain theSameElementsAs expected(("foo()", AlwaysEdge)) + succOf("bar()") should contain theSameElementsAs expected(("test() ? foo() : bar()", AlwaysEdge)) + succOf("foo()") should contain theSameElementsAs expected(("test() ? foo() : bar()", AlwaysEdge)) + succOf("test() ? foo() : bar()") should contain theSameElementsAs expected( + ("_tmp_0 = test() ? foo() : bar()", AlwaysEdge) + ) + succOf("_tmp_0 = test() ? foo() : bar()") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("_tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a") should contain theSameElementsAs expected(("a = _tmp_0.a", AlwaysEdge)) + succOf("a = _tmp_0.a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("_tmp_0.b", AlwaysEdge)) + succOf("_tmp_0.b") should contain theSameElementsAs expected(("b = _tmp_0.b", AlwaysEdge)) + succOf("b = _tmp_0.b") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected( + ("const { a, b } = test() ? foo() : bar()", AlwaysEdge) + ) + succOf("const { a, b } = test() ? foo() : bar()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for object destruction assignment with reassignment" in { implicit val cpg: Cpg = code("var {a: n, b: m} = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("n", AlwaysEdge)) - succOf("n") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("_tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a") shouldBe expected(("n = _tmp_0.a", AlwaysEdge)) - - succOf("n = _tmp_0.a") shouldBe expected(("m", AlwaysEdge)) - succOf("m") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0.b", AlwaysEdge)) - succOf("_tmp_0.b") shouldBe expected(("m = _tmp_0.b", AlwaysEdge)) - succOf("m = _tmp_0.b") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("var {a: n, b: m} = x", AlwaysEdge)) - succOf("var {a: n, b: m} = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("n", AlwaysEdge)) + succOf("n") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a") should contain theSameElementsAs expected(("n = _tmp_0.a", AlwaysEdge)) + + succOf("n = _tmp_0.a") should contain theSameElementsAs expected(("m", AlwaysEdge)) + succOf("m") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0.b", AlwaysEdge)) + succOf("_tmp_0.b") should contain theSameElementsAs expected(("m = _tmp_0.b", AlwaysEdge)) + succOf("m = _tmp_0.b") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var {a: n, b: m} = x", AlwaysEdge)) + succOf("var {a: n, b: m} = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for object destruction assignment with reassignment and defaults" in { implicit val cpg: Cpg = code("var {a: n = 1, b: m = 2} = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - succOf("_tmp_0 = x") shouldBe expected(("n", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("n", AlwaysEdge)) // test statement - succOf("n") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("_tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a") shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("_tmp_0.a === void 0", AlwaysEdge)) + succOf("n") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a") should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("_tmp_0.a === void 0", AlwaysEdge)) // true, false cases - succOf("_tmp_0.a === void 0") shouldBe expected(("1", TrueEdge), ("_tmp_0", 2, FalseEdge)) - succOf("_tmp_0", 2) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("_tmp_0.a", 1, AlwaysEdge)) - succOf("_tmp_0.a", 1) shouldBe expected(("_tmp_0.a === void 0 ? 1 : _tmp_0.a", AlwaysEdge)) - succOf("1") shouldBe expected(("_tmp_0.a === void 0 ? 1 : _tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a === void 0 ? 1 : _tmp_0.a") shouldBe + succOf("_tmp_0.a === void 0") should contain theSameElementsAs expected(("1", TrueEdge), ("_tmp_0", 2, FalseEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("_tmp_0.a", 1, AlwaysEdge)) + succOf("_tmp_0.a", 1) should contain theSameElementsAs expected( + ("_tmp_0.a === void 0 ? 1 : _tmp_0.a", AlwaysEdge) + ) + succOf("1") should contain theSameElementsAs expected(("_tmp_0.a === void 0 ? 1 : _tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a === void 0 ? 1 : _tmp_0.a") should contain theSameElementsAs expected(("n = _tmp_0.a === void 0 ? 1 : _tmp_0.a", AlwaysEdge)) - succOf("n = _tmp_0.a === void 0 ? 1 : _tmp_0.a") shouldBe + succOf("n = _tmp_0.a === void 0 ? 1 : _tmp_0.a") should contain theSameElementsAs expected(("m", AlwaysEdge)) // test statement - succOf("m") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0.b", AlwaysEdge)) - succOf("_tmp_0.b") shouldBe expected(("void 0", 1, AlwaysEdge)) - succOf("void 0", 1) shouldBe expected(("_tmp_0.b === void 0", AlwaysEdge)) + succOf("m") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0.b", AlwaysEdge)) + succOf("_tmp_0.b") should contain theSameElementsAs expected(("void 0", 1, AlwaysEdge)) + succOf("void 0", 1) should contain theSameElementsAs expected(("_tmp_0.b === void 0", AlwaysEdge)) // true, false cases - succOf("_tmp_0.b === void 0") shouldBe expected(("2", TrueEdge), ("_tmp_0", 4, FalseEdge)) - succOf("_tmp_0", 4) shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("_tmp_0.b", 1, AlwaysEdge)) - succOf("_tmp_0.b", 1) shouldBe expected(("_tmp_0.b === void 0 ? 2 : _tmp_0.b", AlwaysEdge)) - succOf("2") shouldBe expected(("_tmp_0.b === void 0 ? 2 : _tmp_0.b", AlwaysEdge)) - succOf("_tmp_0.b === void 0 ? 2 : _tmp_0.b") shouldBe + succOf("_tmp_0.b === void 0") should contain theSameElementsAs expected(("2", TrueEdge), ("_tmp_0", 4, FalseEdge)) + succOf("_tmp_0", 4) should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("_tmp_0.b", 1, AlwaysEdge)) + succOf("_tmp_0.b", 1) should contain theSameElementsAs expected( + ("_tmp_0.b === void 0 ? 2 : _tmp_0.b", AlwaysEdge) + ) + succOf("2") should contain theSameElementsAs expected(("_tmp_0.b === void 0 ? 2 : _tmp_0.b", AlwaysEdge)) + succOf("_tmp_0.b === void 0 ? 2 : _tmp_0.b") should contain theSameElementsAs expected(("m = _tmp_0.b === void 0 ? 2 : _tmp_0.b", AlwaysEdge)) - succOf("m = _tmp_0.b === void 0 ? 2 : _tmp_0.b") shouldBe + succOf("m = _tmp_0.b === void 0 ? 2 : _tmp_0.b") should contain theSameElementsAs expected(("_tmp_0", 5, AlwaysEdge)) - succOf("_tmp_0", 5) shouldBe expected(("var {a: n = 1, b: m = 2} = x", AlwaysEdge)) - succOf("var {a: n = 1, b: m = 2} = x") shouldBe expected(("RET", AlwaysEdge)) + succOf("_tmp_0", 5) should contain theSameElementsAs expected(("var {a: n = 1, b: m = 2} = x", AlwaysEdge)) + succOf("var {a: n = 1, b: m = 2} = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for object destruction assignment with rest" in { implicit val cpg: Cpg = code("var {a, ...rest} = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("_tmp_0.a", AlwaysEdge)) - succOf("_tmp_0.a") shouldBe expected(("a = _tmp_0.a", AlwaysEdge)) - - succOf("a = _tmp_0.a") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("rest", AlwaysEdge)) - succOf("rest") shouldBe expected(("...rest", AlwaysEdge)) - succOf("...rest") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - - succOf("_tmp_0", 3) shouldBe expected(("var {a, ...rest} = x", AlwaysEdge)) - succOf("var {a, ...rest} = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("_tmp_0.a", AlwaysEdge)) + succOf("_tmp_0.a") should contain theSameElementsAs expected(("a = _tmp_0.a", AlwaysEdge)) + + succOf("a = _tmp_0.a") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("rest", AlwaysEdge)) + succOf("rest") should contain theSameElementsAs expected(("...rest", AlwaysEdge)) + succOf("...rest") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var {a, ...rest} = x", AlwaysEdge)) + succOf("var {a, ...rest} = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for object destruction assignment with computed property name" in { implicit val cpg: Cpg = code("var {[propName]: n} = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("n", AlwaysEdge)) - succOf("n") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("propName", AlwaysEdge)) - succOf("propName") shouldBe expected(("_tmp_0.propName", AlwaysEdge)) - succOf("_tmp_0.propName") shouldBe expected(("n = _tmp_0.propName", AlwaysEdge)) - - succOf("n = _tmp_0.propName") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("var {[propName]: n} = x", AlwaysEdge)) - succOf("var {[propName]: n} = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("n", AlwaysEdge)) + succOf("n") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("propName", AlwaysEdge)) + succOf("propName") should contain theSameElementsAs expected(("_tmp_0.propName", AlwaysEdge)) + succOf("_tmp_0.propName") should contain theSameElementsAs expected(("n = _tmp_0.propName", AlwaysEdge)) + + succOf("n = _tmp_0.propName") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("var {[propName]: n} = x", AlwaysEdge)) + succOf("var {[propName]: n} = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested object destruction assignment with defaults as parameter" in { @@ -172,46 +180,50 @@ class MixedCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCpg |function userId({id = {}, b} = {}) { | return id |}""".stripMargin) - succOf("userId", NodeTypes.METHOD) shouldBe expected(("_tmp_1", AlwaysEdge)) - succOf("_tmp_1") shouldBe expected(("param1_0", AlwaysEdge)) - succOf("param1_0") shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("param1_0 === void 0", AlwaysEdge)) - succOf("param1_0 === void 0") shouldBe expected( + succOf("userId", NodeTypes.METHOD) should contain theSameElementsAs expected(("_tmp_1", AlwaysEdge)) + succOf("_tmp_1") should contain theSameElementsAs expected(("param1_0", AlwaysEdge)) + succOf("param1_0") should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("param1_0 === void 0", AlwaysEdge)) + succOf("param1_0 === void 0") should contain theSameElementsAs expected( ("_tmp_0", TrueEdge), // holds {} ("param1_0", 1, FalseEdge) ) - succOf("param1_0", 1) shouldBe expected(("param1_0 === void 0 ? {} : param1_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("param1_0 === void 0 ? {} : param1_0", AlwaysEdge)) - succOf("param1_0 === void 0 ? {} : param1_0") shouldBe expected( + succOf("param1_0", 1) should contain theSameElementsAs expected( + ("param1_0 === void 0 ? {} : param1_0", AlwaysEdge) + ) + succOf("_tmp_0") should contain theSameElementsAs expected(("param1_0 === void 0 ? {} : param1_0", AlwaysEdge)) + succOf("param1_0 === void 0 ? {} : param1_0") should contain theSameElementsAs expected( ("_tmp_1 = param1_0 === void 0 ? {} : param1_0", AlwaysEdge) ) - succOf("_tmp_1 = param1_0 === void 0 ? {} : param1_0") shouldBe expected(("id", AlwaysEdge)) - succOf("id") shouldBe expected(("_tmp_1", 1, AlwaysEdge)) - succOf("_tmp_1", 1) shouldBe expected(("id", 1, AlwaysEdge)) - succOf("id", 1) shouldBe expected(("_tmp_1.id", AlwaysEdge)) - succOf("_tmp_1.id") shouldBe expected(("void 0", 1, AlwaysEdge)) - succOf("void 0", 1) shouldBe expected(("_tmp_1.id === void 0", AlwaysEdge)) - succOf("_tmp_1.id === void 0") shouldBe expected( + succOf("_tmp_1 = param1_0 === void 0 ? {} : param1_0") should contain theSameElementsAs expected( + ("id", AlwaysEdge) + ) + succOf("id") should contain theSameElementsAs expected(("_tmp_1", 1, AlwaysEdge)) + succOf("_tmp_1", 1) should contain theSameElementsAs expected(("id", 1, AlwaysEdge)) + succOf("id", 1) should contain theSameElementsAs expected(("_tmp_1.id", AlwaysEdge)) + succOf("_tmp_1.id") should contain theSameElementsAs expected(("void 0", 1, AlwaysEdge)) + succOf("void 0", 1) should contain theSameElementsAs expected(("_tmp_1.id === void 0", AlwaysEdge)) + succOf("_tmp_1.id === void 0") should contain theSameElementsAs expected( ("_tmp_2", TrueEdge), // holds {} ("_tmp_1", 2, FalseEdge) ) - succOf("_tmp_2") shouldBe expected(("_tmp_1.id === void 0 ? {} : _tmp_1.id", AlwaysEdge)) - succOf("_tmp_1", 2) shouldBe expected(("id", 2, AlwaysEdge)) + succOf("_tmp_2") should contain theSameElementsAs expected(("_tmp_1.id === void 0 ? {} : _tmp_1.id", AlwaysEdge)) + succOf("_tmp_1", 2) should contain theSameElementsAs expected(("id", 2, AlwaysEdge)) - succOf("_tmp_1.id === void 0 ? {} : _tmp_1.id") shouldBe expected( + succOf("_tmp_1.id === void 0 ? {} : _tmp_1.id") should contain theSameElementsAs expected( ("id = _tmp_1.id === void 0 ? {} : _tmp_1.id", AlwaysEdge) ) - succOf("id", 2) shouldBe expected(("_tmp_1.id", 1, AlwaysEdge)) + succOf("id", 2) should contain theSameElementsAs expected(("_tmp_1.id", 1, AlwaysEdge)) - succOf("id = _tmp_1.id === void 0 ? {} : _tmp_1.id") shouldBe expected(("b", AlwaysEdge)) + succOf("id = _tmp_1.id === void 0 ? {} : _tmp_1.id") should contain theSameElementsAs expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_1", 3, AlwaysEdge)) - succOf("_tmp_1", 3) shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("_tmp_1.b", AlwaysEdge)) - succOf("_tmp_1.b") shouldBe expected(("b = _tmp_1.b", AlwaysEdge)) - succOf("b = _tmp_1.b") shouldBe expected(("_tmp_1", 4, AlwaysEdge)) - succOf("_tmp_1", 4) shouldBe expected(("{id = {}, b} = {}", 1, AlwaysEdge)) - succOf("{id = {}, b} = {}", 1) shouldBe expected(("id", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_1", 3, AlwaysEdge)) + succOf("_tmp_1", 3) should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("_tmp_1.b", AlwaysEdge)) + succOf("_tmp_1.b") should contain theSameElementsAs expected(("b = _tmp_1.b", AlwaysEdge)) + succOf("b = _tmp_1.b") should contain theSameElementsAs expected(("_tmp_1", 4, AlwaysEdge)) + succOf("_tmp_1", 4) should contain theSameElementsAs expected(("{id = {}, b} = {}", 1, AlwaysEdge)) + succOf("{id = {}, b} = {}", 1) should contain theSameElementsAs expected(("id", AlwaysEdge)) } @@ -220,151 +232,163 @@ class MixedCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCpg |function userId({id}) { | return id |}""".stripMargin) - succOf("userId", NodeTypes.METHOD) shouldBe expected(("id", AlwaysEdge)) - succOf("id") shouldBe expected(("param1_0", AlwaysEdge)) - succOf("param1_0") shouldBe expected(("id", 1, AlwaysEdge)) - succOf("id", 1) shouldBe expected(("param1_0.id", AlwaysEdge)) - succOf("param1_0.id") shouldBe expected(("id = param1_0.id", AlwaysEdge)) - succOf("id = param1_0.id") shouldBe expected(("id", 2, AlwaysEdge)) - succOf("id", 2) shouldBe expected(("return id", AlwaysEdge)) - succOf("return id") shouldBe expected(("RET", AlwaysEdge)) + succOf("userId", NodeTypes.METHOD) should contain theSameElementsAs expected(("id", AlwaysEdge)) + succOf("id") should contain theSameElementsAs expected(("param1_0", AlwaysEdge)) + succOf("param1_0") should contain theSameElementsAs expected(("id", 1, AlwaysEdge)) + succOf("id", 1) should contain theSameElementsAs expected(("param1_0.id", AlwaysEdge)) + succOf("param1_0.id") should contain theSameElementsAs expected(("id = param1_0.id", AlwaysEdge)) + succOf("id = param1_0.id") should contain theSameElementsAs expected(("id", 2, AlwaysEdge)) + succOf("id", 2) should contain theSameElementsAs expected(("return id", AlwaysEdge)) + succOf("return id") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment with declaration" in { implicit val cpg: Cpg = code("var [a, b] = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("_tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0]") shouldBe expected(("a = _tmp_0[0]", AlwaysEdge)) - - succOf("a = _tmp_0[0]") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("_tmp_0[1]", AlwaysEdge)) - succOf("_tmp_0[1]") shouldBe expected(("b = _tmp_0[1]", AlwaysEdge)) - succOf("b = _tmp_0[1]") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("var [a, b] = x", AlwaysEdge)) - succOf("var [a, b] = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("_tmp_0[0]", AlwaysEdge)) + succOf("_tmp_0[0]") should contain theSameElementsAs expected(("a = _tmp_0[0]", AlwaysEdge)) + + succOf("a = _tmp_0[0]") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("_tmp_0[1]", AlwaysEdge)) + succOf("_tmp_0[1]") should contain theSameElementsAs expected(("b = _tmp_0[1]", AlwaysEdge)) + succOf("b = _tmp_0[1]") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var [a, b] = x", AlwaysEdge)) + succOf("var [a, b] = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment without declaration" in { implicit val cpg: Cpg = code("[a, b] = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("_tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0]") shouldBe expected(("a = _tmp_0[0]", AlwaysEdge)) - - succOf("a = _tmp_0[0]") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("_tmp_0[1]", AlwaysEdge)) - succOf("_tmp_0[1]") shouldBe expected(("b = _tmp_0[1]", AlwaysEdge)) - succOf("b = _tmp_0[1]") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("[a, b] = x", AlwaysEdge)) - succOf("[a, b] = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("_tmp_0[0]", AlwaysEdge)) + succOf("_tmp_0[0]") should contain theSameElementsAs expected(("a = _tmp_0[0]", AlwaysEdge)) + + succOf("a = _tmp_0[0]") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("_tmp_0[1]", AlwaysEdge)) + succOf("_tmp_0[1]") should contain theSameElementsAs expected(("b = _tmp_0[1]", AlwaysEdge)) + succOf("b = _tmp_0[1]") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("[a, b] = x", AlwaysEdge)) + succOf("[a, b] = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment with defaults" in { implicit val cpg: Cpg = code("var [a = 1, b = 2] = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) // test statement - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("_tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0]") shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("_tmp_0[0] === void 0", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("_tmp_0[0]", AlwaysEdge)) + succOf("_tmp_0[0]") should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("_tmp_0[0] === void 0", AlwaysEdge)) // true, false cases - succOf("_tmp_0[0] === void 0") shouldBe expected(("1", TrueEdge), ("_tmp_0", 2, FalseEdge)) - succOf("_tmp_0", 2) shouldBe expected(("0", 1, AlwaysEdge)) - succOf("0", 1) shouldBe expected(("_tmp_0[0]", 1, AlwaysEdge)) - succOf("_tmp_0[0]", 1) shouldBe expected(("_tmp_0[0] === void 0 ? 1 : _tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0] === void 0 ? 1 : _tmp_0[0]") shouldBe expected( + succOf("_tmp_0[0] === void 0") should contain theSameElementsAs expected( + ("1", TrueEdge), + ("_tmp_0", 2, FalseEdge) + ) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("0", 1, AlwaysEdge)) + succOf("0", 1) should contain theSameElementsAs expected(("_tmp_0[0]", 1, AlwaysEdge)) + succOf("_tmp_0[0]", 1) should contain theSameElementsAs expected( + ("_tmp_0[0] === void 0 ? 1 : _tmp_0[0]", AlwaysEdge) + ) + succOf("_tmp_0[0] === void 0 ? 1 : _tmp_0[0]") should contain theSameElementsAs expected( ("a = _tmp_0[0] === void 0 ? 1 : _tmp_0[0]", AlwaysEdge) ) - succOf("a = _tmp_0[0] === void 0 ? 1 : _tmp_0[0]") shouldBe expected(("b", AlwaysEdge)) + succOf("a = _tmp_0[0] === void 0 ? 1 : _tmp_0[0]") should contain theSameElementsAs expected(("b", AlwaysEdge)) // test statement - succOf("b") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("1", 1, AlwaysEdge)) - succOf("1", 1) shouldBe expected(("_tmp_0[1]", AlwaysEdge)) - succOf("_tmp_0[1]") shouldBe expected(("void 0", 1, AlwaysEdge)) - succOf("void 0", 1) shouldBe expected(("_tmp_0[1] === void 0", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("1", 1, AlwaysEdge)) + succOf("1", 1) should contain theSameElementsAs expected(("_tmp_0[1]", AlwaysEdge)) + succOf("_tmp_0[1]") should contain theSameElementsAs expected(("void 0", 1, AlwaysEdge)) + succOf("void 0", 1) should contain theSameElementsAs expected(("_tmp_0[1] === void 0", AlwaysEdge)) // true, false cases - succOf("_tmp_0[1] === void 0") shouldBe expected(("2", TrueEdge), ("_tmp_0", 4, FalseEdge)) - succOf("_tmp_0", 4) shouldBe expected(("1", 2, AlwaysEdge)) - succOf("1", 2) shouldBe expected(("_tmp_0[1]", 1, AlwaysEdge)) - succOf("_tmp_0[1]", 1) shouldBe expected(("_tmp_0[1] === void 0 ? 2 : _tmp_0[1]", AlwaysEdge)) - succOf("_tmp_0[1] === void 0 ? 2 : _tmp_0[1]") shouldBe expected( + succOf("_tmp_0[1] === void 0") should contain theSameElementsAs expected( + ("2", TrueEdge), + ("_tmp_0", 4, FalseEdge) + ) + succOf("_tmp_0", 4) should contain theSameElementsAs expected(("1", 2, AlwaysEdge)) + succOf("1", 2) should contain theSameElementsAs expected(("_tmp_0[1]", 1, AlwaysEdge)) + succOf("_tmp_0[1]", 1) should contain theSameElementsAs expected( + ("_tmp_0[1] === void 0 ? 2 : _tmp_0[1]", AlwaysEdge) + ) + succOf("_tmp_0[1] === void 0 ? 2 : _tmp_0[1]") should contain theSameElementsAs expected( ("b = _tmp_0[1] === void 0 ? 2 : _tmp_0[1]", AlwaysEdge) ) - succOf("b = _tmp_0[1] === void 0 ? 2 : _tmp_0[1]") shouldBe expected(("_tmp_0", 5, AlwaysEdge)) - succOf("_tmp_0", 5) shouldBe expected(("var [a = 1, b = 2] = x", AlwaysEdge)) - succOf("var [a = 1, b = 2] = x") shouldBe expected(("RET", AlwaysEdge)) + succOf("b = _tmp_0[1] === void 0 ? 2 : _tmp_0[1]") should contain theSameElementsAs expected( + ("_tmp_0", 5, AlwaysEdge) + ) + succOf("_tmp_0", 5) should contain theSameElementsAs expected(("var [a = 1, b = 2] = x", AlwaysEdge)) + succOf("var [a = 1, b = 2] = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment with ignores" in { implicit val cpg: Cpg = code("var [a, , b] = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("_tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0]") shouldBe expected(("a = _tmp_0[0]", AlwaysEdge)) - - succOf("a = _tmp_0[0]") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("_tmp_0[2]", AlwaysEdge)) - succOf("_tmp_0[2]") shouldBe expected(("b = _tmp_0[2]", AlwaysEdge)) - succOf("b = _tmp_0[2]") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("var [a, , b] = x", AlwaysEdge)) - succOf("var [a, , b] = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("_tmp_0[0]", AlwaysEdge)) + succOf("_tmp_0[0]") should contain theSameElementsAs expected(("a = _tmp_0[0]", AlwaysEdge)) + + succOf("a = _tmp_0[0]") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("_tmp_0[2]", AlwaysEdge)) + succOf("_tmp_0[2]") should contain theSameElementsAs expected(("b = _tmp_0[2]", AlwaysEdge)) + succOf("b = _tmp_0[2]") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var [a, , b] = x", AlwaysEdge)) + succOf("var [a, , b] = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment with rest" in { implicit val cpg: Cpg = code("var [a, ...rest] = x") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0 = x", AlwaysEdge)) - - succOf("_tmp_0 = x") shouldBe expected(("a", AlwaysEdge)) - - succOf("a") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("_tmp_0[0]", AlwaysEdge)) - succOf("_tmp_0[0]") shouldBe expected(("a = _tmp_0[0]", AlwaysEdge)) - - succOf("a = _tmp_0[0]") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("_tmp_0[1]", AlwaysEdge)) - succOf("_tmp_0[1]") shouldBe expected(("rest", AlwaysEdge)) - succOf("rest") shouldBe expected(("...rest", AlwaysEdge)) - succOf("...rest") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("var [a, ...rest] = x", AlwaysEdge)) - succOf("var [a, ...rest] = x") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0 = x", AlwaysEdge)) + + succOf("_tmp_0 = x") should contain theSameElementsAs expected(("a", AlwaysEdge)) + + succOf("a") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("_tmp_0[0]", AlwaysEdge)) + succOf("_tmp_0[0]") should contain theSameElementsAs expected(("a = _tmp_0[0]", AlwaysEdge)) + + succOf("a = _tmp_0[0]") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("_tmp_0[1]", AlwaysEdge)) + succOf("_tmp_0[1]") should contain theSameElementsAs expected(("rest", AlwaysEdge)) + succOf("rest") should contain theSameElementsAs expected(("...rest", AlwaysEdge)) + succOf("...rest") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("var [a, ...rest] = x", AlwaysEdge)) + succOf("var [a, ...rest] = x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for array destruction assignment as parameter" in { @@ -373,25 +397,25 @@ class MixedCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCpg | return id |} |""".stripMargin) - succOf("userId", NodeTypes.METHOD) shouldBe expected(("id", AlwaysEdge)) - succOf("id") shouldBe expected(("param1_0", AlwaysEdge)) - succOf("param1_0") shouldBe expected(("id", 1, AlwaysEdge)) - succOf("id", 1) shouldBe expected(("param1_0.id", AlwaysEdge)) - succOf("param1_0.id") shouldBe expected(("id = param1_0.id", AlwaysEdge)) - succOf("id = param1_0.id") shouldBe expected(("id", 2, AlwaysEdge)) - succOf("id", 2) shouldBe expected(("return id", AlwaysEdge)) - succOf("return id") shouldBe expected(("RET", AlwaysEdge)) + succOf("userId", NodeTypes.METHOD) should contain theSameElementsAs expected(("id", AlwaysEdge)) + succOf("id") should contain theSameElementsAs expected(("param1_0", AlwaysEdge)) + succOf("param1_0") should contain theSameElementsAs expected(("id", 1, AlwaysEdge)) + succOf("id", 1) should contain theSameElementsAs expected(("param1_0.id", AlwaysEdge)) + succOf("param1_0.id") should contain theSameElementsAs expected(("id = param1_0.id", AlwaysEdge)) + succOf("id = param1_0.id") should contain theSameElementsAs expected(("id", 2, AlwaysEdge)) + succOf("id", 2) should contain theSameElementsAs expected(("return id", AlwaysEdge)) + succOf("return id") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "CFG generation for spread arguments" should { "have correct structure for method spread argument" in { implicit val cpg: Cpg = code("foo(...args)") - succOf(":program") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("args", AlwaysEdge)) - succOf("args") shouldBe expected(("...args", AlwaysEdge)) - succOf("...args") shouldBe expected(("foo(...args)", AlwaysEdge)) - succOf("foo(...args)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("args", AlwaysEdge)) + succOf("args") should contain theSameElementsAs expected(("...args", AlwaysEdge)) + succOf("...args") should contain theSameElementsAs expected(("foo(...args)", AlwaysEdge)) + succOf("foo(...args)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } @@ -400,110 +424,110 @@ class MixedCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCpg "CFG generation for await/async" should { "be correct for await/async" in { implicit val cpg: Cpg = code("async function x(foo) { await foo() }") - succOf("x", NodeTypes.METHOD) shouldBe expected(("foo", AlwaysEdge)) - succOf("foo", NodeTypes.IDENTIFIER) shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("foo()", AlwaysEdge)) - succOf("foo()") shouldBe expected(("await foo()", AlwaysEdge)) - succOf("await foo()") shouldBe expected(("RET", AlwaysEdge)) + succOf("x", NodeTypes.METHOD) should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("foo()", AlwaysEdge)) + succOf("foo()") should contain theSameElementsAs expected(("await foo()", AlwaysEdge)) + succOf("await foo()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } "CFG generation for instanceof/delete" should { "be correct for instanceof" in { implicit val cpg: Cpg = code("x instanceof Foo") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("Foo", AlwaysEdge)) - succOf("Foo") shouldBe expected(("x instanceof Foo", AlwaysEdge)) - succOf("x instanceof Foo", NodeTypes.CALL) shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("Foo", AlwaysEdge)) + succOf("Foo") should contain theSameElementsAs expected(("x instanceof Foo", AlwaysEdge)) + succOf("x instanceof Foo", NodeTypes.CALL) should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for delete" in { implicit val cpg: Cpg = code("delete foo.x") - succOf(":program") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("foo.x", AlwaysEdge)) - succOf("foo.x") shouldBe expected(("delete foo.x", AlwaysEdge)) - succOf("delete foo.x", NodeTypes.CALL) shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("foo.x", AlwaysEdge)) + succOf("foo.x") should contain theSameElementsAs expected(("delete foo.x", AlwaysEdge)) + succOf("delete foo.x", NodeTypes.CALL) should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } "CFG generation for default parameters" should { "be correct for method parameter with default" in { implicit val cpg: Cpg = code("function foo(a = 1) { }") - cpg.method.nameExact("foo").parameter.code.l shouldBe List("this", "a = 1") - - succOf("foo", NodeTypes.METHOD) shouldBe expected(("a", AlwaysEdge)) - succOf("a", NodeTypes.IDENTIFIER) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("a === void 0", AlwaysEdge)) - succOf("a === void 0") shouldBe expected(("1", TrueEdge), ("a", 2, FalseEdge)) - succOf("1") shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a", 2) shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a === void 0 ? 1 : a") shouldBe expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a = a === void 0 ? 1 : a") shouldBe expected(("RET", AlwaysEdge)) + cpg.method.nameExact("foo").parameter.code.l should contain theSameElementsAs List("this", "a = 1") + + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("a === void 0", AlwaysEdge)) + succOf("a === void 0") should contain theSameElementsAs expected(("1", TrueEdge), ("a", 2, FalseEdge)) + succOf("1") should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a", 2) should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a === void 0 ? 1 : a") should contain theSameElementsAs expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a = a === void 0 ? 1 : a") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for multiple method parameters with default" in { implicit val cpg: Cpg = code("function foo(a = 1, b = 2) { }") - cpg.method.nameExact("foo").parameter.code.l shouldBe List("this", "a = 1", "b = 2") - - succOf("foo", NodeTypes.METHOD) shouldBe expected(("a", AlwaysEdge)) - succOf("a", NodeTypes.IDENTIFIER) shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("a === void 0", AlwaysEdge)) - succOf("a === void 0") shouldBe expected(("1", TrueEdge), ("a", 2, FalseEdge)) - succOf("1") shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a", 2) shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a === void 0 ? 1 : a") shouldBe expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a = a === void 0 ? 1 : a") shouldBe expected(("b", AlwaysEdge)) - - succOf("b", NodeTypes.IDENTIFIER) shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("void 0", 1, AlwaysEdge)) - succOf("void 0", 1) shouldBe expected(("b === void 0", AlwaysEdge)) - succOf("b === void 0") shouldBe expected(("2", TrueEdge), ("b", 2, FalseEdge)) - succOf("2") shouldBe expected(("b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b", 2) shouldBe expected(("b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b === void 0 ? 2 : b") shouldBe expected(("b = b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b = b === void 0 ? 2 : b") shouldBe expected(("RET", AlwaysEdge)) + cpg.method.nameExact("foo").parameter.code.l should contain theSameElementsAs List("this", "a = 1", "b = 2") + + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("a === void 0", AlwaysEdge)) + succOf("a === void 0") should contain theSameElementsAs expected(("1", TrueEdge), ("a", 2, FalseEdge)) + succOf("1") should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a", 2) should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a === void 0 ? 1 : a") should contain theSameElementsAs expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a = a === void 0 ? 1 : a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + + succOf("b", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("void 0", 1, AlwaysEdge)) + succOf("void 0", 1) should contain theSameElementsAs expected(("b === void 0", AlwaysEdge)) + succOf("b === void 0") should contain theSameElementsAs expected(("2", TrueEdge), ("b", 2, FalseEdge)) + succOf("2") should contain theSameElementsAs expected(("b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b", 2) should contain theSameElementsAs expected(("b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b === void 0 ? 2 : b") should contain theSameElementsAs expected(("b = b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b = b === void 0 ? 2 : b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for method mixed parameters with default" in { implicit val cpg: Cpg = code("function foo(a, b = 1) { }") - cpg.method.nameExact("foo").parameter.code.l shouldBe List("this", "a", "b = 1") - - succOf("foo", NodeTypes.METHOD) shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("b === void 0", AlwaysEdge)) - succOf("b === void 0") shouldBe expected(("1", TrueEdge), ("b", 2, FalseEdge)) - succOf("1") shouldBe expected(("b === void 0 ? 1 : b", AlwaysEdge)) - succOf("b", 2) shouldBe expected(("b === void 0 ? 1 : b", AlwaysEdge)) - succOf("b === void 0 ? 1 : b") shouldBe expected(("b = b === void 0 ? 1 : b", AlwaysEdge)) - succOf("b = b === void 0 ? 1 : b") shouldBe expected(("RET", AlwaysEdge)) + cpg.method.nameExact("foo").parameter.code.l should contain theSameElementsAs List("this", "a", "b = 1") + + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("b === void 0", AlwaysEdge)) + succOf("b === void 0") should contain theSameElementsAs expected(("1", TrueEdge), ("b", 2, FalseEdge)) + succOf("1") should contain theSameElementsAs expected(("b === void 0 ? 1 : b", AlwaysEdge)) + succOf("b", 2) should contain theSameElementsAs expected(("b === void 0 ? 1 : b", AlwaysEdge)) + succOf("b === void 0 ? 1 : b") should contain theSameElementsAs expected(("b = b === void 0 ? 1 : b", AlwaysEdge)) + succOf("b = b === void 0 ? 1 : b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for multiple method mixed parameters with default" in { implicit val cpg: Cpg = code("function foo(x, a = 1, b = 2) { }") - cpg.method.nameExact("foo").parameter.code.l shouldBe List("this", "x", "a = 1", "b = 2") - - succOf("foo", NodeTypes.METHOD) shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("a", 1, AlwaysEdge)) - succOf("a", 1) shouldBe expected(("void 0", AlwaysEdge)) - succOf("void 0") shouldBe expected(("a === void 0", AlwaysEdge)) - succOf("a === void 0") shouldBe expected(("1", TrueEdge), ("a", 2, FalseEdge)) - succOf("1") shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a", 2) shouldBe expected(("a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a === void 0 ? 1 : a") shouldBe expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) - succOf("a = a === void 0 ? 1 : a") shouldBe expected(("b", AlwaysEdge)) - - succOf("b") shouldBe expected(("b", 1, AlwaysEdge)) - succOf("b", 1) shouldBe expected(("void 0", 1, AlwaysEdge)) - succOf("void 0", 1) shouldBe expected(("b === void 0", AlwaysEdge)) - succOf("b === void 0") shouldBe expected(("2", TrueEdge), ("b", 2, FalseEdge)) - succOf("2") shouldBe expected(("b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b", 2) shouldBe expected(("b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b === void 0 ? 2 : b") shouldBe expected(("b = b === void 0 ? 2 : b", AlwaysEdge)) - succOf("b = b === void 0 ? 2 : b") shouldBe expected(("RET", AlwaysEdge)) + cpg.method.nameExact("foo").parameter.code.l should contain theSameElementsAs List("this", "x", "a = 1", "b = 2") + + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("a", 1, AlwaysEdge)) + succOf("a", 1) should contain theSameElementsAs expected(("void 0", AlwaysEdge)) + succOf("void 0") should contain theSameElementsAs expected(("a === void 0", AlwaysEdge)) + succOf("a === void 0") should contain theSameElementsAs expected(("1", TrueEdge), ("a", 2, FalseEdge)) + succOf("1") should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a", 2) should contain theSameElementsAs expected(("a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a === void 0 ? 1 : a") should contain theSameElementsAs expected(("a = a === void 0 ? 1 : a", AlwaysEdge)) + succOf("a = a === void 0 ? 1 : a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + + succOf("b") should contain theSameElementsAs expected(("b", 1, AlwaysEdge)) + succOf("b", 1) should contain theSameElementsAs expected(("void 0", 1, AlwaysEdge)) + succOf("void 0", 1) should contain theSameElementsAs expected(("b === void 0", AlwaysEdge)) + succOf("b === void 0") should contain theSameElementsAs expected(("2", TrueEdge), ("b", 2, FalseEdge)) + succOf("2") should contain theSameElementsAs expected(("b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b", 2) should contain theSameElementsAs expected(("b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b === void 0 ? 2 : b") should contain theSameElementsAs expected(("b = b === void 0 ? 2 : b", AlwaysEdge)) + succOf("b = b === void 0 ? 2 : b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/SimpleCfgCreationPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/SimpleCfgCreationPassTests.scala index 9a290cbb5931..35caf9728ac7 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/SimpleCfgCreationPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/cfg/SimpleCfgCreationPassTests.scala @@ -11,85 +11,99 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp "CFG generation for simple fragments" should { "have correct structure for block expression" in { implicit val cpg: Cpg = code("let x = (class Foo {}, bar())") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("class Foo", AlwaysEdge)) - succOf("class Foo") shouldBe expected(("bar", AlwaysEdge)) - succOf("bar") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("bar()", AlwaysEdge)) - succOf("bar()") shouldBe expected(("class Foo {}, bar()", AlwaysEdge)) - succOf("class Foo {}, bar()") shouldBe expected(("let x = (class Foo {}, bar())", AlwaysEdge)) - succOf("let x = (class Foo {}, bar())") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("class Foo", AlwaysEdge)) + succOf("class Foo") should contain theSameElementsAs expected(("bar", AlwaysEdge)) + succOf("bar") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("bar()", AlwaysEdge)) + succOf("bar()") should contain theSameElementsAs expected(("class Foo {}, bar()", AlwaysEdge)) + succOf("class Foo {}, bar()") should contain theSameElementsAs expected( + ("let x = (class Foo {}, bar())", AlwaysEdge) + ) + succOf("let x = (class Foo {}, bar())") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "have correct structure for empty array literal" in { implicit val cpg: Cpg = code("var x = []") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("__ecma.Array.factory()", AlwaysEdge)) - succOf("__ecma.Array.factory()") shouldBe expected(("var x = []", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("__ecma.Array.factory()", AlwaysEdge)) + succOf("__ecma.Array.factory()") should contain theSameElementsAs expected(("var x = []", AlwaysEdge)) } "have correct structure for array literal with values" in { implicit val cpg: Cpg = code("var x = [1, 2]") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("__ecma.Array.factory()", AlwaysEdge)) - succOf("__ecma.Array.factory()") shouldBe expected(("_tmp_0 = __ecma.Array.factory()", AlwaysEdge)) - - succOf("_tmp_0 = __ecma.Array.factory()") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("push", AlwaysEdge)) - succOf("push") shouldBe expected(("_tmp_0.push", AlwaysEdge)) - succOf("_tmp_0.push") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("_tmp_0.push(1)", AlwaysEdge)) - - succOf("_tmp_0.push(1)") shouldBe expected(("_tmp_0", 3, AlwaysEdge)) - succOf("_tmp_0", 3) shouldBe expected(("push", 1, AlwaysEdge)) - succOf("push", 1) shouldBe expected(("_tmp_0.push", 1, AlwaysEdge)) - succOf("_tmp_0.push", 1) shouldBe expected(("_tmp_0", 4, AlwaysEdge)) - succOf("_tmp_0", 4) shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("_tmp_0.push(2)", AlwaysEdge)) - - succOf("_tmp_0.push(2)") shouldBe expected(("_tmp_0", 5, AlwaysEdge)) - succOf("_tmp_0", 5) shouldBe expected(("[1, 2]", AlwaysEdge)) - succOf("[1, 2]") shouldBe expected(("var x = [1, 2]", AlwaysEdge)) - succOf("var x = [1, 2]") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("__ecma.Array.factory()", AlwaysEdge)) + succOf("__ecma.Array.factory()") should contain theSameElementsAs expected( + ("_tmp_0 = __ecma.Array.factory()", AlwaysEdge) + ) + + succOf("_tmp_0 = __ecma.Array.factory()") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("push", AlwaysEdge)) + succOf("push") should contain theSameElementsAs expected(("_tmp_0.push", AlwaysEdge)) + succOf("_tmp_0.push") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("_tmp_0.push(1)", AlwaysEdge)) + + succOf("_tmp_0.push(1)") should contain theSameElementsAs expected(("_tmp_0", 3, AlwaysEdge)) + succOf("_tmp_0", 3) should contain theSameElementsAs expected(("push", 1, AlwaysEdge)) + succOf("push", 1) should contain theSameElementsAs expected(("_tmp_0.push", 1, AlwaysEdge)) + succOf("_tmp_0.push", 1) should contain theSameElementsAs expected(("_tmp_0", 4, AlwaysEdge)) + succOf("_tmp_0", 4) should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("_tmp_0.push(2)", AlwaysEdge)) + + succOf("_tmp_0.push(2)") should contain theSameElementsAs expected(("_tmp_0", 5, AlwaysEdge)) + succOf("_tmp_0", 5) should contain theSameElementsAs expected(("[1, 2]", AlwaysEdge)) + succOf("[1, 2]") should contain theSameElementsAs expected(("var x = [1, 2]", AlwaysEdge)) + succOf("var x = [1, 2]") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "have correct structure for untagged runtime node in call" in { implicit val cpg: Cpg = code(s"foo(`Hello $${world}!`)") - succOf(":program") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("\"Hello \"", AlwaysEdge)) - succOf("\"Hello \"") shouldBe expected(("world", AlwaysEdge)) - succOf("world") shouldBe expected(("\"!\"", AlwaysEdge)) - succOf("\"!\"") shouldBe expected((s"${Operators.formatString}(\"Hello \", world, \"!\")", AlwaysEdge)) - succOf(s"${Operators.formatString}(\"Hello \", world, \"!\")") shouldBe expected( + succOf(":program") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("\"Hello \"", AlwaysEdge)) + succOf("\"Hello \"") should contain theSameElementsAs expected(("world", AlwaysEdge)) + succOf("world") should contain theSameElementsAs expected(("\"!\"", AlwaysEdge)) + succOf("\"!\"") should contain theSameElementsAs expected( + (s"${Operators.formatString}(\"Hello \", world, \"!\")", AlwaysEdge) + ) + succOf(s"${Operators.formatString}(\"Hello \", world, \"!\")") should contain theSameElementsAs expected( (s"foo(`Hello $${world}!`)", AlwaysEdge) ) - succOf(s"foo(`Hello $${world}!`)") shouldBe expected(("RET", AlwaysEdge)) + succOf(s"foo(`Hello $${world}!`)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "have correct structure for untagged runtime node" in { implicit val cpg: Cpg = code(s"`$${x + 1}`") - succOf(":program") shouldBe expected(("\"\"", AlwaysEdge)) - succOf("\"\"") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x + 1", AlwaysEdge)) - succOf("x + 1") shouldBe expected(("\"\"", 1, AlwaysEdge)) - succOf("\"\"", 1) shouldBe expected((s"${Operators.formatString}(\"\", x + 1, \"\")", AlwaysEdge)) - succOf(s"${Operators.formatString}(\"\", x + 1, \"\")") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("\"\"", AlwaysEdge)) + succOf("\"\"") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x + 1", AlwaysEdge)) + succOf("x + 1") should contain theSameElementsAs expected(("\"\"", 1, AlwaysEdge)) + succOf("\"\"", 1) should contain theSameElementsAs expected( + (s"${Operators.formatString}(\"\", x + 1, \"\")", AlwaysEdge) + ) + succOf(s"${Operators.formatString}(\"\", x + 1, \"\")") should contain theSameElementsAs expected( + ("RET", AlwaysEdge) + ) } "have correct structure for tagged runtime node" in { implicit val cpg: Cpg = code(s"String.raw`../$${42}\\..`") - succOf(":program") shouldBe expected(("\"../\"", AlwaysEdge)) - succOf("\"../\"") shouldBe expected(("42", AlwaysEdge)) - succOf("42") shouldBe expected(("\"\\..\"", AlwaysEdge)) - succOf("\"\\..\"") shouldBe expected((s"${Operators.formatString}(\"../\", 42, \"\\..\")", AlwaysEdge)) - succOf(s"${Operators.formatString}(\"../\", 42, \"\\..\")") shouldBe expected( + succOf(":program") should contain theSameElementsAs expected(("\"../\"", AlwaysEdge)) + succOf("\"../\"") should contain theSameElementsAs expected(("42", AlwaysEdge)) + succOf("42") should contain theSameElementsAs expected(("\"\\..\"", AlwaysEdge)) + succOf("\"\\..\"") should contain theSameElementsAs expected( + (s"${Operators.formatString}(\"../\", 42, \"\\..\")", AlwaysEdge) + ) + succOf(s"${Operators.formatString}(\"../\", 42, \"\\..\")") should contain theSameElementsAs expected( (s"String.raw(${Operators.formatString}(\"../\", 42, \"\\..\"))", AlwaysEdge) ) - succOf(s"String.raw(${Operators.formatString}(\"../\", 42, \"\\..\"))") shouldBe expected(("RET", AlwaysEdge)) + succOf(s"String.raw(${Operators.formatString}(\"../\", 42, \"\\..\"))") should contain theSameElementsAs expected( + ("RET", AlwaysEdge) + ) } "be correct for try" in { @@ -102,13 +116,13 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp | close() |} |""".stripMargin) - succOf(":program") shouldBe expected(("open", AlwaysEdge)) - succOf("open") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("open()", AlwaysEdge)) - succOf("open()") shouldBe expected(("err", AlwaysEdge), ("close", AlwaysEdge)) - succOf("err") shouldBe expected(("handle", AlwaysEdge)) - succOf("handle()") shouldBe expected(("close", AlwaysEdge)) - succOf("close()") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("open", AlwaysEdge)) + succOf("open") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("open()", AlwaysEdge)) + succOf("open()") should contain theSameElementsAs expected(("err", AlwaysEdge), ("close", AlwaysEdge)) + succOf("err") should contain theSameElementsAs expected(("handle", AlwaysEdge)) + succOf("handle()") should contain theSameElementsAs expected(("close", AlwaysEdge)) + succOf("close()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for try with multiple CFG exit nodes in try block" in { @@ -125,14 +139,14 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp | close() |} |""".stripMargin) - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("doA", TrueEdge), ("doB", FalseEdge)) - succOf("doA()") shouldBe expected(("err", AlwaysEdge), ("close", AlwaysEdge)) - succOf("err") shouldBe expected(("handle", AlwaysEdge)) - succOf("doB()") shouldBe expected(("err", AlwaysEdge), ("close", AlwaysEdge)) - succOf("err") shouldBe expected(("handle", AlwaysEdge)) - succOf("handle()") shouldBe expected(("close", AlwaysEdge)) - succOf("close()") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("doA", TrueEdge), ("doB", FalseEdge)) + succOf("doA()") should contain theSameElementsAs expected(("err", AlwaysEdge), ("close", AlwaysEdge)) + succOf("err") should contain theSameElementsAs expected(("handle", AlwaysEdge)) + succOf("doB()") should contain theSameElementsAs expected(("err", AlwaysEdge), ("close", AlwaysEdge)) + succOf("err") should contain theSameElementsAs expected(("handle", AlwaysEdge)) + succOf("handle()") should contain theSameElementsAs expected(("close", AlwaysEdge)) + succOf("close()") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for 1 object with simple values" in { @@ -142,133 +156,135 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp | key2: 2 |} |""".stripMargin) - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("key1", AlwaysEdge)) - succOf("key1") shouldBe expected(("_tmp_0.key1", AlwaysEdge)) - succOf("_tmp_0.key1") shouldBe expected(("\"value\"", AlwaysEdge)) - succOf("\"value\"") shouldBe expected(("_tmp_0.key1 = \"value\"", AlwaysEdge)) - - succOf("_tmp_0.key1 = \"value\"") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("key2", AlwaysEdge)) - succOf("key2") shouldBe expected(("_tmp_0.key2", AlwaysEdge)) - succOf("_tmp_0.key2") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("_tmp_0.key2 = 2", AlwaysEdge)) - - succOf("_tmp_0.key2 = 2") shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("{\n key1: \"value\",\n key2: 2\n}", AlwaysEdge)) - succOf("{\n key1: \"value\",\n key2: 2\n}") shouldBe expected( + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("key1", AlwaysEdge)) + succOf("key1") should contain theSameElementsAs expected(("_tmp_0.key1", AlwaysEdge)) + succOf("_tmp_0.key1") should contain theSameElementsAs expected(("\"value\"", AlwaysEdge)) + succOf("\"value\"") should contain theSameElementsAs expected(("_tmp_0.key1 = \"value\"", AlwaysEdge)) + + succOf("_tmp_0.key1 = \"value\"") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("key2", AlwaysEdge)) + succOf("key2") should contain theSameElementsAs expected(("_tmp_0.key2", AlwaysEdge)) + succOf("_tmp_0.key2") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("_tmp_0.key2 = 2", AlwaysEdge)) + + succOf("_tmp_0.key2 = 2") should contain theSameElementsAs expected(("_tmp_0", 2, AlwaysEdge)) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("{\n key1: \"value\",\n key2: 2\n}", AlwaysEdge)) + succOf("{\n key1: \"value\",\n key2: 2\n}") should contain theSameElementsAs expected( ("var x = {\n key1: \"value\",\n key2: 2\n}", AlwaysEdge) ) - succOf("var x = {\n key1: \"value\",\n key2: 2\n}") shouldBe expected(("RET", AlwaysEdge)) + succOf("var x = {\n key1: \"value\",\n key2: 2\n}") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for member access used in an assignment (chained)" in { implicit val cpg: Cpg = code("a.b = c.z;") - succOf(":program") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("a.b", AlwaysEdge)) - succOf("a.b") shouldBe expected(("c", AlwaysEdge)) - succOf("c") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("c.z", AlwaysEdge)) - succOf("c.z") shouldBe expected(("a.b = c.z", AlwaysEdge)) - succOf("a.b = c.z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("a.b", AlwaysEdge)) + succOf("a.b") should contain theSameElementsAs expected(("c", AlwaysEdge)) + succOf("c") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("c.z", AlwaysEdge)) + succOf("c.z") should contain theSameElementsAs expected(("a.b = c.z", AlwaysEdge)) + succOf("a.b = c.z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for decl statement with assignment" in { implicit val cpg: Cpg = code("var x = 1;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("var x = 1", AlwaysEdge)) - succOf("var x = 1") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("var x = 1", AlwaysEdge)) + succOf("var x = 1") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested expression" in { implicit val cpg: Cpg = code("x = y + 1;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y + 1", AlwaysEdge)) - succOf("y + 1") shouldBe expected(("x = y + 1", AlwaysEdge)) - succOf("x = y + 1") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y + 1", AlwaysEdge)) + succOf("y + 1") should contain theSameElementsAs expected(("x = y + 1", AlwaysEdge)) + succOf("x = y + 1") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for return statement" in { implicit val cpg: Cpg = code("function foo(x) { return x; }") - succOf("foo", NodeTypes.METHOD) shouldBe expected(("x", AlwaysEdge)) - succOf("x", NodeTypes.IDENTIFIER) shouldBe expected(("return x", AlwaysEdge)) - succOf("return x") shouldBe expected(("RET", AlwaysEdge)) + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("return x", AlwaysEdge)) + succOf("return x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for consecutive return statements" in { implicit val cpg: Cpg = code("function foo(x, y) { return x; return y; }") - succOf("foo", NodeTypes.METHOD) shouldBe expected(("x", AlwaysEdge)) - succOf("x", NodeTypes.IDENTIFIER) shouldBe expected(("return x", AlwaysEdge)) - succOf("y", NodeTypes.IDENTIFIER) shouldBe expected(("return y", AlwaysEdge)) - succOf("return x") shouldBe expected(("RET", AlwaysEdge)) - succOf("return y") shouldBe expected(("RET", AlwaysEdge)) + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("return x", AlwaysEdge)) + succOf("y", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("return y", AlwaysEdge)) + succOf("return x") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("return y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for outer program function which declares foo function object" in { implicit val cpg: Cpg = code("function foo(x, y) { return; }") - succOf(":program", NodeTypes.METHOD) shouldBe expected(("foo", 2, AlwaysEdge)) - succOf("foo", NodeTypes.IDENTIFIER) shouldBe expected(("foo", 3, AlwaysEdge)) - succOf("foo", NodeTypes.METHOD_REF) shouldBe expected( + succOf(":program", NodeTypes.METHOD) should contain theSameElementsAs expected(("foo", 2, AlwaysEdge)) + succOf("foo", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("foo", 3, AlwaysEdge)) + succOf("foo", NodeTypes.METHOD_REF) should contain theSameElementsAs expected( ("function foo = function foo(x, y) { return; }", AlwaysEdge) ) - succOf("function foo = function foo(x, y) { return; }") shouldBe expected(("RET", AlwaysEdge)) + succOf("function foo = function foo(x, y) { return; }") should contain theSameElementsAs expected( + ("RET", AlwaysEdge) + ) } "be correct for void return statement" in { implicit val cpg: Cpg = code("function foo() { return; }") - succOf("foo", NodeTypes.METHOD) shouldBe expected(("return", AlwaysEdge)) - succOf("return") shouldBe expected(("RET", AlwaysEdge)) + succOf("foo", NodeTypes.METHOD) should contain theSameElementsAs expected(("return", AlwaysEdge)) + succOf("return") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for call expression" in { implicit val cpg: Cpg = code("foo(a + 1, b);") - succOf(":program") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("this", AlwaysEdge)) - succOf("this", NodeTypes.IDENTIFIER) shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("a + 1", AlwaysEdge)) - succOf("a + 1") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("foo(a + 1, b)", AlwaysEdge)) - succOf("foo(a + 1, b)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("this", AlwaysEdge)) + succOf("this", NodeTypes.IDENTIFIER) should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("a + 1", AlwaysEdge)) + succOf("a + 1") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("foo(a + 1, b)", AlwaysEdge)) + succOf("foo(a + 1, b)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for chained calls" in { implicit val cpg: Cpg = code("x.foo(y).bar(z)") - succOf(":program") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("x.foo", AlwaysEdge)) - succOf("x.foo") shouldBe expected(("x", 1, AlwaysEdge)) - succOf("x", 1) shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("x.foo(y)", AlwaysEdge)) - succOf("x.foo(y)") shouldBe expected(("(_tmp_0 = x.foo(y))", AlwaysEdge)) - succOf("(_tmp_0 = x.foo(y))") shouldBe expected(("bar", AlwaysEdge)) - succOf("bar") shouldBe expected(("(_tmp_0 = x.foo(y)).bar", AlwaysEdge)) - succOf("(_tmp_0 = x.foo(y)).bar") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("x.foo(y).bar(z)", AlwaysEdge)) - succOf("x.foo(y).bar(z)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("x.foo", AlwaysEdge)) + succOf("x.foo") should contain theSameElementsAs expected(("x", 1, AlwaysEdge)) + succOf("x", 1) should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x.foo(y)", AlwaysEdge)) + succOf("x.foo(y)") should contain theSameElementsAs expected(("(_tmp_0 = x.foo(y))", AlwaysEdge)) + succOf("(_tmp_0 = x.foo(y))") should contain theSameElementsAs expected(("bar", AlwaysEdge)) + succOf("bar") should contain theSameElementsAs expected(("(_tmp_0 = x.foo(y)).bar", AlwaysEdge)) + succOf("(_tmp_0 = x.foo(y)).bar") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("x.foo(y).bar(z)", AlwaysEdge)) + succOf("x.foo(y).bar(z)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for unary expression '++'" in { implicit val cpg: Cpg = code("x++") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("x++", AlwaysEdge)) - succOf("x++") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("x++", AlwaysEdge)) + succOf("x++") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for conditional expression" in { implicit val cpg: Cpg = code("x ? y : z;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("z", FalseEdge)) - succOf("y") shouldBe expected(("x ? y : z", AlwaysEdge)) - succOf("z") shouldBe expected(("x ? y : z", AlwaysEdge)) - succOf("x ? y : z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("z", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("x ? y : z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("x ? y : z", AlwaysEdge)) + succOf("x ? y : z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for labeled expressions with continue" in { @@ -283,98 +299,101 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp | } |} |""".stripMargin) - succOf(":program") shouldBe expected(("var i, j;", AlwaysEdge)) - succOf("loop1:") shouldBe expected(("i", AlwaysEdge)) - succOf("i") shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("i = 0", AlwaysEdge)) - succOf("i = 0") shouldBe expected(("i", 1, AlwaysEdge)) - succOf("i", 1) shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("i < 3", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("var i, j;", AlwaysEdge)) + succOf("loop1:") should contain theSameElementsAs expected(("i", AlwaysEdge)) + succOf("i") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("i = 0", AlwaysEdge)) + succOf("i = 0") should contain theSameElementsAs expected(("i", 1, AlwaysEdge)) + succOf("i", 1) should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("i < 3", AlwaysEdge)) import io.shiftleft.semanticcpg.language._ val codeStr = cpg.method.ast.code(".*loop1:.*").code.head - succOf("i < 3") shouldBe expected(("loop2:", AlwaysEdge), (codeStr, AlwaysEdge)) - succOf(codeStr) shouldBe expected(("RET", AlwaysEdge)) + succOf("i < 3") should contain theSameElementsAs expected(("loop2:", AlwaysEdge), (codeStr, AlwaysEdge)) + succOf(codeStr) should contain theSameElementsAs expected(("RET", AlwaysEdge)) - succOf("loop2:") shouldBe expected(("j", AlwaysEdge)) - succOf("j") shouldBe expected(("0", 1, AlwaysEdge)) - succOf("0", 1) shouldBe expected(("j = 0", AlwaysEdge)) - succOf("j = 0") shouldBe expected(("j", 1, AlwaysEdge)) - succOf("j", 1) shouldBe expected(("3", 1, AlwaysEdge)) - succOf("3", 1) shouldBe expected(("j < 3", AlwaysEdge)) + succOf("loop2:") should contain theSameElementsAs expected(("j", AlwaysEdge)) + succOf("j") should contain theSameElementsAs expected(("0", 1, AlwaysEdge)) + succOf("0", 1) should contain theSameElementsAs expected(("j = 0", AlwaysEdge)) + succOf("j = 0") should contain theSameElementsAs expected(("j", 1, AlwaysEdge)) + succOf("j", 1) should contain theSameElementsAs expected(("3", 1, AlwaysEdge)) + succOf("3", 1) should contain theSameElementsAs expected(("j < 3", AlwaysEdge)) val code2 = cpg.method.ast.isBlock.code("loop2: for.*").code.head - succOf("j < 3") shouldBe expected((code2, AlwaysEdge), ("i", 2, AlwaysEdge)) - succOf(code2) shouldBe expected(("i", 2, AlwaysEdge)) - - succOf("i", 2) shouldBe expected(("i++", AlwaysEdge)) - succOf("i++") shouldBe expected(("i", 3, AlwaysEdge)) - succOf("i", 3) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("i === 1", AlwaysEdge)) - succOf("i === 1") shouldBe expected(("j", AlwaysEdge), ("i === 1 && j === 1", AlwaysEdge)) - succOf("i === 1 && j === 1") shouldBe expected(("continue loop1;", AlwaysEdge), ("console", AlwaysEdge)) - succOf("continue loop1;") shouldBe expected(("loop1:", AlwaysEdge)) - succOf("console") shouldBe expected(("log", AlwaysEdge)) - succOf("log") shouldBe expected(("console.log", AlwaysEdge)) + succOf("j < 3") should contain theSameElementsAs expected((code2, AlwaysEdge), ("i", 2, AlwaysEdge)) + succOf(code2) should contain theSameElementsAs expected(("i", 2, AlwaysEdge)) + + succOf("i", 2) should contain theSameElementsAs expected(("i++", AlwaysEdge)) + succOf("i++") should contain theSameElementsAs expected(("i", 3, AlwaysEdge)) + succOf("i", 3) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("i === 1", AlwaysEdge)) + succOf("i === 1") should contain theSameElementsAs expected(("j", AlwaysEdge), ("i === 1 && j === 1", AlwaysEdge)) + succOf("i === 1 && j === 1") should contain theSameElementsAs expected( + ("continue loop1;", AlwaysEdge), + ("console", AlwaysEdge) + ) + succOf("continue loop1;") should contain theSameElementsAs expected(("loop1:", AlwaysEdge)) + succOf("console") should contain theSameElementsAs expected(("log", AlwaysEdge)) + succOf("log") should contain theSameElementsAs expected(("console.log", AlwaysEdge)) } "be correct for plain while loop" in { implicit val cpg: Cpg = code("while (x < 1) { y = 2; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y = 2", AlwaysEdge)) - succOf("y = 2") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y = 2", AlwaysEdge)) + succOf("y = 2") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) } "be correct for plain while loop with break" in { implicit val cpg: Cpg = code("while (x < 1) { break; y; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct for plain while loop with continue" in { implicit val cpg: Cpg = code("while (x < 1) { continue; y; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) - succOf("continue;") shouldBe expected(("x", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("continue;") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) } "be correct for nested while loop" in { implicit val cpg: Cpg = code("while (x) {while(y) {z;}}") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("z", TrueEdge), ("x", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("z", TrueEdge), ("x", FalseEdge)) } "be correct for nested while loop with break" in { implicit val cpg: Cpg = code("while (x) { while(y) { break; z;} a;} b;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("b", FalseEdge)) - succOf("y") shouldBe expected(("break;", TrueEdge), ("a", FalseEdge)) - succOf("a") shouldBe expected(("x", AlwaysEdge)) - succOf("b") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("b", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", TrueEdge), ("a", FalseEdge)) + succOf("a") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for another nested while loop with break" in { implicit val cpg: Cpg = code("while (x) { while(y) { break; z;} a; break; b; } c;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("c", FalseEdge)) - succOf("y") shouldBe expected(("break;", TrueEdge), ("a", FalseEdge)) - succOf("break;") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("break;", 1, AlwaysEdge)) - succOf("break;", 1) shouldBe expected(("c", AlwaysEdge)) - succOf("c") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("c", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", TrueEdge), ("a", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("break;", 1, AlwaysEdge)) + succOf("break;", 1) should contain theSameElementsAs expected(("c", AlwaysEdge)) + succOf("c") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "nested while loop with conditional break" in { @@ -388,134 +407,134 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp | } |} """.stripMargin) - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("break;", TrueEdge), ("z", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("break;", 1) shouldBe expected(("x", AlwaysEdge)) - succOf("z") shouldBe expected(("break;", 1, TrueEdge), ("x", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", TrueEdge), ("z", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("break;", 1) should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("break;", 1, TrueEdge), ("x", FalseEdge)) } // DO-WHILE Loops "be correct for plain do-while loop" in { implicit val cpg: Cpg = code("do { y = 2; } while (x < 1);") - succOf(":program") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y = 2", AlwaysEdge)) - succOf("y = 2") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y = 2", AlwaysEdge)) + succOf("y = 2") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) } "be correct for plain do-while loop with break" in { implicit val cpg: Cpg = code("do { break; y; } while (x < 1);") - succOf(":program") shouldBe expected(("break;", AlwaysEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("break;", AlwaysEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) } "be correct for plain do-while loop with continue" in { implicit val cpg: Cpg = code("do { continue; y; } while (x < 1);") - succOf(":program") shouldBe expected(("continue;", AlwaysEdge)) - succOf("continue;") shouldBe expected(("x", AlwaysEdge)) - succOf("y") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("continue;", AlwaysEdge)) + succOf("continue;") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) } "be correct for nested do-while loop with continue" in { implicit val cpg: Cpg = code("do { do { x; } while (y); } while (z);") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("x", TrueEdge), ("z", FalseEdge)) - succOf("z") shouldBe expected(("x", TrueEdge), ("RET", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("x", TrueEdge), ("z", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("x", TrueEdge), ("RET", FalseEdge)) } "be correct for nested while/do-while loops with break" in { implicit val cpg: Cpg = code("while (x) { do { while(y) { break; a; } z; } while (x < 1); } c;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("c", FalseEdge)) - succOf("y") shouldBe expected(("break;", TrueEdge), ("z", FalseEdge)) - succOf("break;") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("x", 1, AlwaysEdge)) - succOf("x", 1) shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("x < 1", AlwaysEdge)) - succOf("x < 1") shouldBe expected(("y", TrueEdge), ("x", FalseEdge)) - succOf("c") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("c", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", TrueEdge), ("z", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("x", 1, AlwaysEdge)) + succOf("x", 1) should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("x < 1", AlwaysEdge)) + succOf("x < 1") should contain theSameElementsAs expected(("y", TrueEdge), ("x", FalseEdge)) + succOf("c") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested while/do-while loops with break and continue" in { implicit val cpg: Cpg = code("while(x) { do { break; } while (y) } o;") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("break;", TrueEdge), ("o", FalseEdge)) - succOf("break;") shouldBe expected(("x", AlwaysEdge)) - succOf("o") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("break;", TrueEdge), ("o", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("o") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for two nested while loop with inner break" in { implicit val cpg: Cpg = code("while(y) { while(z) { break; x; } }") - succOf(":program") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("z", TrueEdge), ("RET", FalseEdge)) - succOf("z") shouldBe expected(("break;", TrueEdge), ("y", FalseEdge)) - succOf("break;") shouldBe expected(("y", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("z", TrueEdge), ("RET", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("break;", TrueEdge), ("y", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("y", AlwaysEdge)) } // FOR Loops "be correct for plain for-loop" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { a = 3; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("x = 0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("a", TrueEdge), ("RET", FalseEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("x = 0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("a", TrueEdge), ("RET", FalseEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct for plain for-loop with break" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { break; a = 3; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct for plain for-loop with continue" in { implicit val cpg: Cpg = code("for (x = 0; y < 1; z += 2) { continue; a = 3; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("0", AlwaysEdge)) - succOf("0") shouldBe expected(("x = 0", AlwaysEdge)) - succOf("x = 0") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y < 1", AlwaysEdge)) - succOf("y < 1") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) - succOf("continue;") shouldBe expected(("z", AlwaysEdge)) - succOf("a") shouldBe expected(("3", AlwaysEdge)) - succOf("3") shouldBe expected(("a = 3", AlwaysEdge)) - succOf("a = 3") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z += 2", AlwaysEdge)) - succOf("z += 2") shouldBe expected(("y", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("0", AlwaysEdge)) + succOf("0") should contain theSameElementsAs expected(("x = 0", AlwaysEdge)) + succOf("x = 0") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y < 1", AlwaysEdge)) + succOf("y < 1") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("continue;") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("3", AlwaysEdge)) + succOf("3") should contain theSameElementsAs expected(("a = 3", AlwaysEdge)) + succOf("a = 3") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z += 2", AlwaysEdge)) + succOf("z += 2") should contain theSameElementsAs expected(("y", AlwaysEdge)) } "be correct for for-loop with for-in" in { @@ -530,193 +549,214 @@ class SimpleCfgCreationPassTests extends CfgTestFixture(() => new JsSrcCfgTestCp "be correct for nested for-loop" in { implicit val cpg: Cpg = code("for (x; y; z) { for (a; b; c) { u; } }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("a", TrueEdge), ("RET", FalseEdge)) - succOf("z") shouldBe expected(("y", AlwaysEdge)) - succOf("a") shouldBe expected(("b", AlwaysEdge)) - succOf("b") shouldBe expected(("u", TrueEdge), ("z", FalseEdge)) - succOf("c") shouldBe expected(("b", AlwaysEdge)) - succOf("u") shouldBe expected(("c", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("a", TrueEdge), ("RET", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("b") should contain theSameElementsAs expected(("u", TrueEdge), ("z", FalseEdge)) + succOf("c") should contain theSameElementsAs expected(("b", AlwaysEdge)) + succOf("u") should contain theSameElementsAs expected(("c", AlwaysEdge)) } "be correct for for-loop with empty condition" in { implicit val cpg: Cpg = code("for (;;) { a = 1; }") - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("a", TrueEdge), ("RET", FalseEdge)) - succOf("a") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("a = 1", AlwaysEdge)) - succOf("a = 1") shouldBe expected(("true", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("a", TrueEdge), ("RET", FalseEdge)) + succOf("a") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("a = 1", AlwaysEdge)) + succOf("a = 1") should contain theSameElementsAs expected(("true", AlwaysEdge)) } "be correct for for-loop with empty condition and break" in { implicit val cpg: Cpg = code("for (;;) { break; }") - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("break;", TrueEdge), ("RET", FalseEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("break;", TrueEdge), ("RET", FalseEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for for-loop with empty condition and continue" in { implicit val cpg: Cpg = code("for (;;) { continue; }") - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("continue;", TrueEdge), ("RET", FalseEdge)) - succOf("continue;") shouldBe expected(("true", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("continue;", TrueEdge), ("RET", FalseEdge)) + succOf("continue;") should contain theSameElementsAs expected(("true", AlwaysEdge)) } "be correct with empty condition with nested empty for-loop" in { implicit val cpg: Cpg = code("for (;;) { for (;;) { x; } }") - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("true", 1, TrueEdge), ("RET", FalseEdge)) - succOf("true", 1) shouldBe expected(("x", TrueEdge), ("true", 0, FalseEdge)) - succOf("x") shouldBe expected(("true", 1, AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("true", 1, TrueEdge), ("RET", FalseEdge)) + succOf("true", 1) should contain theSameElementsAs expected(("x", TrueEdge), ("true", 0, FalseEdge)) + succOf("x") should contain theSameElementsAs expected(("true", 1, AlwaysEdge)) } "be correct for for-loop with empty block" in { implicit val cpg: Cpg = code("for (;;) ;") - succOf(":program") shouldBe expected(("true", AlwaysEdge)) - succOf("true") shouldBe expected(("true", TrueEdge), ("RET", FalseEdge)) + succOf(":program") should contain theSameElementsAs expected(("true", AlwaysEdge)) + succOf("true") should contain theSameElementsAs expected(("true", TrueEdge), ("RET", FalseEdge)) } "be correct for simple if statement" in { implicit val cpg: Cpg = code("if (x) { y; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for simple if statement with else block" in { implicit val cpg: Cpg = code("if (x) { y; } else { z; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("z", FalseEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("z", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested if statement" in { implicit val cpg: Cpg = code("if (x) { if (y) { z; } }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("y", TrueEdge), ("RET", FalseEdge)) - succOf("y") shouldBe expected(("z", TrueEdge), ("RET", FalseEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("y", TrueEdge), ("RET", FalseEdge)) + succOf("y") should contain theSameElementsAs expected(("z", TrueEdge), ("RET", FalseEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for nested if statement with else-if chains" in { implicit val cpg: Cpg = code("if (a) { b; } else if (c) { d;} else { e; }") - succOf(":program") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("b", TrueEdge), ("c", FalseEdge)) - succOf("b") shouldBe expected(("RET", AlwaysEdge)) - succOf("c") shouldBe expected(("d", TrueEdge), ("e", FalseEdge)) - succOf("d") shouldBe expected(("RET", AlwaysEdge)) - succOf("e") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("a", AlwaysEdge)) + succOf("a") should contain theSameElementsAs expected(("b", TrueEdge), ("c", FalseEdge)) + succOf("b") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("c") should contain theSameElementsAs expected(("d", TrueEdge), ("e", FalseEdge)) + succOf("d") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("e") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch-case with single case" in { implicit val cpg: Cpg = code("switch (x) { case 1: y;}") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("case 1:", CaseEdge), ("RET", CaseEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch-case with multiple cases" in { implicit val cpg: Cpg = code("switch (x) { case 1: y; case 2: z;}") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("case 2:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("case 2:", AlwaysEdge)) - succOf("case 2:") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected( + ("case 1:", CaseEdge), + ("case 2:", CaseEdge), + ("RET", CaseEdge) + ) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("case 2:", AlwaysEdge)) + succOf("case 2:") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch-case with multiple cases on the same spot" in { implicit val cpg: Cpg = code("switch (x) { case 1: case 2: y; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("case 2:", CaseEdge), ("RET", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("case 2:", AlwaysEdge)) - succOf("case 2:") shouldBe expected(("2", AlwaysEdge)) - succOf("2") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected( + ("case 1:", CaseEdge), + ("case 2:", CaseEdge), + ("RET", CaseEdge) + ) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("case 2:", AlwaysEdge)) + succOf("case 2:") should contain theSameElementsAs expected(("2", AlwaysEdge)) + succOf("2") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch-case with default case" in { implicit val cpg: Cpg = code("switch (x) { default: y; }") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("default:", CaseEdge)) - succOf("default:") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("default:", CaseEdge)) + succOf("default:") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for switch-case with multiple cases and default combined" in { implicit val cpg: Cpg = code("switch (x) { case 1: y; break; default: z;}") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("case 1:", CaseEdge), ("default:", CaseEdge)) - succOf("case 1:") shouldBe expected(("1", AlwaysEdge)) - succOf("1") shouldBe expected(("y", AlwaysEdge)) - succOf("y") shouldBe expected(("break;", AlwaysEdge)) - succOf("break;") shouldBe expected(("RET", AlwaysEdge)) - succOf("default:") shouldBe expected(("z", AlwaysEdge)) - succOf("z") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("case 1:", CaseEdge), ("default:", CaseEdge)) + succOf("case 1:") should contain theSameElementsAs expected(("1", AlwaysEdge)) + succOf("1") should contain theSameElementsAs expected(("y", AlwaysEdge)) + succOf("y") should contain theSameElementsAs expected(("break;", AlwaysEdge)) + succOf("break;") should contain theSameElementsAs expected(("RET", AlwaysEdge)) + succOf("default:") should contain theSameElementsAs expected(("z", AlwaysEdge)) + succOf("z") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for constructor call with new" in { implicit val cpg: Cpg = code("""var x = new MyClass(arg1, arg2)""") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("_tmp_0", AlwaysEdge)) - succOf("_tmp_0") shouldBe expected((".alloc", AlwaysEdge)) - succOf(".alloc") shouldBe expected(("_tmp_0 = .alloc", AlwaysEdge)) - succOf("_tmp_0 = .alloc") shouldBe expected(("MyClass", AlwaysEdge)) - succOf("MyClass") shouldBe expected(("_tmp_0", 1, AlwaysEdge)) - succOf("_tmp_0", 1) shouldBe expected(("arg1", AlwaysEdge)) - succOf("arg1") shouldBe expected(("arg2", AlwaysEdge)) - succOf("arg2") shouldBe expected(("new MyClass(arg1, arg2)", AlwaysEdge)) - succOf("new MyClass(arg1, arg2)", NodeTypes.CALL) shouldBe expected(("_tmp_0", 2, AlwaysEdge)) - succOf("_tmp_0", 2) shouldBe expected(("new MyClass(arg1, arg2)", AlwaysEdge)) - succOf("new MyClass(arg1, arg2)") shouldBe expected(("var x = new MyClass(arg1, arg2)", AlwaysEdge)) - succOf("var x = new MyClass(arg1, arg2)") shouldBe expected(("RET", AlwaysEdge)) + succOf(":program") should contain theSameElementsAs expected(("x", AlwaysEdge)) + succOf("x") should contain theSameElementsAs expected(("_tmp_0", AlwaysEdge)) + succOf("_tmp_0") should contain theSameElementsAs expected((".alloc", AlwaysEdge)) + succOf(".alloc") should contain theSameElementsAs expected(("_tmp_0 = .alloc", AlwaysEdge)) + succOf("_tmp_0 = .alloc") should contain theSameElementsAs expected(("MyClass", AlwaysEdge)) + succOf("MyClass") should contain theSameElementsAs expected(("_tmp_0", 1, AlwaysEdge)) + succOf("_tmp_0", 1) should contain theSameElementsAs expected(("arg1", AlwaysEdge)) + succOf("arg1") should contain theSameElementsAs expected(("arg2", AlwaysEdge)) + succOf("arg2") should contain theSameElementsAs expected(("new MyClass(arg1, arg2)", AlwaysEdge)) + succOf("new MyClass(arg1, arg2)", NodeTypes.CALL) should contain theSameElementsAs expected( + ("_tmp_0", 2, AlwaysEdge) + ) + succOf("_tmp_0", 2) should contain theSameElementsAs expected(("new MyClass(arg1, arg2)", AlwaysEdge)) + succOf("new MyClass(arg1, arg2)") should contain theSameElementsAs expected( + ("var x = new MyClass(arg1, arg2)", AlwaysEdge) + ) + succOf("var x = new MyClass(arg1, arg2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } private def testForInOrOf()(implicit cpg: Cpg): Unit = { - succOf(":program") shouldBe expected(("_iterator_0", AlwaysEdge)) - succOf("_iterator_0") shouldBe expected(("arr", AlwaysEdge)) - succOf("arr") shouldBe expected((".iterator(arr)", AlwaysEdge)) - succOf(".iterator(arr)") shouldBe expected(("_iterator_0 = .iterator(arr)", AlwaysEdge)) - succOf("_iterator_0 = .iterator(arr)") shouldBe expected(("_result_0", AlwaysEdge)) - succOf("_result_0") shouldBe expected(("i", AlwaysEdge)) - succOf("i") shouldBe expected(("_result_0", 1, AlwaysEdge)) - succOf("_result_0", 1) shouldBe expected(("_iterator_0", 1, AlwaysEdge)) - succOf("_iterator_0", 1) shouldBe expected(("next", AlwaysEdge)) - succOf("next") shouldBe expected(("_iterator_0.next", AlwaysEdge)) - succOf("_iterator_0.next") shouldBe expected(("_iterator_0", 2, AlwaysEdge)) - succOf("_iterator_0", 2) shouldBe expected(("_iterator_0.next()", AlwaysEdge)) - succOf("_iterator_0.next()") shouldBe expected(("(_result_0 = _iterator_0.next())", AlwaysEdge)) - succOf("(_result_0 = _iterator_0.next())") shouldBe expected(("done", AlwaysEdge)) - succOf("done") shouldBe expected(("(_result_0 = _iterator_0.next()).done", AlwaysEdge)) - succOf("(_result_0 = _iterator_0.next()).done") shouldBe expected( + succOf(":program") should contain theSameElementsAs expected(("_iterator_0", AlwaysEdge)) + succOf("_iterator_0") should contain theSameElementsAs expected(("arr", AlwaysEdge)) + succOf("arr") should contain theSameElementsAs expected((".iterator(arr)", AlwaysEdge)) + succOf(".iterator(arr)") should contain theSameElementsAs expected( + ("_iterator_0 = .iterator(arr)", AlwaysEdge) + ) + succOf("_iterator_0 = .iterator(arr)") should contain theSameElementsAs expected( + ("_result_0", AlwaysEdge) + ) + succOf("_result_0") should contain theSameElementsAs expected(("i", AlwaysEdge)) + succOf("i") should contain theSameElementsAs expected(("_result_0", 1, AlwaysEdge)) + succOf("_result_0", 1) should contain theSameElementsAs expected(("_iterator_0", 1, AlwaysEdge)) + succOf("_iterator_0", 1) should contain theSameElementsAs expected(("next", AlwaysEdge)) + succOf("next") should contain theSameElementsAs expected(("_iterator_0.next", AlwaysEdge)) + succOf("_iterator_0.next") should contain theSameElementsAs expected(("_iterator_0", 2, AlwaysEdge)) + succOf("_iterator_0", 2) should contain theSameElementsAs expected(("_iterator_0.next()", AlwaysEdge)) + succOf("_iterator_0.next()") should contain theSameElementsAs expected( + ("(_result_0 = _iterator_0.next())", AlwaysEdge) + ) + succOf("(_result_0 = _iterator_0.next())") should contain theSameElementsAs expected(("done", AlwaysEdge)) + succOf("done") should contain theSameElementsAs expected(("(_result_0 = _iterator_0.next()).done", AlwaysEdge)) + succOf("(_result_0 = _iterator_0.next()).done") should contain theSameElementsAs expected( ("!(_result_0 = _iterator_0.next()).done", AlwaysEdge) ) import io.shiftleft.semanticcpg.language._ val code = cpg.method.ast.isBlock.code("for \\(var i.*foo.*}").code.head - succOf("!(_result_0 = _iterator_0.next()).done") shouldBe expected(("i", 1, TrueEdge), (code, FalseEdge)) - succOf(code) shouldBe expected(("RET", AlwaysEdge)) - - succOf("i", 1) shouldBe expected(("_result_0", 2, AlwaysEdge)) - succOf("_result_0", 2) shouldBe expected(("value", AlwaysEdge)) - succOf("value") shouldBe expected(("_result_0.value", AlwaysEdge)) - succOf("_result_0.value") shouldBe expected(("i = _result_0.value", AlwaysEdge)) - succOf("i = _result_0.value") shouldBe expected(("foo", AlwaysEdge)) - succOf("foo") shouldBe expected(("this", 1, AlwaysEdge)) - succOf("this", 1) shouldBe expected(("i", 2, AlwaysEdge)) - succOf("i", 2) shouldBe expected(("foo(i)", AlwaysEdge)) + succOf("!(_result_0 = _iterator_0.next()).done") should contain theSameElementsAs expected( + ("i", 1, TrueEdge), + (code, FalseEdge) + ) + succOf(code) should contain theSameElementsAs expected(("RET", AlwaysEdge)) + + succOf("i", 1) should contain theSameElementsAs expected(("_result_0", 2, AlwaysEdge)) + succOf("_result_0", 2) should contain theSameElementsAs expected(("value", AlwaysEdge)) + succOf("value") should contain theSameElementsAs expected(("_result_0.value", AlwaysEdge)) + succOf("_result_0.value") should contain theSameElementsAs expected(("i = _result_0.value", AlwaysEdge)) + succOf("i = _result_0.value") should contain theSameElementsAs expected(("foo", AlwaysEdge)) + succOf("foo") should contain theSameElementsAs expected(("this", 1, AlwaysEdge)) + succOf("this", 1) should contain theSameElementsAs expected(("i", 2, AlwaysEdge)) + succOf("i", 2) should contain theSameElementsAs expected(("foo(i)", AlwaysEdge)) val code2 = "{ foo(i) }" - succOf("foo(i)") shouldBe expected((code2, AlwaysEdge)) - succOf(code2) shouldBe expected(("_result_0", 1, AlwaysEdge)) + succOf("foo(i)") should contain theSameElementsAs expected((code2, AlwaysEdge)) + succOf(code2) should contain theSameElementsAs expected(("_result_0", 1, AlwaysEdge)) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/preprocessing/EjsPassTests.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/preprocessing/EjsPassTests.scala index 7eda70ca7a15..a1c8f9593d58 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/preprocessing/EjsPassTests.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/preprocessing/EjsPassTests.scala @@ -7,7 +7,7 @@ class EjsPassTests extends AstJsSrc2CpgSuite { "ejs files" should { - "be renamed correctly " in { + "be renamed correctly" in { val cpg = code( """ | @@ -20,6 +20,20 @@ class EjsPassTests extends AstJsSrc2CpgSuite { cpg.call.code.l.sorted shouldBe List("user.name") } + "be ignored at folders excluded by default" in { + val codeString = """ + | + |

Welcome <%= user.name %>

+ | + |""".stripMargin + val cpg = code(codeString, "index.js.ejs") + .moreCode(codeString, "node_modules/foo.js.ejs") + .moreCode(codeString, "vendor/bar.js.ejs") + .moreCode(codeString, "www/baz.js.ejs") + cpg.file.name.l shouldBe List("index.js.ejs") + cpg.call.code.l.sorted shouldBe List("user.name") + } + "be handled correctly" in { val cpg = code( """ diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala index eb93c807ba97..3e6a9fdaab97 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala @@ -31,7 +31,7 @@ class DataFlowCodeToCpgSuite extends Code2CpgFixture(() => new DataFlowTestCpg() protected implicit val context: EngineContext = EngineContext() protected def flowToResultPairs(path: Path): List[(String, Integer)] = - path.resultPairs().collect { case (firstElement: String, secondElement: Option[Integer]) => + path.resultPairs().collect { case (firstElement: String, secondElement) => (firstElement, secondElement.getOrElse(-1)) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/JsSrc2CpgSuite.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/JsSrc2CpgSuite.scala index f188520b43c9..43cbdc216aaa 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/JsSrc2CpgSuite.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/testfixtures/JsSrc2CpgSuite.scala @@ -1,16 +1,17 @@ package io.joern.jssrc2cpg.testfixtures -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.x2cpg.testfixtures.Code2CpgFixture class JsSrc2CpgSuite( fileSuffix: String = ".js", withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), withPostProcessing: Boolean = false ) extends Code2CpgFixture(() => new JsSrcDefaultTestCpg(fileSuffix) .withOssDataflow(withOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) diff --git a/joern-cli/frontends/kotlin2cpg/build.sbt b/joern-cli/frontends/kotlin2cpg/build.sbt index 83032b7c6024..825505cc8cea 100644 --- a/joern-cli/frontends/kotlin2cpg/build.sbt +++ b/joern-cli/frontends/kotlin2cpg/build.sbt @@ -23,4 +23,5 @@ libraryDependencies ++= Seq( enablePlugins(JavaAppPackaging, LauncherJarPlugin) trapExit := false -Test / fork := false +Test / fork := true +Test / javaOptions ++= Seq("-ea") diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala index 4d20c13c1e90..98bcf4053e47 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala @@ -2,6 +2,7 @@ package io.joern.kotlin2cpg import io.joern.kotlin2cpg.Frontend.* import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser case class DefaultContentRootJarPath(path: String, isResource: Boolean) @@ -96,8 +97,12 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new Kotlin2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new Kotlin2Cpg()) with FrontendHTTPServer[Config, Kotlin2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, kotlin2cpg: Kotlin2Cpg): Unit = { - kotlin2cpg.run(config) + if (config.serverMode) { startup() } + else { kotlin2cpg.run(config) } } } diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala index 11bfd8af9afd..dd0c80b012b4 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstCreator.scala @@ -13,11 +13,11 @@ import io.joern.x2cpg.Defines import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.datastructures.Stack.* +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.jetbrains.kotlin.com.intellij.psi.PsiElement @@ -28,7 +28,7 @@ import org.jetbrains.kotlin.lexer.KtTokens import org.jetbrains.kotlin.psi.* import org.slf4j.Logger import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import java.io.PrintWriter import java.io.StringWriter diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForExpressionsCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForExpressionsCreator.scala index 6a6ee736ad36..12032a94d34c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForExpressionsCreator.scala @@ -11,9 +11,11 @@ import io.joern.x2cpg.utils.NodeBuilders import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.NewMethodRef +import org.jetbrains.kotlin.descriptors.{DescriptorVisibilities, FunctionDescriptor} import org.jetbrains.kotlin.lexer.KtToken import org.jetbrains.kotlin.lexer.KtTokens import org.jetbrains.kotlin.psi.* +import org.jetbrains.kotlin.resolve.BindingContext import scala.jdk.CollectionConverters.* @@ -341,14 +343,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val callKind = typeInfoProvider.bindingKind(expr) val isExtensionCall = callKind == CallKind.ExtensionCall - val hasThisSuperOrNameRefReceiver = expr.getReceiverExpression match { - case _: KtThisExpression => true - case _: KtNameReferenceExpression => true - case _: KtSuperExpression => true - case _ => false - } val hasNameRefSelector = expr.getSelectorExpression.isInstanceOf[KtNameReferenceExpression] - val isFieldAccessCall = hasThisSuperOrNameRefReceiver && hasNameRefSelector val isCallToSuper = expr.getReceiverExpression match { case _: KtSuperExpression => true case _ => false @@ -366,10 +361,10 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val outAst = if (isCtorCtorCall.getOrElse(false)) { astForQualifiedExpressionCtor(expr, argIdx, argNameMaybe) - } else if (isFieldAccessCall) { - astForQualifiedExpressionFieldAccess(expr, argIdx, argNameMaybe) } else if (isExtensionCall) { astForQualifiedExpressionExtensionCall(expr, argIdx, argNameMaybe) + } else if (hasNameRefSelector) { + astForQualifiedExpressionFieldAccess(expr, argIdx, argNameMaybe) } else if (isCallToSuper) { astForQualifiedExpressionCallToSuper(expr, argIdx, argNameMaybe) } else if (noAstForReceiver) { @@ -463,21 +458,70 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { else s"$methodFqName:$explicitSignature" val (fullName, signature) = typeInfoProvider.fullNameWithSignature(expr, (explicitFullName, explicitSignature)) + val bindingContext = typeInfoProvider.bindingContext + val call = bindingContext.get(BindingContext.CALL, expr.getCalleeExpression) + val resolvedCall = bindingContext.get(BindingContext.RESOLVED_CALL, call) + + val (dispatchType, instanceAsArgument) = + if (resolvedCall == null) { + (DispatchTypes.STATIC_DISPATCH, false) + } else { + if (resolvedCall.getDispatchReceiver == null) { + (DispatchTypes.STATIC_DISPATCH, false) + } else { + resolvedCall.getResultingDescriptor match { + case functionDescriptor: FunctionDescriptor + if functionDescriptor.getVisibility == DescriptorVisibilities.PRIVATE => + (DispatchTypes.STATIC_DISPATCH, true) + case _ => + (DispatchTypes.DYNAMIC_DISPATCH, true) + } + } + } + // TODO: add test case to confirm whether the ANY fallback makes sense (could be void) val returnType = registerType(typeInfoProvider.expressionType(expr, TypeConstants.any)) - val node = callNode( - expr, - expr.getText, - referencedName, - fullName, - DispatchTypes.STATIC_DISPATCH, - Some(signature), - Some(returnType) - ) + val node = callNode(expr, expr.getText, referencedName, fullName, dispatchType, Some(signature), Some(returnType)) + val annotationsAsts = annotations.map(astForAnnotationEntry) val astWithAnnotations = - callAst(withArgumentIndex(node, argIdx).argumentName(argNameMaybe), argAsts.toList) - .withChildren(annotationsAsts) + if (dispatchType == DispatchTypes.STATIC_DISPATCH) { + val compoundArgAsts = + if (instanceAsArgument) { + val instanceArgument = identifierNode( + expr, + Constants.this_, + Constants.this_, + typeInfoProvider.typeFullName(resolvedCall.getDispatchReceiver.getType) + ) + val args = argAsts.prepended(Ast(instanceArgument)) + setArgumentIndices(args, 0) + args + } else { + setArgumentIndices(argAsts, 1) + argAsts + } + + Ast(withArgumentIndex(node, argIdx).argumentName(argNameMaybe)) + .withChildren(compoundArgAsts) + .withArgEdges(node, compoundArgAsts.flatMap(_.root)) + .withChildren(annotationsAsts) + } else { + val receiverNode = identifierNode( + expr, + Constants.this_, + Constants.this_, + typeInfoProvider.typeFullName(resolvedCall.getDispatchReceiver.getType) + ) + + callAst( + withArgumentIndex(node, argIdx).argumentName(argNameMaybe), + argAsts.toList, + base = Some(Ast(receiverNode)) + ) + .withChildren(annotationsAsts) + } + List(astWithAnnotations) } diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForPrimitivesCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForPrimitivesCreator.scala index e6d85bb26cb9..7ceb7de5f4e3 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForPrimitivesCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForPrimitivesCreator.scala @@ -256,8 +256,9 @@ trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { case _ => typeInfoProvider .typeFromImports(entry.getShortName.toString, entry.getContainingKtFile) - .getOrElse(TypeConstants.any) + .getOrElse(s"${Defines.UnresolvedNamespace}.${entry.getShortName.toString}") }) + val node = NewAnnotation() .code(entry.getText) diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala index 65ce497f60a9..f083d75d63af 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/ast/AstForStatementsCreator.scala @@ -62,7 +62,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { // private def astForForWithDestructuringLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = { val loopRangeText = expr.getLoopRange.getText - val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}" + val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}" val localForIterator = localNode(expr, iteratorName, iteratorName, TypeConstants.any) val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any) val iteratorLocalAst = Ast(localForIterator).withRefEdge(iteratorAssignmentLhs, localForIterator) @@ -182,7 +182,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { // private def astForForWithSimpleVarLHS(expr: KtForExpression)(implicit typeInfoProvider: TypeInfoProvider): Ast = { val loopRangeText = expr.getLoopRange.getText - val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next()}" + val iteratorName = s"${Constants.iteratorPrefix}${iteratorKeyPool.next}" val iteratorLocal = localNode(expr, iteratorName, iteratorName, TypeConstants.any) val iteratorAssignmentLhs = newIdentifierNode(iteratorName, TypeConstants.any) val iteratorLocalAst = Ast(iteratorLocal).withRefEdge(iteratorAssignmentLhs, iteratorLocal) diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeHintCallLinker.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeHintCallLinker.scala index 5c688e3bbc79..31ed0b375ce5 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeHintCallLinker.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeHintCallLinker.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.Defines import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeRecoveryPassGenerator.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeRecoveryPassGenerator.scala index 9b0960fcc2e3..e4614d139906 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeRecoveryPassGenerator.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/passes/KotlinTypeRecoveryPassGenerator.scala @@ -7,7 +7,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class KotlinTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) extends XTypeRecoveryPassGenerator[File](cpg, config) { diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala index 523b5cb0c9e1..8abe25862090 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala @@ -52,7 +52,7 @@ import org.jetbrains.kotlin.resolve.DescriptorUtils import org.jetbrains.kotlin.resolve.DescriptorUtils.getSuperclassDescriptors import org.jetbrains.kotlin.resolve.`lazy`.descriptors.LazyClassDescriptor import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor -import org.jetbrains.kotlin.types.TypeUtils +import org.jetbrains.kotlin.types.{KotlinType, TypeUtils} import org.jetbrains.kotlin.types.error.ErrorType import org.slf4j.LoggerFactory @@ -323,6 +323,10 @@ class DefaultTypeInfoProvider(environment: KotlinCoreEnvironment, typeRenderer: .getOrElse(defaultValue) } + def typeFullName(typ: KotlinType): String = { + typeRenderer.render(typ) + } + def expressionType(expr: KtExpression, defaultValue: String): String = { Option(bindingContext.get(BindingContext.EXPRESSION_TYPE_INFO, expr)) .flatMap(tpeInfo => Option(tpeInfo.getType)) @@ -403,11 +407,8 @@ class DefaultTypeInfoProvider(environment: KotlinCoreEnvironment, typeRenderer: val relevantDesc = originalDesc match { case typedDesc: TypeAliasConstructorDescriptorImpl => typedDesc.getUnderlyingConstructorDescriptor - case typedDesc: FunctionDescriptor if !typedDesc.isActual => - val overriddenDescriptors = typedDesc.getOverriddenDescriptors.asScala.toList - if (overriddenDescriptors.nonEmpty) overriddenDescriptors.head - else typedDesc - case _ => originalDesc + case _ => + originalDesc } val returnTypeFullName = if (isConstructorCall(expr).getOrElse(false)) TypeConstants.void diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeInfoProvider.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeInfoProvider.scala index 7f1a21abcf7b..4f293c6e597c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeInfoProvider.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeInfoProvider.scala @@ -13,8 +13,8 @@ import org.jetbrains.kotlin.psi.{ KtExpression, KtFile, KtLambdaExpression, - KtNamedFunction, KtNameReferenceExpression, + KtNamedFunction, KtParameter, KtPrimaryConstructor, KtProperty, @@ -23,10 +23,13 @@ import org.jetbrains.kotlin.psi.{ KtTypeAlias, KtTypeReference } +import org.jetbrains.kotlin.resolve.BindingContext +import org.jetbrains.kotlin.types.KotlinType case class AnonymousObjectContext(declaration: KtElement) trait TypeInfoProvider(val typeRenderer: TypeRenderer = new TypeRenderer()) { + val bindingContext: BindingContext def isExtensionFn(fn: KtNamedFunction): Boolean def usedAsExpression(expr: KtExpression): Option[Boolean] @@ -113,6 +116,8 @@ trait TypeInfoProvider(val typeRenderer: TypeRenderer = new TypeRenderer()) { def typeFullName(expr: KtParameter, defaultValue: String): String + def typeFullName(typ: KotlinType): String + def typeFullName(expr: KtDestructuringDeclarationEntry, defaultValue: String): String def hasStaticDesc(expr: KtQualifiedExpression): Boolean diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeRenderer.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeRenderer.scala index 1660485f84e8..5b5d79522b43 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeRenderer.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/TypeRenderer.scala @@ -44,6 +44,7 @@ class TypeRenderer(val keepTypeArguments: Boolean = false) { case _: ErrorType => cpgUnresolvedType case t => t } + opts.lock() new DescriptorRendererImpl(opts) } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala index d7c78defaa52..e8dd77a86bb6 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala @@ -77,8 +77,9 @@ class CompilerAPITests extends AnyFreeSpec with Matchers { "should not contain methods with unresolved types/namespaces" in { val command = - if (scala.util.Properties.isWin) "cmd.exe /C gradlew.bat gatherDependencies" else "./gradlew gatherDependencies" - ExternalCommand.run(command, projectDirPath) shouldBe Symbol("success") + if (scala.util.Properties.isWin) Seq("cmd.exe", "/C", "gradlew.bat", "gatherDependencies") + else Seq("./gradlew", "gatherDependencies") + ExternalCommand.run(command, projectDirPath).toTry shouldBe Symbol("success") val config = Config(classpath = Set(projectDependenciesPath.toString)) val cpg = new Kotlin2Cpg().createCpg(projectDirPath)(config).getOrElse { fail("Could not create a CPG!") diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/JavaInteroperabilityTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/JavaInteroperabilityTests.scala index 580b2056da24..55f047956f61 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/JavaInteroperabilityTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/JavaInteroperabilityTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.compiler import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class JavaInteroperabilityTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with Java interop" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/CollectionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/CollectionsTests.scala index c9bb114aba6a..baaa894378a8 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/CollectionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/CollectionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CollectionsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ControlExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ControlExpressionsTests.scala index 83742fc2916a..99031a2c5510 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ControlExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ControlExpressionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ControlExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/DestructuringTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/DestructuringTests.scala index 23d0a75e2e71..108ae3e4fab1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/DestructuringTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/DestructuringTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DestructuringTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ExtensionFnsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ExtensionFnsTests.scala index ef42c65d7aac..d6df51061783 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ExtensionFnsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ExtensionFnsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ExtensionFnsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ForTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ForTests.scala index 5b52128e351f..a810b2f050f9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ForTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ForTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ForTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/FunctionCallTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/FunctionCallTests.scala index bffc61e47053..f5e138aec00c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/FunctionCallTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/FunctionCallTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class FunctionCallTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/GenericsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/GenericsTests.scala index 6510a41d156f..7c4528652d61 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/GenericsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/GenericsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class GenericsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/IfTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/IfTests.scala index 39b9a9a03f59..3c2b48e764b2 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/IfTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/IfTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class IfTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/InterproceduralTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/InterproceduralTests.scala index 6d0d9ea0f607..474fd0b25863 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/InterproceduralTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/InterproceduralTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class InterproceduralTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/JavaInteroperabilityTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/JavaInteroperabilityTests.scala index 3219e96572f6..b6e998601691 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/JavaInteroperabilityTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/JavaInteroperabilityTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class JavaInteroperabilityTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/LambdaTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/LambdaTests.scala index 3946446c2657..c13308cd1448 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/LambdaTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/LambdaTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LambdaTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ObjectExpressionsAndDeclarationsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ObjectExpressionsAndDeclarationsTests.scala index 7ab56c2b3df4..f1a7e87f961d 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ObjectExpressionsAndDeclarationsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ObjectExpressionsAndDeclarationsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ObjectExpressionsAndDeclarationsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/OperatorTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/OperatorTests.scala index 927c9dcf8069..8e095ce30420 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/OperatorTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/OperatorTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class OperatorTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ScopeFunctionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ScopeFunctionsTests.scala index 7e70895c5772..690d7eaec9bd 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ScopeFunctionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/ScopeFunctionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ScopeFunctionsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/SimpleDataFlowTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/SimpleDataFlowTests.scala index ac54ea19e525..987bd4bd9c10 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/SimpleDataFlowTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/SimpleDataFlowTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SimpleDataFlowTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/TryTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/TryTests.scala index 7002f4da3b0e..d25e5e83c117 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/TryTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/TryTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TryTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhenTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhenTests.scala index 0c94cf520002..021fc3198813 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhenTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhenTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class WhenTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhileTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhileTests.scala index 74beb35f10fd..aa7e419eabda 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhileTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/dataflow/WhileTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.dataflow import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class WhileTests extends KotlinCode2CpgFixture(withOssDataflow = true) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..2ceaa95affa3 --- /dev/null +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala @@ -0,0 +1,84 @@ +package io.joern.kotlin2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class Kotlin2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("kotlin2cpgTestsHttpTest") + val file = dir / "main.kt" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |package mypkg + |fun main(args : Array) { + | println($indexStr) + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.kotlin2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.kotlin2cpg.Main.stop() + } + + "Using kotlin2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("kotlin2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List("println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("kotlin2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List(s"println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala similarity index 94% rename from joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala rename to joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala index 730b34aa9ee4..fd252128c740 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala @@ -1,7 +1,6 @@ -package io.joern.kotlin2cpg +package io.joern.kotlin2cpg.io import io.joern.kotlin2cpg.files.SourceFilesPicker - import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers import org.scalatest.BeforeAndAfterAll diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/postProcessing/TypeRecoveryPassTest.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/postProcessing/TypeRecoveryPassTest.scala index 6e241e7e2afd..06276c0ec95f 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/postProcessing/TypeRecoveryPassTest.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/postProcessing/TypeRecoveryPassTest.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.postProcessing import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TypeRecoveryPassTest extends KotlinCode2CpgFixture(withPostProcessing = true) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnnotationsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnnotationsTests.scala index 71d1ee17cc51..744e0aa100f5 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnnotationsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnnotationsTests.scala @@ -1,8 +1,9 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture +import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, AnnotationLiteral} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AnnotationsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with two identical calls, one annotated and one not" should { @@ -606,7 +607,7 @@ class AnnotationsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { } } - "CPG for code with a custom annotation" should { + "CPG for code with a custom annotation and fallback handling" should { val cpg = code(""" |package mypkg |import retrofit2.http.POST @@ -618,12 +619,16 @@ class AnnotationsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { |} |""".stripMargin) - "contain an ANNOTATION node" in { - cpg.all.collectAll[Annotation].codeExact("@POST(\"/name\")").size shouldBe 1 - } - "the ANNOTATION node should have correct full name" in { - cpg.all.collectAll[Annotation].codeExact("@POST(\"/name\")").fullName.head shouldBe "retrofit2.http.POST" + inside(cpg.annotation.codeExact("@POST(\"/name\")").l) { case List(annotation) => + annotation.fullName shouldBe "retrofit2.http.POST" + } + inside(cpg.annotation.code(".*Headers.*").l) { case List(annotation) => + annotation.fullName shouldBe s"${Defines.UnresolvedNamespace}.Headers" + } + inside(cpg.annotation.code(".*Body.*").l) { case List(annotation) => + annotation.fullName shouldBe s"${Defines.UnresolvedNamespace}.Body" + } } } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnonymousFunctionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnonymousFunctionsTests.scala index fad29eeaf8b4..49c85dae7c6c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnonymousFunctionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AnonymousFunctionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Ignore import scala.annotation.unused diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArithmeticOperationsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArithmeticOperationsTests.scala index 1116622eaeaa..80f1d9695263 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArithmeticOperationsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArithmeticOperationsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ArithmeticOperationsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArrayAccessExprsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArrayAccessExprsTests.scala index 5a9b5767607c..84632d3f4acf 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArrayAccessExprsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ArrayAccessExprsTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ArrayAccessExprsTests extends KotlinCode2CpgFixture(withOssDataflow = true) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AssignmentTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AssignmentTests.scala index 25ddaea2e3ef..4ef82a053da0 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AssignmentTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/AssignmentTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AssignmentTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/BooleanLogicTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/BooleanLogicTests.scala index 3e6ed13c3b82..922a64926e35 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/BooleanLogicTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/BooleanLogicTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class BooleanLogicTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallGraphTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallGraphTests.scala index 794edfec1c3e..ddd91f3f1f80 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallGraphTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallGraphTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallGraphTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallTests.scala index a87bf5df6fc5..6d46901359f1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallTests.scala @@ -1,7 +1,8 @@ package io.joern.kotlin2cpg.querying +import io.joern.kotlin2cpg.Constants import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.semanticcpg.language.* @@ -595,4 +596,91 @@ class CallTests extends KotlinCode2CpgFixture(withOssDataflow = false) { c.argument.map(_.argumentName).flatten.l shouldBe List("two", "one") } } + + "have correct call to overriden base class" in { + val cpg = code(""" + |package somePackage + |class A: java.io.Closeable { + | fun foo() { + | close() + | } + | override fun close() { + | } + |} + |""".stripMargin) + + inside(cpg.call.nameExact("close").l) { case List(call) => + call.methodFullName shouldBe "somePackage.A.close:void()" + call.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + inside(call.receiver.l) { case List(receiver: Identifier) => + receiver.name shouldBe Constants.this_ + receiver.typeFullName shouldBe "somePackage.A" + } + inside(call.argument.l) { case List(argument: Identifier) => + argument.name shouldBe Constants.this_ + argument.argumentIndex shouldBe 0 + } + } + } + + "have correct call to kotlin standard library function" in { + val cpg = code(""" + |fun method() { + | println("test") + |} + |""".stripMargin) + + inside(cpg.call.nameExact("println").l) { case List(call) => + call.methodFullName shouldBe "kotlin.io.println:void(java.lang.Object)" + call.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + call.receiver.isEmpty shouldBe true + inside(call.argument.l) { case List(argument: Literal) => + argument.code shouldBe "\"test\"" + argument.argumentIndex shouldBe 1 + } + } + } + + "have correct call to custom top level function" in { + val cpg = code(""" + |package somePackage + |fun topLevelFunc() { + |} + |fun method() { + | topLevelFunc() + |} + |""".stripMargin) + + inside(cpg.call.nameExact("topLevelFunc").l) { case List(call) => + call.methodFullName shouldBe "somePackage.topLevelFunc:void()" + call.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + + call.receiver.isEmpty shouldBe true + call.argument.isEmpty shouldBe true + } + } + + "have correct call to private class method" in { + val cpg = code(""" + |package somePackage + |class A { + | private fun func1() { + | } + | fun func2() { + | func1() + | } + |} + |""".stripMargin) + + inside(cpg.call.nameExact("func1").l) { case List(call) => + call.methodFullName shouldBe "somePackage.A.func1:void()" + call.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + + call.receiver.isEmpty shouldBe true + inside(call.argument.l) { case List(argument: Identifier) => + argument.name shouldBe "this" + argument.argumentIndex shouldBe 0 + } + } + } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallableReferenceTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallableReferenceTests.scala index e5edd465e017..da0b64126080 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallableReferenceTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallableReferenceTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallableReferenceTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallbackTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallbackTests.scala index d6c2814bafc0..9b1a3eb9c00d 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallbackTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallbackTests.scala @@ -2,7 +2,8 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, MethodRef} +import io.shiftleft.semanticcpg.language.* class CallbackTests extends KotlinCode2CpgFixture(withOssDataflow = false) { @@ -99,7 +100,12 @@ class CallbackTests extends KotlinCode2CpgFixture(withOssDataflow = false) { c.lineNumber shouldBe Some(10) c.columnNumber shouldBe Some(8) c.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - c.argument.size shouldBe 1 + c.argument.size shouldBe 2 + inside(c.argument.l) { case List(arg1: Identifier, arg2: MethodRef) => + arg1.name shouldBe "this" + arg1.argumentIndex shouldBe 0 + arg2.argumentIndex shouldBe 1 + } } } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToConstructorTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToConstructorTests.scala index 49335321c273..aeb59738be64 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToConstructorTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToConstructorTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal, Local} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallsToConstructorTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToFieldAccessTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToFieldAccessTests.scala index 95c3ee8c49e3..99a7b32292e3 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToFieldAccessTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CallsToFieldAccessTests.scala @@ -139,4 +139,21 @@ class CallsToFieldAccessTests extends KotlinCode2CpgFixture(withOssDataflow = fa x.methodFullName shouldBe "mypkg.AClass.printX:void()" } } + + "Field access after array/map access" should { + "have correct arguments" in { + val cpg = code(""" + |val m = LinkedHashMap() + |val x = m[1].aaa + |""".stripMargin) + + inside(cpg.call.methodFullNameExact(Operators.fieldAccess).argument.l) { case List(arg1, arg2) => + arg1.code shouldBe "m[1]" + arg1.argumentIndex shouldBe 1 + arg2.code shouldBe "aaa" + arg2.argumentIndex shouldBe 2 + } + } + } + } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CfgTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CfgTests.scala index 3ca374924968..8455d8aeb781 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CfgTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CfgTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CfgTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ClassLiteralTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ClassLiteralTests.scala index 0fe33a3b79ee..77a1fb225fb1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ClassLiteralTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ClassLiteralTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ClassLiteralTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CollectionAccessTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CollectionAccessTests.scala index 3d30027b02e5..aaddb05e6db2 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CollectionAccessTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CollectionAccessTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CollectionAccessTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CompanionObjectTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CompanionObjectTests.scala index bcb0948837db..6916bbe07d10 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CompanionObjectTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/CompanionObjectTests.scala @@ -4,7 +4,7 @@ import io.joern.kotlin2cpg.Constants import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Member} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CompanionObjectTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComparisonOperatorTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComparisonOperatorTests.scala index 9057c973301e..6fc007434046 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComparisonOperatorTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComparisonOperatorTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ComparisonOperatorTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala index 7e31ba806ead..b469e6f84ca1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ComplexExpressionsTests.scala @@ -3,8 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.edges.Argument -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.jIteratortoTraversal +import io.shiftleft.semanticcpg.language.* class ComplexExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with _and_/_or_ operator and try-catch as one of the arguments" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConfigFileTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConfigFileTests.scala index e0fdc9238657..99fca7000164 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConfigFileTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConfigFileTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ConfigFileTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConstructorTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConstructorTests.scala index ecf3222d003b..09e8df5cb759 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConstructorTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ConstructorTests.scala @@ -4,7 +4,7 @@ import io.joern.kotlin2cpg.Constants import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, MethodParameterIn} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ConstructorTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ControlStructureTests.scala index 4c14ced555f7..367c7e7f0fd3 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ControlStructureTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, ControlStructure, Identifier, Local} import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ControlStructureTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple if-else" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DataClassTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DataClassTests.scala index b5bc553d1b4e..bee57af468ad 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DataClassTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DataClassTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DataClassTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple data class" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DefaultContentRootsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DefaultContentRootsTests.scala index c53f42ab2b96..02b442bbbaf8 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DefaultContentRootsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DefaultContentRootsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DefaultContentRootsTests extends KotlinCode2CpgFixture(withOssDataflow = false, withDefaultJars = true) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DelegatedPropertiesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DelegatedPropertiesTests.scala index 7fc0b7dc4649..9f53a65e7f21 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DelegatedPropertiesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DelegatedPropertiesTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DelegatedPropertiesTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DestructuringTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DestructuringTests.scala index c0d046f962ed..96d03e1a589c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DestructuringTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/DestructuringTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DestructuringTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/EnumTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/EnumTests.scala index 8feb1ba5df25..3d4e6ad56dea 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/EnumTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/EnumTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class EnumTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ExtensionTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ExtensionTests.scala index 7c1e535826ea..2dd7156c2f5c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ExtensionTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ExtensionTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ExtensionTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple extension function declarations" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FieldAccessTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FieldAccessTests.scala index fd8521787396..df05efd93966 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FieldAccessTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FieldAccessTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{FieldIdentifier, Identifier} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class FieldAccessTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FileTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FileTests.scala index 098c7f9329ba..4dae7d002d57 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FileTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/FileTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal import java.io.File diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GenericsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GenericsTests.scala index d0a78b7ec837..55100727bac4 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GenericsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GenericsTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class GenericsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GlobalsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GlobalsTests.scala index ff11e7d51c86..8e3eed81a92f 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GlobalsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/GlobalsTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class GlobalsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code simple global declaration" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/IdentifierTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/IdentifierTests.scala index 0650814cb889..6c4c0745afa4 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/IdentifierTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/IdentifierTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class IdentifierTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with two simple methods" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ImportTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ImportTests.scala index 00b79d34b1ea..38bdb3aba291 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ImportTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ImportTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ImportTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/InnerClassesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/InnerClassesTests.scala index 7bc06f15d41e..b81c62b036ca 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/InnerClassesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/InnerClassesTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.Ignore class InnerClassesTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LabeledExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LabeledExpressionsTests.scala index b4243faf88c0..dc0fb0b566f9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LabeledExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LabeledExpressionsTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LabeledExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala index 6de2838c017e..3a681cef31b0 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LambdaTests.scala @@ -17,7 +17,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.MethodRef import io.shiftleft.codepropertygraph.generated.nodes.Return import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.jIteratortoTraversal class LambdaTests extends KotlinCode2CpgFixture(withOssDataflow = false, withDefaultJars = true) { "CPG for code with a simple lambda which captures a method parameter" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LiteralTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LiteralTests.scala index ba4288736d87..5a5c8475318a 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LiteralTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LiteralTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LiteralTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalClassesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalClassesTests.scala index bb57eb21932b..42a261fe5f83 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalClassesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalClassesTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocalClassesTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalTests.scala index a0c49d8842ca..07635e8dd1d4 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/LocalTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocalTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code simple local declarations" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MemberTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MemberTests.scala index 19457372c82d..4d0db375a81f 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MemberTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MemberTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MemberTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MetaDataTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MetaDataTests.scala index 40695e336c24..1f54baae7bc2 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MetaDataTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MetaDataTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MetaDataTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodParameterTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodParameterTests.scala index 5e5db29df2dc..3a610e47ab1a 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodParameterTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodParameterTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodParameterTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodReturnTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodReturnTests.scala index d34372b3d2d9..fc332dade842 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodReturnTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodReturnTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.EvaluationStrategies -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodReturnTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala index efc0e016c1e9..f6c472a5b0a9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/MethodTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Return} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MethodTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple method defined at package-level" should { @@ -174,7 +174,7 @@ class MethodTests extends KotlinCode2CpgFixture(withOssDataflow = false) { |""".stripMargin) "pass the lambda to a `sortedWith` call which is then under the method `sorted`" in { - inside(cpg.methodRef(".*.*").inCall.l) { + inside(cpg.methodRefWithName(".*.*").inCall.l) { case sortedWith :: Nil => sortedWith.name shouldBe "sortedWith" sortedWith.method.name shouldBe "sorted" diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ModifierTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ModifierTests.scala index 3b686142e41b..605185f136b9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ModifierTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ModifierTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ModifierTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with various modifiers applied to various functions" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/NamespaceBlockTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/NamespaceBlockTests.scala index 33cf5c4be855..200ff2a9a41b 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/NamespaceBlockTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/NamespaceBlockTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NamespaceBlockTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple namespace declaration" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ObjectDeclarationsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ObjectDeclarationsTests.scala index 81118e41fe2a..310aa4086717 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ObjectDeclarationsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ObjectDeclarationsTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ObjectDeclarationsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple object declaration" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ParenthesizedExpressionTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ParenthesizedExpressionTests.scala index da0f5858545c..bd1548c9e909 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ParenthesizedExpressionTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ParenthesizedExpressionTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ParenthesizedExpressionTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/QualifiedExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/QualifiedExpressionsTests.scala index 824698ecce4a..570ec15821b2 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/QualifiedExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/QualifiedExpressionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class QualifiedExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with qualified expression with QE as a receiver" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ResolutionErrorsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ResolutionErrorsTests.scala index c4e676596e97..652e2f7e2253 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ResolutionErrorsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ResolutionErrorsTests.scala @@ -4,7 +4,7 @@ import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.kotlin2cpg.types.TypeConstants import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ResolutionErrorsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with QE of receiver for which the type cannot be inferred" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SafeQualifiedExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SafeQualifiedExpressionsTests.scala index 28267aa5ef34..f340e39dd835 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SafeQualifiedExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SafeQualifiedExpressionsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SafeQualifiedExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ScopeFunctionTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ScopeFunctionTests.scala index 8d5a75d91e27..2881fdf26380 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ScopeFunctionTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ScopeFunctionTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Block, Return} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ScopeFunctionTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code call to `also` scope function without an explicitly-defined parameter" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SpecialOperatorsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SpecialOperatorsTests.scala index a307d4b01dda..78078f2cf511 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SpecialOperatorsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SpecialOperatorsTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SpecialOperatorsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StdLibTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StdLibTests.scala index 72498ccfea49..4dd6884d94ea 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StdLibTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StdLibTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class StdLibTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with call to `takeIf`" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StringInterpolationTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StringInterpolationTests.scala index 644435182656..7253fbbf3221 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StringInterpolationTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/StringInterpolationTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class StringInterpolationTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SuperTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SuperTests.scala index 79de86e710b2..00d832760d29 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SuperTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/SuperTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SuperTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple call using _super_" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ThisTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ThisTests.scala index 8af6d9af6548..d976133041d6 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ThisTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/ThisTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ThisTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with calls to functions of same name, but different scope" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TryExpressionsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TryExpressionsTests.scala index ce5b2fc7ce02..473aa0e84c5c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TryExpressionsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TryExpressionsTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TryExpressionsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with simple `try`-expression" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeAliasTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeAliasTests.scala index d76b92786d78..3af134f92b59 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeAliasTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeAliasTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.x2cpg.Defines -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class TypeAliasTests extends KotlinCode2CpgFixture(withOssDataflow = false, withDefaultJars = true) { "CPG for code with simple typealias to Int" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeDeclTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeDeclTests.scala index e05da10d7311..3ee7dc51b82d 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeDeclTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/TypeDeclTests.scala @@ -11,7 +11,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ MethodParameterIn } import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal class TypeDeclTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/UnaryOpTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/UnaryOpTests.scala index 7c6aa3617877..4ca47bfc65aa 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/UnaryOpTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/querying/UnaryOpTests.scala @@ -2,7 +2,7 @@ package io.joern.kotlin2cpg.querying import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class UnaryOpTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/testfixtures/KotlinCodeToCpgFixture.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/testfixtures/KotlinCodeToCpgFixture.scala index 486654812a20..713bcfd868dc 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/testfixtures/KotlinCodeToCpgFixture.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/testfixtures/KotlinCodeToCpgFixture.scala @@ -1,8 +1,9 @@ package io.joern.kotlin2cpg.testfixtures import better.files.File as BFile +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.* -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.SemanticCpgTestFixture import io.joern.dataflowengineoss.testfixtures.SemanticTestCpg import io.joern.kotlin2cpg.Config @@ -57,14 +58,14 @@ class KotlinCode2CpgFixture( withOssDataflow: Boolean = false, withDefaultJars: Boolean = false, withPostProcessing: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty + semantics: Semantics = DefaultSemantics() ) extends Code2CpgFixture(() => new KotlinTestCpg(withDefaultJars) .withOssDataflow(withOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) - with SemanticCpgTestFixture(extraFlows) { + with SemanticCpgTestFixture(semantics) { protected def flowToResultPairs(path: Path): List[(String, Option[Int])] = path.resultPairs() } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala similarity index 85% rename from joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala rename to joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala index 1978c0e3d840..6219f73c175a 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala @@ -1,7 +1,7 @@ -package io.joern.kotlin2cpg +package io.joern.kotlin2cpg.types import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DefaultRegisteredTypesTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/DefaultImportsTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/DefaultImportsTests.scala index 2e381352d259..607313386ce5 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/DefaultImportsTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/DefaultImportsTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.validation import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DefaultImportsTests extends KotlinCode2CpgFixture(withOssDataflow = false) { // It tests if we take into consideration default imports: https://kotlinlang.org/docs/packages.html#default-imports diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/IdentifierReferencesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/IdentifierReferencesTests.scala index c24e30d6d6b7..02e94913b2ae 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/IdentifierReferencesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/IdentifierReferencesTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.validation import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Local, MethodParameterIn} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* // TODO: also add test with refs inside TYPE_DECL diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/MissingTypeInformationTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/MissingTypeInformationTests.scala index c435499d3310..14e375b0d9ac 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/MissingTypeInformationTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/MissingTypeInformationTests.scala @@ -3,7 +3,7 @@ package io.joern.kotlin2cpg.validation import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class MissingTypeInformationTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with CALL to Java stdlib fn with argument of unknown type" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/PrimitiveArrayTypeMappingTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/PrimitiveArrayTypeMappingTests.scala index afb57d324c9f..e4191018c6ad 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/PrimitiveArrayTypeMappingTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/PrimitiveArrayTypeMappingTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.validation import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class PrimitiveArrayTypeMappingTests extends KotlinCode2CpgFixture(withOssDataflow = false) { "CPG for code with usage of `kotlin.BooleanArray`" should { diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/UnitTypeMappingTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/UnitTypeMappingTests.scala index dc2f0f467378..8ee9dde0d2e1 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/UnitTypeMappingTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/validation/UnitTypeMappingTests.scala @@ -1,7 +1,7 @@ package io.joern.kotlin2cpg.validation import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class UnitTypeMappingTests extends KotlinCode2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala index db3ade3308b1..afe2947167a6 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.php2cpg import io.joern.php2cpg.Frontend.* import io.joern.x2cpg.passes.frontend.* import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser /** Command line configuration parameters @@ -51,8 +52,12 @@ object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new Php2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new Php2Cpg()) with FrontendHTTPServer[Config, Php2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, php2Cpg: Php2Cpg): Unit = { - php2Cpg.run(config) + if (config.serverMode) { startup() } + else { php2Cpg.run(config) } } } diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala index d0230fd616d5..c2b0b759be25 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala @@ -22,7 +22,7 @@ class Php2Cpg extends X2CpgFrontend[Config] { private val PhpVersionRegex = new Regex("^PHP ([78]\\.[1-9]\\.[0-9]|[9-9]\\d\\.\\d\\.\\d)") private def isPhpVersionSupported: Boolean = { - val result = ExternalCommand.run("php --version", ".") + val result = ExternalCommand.run(Seq("php", "--version"), ".").toTry result match { case Success(listString) => val phpVersionStr = listString.headOption.getOrElse("") diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala index c821740b5c9c..8892100efc75 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/astcreation/AstCreator.scala @@ -8,30 +8,36 @@ import io.joern.php2cpg.utils.Scope import io.joern.x2cpg.Ast.storeInDiffGraph import io.joern.x2cpg.Defines.{StaticInitMethodName, UnresolvedNamespace, UnresolvedSignature} import io.joern.x2cpg.utils.AstPropertiesUtil.RootProperties +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.* -import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} +import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, Defines, ValidationMode} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.collection.mutable -class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], disableFileContent: Boolean)(implicit +class AstCreator(relativeFileName: String, fileName: String, phpAst: PhpFile, disableFileContent: Boolean)(implicit withSchemaValidation: ValidationMode -) extends AstCreatorBase(filename) +) extends AstCreatorBase(relativeFileName) with AstNodeBuilder[PhpNode, AstCreator] { private val logger = LoggerFactory.getLogger(AstCreator.getClass) private val scope = new Scope()(() => nextClosureName()) private val tmpKeyPool = new IntervalKeyPool(first = 0, last = Long.MaxValue) private val globalNamespace = globalNamespaceBlock() + private var fileContent = Option.empty[String] private def getNewTmpName(prefix: String = "tmp"): String = s"$prefix${tmpKeyPool.next.toString}" - override def createAst(): BatchedUpdate.DiffGraphBuilder = { + override def createAst(): DiffGraphBuilder = { + if (!disableFileContent) { + fileContent = Some(Files.readString(Path.of(fileName))) + } + val ast = astForPhpFile(phpAst) storeInDiffGraph(ast, diffGraph) diffGraph @@ -51,7 +57,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], file, globalNamespace.name, globalNamespace.fullName, - filename, + relativeFileName, globalNamespace.code, NodeTypes.NAMESPACE_BLOCK, globalNamespace.fullName @@ -75,7 +81,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], } private def astForPhpFile(file: PhpFile): Ast = { - val fileNode = NewFile().name(filename) + val fileNode = NewFile().name(relativeFileName) fileContent.foreach(fileNode.content(_)) scope.pushNewScope(globalNamespace) @@ -135,7 +141,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], case enumCase: PhpEnumCaseStmt => astForEnumCase(enumCase) :: Nil case staticStmt: PhpStaticStmt => astsForStaticStmt(staticStmt) case unhandled => - logger.error(s"Unhandled stmt $unhandled in $filename") + logger.error(s"Unhandled stmt $unhandled in $relativeFileName") ??? } } @@ -231,7 +237,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], } val methodCode = s"${modifierString}function $methodName(${parameters.map(_.rootCodeOrEmpty).mkString(",")})" - val method = methodNode(decl, methodName, methodCode, fullName, Some(signature), filename) + val method = methodNode(decl, methodName, methodCode, fullName, Some(signature), relativeFileName) scope.pushNewScope(method) @@ -477,7 +483,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], private def astForNamespaceStmt(stmt: PhpNamespaceStmt): Ast = { val name = stmt.name.map(_.name).getOrElse(NameConstants.Unknown) - val fullName = s"$filename:$name" + val fullName = s"$relativeFileName:$name" val namespaceBlock = NewNamespaceBlock() .name(name) @@ -574,6 +580,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], private def astForForeachStmt(stmt: PhpForeachStmt): Ast = { val iterIdentifier = getTmpIdentifier(stmt, maybeTypeFullName = None, prefix = "iter_") + // keep this just used to construct the `code` field val assignItemTargetAst = stmt.keyVar match { case Some(key) => astForKeyValPair(key, stmt.valueVar, line(stmt)) case None => astForExpr(stmt.valueVar) @@ -585,7 +592,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val iteratorAssignAst = simpleAssignAst(Ast(iterIdentifier), iterValue, line(stmt)) // - Assigned item assign - val itemInitAst = getItemAssignAstForForeach(stmt, assignItemTargetAst, iterIdentifier.copy) + val itemInitAst = getItemAssignAstForForeach(stmt, iterIdentifier.copy) // Condition ast val isNullName = PhpOperators.isNull @@ -614,7 +621,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val itemUpdateAst = itemInitAst.root match { case Some(initRoot: AstNodeNew) => itemInitAst.subTreeCopy(initRoot) case _ => - logger.warn(s"Could not copy foreach init ast in $filename") + logger.warn(s"Could not copy foreach init ast in $relativeFileName") Ast() } @@ -631,34 +638,63 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], .withConditionEdges(foreachNode, conditionAst.root.toList) } - private def getItemAssignAstForForeach( - stmt: PhpForeachStmt, - assignItemTargetAst: Ast, - iteratorIdentifier: NewIdentifier - ): Ast = { - val iteratorIdentifierAst = Ast(iteratorIdentifier) - val currentCallSignature = s"$UnresolvedSignature(0)" - val currentCallCode = s"${iteratorIdentifierAst.rootCodeOrEmpty}->current()" - val currentCallNode = callNode( - stmt, - currentCallCode, - "current", - "Iterator.current", - DispatchTypes.DYNAMIC_DISPATCH, - Some(currentCallSignature), - Some(TypeConstants.Any) + private def getItemAssignAstForForeach(stmt: PhpForeachStmt, iteratorIdentifier: NewIdentifier): Ast = { + // create assignment for value-part + val valueAssign = { + val iteratorIdentifierAst = Ast(iteratorIdentifier) + val currentCallSignature = s"$UnresolvedSignature(0)" + val currentCallCode = s"${iteratorIdentifierAst.rootCodeOrEmpty}->current()" + // `current` function is used to get the current element of given array + // see https://www.php.net/manual/en/function.current.php & https://www.php.net/manual/en/iterator.current.php + val currentCallNode = callNode( + stmt, + currentCallCode, + "current", + "Iterator.current", + DispatchTypes.DYNAMIC_DISPATCH, + Some(currentCallSignature), + Some(TypeConstants.Any) + ) + val currentCallAst = callAst(currentCallNode, base = Option(iteratorIdentifierAst)) + + val valueAst = if (stmt.assignByRef) { + val addressOfCode = s"&${currentCallAst.rootCodeOrEmpty}" + val addressOfCall = newOperatorCallNode(Operators.addressOf, addressOfCode, line = line(stmt)) + callAst(addressOfCall, currentCallAst :: Nil) + } else { + currentCallAst + } + simpleAssignAst(astForExpr(stmt.valueVar), valueAst, line(stmt)) + } + + // try to create assignment for key-part + val keyAssignOption = stmt.keyVar.map(keyVar => + val iteratorIdentifierAst = Ast(iteratorIdentifier.copy) + val keyCallSignature = s"$UnresolvedSignature(0)" + val keyCallCode = s"${iteratorIdentifierAst.rootCodeOrEmpty}->key()" + // `key` function is used to get the key of the current element + // see https://www.php.net/manual/en/function.key.php & https://www.php.net/manual/en/iterator.key.php + val keyCallNode = callNode( + stmt, + keyCallCode, + "key", + "Iterator.key", + DispatchTypes.DYNAMIC_DISPATCH, + Some(keyCallSignature), + Some(TypeConstants.Any) + ) + val keyCallAst = callAst(keyCallNode, base = Option(iteratorIdentifierAst)) + simpleAssignAst(astForExpr(keyVar), keyCallAst, line(stmt)) ) - val currentCallAst = callAst(currentCallNode, base = Option(iteratorIdentifierAst)) - val valueAst = if (stmt.assignByRef) { - val addressOfCode = s"&${currentCallAst.rootCodeOrEmpty}" - val addressOfCall = newOperatorCallNode(Operators.addressOf, addressOfCode, line = line(stmt)) - callAst(addressOfCall, currentCallAst :: Nil) - } else { - currentCallAst + keyAssignOption match { + case Some(keyAssign) => + Ast(blockNode(stmt)) + .withChild(keyAssign) + .withChild(valueAssign) + case None => + valueAssign } - - simpleAssignAst(assignItemTargetAst, valueAst, line(stmt)) } private def simpleAssignAst(target: Ast, source: Ast, lineNo: Option[Int]): Ast = { @@ -716,7 +752,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], Ast(local) :: assignmentAst.toList case other => - logger.warn(s"Unexpected static variable type $other in $filename") + logger.warn(s"Unexpected static variable type $other in $relativeFileName") Nil } } @@ -753,7 +789,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], prependNamespacePrefix(name.name) } - val typeDecl = typeDeclNode(stmt, name.name, fullName, filename, code, inherits = inheritsFrom) + val typeDecl = typeDeclNode(stmt, name.name, fullName, relativeFileName, code, inherits = inheritsFrom) val createDefaultConstructor = stmt.hasConstructor scope.pushNewScope(typeDecl) @@ -772,7 +808,16 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], case inits => val signature = s"${TypeConstants.Void}()" val fullName = composeMethodFullName(StaticInitMethodName, isStatic = true) - val ast = staticInitMethodAst(inits, fullName, Option(signature), TypeConstants.Void, fileName = Some(filename)) + val ast = + staticInitMethodAst(inits, fullName, Option(signature), TypeConstants.Void, fileName = Some(relativeFileName)) + + for { + method <- ast.root.collect { case method: NewMethod => method } + content <- fileContent + } { + method.offset(0) + method.offsetEnd(content.length) + } Option(ast) } @@ -837,7 +882,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val thisParam = thisParamAstForMethod(originNode) - val method = methodNode(originNode, ConstructorMethodName, fullName, fullName, Some(signature), filename) + val method = methodNode(originNode, ConstructorMethodName, fullName, fullName, Some(signature), relativeFileName) val methodBody = blockAst(blockNode(originNode), scope.getFieldInits) @@ -855,11 +900,15 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val fieldIdentifier = newFieldIdentifierNode(memberNode.name, memberNode.lineNumber) callAst(fieldAccessNode, List(identifier, fieldIdentifier).map(Ast(_))).withRefEdges(identifier, thisParam.toList) } else { - val identifierCode = memberNode.code.replaceAll("const ", "").replaceAll("case ", "") - val typeFullName = Option(memberNode.typeFullName) - val identifier = newIdentifierNode(memberNode.name, typeFullName.getOrElse("ANY")) - .code(identifierCode) - Ast(identifier).withRefEdge(identifier, memberNode) + val selfIdentifier = { + val name = "self" + val typ = scope.getEnclosingTypeDeclTypeName + newIdentifierNode(name, typ.getOrElse(Defines.Any), typ.toList, memberNode.lineNumber).code(name) + } + val fieldIdentifier = newFieldIdentifierNode(memberNode.name, memberNode.lineNumber) + val code = s"self::${memberNode.code.replaceAll("(static|case|const) ", "")}" + val fieldAccessNode = newOperatorCallNode(Operators.fieldAccess, code, line = memberNode.lineNumber) + callAst(fieldAccessNode, List(selfIdentifier, fieldIdentifier).map(Ast(_))) } val value = astForExpr(valueExpr) @@ -878,7 +927,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val name = constDecl.name.name val code = s"const $name" val someValue = Option(constDecl.value) - astForConstOrFieldValue(stmt, name, code, someValue, scope.addConstOrStaticInitToScope, isField = false) + astForConstOrStaticOrFieldValue(stmt, name, code, someValue, scope.addConstOrStaticInitToScope, isField = false) .withChildren(modifierAsts) } } @@ -889,21 +938,35 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val name = stmt.name.name val code = s"case $name" - astForConstOrFieldValue(stmt, name, code, stmt.expr, scope.addConstOrStaticInitToScope, isField = false) + astForConstOrStaticOrFieldValue(stmt, name, code, stmt.expr, scope.addConstOrStaticInitToScope, isField = false) .withChild(finalModifier) } private def astsForPropertyStmt(stmt: PhpPropertyStmt): List[Ast] = { stmt.variables.map { varDecl => - val modifierAsts = stmt.modifiers.map(newModifierNode).map(Ast(_)) + val modifiers = stmt.modifiers + val modifierAsts = modifiers.map(newModifierNode).map(Ast(_)) val name = varDecl.name.name - astForConstOrFieldValue(stmt, name, s"$$$name", varDecl.defaultValue, scope.addFieldInitToScope, isField = true) - .withChildren(modifierAsts) + val ast = if (modifiers.contains(ModifierTypes.STATIC)) { + // A static member belongs to a class, not an instance + val memberCode = s"static $$$name" + astForConstOrStaticOrFieldValue( + stmt, + name, + memberCode, + varDecl.defaultValue, + scope.addConstOrStaticInitToScope, + false + ) + } else + astForConstOrStaticOrFieldValue(stmt, name, s"$$$name", varDecl.defaultValue, scope.addFieldInitToScope, true) + + ast.withChildren(modifierAsts) } } - private def astForConstOrFieldValue( + private def astForConstOrStaticOrFieldValue( originNode: PhpNode, name: String, code: String, @@ -1087,7 +1150,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], .sortBy(_.argumentIndex) if (args.size != 2) { - val position = s"${line(assignment).getOrElse("")}:$filename" + val position = s"${line(assignment).getOrElse("")}:$relativeFileName" logger.warn(s"Expected 2 call args for emptyArrayDimAssign. Not resetting code: $position") } else { val codeOverride = s"${args.head.code}[] = ${args.last.code}" @@ -1097,12 +1160,103 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], arrayPushAst } + /** Lower the array/list unpack. For example `[$a, $b] = $arr;` will be lowered to `$a = $arr[0]; $b = $arr[1];` + */ + private def astForArrayUnpack(assignment: PhpAssignment, target: PhpArrayExpr | PhpListExpr): Ast = { + val loweredAssignNodes = mutable.ListBuffer.empty[Ast] + + // create a Identifier ast for given name + def createIdentifier(name: String): Ast = Ast(identifierNode(assignment, name, s"$$$name", TypeConstants.Any)) + + def createIndexAccessChain( + targetAst: Ast, + sourceAst: Ast, + idxTracker: ArrayIndexTracker, + item: PhpArrayItem + ): Ast = { + // copy from `assignForArrayItem` to handle the case where key exists, such as `list("id" => $a, "name" => $b) = $arr;` + val dimension = item.key match { + case Some(key: PhpSimpleScalar) => dimensionFromSimpleScalar(key, idxTracker) + case Some(key) => key + case None => PhpInt(idxTracker.next, item.attributes) + } + val dimensionAst = astForExpr(dimension) + val indexAccessCode = s"${sourceAst.rootCodeOrEmpty}[${dimensionAst.rootCodeOrEmpty}]" + // .indexAccess(sourceAst, index) + val indexAccessNode = callAst( + newOperatorCallNode(Operators.indexAccess, indexAccessCode, line = line(item)), + sourceAst :: dimensionAst :: Nil + ) + val assignCode = s"${targetAst.rootCodeOrEmpty} = $indexAccessCode" + val assignNode = newOperatorCallNode(Operators.assignment, assignCode, line = line(item)) + // targetAst = .indexAccess(sourceAst, index) + callAst(assignNode, targetAst :: indexAccessNode :: Nil) + } + + // Take `[[$a, $b], $c] = $arr;` as an example + def handleUnpackLowering( + target: PhpArrayExpr | PhpListExpr, + itemsOf: PhpArrayExpr | PhpListExpr => List[Option[PhpArrayItem]], + sourceAst: Ast + ): Unit = { + val idxTracker = new ArrayIndexTracker + + // create an alias identifier of $arr + val sourceAliasName = getNewTmpName() + val sourceAliasIdentifier = createIdentifier(sourceAliasName) + val assignCode = s"${sourceAliasIdentifier.rootCodeOrEmpty} = ${sourceAst.rootCodeOrEmpty}" + val assignNode = newOperatorCallNode(Operators.assignment, assignCode, line = line(assignment)) + loweredAssignNodes += callAst(assignNode, sourceAliasIdentifier :: sourceAst :: Nil) + + itemsOf(target).foreach { + case Some(item) => + item.value match { + case nested: (PhpArrayExpr | PhpListExpr) => // item is [$a, $b] + // create tmp variable for [$a, $b] to receive the result of .indexAccess($arr, 0) + val tmpIdentifierName = getNewTmpName() + // tmpVar = .indexAccess($arr, 0) + val targetAssignNode = + createIndexAccessChain( + createIdentifier(tmpIdentifierName), + createIdentifier(sourceAliasName), + idxTracker, + item + ) + loweredAssignNodes += targetAssignNode + handleUnpackLowering(nested, itemsOf, createIdentifier(tmpIdentifierName)) + case phpVar: PhpVariable => // item is $c + val identifier = astForExpr(phpVar) + // $c = .indexAccess($arr, 1) + val targetAssignNode = + createIndexAccessChain(identifier, createIdentifier(sourceAliasName), idxTracker, item) + loweredAssignNodes += targetAssignNode + case _ => + // unknown case + idxTracker.next + } + case None => + idxTracker.next + } + } + + val sourceAst = astForExpr(assignment.source) + val itemsOf = (exp: PhpArrayExpr | PhpListExpr) => + exp match { + case x: PhpArrayExpr => x.items + case x: PhpListExpr => x.items + } + handleUnpackLowering(target, itemsOf, sourceAst) + Ast(blockNode(assignment)) + .withChildren(loweredAssignNodes.toList) + } + private def astForAssignment(assignment: PhpAssignment): Ast = { assignment.target match { case arrayDimFetch: PhpArrayDimFetchExpr if arrayDimFetch.dimension.isEmpty => // Rewrite `$xs[] = ` as `array_push($xs, )` to simplify finding dataflows. astForEmptyArrayDimAssign(assignment, arrayDimFetch) - + case arrayExpr: (PhpArrayExpr | PhpListExpr) => + astForArrayUnpack(assignment, arrayExpr) case _ => val operatorName = assignment.assignOp @@ -1293,10 +1447,30 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], private def astForArrayExpr(expr: PhpArrayExpr): Ast = { val idxTracker = new ArrayIndexTracker - val tmpIdentifier = getTmpIdentifier(expr, Some(TypeConstants.Array)) + val tmpName = getNewTmpName() + + def newTmpIdentifier: Ast = Ast(identifierNode(expr, tmpName, s"$$$tmpName", TypeConstants.Array)) + + val tmpIdentifierAssignNode = { + // use array() function to create an empty array. see https://www.php.net/manual/zh/function.array.php + val initArrayNode = callNode( + expr, + "array()", + "array", + "array", + DispatchTypes.STATIC_DISPATCH, + Some("array()"), + Some(TypeConstants.Array) + ) + val initArrayCallAst = callAst(initArrayNode) + + val assignCode = s"$$$tmpName = ${initArrayCallAst.rootCodeOrEmpty}" + val assignNode = newOperatorCallNode(Operators.assignment, assignCode, line = line(expr)) + callAst(assignNode, newTmpIdentifier :: initArrayCallAst :: Nil) + } val itemAssignments = expr.items.flatMap { - case Some(item) => Option(assignForArrayItem(item, tmpIdentifier.name, idxTracker)) + case Some(item) => Option(assignForArrayItem(item, tmpName, idxTracker)) case None => idxTracker.next // Skip an index None @@ -1304,8 +1478,9 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], val arrayBlock = blockNode(expr) Ast(arrayBlock) + .withChild(tmpIdentifierAssignNode) .withChildren(itemAssignments) - .withChild(Ast(tmpIdentifier)) + .withChild(newTmpIdentifier) } private def astForListExpr(expr: PhpListExpr): Ast = { @@ -1419,14 +1594,14 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], Some(localNode(closureExpr, name, s"$byRefPrefix$$$name", typeFullName)) case other => - logger.warn(s"Found incorrect closure use variable '$other' in $filename") + logger.warn(s"Found incorrect closure use variable '$other' in $relativeFileName") None } } // Add closure bindings to diffgraph localsForUses.foreach { local => - val closureBindingId = s"$filename:$methodName:${local.name}" + val closureBindingId = s"$relativeFileName:$methodName:${local.name}" local.closureBindingId(closureBindingId) scope.addToScope(local.name, local) @@ -1607,7 +1782,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], callAst(accessNode, variableAst :: dimensionAst :: Nil) case None => - val errorPosition = s"$variableCode:${line(expr).getOrElse("")}:$filename" + val errorPosition = s"$variableCode:${line(expr).getOrElse("")}:$relativeFileName" logger.error(s"ArrayDimFetchExpr without dimensions should be handled in assignment: $errorPosition") Ast() } @@ -1681,7 +1856,7 @@ class AstCreator(filename: String, phpAst: PhpFile, fileContent: Option[String], .getOrElse(nameExpr.name) case expr => - logger.warn(s"Unexpected expression as class name in ::class expression: $filename") + logger.warn(s"Unexpected expression as class name in ::class expression: $relativeFileName") NameConstants.Unknown } diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala index 0ac7a485d9b0..7110da272d71 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala @@ -3,7 +3,6 @@ import better.files.File import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory -import scala.collection.immutable.LazyList.from import scala.io.Source import scala.util.{Failure, Success, Try, Using} import upickle.default.* @@ -26,11 +25,12 @@ class ClassParser(targetDir: File) { f } - private lazy val phpClassParseCommand: String = s"php ${classParserScript.pathAsString} ${targetDir.pathAsString}" + private lazy val phpClassParseCommand: Seq[String] = + Seq("php", classParserScript.pathAsString, targetDir.pathAsString) def parse(): Try[List[ClassParserClass]] = Try { val inputDirectory = targetDir.parent.canonicalPath - ExternalCommand.run(phpClassParseCommand, inputDirectory).map(_.reverse) match { + ExternalCommand.run(phpClassParseCommand, inputDirectory).toTry.map(_.reverse) match { case Success(output) => read[List[ClassParserClass]](output.mkString("\n")) case Failure(exception) => diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala index 45017300cafa..583000ec8fed 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala @@ -6,7 +6,10 @@ import io.joern.php2cpg.parser.Domain.PhpFile import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory +import java.nio.file.Path import java.nio.file.Paths +import java.util.regex.Pattern +import scala.collection.mutable import scala.io.Source import scala.util.{Failure, Success, Try} @@ -14,56 +17,113 @@ class PhpParser private (phpParserPath: String, phpIniPath: String, disableFileC private val logger = LoggerFactory.getLogger(this.getClass) - private def phpParseCommand(filename: String): String = { - val phpParserCommands = "--with-recovery --resolve-names --json-dump" - s"php --php-ini $phpIniPath $phpParserPath $phpParserCommands $filename" + private def phpParseCommand(filenames: collection.Seq[String]): Seq[String] = { + val phpParserCommands = Seq("--with-recovery", "--resolve-names", "--json-dump") + Seq("php", "--php-ini", phpIniPath, phpParserPath) ++ phpParserCommands ++ filenames } - def parseFile(inputPath: String): Option[(PhpFile, Option[String])] = { - val inputFile = File(inputPath) - val inputFilePath = inputFile.canonicalPath - val inputDirectory = inputFile.parent.canonicalPath - val command = phpParseCommand(inputFilePath) - ExternalCommand.run(command, inputDirectory) match { - case Success(output) => - val content = Option.unless(disableFileContent)(inputFile.contentAsString) - processParserOutput(output, inputFilePath).map((_, content)) - case Failure(exception) => - logger.error(s"Failure running php-parser with $command", exception.getMessage) - None + def parseFiles(inputPaths: collection.Seq[String]): collection.Seq[(String, Option[PhpFile], String)] = { + // We need to keep a map between the input path and its canonical representation in + // order to map back the canonical file name we get from the php parser. + // Otherwise later on file name/path processing might get confused because the returned + // file paths are in no relation to the input paths. + val canonicalToInputPath = mutable.HashMap.empty[String, String] + + inputPaths.foreach { inputPath => + val canonicalPath = Path.of(inputPath).toFile.getCanonicalPath + canonicalToInputPath.put(canonicalPath, inputPath) } - } - private def processParserOutput(output: Seq[String], filename: String): Option[PhpFile] = { - val maybeJson = linesToJsonValue(output, filename) - maybeJson.flatMap(jsonValueToPhpFile(_, filename)) + val command = phpParseCommand(inputPaths) + + val result = ExternalCommand.run(command, ".", mergeStdErrInStdOut = true) + result match { + case ExternalCommand.ExternalCommandResult(0, stdOut, _) => + val asJson = linesToJsonValues(stdOut) + val asPhpFile = asJson.map { case (filename, jsonObjectOption, infoLines) => + (filename, jsonToPhpFile(jsonObjectOption, filename), infoLines) + } + val withRemappedFileName = asPhpFile.map { case (filename, phpFileOption, infoLines) => + (canonicalToInputPath.apply(filename), phpFileOption, infoLines) + } + withRemappedFileName + case ExternalCommand.ExternalCommandResult(exitCode, _, _) => + logger.error(s"Failure running php-parser with ${command.mkString(" ")}, exit code $exitCode") + Nil + } } - private def linesToJsonValue(lines: Seq[String], filename: String): Option[ujson.Value] = { - if (lines.exists(_.startsWith("["))) { - val jsonString = lines.dropWhile(_.charAt(0) != '[').mkString - Try(Option(ujson.read(jsonString))) match { - case Success(Some(value)) => Some(value) - case Success(None) => - logger.error(s"Parsing json string for $filename resulted in null return value") - None - case Failure(exception) => - logger.error(s"Parsing json string for $filename failed with exception", exception) + private def jsonToPhpFile(jsonObject: Option[ujson.Value], filename: String): Option[PhpFile] = { + val phpFile = jsonObject.flatMap { jsonObject => + Try(Domain.fromJson(jsonObject)) match { + case Success(phpFile) => + Some(phpFile) + case Failure(e) => + logger.error(s"Failed to generate intermediate AST for $filename", e) None } - } else { - logger.warn(s"No JSON output for $filename") - None } + phpFile } - private def jsonValueToPhpFile(json: ujson.Value, filename: String): Option[PhpFile] = { - Try(Domain.fromJson(json)) match { - case Success(phpFile) => Some(phpFile) - case Failure(e) => - logger.error(s"Failed to generate intermediate AST for $filename", e) - None + enum PARSE_MODE { + case PARSE_INFO, PARSE_JSON, SKIP_TRAILER + } + + private def linesToJsonValues( + lines: collection.Seq[String] + ): collection.Seq[(String, Option[ujson.Value], String)] = { + val filePrefix = "====> File " + val filenameRegex = Pattern.compile(s"$filePrefix(.*):") + val result = mutable.ArrayBuffer.empty[(String, Option[ujson.Value], String)] + + var filename = "" + val infoLines = mutable.ArrayBuffer.empty[String] + val jsonLines = mutable.ArrayBuffer.empty[String] + + var mode = PARSE_MODE.SKIP_TRAILER + val linesIt = lines.iterator + while (linesIt.hasNext) { + val line = linesIt.next + mode match { + case PARSE_MODE.PARSE_INFO => + if (line != "==> JSON dump:") { + infoLines.append(line) + } else { + mode = PARSE_MODE.PARSE_JSON + } + case PARSE_MODE.PARSE_JSON => + jsonLines.append(line) + if (line.startsWith("]") || line == "[]") { + val jsonString = jsonLines.mkString + + Try(Option(ujson.read(jsonString))) match { + case Success(option) => + result.append((filename, option, infoLines.mkString)) + if (option.isEmpty) { + logger.error(s"Parsing json string for $filename resulted in null return value") + } + case Failure(exception) => + result.append((filename, None, infoLines.mkString)) + logger.error(s"Parsing json string for $filename failed with exception", exception) + } + + mode = PARSE_MODE.SKIP_TRAILER + } + case _ => + } + + if (line.startsWith(filePrefix)) { + val matcher = filenameRegex.matcher(line) + if (matcher.find()) { + filename = matcher.group(1) + infoLines.clear() + jsonLines.clear() + mode = PARSE_MODE.PARSE_INFO + } + } } + result } } @@ -79,9 +139,11 @@ object PhpParser { } private def defaultPhpParserBin: String = { - val dir = Paths.get(this.getClass.getProtectionDomain.getCodeSource.getLocation.toURI).toAbsolutePath.toString - val fixedDir = new java.io.File(dir.substring(0, dir.indexOf("php2cpg"))).toString - Paths.get(fixedDir, "php2cpg", "bin", "php-parser", "php-parser.php").toAbsolutePath.toString + val packagePath = Paths.get(this.getClass.getProtectionDomain.getCodeSource.getLocation.toURI) + ExternalCommand + .executableDir(packagePath) + .resolve("php-parser/php-parser.php") + .toString } private def configOverrideOrDefaultPath( diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala index 8f9c618a3333..c65478137eb0 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AnyTypePass.scala @@ -6,14 +6,14 @@ import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.AstNode import io.shiftleft.codepropertygraph.generated.nodes.Call.PropertyDefaults import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* // TODO This is a hack for a customer issue. Either extend this to handle type full names properly, // or do it elsewhere. class AnyTypePass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) { override def generateParts(): Array[AstNode] = { - cpg.has(PropertyNames.TYPE_FULL_NAME, PropertyDefaults.TypeFullName).collectAll[AstNode].toArray + cpg.graph.nodesWithProperty(PropertyNames.TYPE_FULL_NAME, PropertyDefaults.TypeFullName).collectAll[AstNode].toArray } override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = { diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstCreationPass.scala index 7d8e3f0c15f5..01d80874a7cc 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstCreationPass.scala @@ -12,36 +12,53 @@ import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* class AstCreationPass(config: Config, cpg: Cpg, parser: PhpParser)(implicit withSchemaValidation: ValidationMode) - extends ForkJoinParallelCpgPass[String](cpg) { + extends ForkJoinParallelCpgPass[Array[String]](cpg) { private val logger = LoggerFactory.getLogger(this.getClass) val PhpSourceFileExtensions: Set[String] = Set(".php") - override def generateParts(): Array[String] = SourceFiles - .determine( - config.inputPath, - PhpSourceFileExtensions, - ignoredFilesRegex = Option(config.ignoredFilesRegex), - ignoredFilesPath = Option(config.ignoredFiles) - ) - .toArray - - override def runOnPart(diffGraph: DiffGraphBuilder, filename: String): Unit = { - val relativeFilename = if (filename == config.inputPath) { - File(filename).name - } else { - File(config.inputPath).relativize(File(filename)).toString - } - parser.parseFile(filename) match { - case Some((parseResult, fileContent)) => - diffGraph.absorb( - new AstCreator(relativeFilename, parseResult, fileContent, config.disableFileContent)(config.schemaValidation) - .createAst() - ) - - case None => - logger.warn(s"Could not parse file $filename. Results will be missing!") + override def generateParts(): Array[Array[String]] = { + val sourceFiles = SourceFiles + .determine( + config.inputPath, + PhpSourceFileExtensions, + ignoredFilesRegex = Option(config.ignoredFilesRegex), + ignoredFilesPath = Option(config.ignoredFiles) + ) + .toArray + + // We need to feed the php parser big groups of file in order + // to speed up the parsing. Apparently it is some sort of slow + // startup phase which makes single file processing prohibitively + // slow. + // On the other hand we need to be careful to not choose too big + // chunks because: + // 1. The argument length to the php executable has system + // dependent limits + // 2. We want to make use of multiple CPU cores for the rest + // of the CPG creation. + // + val parts = sourceFiles.grouped(20).toArray + parts + } + + override def runOnPart(diffGraph: DiffGraphBuilder, filenames: Array[String]): Unit = { + parser.parseFiles(filenames).foreach { case (filename, parseResult, infoLines) => + parseResult match { + case Some(parseResult) => + val relativeFilename = if (filename == config.inputPath) { + File(filename).name + } else { + File(config.inputPath).relativize(File(filename)).toString + } + diffGraph.absorb( + new AstCreator(relativeFilename, filename, parseResult, config.disableFileContent)(config.schemaValidation) + .createAst() + ) + case None => + logger.warn(s"Could not parse file $filename. Results will be missing!") + } } } } diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala index 8594efb2a949..701b7523d033 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/AstParentInfoPass.scala @@ -1,10 +1,9 @@ package io.joern.php2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, NamespaceBlock, Method, TypeDecl} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AstParentInfoPass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) { @@ -15,7 +14,7 @@ class AstParentInfoPass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) override def runOnPart(diffGraph: DiffGraphBuilder, node: AstNode): Unit = { findParent(node).foreach { parentNode => val astParentType = parentNode.label - val astParentFullName = parentNode.property(PropertyNames.FULL_NAME) + val astParentFullName = parentNode.property(Properties.FullName) diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_TYPE, astParentType) diffGraph.setNodeProperty(node, PropertyNames.AST_PARENT_FULL_NAME, astParentFullName) diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala index ac790b9fb438..c5c2c289a54c 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/ClosureRefPass.scala @@ -3,7 +3,7 @@ package io.joern.php2cpg.passes import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{ClosureBinding, Method, MethodRef} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.AstNode @@ -22,7 +22,7 @@ class ClosureRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[ClosureBinding](c * that is the scope in which the closure would have originally been created. */ override def runOnPart(diffGraph: DiffGraphBuilder, closureBinding: ClosureBinding): Unit = { - closureBinding.captureIn.collectAll[MethodRef].toList match { + closureBinding._methodRefViaCaptureIn.toList match { case Nil => logger.error(s"No MethodRef corresponding to closureBinding ${closureBinding.closureBindingId}") diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/DependencyPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/DependencyPass.scala index 003efcc8d33c..8b56de0fa15e 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/DependencyPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/DependencyPass.scala @@ -6,7 +6,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.{NewDependency, NewTag} import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} import io.shiftleft.passes.ForkJoinParallelCpgPass import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate import upickle.default.* import scala.annotation.targetName diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala index 89ed00ffe137..2e113b80ae0a 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/passes/LocalCreationPass.scala @@ -13,7 +13,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ NewNode, TypeDecl } -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.php2cpg.astcreation.AstCreator import io.joern.php2cpg.parser.Domain import io.joern.php2cpg.parser.Domain.PhpOperators @@ -96,7 +96,7 @@ abstract class LocalCreationPass[ScopeType <: AstNode](cpg: Cpg) ): Unit = { val identifierMap = getIdentifiersInScope(bodyNode) - .filter(_.refOut.isEmpty) + .filter(_._refOut.isEmpty) .filterNot(excludeIdentifierFn) .groupBy(_.name) diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/dataflow/IntraMethodDataflowTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/dataflow/IntraMethodDataflowTests.scala index 737898f2bc05..7d680770be71 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/dataflow/IntraMethodDataflowTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/dataflow/IntraMethodDataflowTests.scala @@ -1,8 +1,8 @@ package io.joern.php2cpg.dataflow import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* class IntraMethodDataflowTests extends PhpCode2CpgFixture(runOssDataflow = true) { "flows from parameters to corresponding identifiers should be found" in { @@ -29,4 +29,50 @@ class IntraMethodDataflowTests extends PhpCode2CpgFixture(runOssDataflow = true) flows.size shouldBe 1 } + + "flow from single layer array unpacking should be found" in { + val cpg = code(""" $value) { + | echo $key; + | echo $value; + |} + |""".stripMargin) + val source = cpg.identifier("arr") + val sink = cpg.call("echo").argument(1) + val flows = sink.reachableByFlows(source) + flows.size shouldBe 2 + } } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..ddadfc9d0622 --- /dev/null +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala @@ -0,0 +1,79 @@ +package io.joern.php2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class Php2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("php2cpgTestsHttpTest") + val file = dir / "main.php" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse(""""Hello, World!"""") + file.writeText(s""" fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.call.code.l shouldBe List("""print("Hello, World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("php2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.call.code.l shouldBe List(s"print($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/CfgCreationPassTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/CfgCreationPassTests.scala index a4d244a01c48..b602b6c82784 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/CfgCreationPassTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/CfgCreationPassTests.scala @@ -29,7 +29,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("break(1)") shouldBe expected(("$i", AlwaysEdge)) + succOf("break(1)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } "be correct for break with level 2" in { @@ -40,7 +40,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("break(2)") shouldBe expected(("RET", AlwaysEdge)) + succOf("break(2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for continue with level 1" in { @@ -51,7 +51,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("continue(1)") shouldBe expected(("$j", AlwaysEdge)) + succOf("continue(1)") should contain theSameElementsAs expected(("$j", AlwaysEdge)) } "be correct for continue with level 2" in { @@ -62,7 +62,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("continue(2)") shouldBe expected(("$i", AlwaysEdge)) + succOf("continue(2)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } } @@ -75,7 +75,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } while ($j < 1); |} while ($i < 1) |""".stripMargin) - succOf("break(1)") shouldBe expected(("$i", AlwaysEdge)) + succOf("break(1)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } "be correct for break with level 2" in { @@ -86,7 +86,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } while ($j < 1); |} while ($i < 1) |""".stripMargin) - succOf("break(2)") shouldBe expected(("RET", AlwaysEdge)) + succOf("break(2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for continue with level 1" in { @@ -97,7 +97,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } while ($j < 1); |} while ($i < 1) |""".stripMargin) - succOf("continue(1)") shouldBe expected(("$j", AlwaysEdge)) + succOf("continue(1)") should contain theSameElementsAs expected(("$j", AlwaysEdge)) } "be correct for continue with level 2" in { @@ -108,7 +108,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } while ($j < 1); |} while ($i < 1) |""".stripMargin) - succOf("continue(2)") shouldBe expected(("$i", AlwaysEdge)) + succOf("continue(2)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } } @@ -121,7 +121,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("break(1)") shouldBe expected(("$i", AlwaysEdge)) + succOf("break(1)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } "be correct for break with level 2" in { @@ -132,7 +132,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("break(2)") shouldBe expected(("RET", AlwaysEdge)) + succOf("break(2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } "be correct for continue with level 1" in { @@ -143,7 +143,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("continue(1)") shouldBe expected(("$j", AlwaysEdge)) + succOf("continue(1)") should contain theSameElementsAs expected(("$j", AlwaysEdge)) } "be correct for continue with level 2" in { @@ -154,7 +154,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | } |} |""".stripMargin) - succOf("continue(2)") shouldBe expected(("$i", AlwaysEdge)) + succOf("continue(2)") should contain theSameElementsAs expected(("$i", AlwaysEdge)) } } @@ -170,7 +170,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | $k; |} |""".stripMargin) - succOf("break(1)") shouldBe expected(("$k", AlwaysEdge)) + succOf("break(1)") should contain theSameElementsAs expected(("$k", AlwaysEdge)) } "be correct for break with level 2" in { @@ -184,7 +184,7 @@ class CfgCreationPassTests extends CfgTestFixture(() => new PhpCfgTestCpg) { | $k; |} |""".stripMargin) - succOf("break(2)") shouldBe expected(("RET", AlwaysEdge)) + succOf("break(2)") should contain theSameElementsAs expected(("RET", AlwaysEdge)) } } } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala index d5d19ab7c5bd..0384627d1110 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/passes/PhpTypeRecoveryPassTests.scala @@ -525,7 +525,7 @@ class PhpTypeRecoveryPassTests extends PhpCode2CpgFixture() { "propagate this QueryBuilder type to the identifier assigned to the inherited call for the wrapped `createQueryBuilder`" in { cpg.method .nameExact("findSomething") - ._containsOut + .containsOut .collectAll[Identifier] .nameExact("queryBuilder") .typeFullName diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ArrayTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ArrayTests.scala index 7e671a914683..bf213794b2c6 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ArrayTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ArrayTests.scala @@ -3,7 +3,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal, Local} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ArrayTests extends PhpCode2CpgFixture { "array accesses with variable keys should be represented as index accesses" in { @@ -77,16 +77,20 @@ class ArrayTests extends PhpCode2CpgFixture { tmpLocal.name shouldBe "tmp0" tmpLocal.code shouldBe "$tmp0" - inside(arrayBlock.astChildren.l) { case List(aAssign: Call, bAssign: Call, tmpIdent: Identifier) => - aAssign.code shouldBe "$tmp0[\"A\"] = 1" - aAssign.lineNumber shouldBe Some(3) + inside(arrayBlock.astChildren.l) { + case List(initAssign: Call, aAssign: Call, bAssign: Call, tmpIdent: Identifier) => + initAssign.code shouldBe "$tmp0 = array()" + initAssign.lineNumber shouldBe Some(2) - bAssign.code shouldBe "$tmp0[\"B\"] = 2" - bAssign.lineNumber shouldBe Some(4) + aAssign.code shouldBe "$tmp0[\"A\"] = 1" + aAssign.lineNumber shouldBe Some(3) - tmpIdent.name shouldBe "tmp0" - tmpIdent.code shouldBe "$tmp0" - tmpIdent._localViaRefOut should contain(tmpLocal) + bAssign.code shouldBe "$tmp0[\"B\"] = 2" + bAssign.lineNumber shouldBe Some(4) + + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) } } } @@ -103,16 +107,20 @@ class ArrayTests extends PhpCode2CpgFixture { tmpLocal.name shouldBe "tmp0" tmpLocal.code shouldBe "$tmp0" - inside(arrayBlock.astChildren.l) { case List(aAssign: Call, bAssign: Call, tmpIdent: Identifier) => - aAssign.code shouldBe "$tmp0[0] = \"A\"" - aAssign.lineNumber shouldBe Some(3) + inside(arrayBlock.astChildren.l) { + case List(initAssign: Call, aAssign: Call, bAssign: Call, tmpIdent: Identifier) => + initAssign.code shouldBe "$tmp0 = array()" + initAssign.lineNumber shouldBe Some(2) + + aAssign.code shouldBe "$tmp0[0] = \"A\"" + aAssign.lineNumber shouldBe Some(3) - bAssign.code shouldBe "$tmp0[1] = \"B\"" - bAssign.lineNumber shouldBe Some(4) + bAssign.code shouldBe "$tmp0[1] = \"B\"" + bAssign.lineNumber shouldBe Some(4) - tmpIdent.name shouldBe "tmp0" - tmpIdent.code shouldBe "$tmp0" - tmpIdent._localViaRefOut should contain(tmpLocal) + tmpIdent.name shouldBe "tmp0" + tmpIdent.code shouldBe "$tmp0" + tmpIdent._localViaRefOut should contain(tmpLocal) } } } @@ -128,7 +136,10 @@ class ArrayTests extends PhpCode2CpgFixture { tmpLocal.name shouldBe "tmp0" tmpLocal.code shouldBe "$tmp0" - inside(arrayBlock.astChildren.l) { case List(assign: Call, tmpIdent: Identifier) => + inside(arrayBlock.astChildren.l) { case List(initAssign: Call, assign: Call, tmpIdent: Identifier) => + initAssign.code shouldBe "$tmp0 = array()" + initAssign.lineNumber shouldBe Some(2) + assign.code shouldBe "$tmp0[2] = \"A\"" inside(assign.argument.collectAll[Call].argument.l) { case List(array: Identifier, index: Literal) => array.name shouldBe "tmp0" @@ -164,6 +175,7 @@ class ArrayTests extends PhpCode2CpgFixture { inside(arrayBlock.astChildren.l) { case List( + initAssign: Call, aAssign: Call, cAssign: Call, fourAssign: Call, @@ -173,6 +185,9 @@ class ArrayTests extends PhpCode2CpgFixture { eightAssign: Call, tmpIdent: Identifier ) => + initAssign.code shouldBe "$tmp0 = array()" + initAssign.lineNumber shouldBe Some(2) + aAssign.code shouldBe "$tmp0[\"A\"] = \"B\"" cAssign.code shouldBe "$tmp0[0] = \"C\"" fourAssign.code shouldBe "$tmp0[4] = \"D\"" diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CallTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CallTests.scala index 9e554bb19725..7297935b2db3 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CallTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CallTests.scala @@ -5,7 +5,7 @@ import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CallTests extends PhpCode2CpgFixture { "variable call arguments with names matching methods should not have a methodref" in { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CfgTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CfgTests.scala index c630338e7794..49f9c79a2708 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CfgTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CfgTests.scala @@ -3,7 +3,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.parser.Domain.PhpOperators import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Call, JumpTarget} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CfgTests extends PhpCode2CpgFixture { "the CFG for match constructs" when { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CommentTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CommentTests.scala index c7b490ff6922..26728291ee45 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CommentTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/CommentTests.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CommentTests extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ControlStructureTests.scala index c67fe7bbc2a4..419d1d83d446 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ControlStructureTests.scala @@ -14,7 +14,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ Literal, Local } -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.AstNode import scala.util.Try @@ -1261,17 +1261,24 @@ class ControlStructureTests extends PhpCode2CpgFixture { (initAsts, updateAsts, body) } - inside(initAsts.astChildren.l) { case List(_: Call, valInit: Call) => - valInit.name shouldBe Operators.assignment - valInit.code shouldBe "$key => $val = $iter_tmp0->current()" - inside(valInit.argument.l) { case List(valPair: Call, currentCall: Call) => - valPair.name shouldBe PhpOperators.doubleArrow - valPair.code shouldBe "$key => $val" - inside(valPair.argument.l) { case List(keyId: Identifier, valId: Identifier) => - keyId.name shouldBe "key" - valId.name shouldBe "val" + inside(initAsts.assignment.l) { case List(_: Call, keyInit: Call, valInit: Call) => + keyInit.name shouldBe Operators.assignment + keyInit.code shouldBe "$key = $iter_tmp0->key()" + inside(keyInit.argument.l) { case List(target: Identifier, keyCall: Call) => + target.name shouldBe "key" + keyCall.name shouldBe "key" + keyCall.methodFullName shouldBe s"Iterator.key" + keyCall.code shouldBe "$iter_tmp0->key()" + inside(keyCall.argument(0).start.l) { case List(iterRecv: Identifier) => + iterRecv.name shouldBe "iter_tmp0" + iterRecv.argumentIndex shouldBe 0 } + } + valInit.name shouldBe Operators.assignment + valInit.code shouldBe "$val = $iter_tmp0->current()" + inside(valInit.argument.l) { case List(target: Identifier, currentCall: Call) => + target.name shouldBe "val" currentCall.name shouldBe "current" currentCall.methodFullName shouldBe s"Iterator.current" currentCall.code shouldBe "$iter_tmp0->current()" @@ -1282,9 +1289,12 @@ class ControlStructureTests extends PhpCode2CpgFixture { } } - inside(updateAsts.astChildren.l) { case List(_: Call, valAssign: Call) => - valAssign.name shouldBe Operators.assignment - valAssign.code shouldBe "$key => $val = $iter_tmp0->current()" + inside(updateAsts.astChildren.l) { case List(_: Call, updateBlock: Block) => + val tmp = updateBlock.astChildren.l + inside(updateBlock.assignment.l) { case List(keyInit: Call, valInit: Call) => + keyInit.code shouldBe "$key = $iter_tmp0->key()" + valInit.code shouldBe "$val = $iter_tmp0->current()" + } } inside(body.astChildren.l) { case List(echoCall: Call) => diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/FieldAccessTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/FieldAccessTests.scala index c016b8c0fadb..3925ebd2d799 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/FieldAccessTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/FieldAccessTests.scala @@ -3,7 +3,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{FieldIdentifier, Identifier} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class FieldAccessTests extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/LocalTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/LocalTests.scala index 322bd1a5a79d..08a730754cc0 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/LocalTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/LocalTests.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class LocalTests extends PhpCode2CpgFixture { @@ -71,7 +71,6 @@ class LocalTests extends PhpCode2CpgFixture { | } |} |""".stripMargin) - println(cpg.local.name.l) inside(cpg.local.l) { case List(xLocal) => xLocal.name shouldBe "x" xLocal.code shouldBe "static $x" diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/MemberTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/MemberTests.scala index 087fb9e578c8..f6ff1585ab70 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/MemberTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/MemberTests.scala @@ -1,21 +1,24 @@ package io.joern.php2cpg.querying +import io.joern.php2cpg.Config import io.joern.php2cpg.parser.Domain import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, Literal, Local} +import io.shiftleft.semanticcpg.language.* class MemberTests extends PhpCode2CpgFixture { "class constants" should { - val cpg = code(""" @@ -40,11 +43,20 @@ class MemberTests extends PhpCode2CpgFixture { "have a clinit method with the constant initializers" in { inside(cpg.method.nameExact(Defines.StaticInitMethodName).l) { case List(clinitMethod) => - inside(clinitMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call, cAssign: Call) => + inside(clinitMethod.body.astChildren.l) { case List(self: Local, aAssign: Call, bAssign: Call, cAssign: Call) => + self.name shouldBe "self" checkConstAssign(aAssign, "A") checkConstAssign(bAssign, "B") checkConstAssign(cAssign, "C") } + clinitMethod.isExternal shouldBe false + clinitMethod.offset shouldBe Some(0) + clinitMethod.offsetEnd shouldBe Some(source.length) + cpg.file + .name("foo.php") + .content + .map(_.substring(clinitMethod.offset.get, clinitMethod.offsetEnd.get)) + .l shouldBe List(source) } } } @@ -213,9 +225,14 @@ class MemberTests extends PhpCode2CpgFixture { assign.name shouldBe Operators.assignment assign.methodFullName shouldBe Operators.assignment - inside(assign.argument.l) { case List(target: Identifier, source: Literal) => - target.name shouldBe expectedValue - target.code shouldBe expectedValue + inside(assign.argument.l) { case List(target: Call, source: Literal) => + inside(target.argument.l) { case List(base: Identifier, field: FieldIdentifier) => + base.name shouldBe "self" + field.code shouldBe expectedValue + } + + target.name shouldBe Operators.fieldAccess + target.code shouldBe s"self::$expectedValue" target.argumentIndex shouldBe 1 source.code shouldBe s"\"$expectedValue\"" diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/NamespaceTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/NamespaceTests.scala index 4b5ff757d74b..b8e062767765 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/NamespaceTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/NamespaceTests.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.Method class NamespaceTests extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala index cc0c46830db0..1d5b184888a1 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/OperatorTests.scala @@ -4,10 +4,10 @@ import io.joern.php2cpg.astcreation.AstCreator.TypeConstants import io.joern.php2cpg.parser.Domain.{PhpDomainTypeConstants, PhpOperators} import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.joern.x2cpg.utils.IntervalKeyPool +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, NodeTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal, TypeRef} -import io.shiftleft.passes.IntervalKeyPool -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal class OperatorTests extends PhpCode2CpgFixture { @@ -497,29 +497,6 @@ class OperatorTests extends PhpCode2CpgFixture { } } - "temporary list implementation should work" in { - // TODO This is a simple placeholder implementation that represents most of the useful information - // in the AST, while being pretty much unusable for dataflow. A better implementation needs to follow. - val cpg = code(""" - listCall.methodFullName shouldBe PhpOperators.listFunc - listCall.code shouldBe "list($a,$b)" - listCall.lineNumber shouldBe Some(2) - inside(listCall.argument.l) { case List(aArg: Identifier, bArg: Identifier) => - aArg.name shouldBe "a" - aArg.code shouldBe "$a" - aArg.lineNumber shouldBe Some(2) - - bArg.name shouldBe "b" - bArg.code shouldBe "$b" - bArg.lineNumber shouldBe Some(2) - } - } - } - "include calls" should { "be correctly represented for normal includes" in { val cpg = code(""" $d) = $arr; + |""".stripMargin) + // finds the block containing the assignments + val block = cpg.all.collect { case block: Block if block.lineNumber.contains(2) => block }.head + inside(block.astChildren.assignment.l) { case tmp0 :: tmp1 :: tmp2 :: a :: b :: c :: d :: Nil => + tmp0.code shouldBe "$tmp0 = $arr" + tmp0.source.label shouldBe NodeTypes.IDENTIFIER + tmp0.source.code shouldBe "$arr" + tmp0.target.label shouldBe NodeTypes.IDENTIFIER + tmp0.target.code shouldBe "$tmp0" + + tmp1.code shouldBe "$tmp1 = $tmp0[0]" + tmp1.source.label shouldBe NodeTypes.CALL + tmp1.source.asInstanceOf[Call].name shouldBe Operators.indexAccess + tmp1.source.code shouldBe "$tmp0[0]" + tmp1.target.label shouldBe NodeTypes.IDENTIFIER + tmp1.target.code shouldBe "$tmp1" + + tmp2.code shouldBe "$tmp2 = $tmp1" + tmp2.source.label shouldBe NodeTypes.IDENTIFIER + tmp2.source.code shouldBe "$tmp1" + tmp2.target.label shouldBe NodeTypes.IDENTIFIER + tmp2.target.code shouldBe "$tmp2" + + a.code shouldBe "$a = $tmp2[0]" + a.source.label shouldBe NodeTypes.CALL + a.source.asInstanceOf[Call].name shouldBe Operators.indexAccess + a.source.code shouldBe "$tmp2[0]" + a.target.label shouldBe NodeTypes.IDENTIFIER + a.target.code shouldBe "$a" + + b.code shouldBe "$b = $tmp2[1]" + b.source.label shouldBe NodeTypes.CALL + b.source.asInstanceOf[Call].name shouldBe Operators.indexAccess + b.source.code shouldBe "$tmp2[1]" + b.target.label shouldBe NodeTypes.IDENTIFIER + b.target.code shouldBe "$b" + + c.code shouldBe "$c = $tmp0[1]" + c.source.label shouldBe NodeTypes.CALL + c.source.asInstanceOf[Call].name shouldBe Operators.indexAccess + c.source.code shouldBe "$tmp0[1]" + c.target.label shouldBe NodeTypes.IDENTIFIER + c.target.code shouldBe "$c" + + d.code shouldBe "$d = $tmp0[\"d\"]" + d.source.label shouldBe NodeTypes.CALL + d.source.asInstanceOf[Call].name shouldBe Operators.indexAccess + d.source.code shouldBe "$tmp0[\"d\"]" + d.target.label shouldBe NodeTypes.IDENTIFIER + d.target.code shouldBe "$d" + } + } } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/PocTest.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/PocTest.scala index 83a8b537fce1..ca17c9c06936 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/PocTest.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/PocTest.scala @@ -4,7 +4,7 @@ import io.joern.php2cpg.astcreation.AstCreator.TypeConstants import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class PocTest extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ScalarTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ScalarTests.scala index 0d913e2fa4cd..2f3b4e530e94 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ScalarTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/ScalarTests.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal} class ScalarTests extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeDeclTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeDeclTests.scala index 69d29a30da31..e69f35512630 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeDeclTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeDeclTests.scala @@ -3,12 +3,9 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.Config import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local, Member, Method} import io.shiftleft.semanticcpg.language.* -import io.shiftleft.codepropertygraph.generated.nodes.Block -import io.shiftleft.codepropertygraph.generated.nodes.MethodRef -import io.shiftleft.codepropertygraph.generated.nodes.TypeRef class TypeDeclTests extends PhpCode2CpgFixture { @@ -248,19 +245,27 @@ class TypeDeclTests extends PhpCode2CpgFixture { clinitMethod.filename shouldBe "foo.php" clinitMethod.file.name.l shouldBe List("foo.php") - inside(clinitMethod.body.astChildren.l) { case List(aAssign: Call, bAssign: Call) => - aAssign.code shouldBe "A = \"A\"" - inside(aAssign.astChildren.l) { case List(aIdentifier: Identifier, aLiteral: Literal) => - aIdentifier.name shouldBe "A" - aIdentifier.code shouldBe "A" + inside(clinitMethod.body.astChildren.l) { case List(self: Local, aAssign: Call, bAssign: Call) => + aAssign.code shouldBe "self::A = \"A\"" + inside(aAssign.astChildren.l) { case List(aCall: Call, aLiteral: Literal) => + inside(aCall.argument.l) { case List(aSelf: Identifier, aField: FieldIdentifier) => + aSelf.name shouldBe "self" + aField.code shouldBe "A" + } + aCall.name shouldBe Operators.fieldAccess + aCall.code shouldBe "self::A" aLiteral.code shouldBe "\"A\"" } - bAssign.code shouldBe "B = \"B\"" - inside(bAssign.astChildren.l) { case List(bIdentifier: Identifier, bLiteral: Literal) => - bIdentifier.name shouldBe "B" - bIdentifier.code shouldBe "B" + bAssign.code shouldBe "self::B = \"B\"" + inside(bAssign.astChildren.l) { case List(bCall: Call, bLiteral: Literal) => + inside(bCall.argument.l) { case List(bSelf: Identifier, bField: FieldIdentifier) => + bSelf.name shouldBe "self" + bField.code shouldBe "B" + } + bCall.name shouldBe Operators.fieldAccess + bCall.code shouldBe "self::B" bLiteral.code shouldBe "\"B\"" } @@ -309,4 +314,40 @@ class TypeDeclTests extends PhpCode2CpgFixture { } } } + + "static/const member of class should be put in method" in { + val cpg = code(""" + inside(clinitMethod.body.astChildren.l) { case List(self: Local, bAssign: Call, aAssign: Call) => + self.name shouldBe "self" + inside(aAssign.astChildren.l) { case List(aCall: Call, aLiteral: Literal) => + inside(aCall.argument.l) { case List(aSelf: Identifier, aField: FieldIdentifier) => + aSelf.name shouldBe "self" + aField.code shouldBe "A" + } + aCall.name shouldBe Operators.fieldAccess + aCall.code shouldBe "self::$A" + + aLiteral.code shouldBe "\"A\"" + } + + inside(bAssign.astChildren.l) { case List(bCall: Call, bLiteral: Literal) => + inside(bCall.argument.l) { case List(bSelf: Identifier, bField: FieldIdentifier) => + bSelf.name shouldBe "self" + bField.code shouldBe "B" + } + bCall.name shouldBe Operators.fieldAccess + bCall.code shouldBe "self::B" // Notice there is no `$` in front of the const member + + bLiteral.code shouldBe "\"B\"" + } + } + } + } } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeNodeTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeNodeTests.scala index c26317f4dc3f..2871427aae40 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeNodeTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/TypeNodeTests.scala @@ -4,7 +4,7 @@ import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.{ModifierTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Local, Member, Method} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.Block class TypeNodeTests extends PhpCode2CpgFixture { @@ -29,7 +29,6 @@ class TypeNodeTests extends PhpCode2CpgFixture { |""".stripMargin) "have corresponding type nodes created" in { - println(cpg.literal.toList) cpg.typ.fullName.toSet shouldEqual Set("ANY", "int") } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/UseTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/UseTests.scala index f9c911f8c617..040966272f3c 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/UseTests.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/querying/UseTests.scala @@ -1,7 +1,7 @@ package io.joern.php2cpg.querying import io.joern.php2cpg.testfixtures.PhpCode2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class UseTests extends PhpCode2CpgFixture { diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/testfixtures/PhpCode2CpgFixture.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/testfixtures/PhpCode2CpgFixture.scala index cb848636f420..0cf55b76fc97 100644 --- a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/testfixtures/PhpCode2CpgFixture.scala +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/testfixtures/PhpCode2CpgFixture.scala @@ -1,10 +1,11 @@ package io.joern.php2cpg.testfixtures -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} import io.joern.php2cpg.{Config, Php2Cpg} import io.joern.x2cpg.frontendspecific.php2cpg -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, DefaultTestCpg} +import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} @@ -33,14 +34,14 @@ class PhpTestCpg extends DefaultTestCpg with PhpFrontend with SemanticTestCpg { class PhpCode2CpgFixture( runOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), withPostProcessing: Boolean = true ) extends Code2CpgFixture(() => new PhpTestCpg() .withOssDataflow(runOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) - with SemanticCpgTestFixture(extraFlows) { + with SemanticCpgTestFixture(semantics) { implicit val resolver: ICallResolver = NoResolve } diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala index fdf737a98ecb..088b0cd6638d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ConfigFileCreationPass.scala @@ -19,7 +19,10 @@ class ConfigFileCreationPass(cpg: Cpg, requirementsTxt: String = "requirement.tx // HTM files extensionFilter(".htm"), // Requirements.txt - pathEndFilter(requirementsTxt) + pathEndFilter(requirementsTxt), + // Pipfile + pathEndFilter("Pipfile"), + pathEndFilter("Pipfile.lock") ) } diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/DependenciesFromRequirementsTxtPass.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/DependenciesFromRequirementsTxtPass.scala index 0afff1632080..01a932277c44 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/DependenciesFromRequirementsTxtPass.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/DependenciesFromRequirementsTxtPass.scala @@ -2,7 +2,7 @@ package io.joern.pysrc2cpg import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.{NewDependency} import org.slf4j.{Logger, LoggerFactory} diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/EdgeBuilder.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/EdgeBuilder.scala index fdb409d6ab0b..2973108ba84d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/EdgeBuilder.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/EdgeBuilder.scala @@ -23,7 +23,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ NewUnknown } import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class EdgeBuilder(diffGraph: DiffGraphBuilder) { def astEdge(dstNode: nodes.NewNode, srcNode: nodes.NewNode, order: Int): Unit = { diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala index 2ab54ea2c507..7bf13ec760c1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.pysrc2cpg import io.joern.pysrc2cpg.Frontend.cmdLineParser import io.joern.x2cpg.X2CpgMain import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -38,9 +39,15 @@ private object Frontend { } } -object NewMain extends X2CpgMain(cmdLineParser, new Py2CpgOnFileSystem())(new Py2CpgOnFileSystemConfig()) { +object NewMain + extends X2CpgMain(cmdLineParser, new Py2CpgOnFileSystem())(Py2CpgOnFileSystemConfig()) + with FrontendHTTPServer[Py2CpgOnFileSystemConfig, Py2CpgOnFileSystem] { + + override protected def newDefaultConfig(): Py2CpgOnFileSystemConfig = Py2CpgOnFileSystemConfig() + def run(config: Py2CpgOnFileSystemConfig, frontend: Py2CpgOnFileSystem): Unit = { - frontend.run(config) + if (config.serverMode) { startup() } + else { frontend.run(config) } } def getCmdLineParser = cmdLineParser diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala index e1364856880e..369c192d3532 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/NodeBuilder.scala @@ -7,7 +7,7 @@ import io.joern.x2cpg.Defines import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.joern.x2cpg.utils.NodeBuilders import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, nodes} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class NodeBuilder(diffGraph: DiffGraphBuilder) { diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala index e951442986bb..b8ab7f82051b 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala @@ -2,10 +2,10 @@ package io.joern.pysrc2cpg import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants -import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder} import io.shiftleft.codepropertygraph.generated.Languages -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder +import flatgraph.DiffGraphApplier +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder object Py2Cpg { case class InputPair(content: String, relFileName: String) @@ -45,7 +45,7 @@ class Py2Cpg( val anyTypeDecl = nodeBuilder.typeDeclNode(Constants.ANY, Constants.ANY, "N/A", Nil, LineAndColumn(1, 1, 1, 1, 1, 1)) edgeBuilder.astEdge(anyTypeDecl, globalNamespaceBlock, 0) - BatchedUpdate.applyDiff(outputCpg.graph, diffGraph) + DiffGraphApplier.applyDiff(outputCpg.graph, diffGraph) new CodeToCpg(outputCpg, inputProviders, schemaValidationMode, enableFileContent).createAndApply() new ConfigFileCreationPass(outputCpg, requirementsTxt).createAndApply() new DependenciesFromRequirementsTxtPass(outputCpg).createAndApply() diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala index 58e58dd8da88..391e044dca21 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala @@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory import java.nio.file.* import scala.util.Try -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* case class Py2CpgOnFileSystemConfig( venvDir: Option[Path] = None, diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala index f1b8f7993a4b..fe5053bf0b29 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala @@ -9,7 +9,7 @@ import io.joern.x2cpg.{AstCreatorBase, ValidationMode} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier, NewNode, NewTypeDecl} import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import scala.collection.mutable diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstPrinter.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstPrinter.scala index 18a3cb30d32c..3e8864537c8b 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstPrinter.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstPrinter.scala @@ -1,5 +1,5 @@ package io.joern.pythonparser -import io.joern.pythonparser.ast._ +import io.joern.pythonparser.ast.* import scala.collection.immutable class AstPrinter(indentStr: String) extends AstVisitor[String] { diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstVisitor.scala index 4a9617055bdd..167c2287c689 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstVisitor.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/AstVisitor.scala @@ -1,6 +1,6 @@ package io.joern.pythonparser -import io.joern.pythonparser.ast._ +import io.joern.pythonparser.ast.* trait AstVisitor[T] { def visit(ast: iast): T diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/PyParser.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/PyParser.scala index 36040ca8274b..826960e7f439 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/PyParser.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/PyParser.scala @@ -6,7 +6,7 @@ import io.joern.pythonparser.ast.{ErrorStatement, iast} import java.io.{BufferedReader, ByteArrayInputStream, InputStream, Reader} import java.nio.charset.StandardCharsets -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class PyParser { private var pythonParser: PythonParser = scala.compiletime.uninitialized diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/ast/Ast.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/ast/Ast.scala index 02c5272c4df7..0ba09590deb0 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/ast/Ast.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pythonparser/ast/Ast.scala @@ -3,7 +3,7 @@ package io.joern.pythonparser.ast import io.joern.pythonparser.AstVisitor import java.util -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* // This file describes the AST classes. // It tries to stay as close as possible to the AST defined by CPython at diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala index 1f7ef899b70f..28cce1f5d986 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala index dca961f63eef..2811650be190 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala index e64fb1369e79..21d6ca62a217 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala index 1e2dc03cb42c..7b0917042e1e 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala index 0d5ffa521e3a..c210a453b33c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala index fff6c7d5889c..3dae7d13a396 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala index 1238e6ed089b..4aafd1d33add 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala index 7ec13f219950..3337cc5bf97d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala @@ -1,9 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.NodeOps +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala index aed25d40e211..5740d5ae4947 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture +import io.shiftleft.semanticcpg.language.* class ClassCpgTests extends PySrc2CpgFixture(withOssDataflow = false) { "class" should { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala index f01d5f698aa7..3550933f5e3f 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala @@ -1,9 +1,9 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala index ccda1f361591..ed60926e22c7 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class ContentCpgTests extends PySrc2CpgFixture() { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala index cc1aa72bf86f..a0b02eeed10e 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala index c1698709dd6c..2e996ab2c924 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala @@ -1,10 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.{Py2CpgTestContext, PySrc2CpgFixture} -import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec class DictCpgTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala index 113401a91718..586b04fe7dd2 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala index 8c526ec859bb..d9dde2aa24f2 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.codepropertygraph.generated.ModifierTypes import io.shiftleft.codepropertygraph.generated.nodes.Call @@ -55,12 +55,12 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers { } "test function method ref" in { - cpg.methodRef("func").referencedMethod.fullName.head shouldBe + cpg.methodRefWithName("func").referencedMethod.fullName.head shouldBe "test.py:.func" } "test assignment of method ref to local variable" in { - val assignNode = cpg.methodRef("func").astParent.isCall.head + val assignNode = cpg.methodRefWithName("func").astParent.isCall.head assignNode.code shouldBe "func = def func(...)" } @@ -132,7 +132,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers { |""".stripMargin) "test decorator wrapping of method reference" in { - val (staticMethod: Call) :: Nil = cpg.methodRef("func").astParent.l: @unchecked + val (staticMethod: Call) :: Nil = cpg.methodRefWithName("func").astParent.l: @unchecked staticMethod.code shouldBe "staticmethod(def func(...))" staticMethod.name shouldBe "staticmethod" val (abc: Call) :: Nil = staticMethod.start.astParent.l: @unchecked diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala index cbb02e48d5ba..6647fe7454d1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, nodes} import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala index 122cccd6d386..4d066ba21c45 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala index 5ae76bdb79f7..d3c94e23307d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala index 73f85aa73fab..b02fd1a9c3cc 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala index f8e88367fc3c..448cd17a4f9c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.nodes.Member -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala index 6e03d0808b92..8476473d02a1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala index 57c274c7502d..a35deb9282e9 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala index 75bacc6c1dac..9510fd3d5ac7 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class PatternMatchingTests extends PySrc2CpgFixture() { "pattern matching" should { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala index 58dd9b40fae4..32c14bcb75e9 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala index 631bddb4aa5e..1b17c2a3b039 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala index d23fa0e133af..10f518a9a70a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala index ece135da41e0..f48c201a8aba 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class SliceCpgTests extends PySrc2CpgFixture() { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala index 67c53bd02f5a..0b90acbbfa41 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala index 2cf2a22b2cda..9a3455c7abd3 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala index e926441a1940..feb7aff51640 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala index 134d0bd16608..adf4531c24ea 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala index 103473b7bf72..92a0f02ba0a8 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala index 0c7739cda935..f98cb73ca464 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala index 9da932fb2023..a476f60601a0 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.EvaluationStrategies -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers @@ -149,7 +149,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -185,7 +185,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -223,7 +223,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding of f in g" in { - val methodRefNode = cpg.methodRef("f").head + val methodRefNode = cpg.methodRefWithName("f").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.g.f:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE @@ -240,7 +240,7 @@ class VariableReferencingCpgTests extends AnyFreeSpec with Matchers { } "test method reference closure binding of g in module" in { - val methodRefNode = cpg.methodRef("g").head + val methodRefNode = cpg.methodRefWithName("g").head val closureBinding = methodRefNode._closureBindingViaCaptureOut.next() closureBinding.closureBindingId shouldBe Some("test.py:.g:x") closureBinding.evaluationStrategy shouldBe EvaluationStrategies.BY_REFERENCE diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala index 661c3bf6cb5e..b2c771250c7a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala @@ -1,6 +1,7 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext -import io.shiftleft.semanticcpg.language._ + +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, nodes} import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala index 034b5183c0b7..66afb379fa0b 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala @@ -1,8 +1,16 @@ package io.joern.pysrc2cpg.dataflow +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.toExtendedCfgNode -import io.joern.dataflowengineoss.semanticsloader.{FlowMapping, FlowSemantic, PassThroughMapping} -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.dataflowengineoss.semanticsloader.{ + FlowMapping, + FlowSemantic, + NilSemantics, + NoCrossTaintSemantics, + NoSemantics, + PassThroughMapping +} +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Literal, Member, Method} import io.shiftleft.semanticcpg.language.* @@ -63,7 +71,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List()))) + .withSemantics(DefaultSemantics().after(NilSemantics.where(List("helpers.py:.foo")))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -76,7 +84,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(0, 0))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(0, 0)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -89,7 +97,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(1, 1))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(1, 1)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -101,7 +109,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |from helpers import foo |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List()))) + .withSemantics(DefaultSemantics().after(NilSemantics.where(List("helpers.py:.foo")))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -113,7 +121,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |from helpers import foo |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(0, 0))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(0, 0)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -125,7 +133,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |from helpers import foo |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(1, 1))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("helpers.py:.foo", List(FlowMapping(1, 1)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -140,7 +148,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List()))) + .withSemantics(DefaultSemantics().after(NilSemantics.where(List("Test0.py:.foo")))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -155,7 +163,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(0, 0))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(0, 0)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -170,7 +178,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |a = 20 |print(foo(a)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(1, 1))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(1, 1)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -184,7 +192,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { | |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List()))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("Test0.py:.foo", List())))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -198,7 +206,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { | |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(0, 0))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(0, 0)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l @@ -212,13 +220,26 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { | |print(foo(20)) |""".stripMargin) - .withExtraFlows(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(1, 1))))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic("Test0.py:.foo", List(FlowMapping(1, 1)))))) val source = cpg.literal("20").l val sink = cpg.call("print").argument(1).l val flows = sink.reachableByFlows(source).l flows shouldBe empty } + "don't taint the return value when specifying a named argument" in { + val cpg = code(""" + |import foo + |foo.bar(foo.baz(A=1)) + |""".stripMargin) + // The taint spec for `baz` here says that its argument "A" only taints itself. This is to make sure + // its return value is not tainted even when we are using `-1` in the spec. + .withSemantics(DefaultSemantics().plus(List(FlowSemantic(".*baz", List(FlowMapping(-1, "A", -1, "A")), true)))) + val one = cpg.literal("1") + val bar = cpg.call("bar").argument + bar.reachableByFlows(one).map(flowToResultPairs) shouldBe empty + } + "chained call" in { val cpg: Cpg = code(""" |a = 42 @@ -660,7 +681,7 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { |x = {'x': 10} |print(1, x) |""".stripMargin) - .withExtraFlows(List(FlowSemantic(".*print", List(PassThroughMapping), true))) + .withSemantics(DefaultSemantics().plus(List(FlowSemantic(".*print", List(PassThroughMapping), true)))) def source = cpg.literal def sink = cpg.call("print").argument.argumentIndex(2) @@ -839,21 +860,223 @@ class DataFlowTests extends PySrc2CpgFixture(withOssDataflow = true) { ) } + "flow from literal to an external method's named argument using two same-methodFullNamed semantics" in { + val cpg = code(""" + |import bar + |x = 'foobar' + |bar.foo(Baz=x) + |""".stripMargin) + .withSemantics( + DefaultSemantics().plus( + List( + // Equivalent to a single `FlowSemantic` entry with both FlowMappings + FlowSemantic("bar.py:.foo", List(PassThroughMapping)), + FlowSemantic("bar.py:.foo", List(FlowMapping(0, 0))) + ) + ) + ) + + val source = cpg.literal("'foobar'") + val sink = cpg.call("foo").argument.argumentName("Baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("x = 'foobar'", 3), ("bar.foo(Baz = x)", 4)) + ) + } + +} + +class DefaultSemanticsDataFlowTest1 extends PySrc2CpgFixture(withOssDataflow = true, semantics = DefaultSemantics()) { + + "DefaultSemantics cross-taints arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |bar.foo(b, Z=a) + |bar.baz(b) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("a = 1", 3), ("bar.foo(b, Z = a)", 4), ("bar.baz(b)", 5)) + ) + } + + "DefaultSemantics taints external method call return values" in { + val cpg = code(""" + |import bar + |y = 1 + |x = bar.foo(y) + |bar.baz(x) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("y = 1", 3), ("bar.foo(y)", 4), ("x = bar.foo(y)", 4), ("bar.baz(x)", 5)) + ) + } + +} + +class NoSemanticsDataFlowTest1 extends PySrc2CpgFixture(withOssDataflow = true, semantics = NoSemantics) { + + "NoSemantics cross-taints arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |bar.foo(b, Z=a) + |bar.baz(b) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("a = 1", 3), ("bar.foo(b, Z = a)", 4), ("bar.baz(b)", 5)) + ) + } + + "NoSemantics taints external method call return values" in { + val cpg = code(""" + |import bar + |y = 1 + |x = bar.foo(y) + |bar.baz(x) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("y = 1", 3), ("bar.foo(y)", 4), ("x = bar.foo(y)", 4), ("bar.baz(x)", 5)) + ) + } +} + +class NilSemanticsDataFlowTest1 + extends PySrc2CpgFixture(withOssDataflow = true, semantics = NilSemantics().after(DefaultSemantics())) { + + "NilSemantics does not cross-taint arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |bar.foo(b, Z=a) + |bar.baz(b) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs) shouldBe empty + } + + "NilSemantics does not taint external method call return values" in { + val cpg = code(""" + |import bar + |y = 1 + |x = bar.foo(y) + |bar.baz(x) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz") + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List() + } +} + +class NoCrossTaintDataFlowTest1 + extends PySrc2CpgFixture( + withOssDataflow = true, + semantics = NoCrossTaintSemantics.where(_.fullName.contains("bar.py")).after(DefaultSemantics()) + ) { + + "NoCrossTaintSemantics prevents cross-tainting arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |bar.foo(b, Z=a) + |bar.baz(b) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("baz").argument.argumentIndex(1) + sink.reachableByFlows(source).map(flowToResultPairs) shouldBe empty + } + + "NoCrossTaintSemantics prevents cross-tainting same-call named-arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |b = 2 + |bar.foo(X=a, Y=b) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("foo").argument.argumentName("Y") + sink.reachableByFlows(source) shouldBe empty + } + + "NoCrossTaintSemantics prevents cross-tainting same-call arguments to external method calls" in { + val cpg = code(""" + |import bar + |a = 1 + |b = 2 + |bar.foo(A=b, a) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call("foo").argument.argumentName("A") + sink.reachableByFlows(source) shouldBe empty + } + + "NoCrossTaintSemantics taints return values" in { + val cpg = code(""" + |import bar + |a = 1 + |b = 2 + |c = bar.foo(X=a, b) + |print(c) + |""".stripMargin) + val source = cpg.literal.lineNumber(3, 4) + val sink = cpg.call("print").argument + sink.reachableByFlows(source).map(flowToResultPairs).sorted shouldBe List( + List(("a = 1", 3), ("bar.foo(b, X = a)", 5), ("c = bar.foo(b, X = a)", 5), ("print(c)", 6)), + List(("b = 2", 4), ("bar.foo(b, X = a)", 5), ("c = bar.foo(b, X = a)", 5), ("print(c)", 6)) + ) + } +} + +class NoCrossTaintDataFlowTest2 + extends PySrc2CpgFixture( + withOssDataflow = true, + semantics = NoCrossTaintSemantics.where(_.fullName.contains("foo")).after(DefaultSemantics()) + ) { + + "NoCrossTaintSemantics works for specific external method call" in { + val cpg = code(""" + |import bar + |a = 1 + |bar.foo(a,b) # foo has no-cross-taint semantics, so b is not tainted by a + |bar.baz(a,c) # however, baz has default semantics, so c is tainted by a + |print(b) + |print(c) + |""".stripMargin) + val source = cpg.literal("1") + val sink = cpg.call.name("print").argument.argumentIndex(1) + // Note: it's unfortunate that `(bar.foo(a, b), 4)` still shows up in this flow. + // However, we can check that NoCrossTaintSemantics is doing its job, as otherwise + // we'd also have a `print(b)` sink. + sink.reachableByFlows(source).map(flowToResultPairs).l shouldBe List( + List(("a = 1", 3), ("bar.foo(a, b)", 4), ("bar.baz(a, c)", 5), ("print(c)", 7)) + ) + } + } class RegexDefinedFlowsDataFlowTests extends PySrc2CpgFixture( withOssDataflow = true, - extraFlows = List( - FlowSemantic.from("^path.*\\.sanitizer$", List((0, 0), (1, 1)), regex = true), - FlowSemantic.from("^foo.*\\.sanitizer.*", List((0, 0), (1, 1)), regex = true), - FlowSemantic.from("^foo.*\\.create_sanitizer\\.\\.sanitize", List((0, 0), (1, 1)), regex = true), - FlowSemantic - .from( - "requests.py:.post", - List((0, 0), (1, "url", -1), (2, "body", -1), (1, "url", 1, "url"), (2, "body", 2, "body")) - ), - FlowSemantic.from("cross_taint.py:.go", List((0, 0), (1, 1), (1, "a", 2, "b"))) + semantics = DefaultSemantics().plus( + List( + FlowSemantic.from("^path.*\\.sanitizer$", List((0, 0), (1, 1)), regex = true), + FlowSemantic.from("^foo.*\\.sanitizer.*", List((0, 0), (1, 1)), regex = true), + FlowSemantic.from("^foo.*\\.create_sanitizer\\.\\.sanitize", List((0, 0), (1, 1)), regex = true), + FlowSemantic + .from( + "requests.py:.post", + List((0, 0), (1, "url", -1), (2, "body", -1), (1, "url", 1, "url"), (2, "body", 2, "body")) + ), + FlowSemantic.from("cross_taint.py:.go", List((0, 0), (1, 1), (1, "a", 2, "b"))) + ) ) ) { @@ -969,8 +1192,8 @@ class RegexDefinedFlowsDataFlowTests |print(Foo.func()) |""".stripMargin) "be found" in { - val src = cpg.call.code("Foo.func").l - val snk = cpg.call("print").l + val src = cpg.call.code("Foo.func") + val snk = cpg.call("print") snk.argument.reachableByFlows(src).size shouldBe 1 } } diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..732674ecdeb0 --- /dev/null +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala @@ -0,0 +1,82 @@ +package io.joern.pysrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class PySrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("pysrc2cpgTestsHttpTest") + val file = dir / "main.py" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |def main(): + | print($indexStr) + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.pysrc2cpg.NewMain.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.pysrc2cpg.NewMain.stop() + } + + "Using pysrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("pysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("print()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("pysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"print($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala index 930affce1d65..c8c993f5d995 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture +import io.shiftleft.semanticcpg.language.* class ConfigPassTests extends PySrc2CpgFixture(withOssDataflow = false) { @@ -12,7 +12,51 @@ class ConfigPassTests extends PySrc2CpgFixture(withOssDataflow = false) { c.content shouldBe "Flask==1.1.2" c.name shouldBe "requirements.txt" } + } + + "Pipfile should be included" in { + val cpg = code( + """ + |[[source]] + |url = "https://pypi.org/simple" + |verify_ssl = true + |name = "pypi" + |""".stripMargin, + "Pipfile" + ) + + val config = cpg.configFile.name("Pipfile").head + config.content should include("verify_ssl = true") + } + + "Pipfile.lock should be included" in { + val cpg = code( + """ + |{ + | "_meta": { + | "hash": { + | "sha256": "293ad83ead15eb7bfef8a768f1853fc4cfa31b32ab85ae6962a2630b57cf569b" + | }, + | "pipfile-spec": 6, + | "requires": { + | "python_full_version": "3.8.18", + | "python_version": "3.8" + | }, + | "sources": [ + | { + | "name": "pypi", + | "url": "https://pypi.org/simple", + | "verify_ssl": true + | } + | ] + | } + |} + |""".stripMargin, + "Pipfile.lock" + ) + val config = cpg.configFile.name("Pipfile.lock").head + config.content should include("\"name\": \"pypi\"") } } diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala index 284fb6b27408..8db531070350 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala index 1a8f338cb498..8cb67c3aee6a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture +import io.shiftleft.semanticcpg.language.* class ImportsPassTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala index 71e90af7761e..10611836281c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture -import io.shiftleft.semanticcpg.language._ +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture +import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala index 8ae43b3361ba..a564993e2571 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Member} @@ -1051,7 +1051,7 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) { "assert the method properties in RedisDB, especially quoted type hints" in { val Some(redisDB) = cpg.typeDecl.nameExact("RedisDB").method.nameExact("").headOption: @unchecked - val List(instanceM, getRedisM, setM) = redisDB.astOut.isMethod.nameExact("instance", "get_redis", "set").l + val List(instanceM, getRedisM, setM) = redisDB.astChildren.isMethod.nameExact("instance", "get_redis", "set").l instanceM.methodReturn.typeFullName shouldBe Seq("db", "redis.py:.RedisDB").mkString(File.separator) getRedisM.methodReturn.typeFullName shouldBe "aioredis.py:.Redis" @@ -1324,12 +1324,9 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) { val variables = cpg.moduleVariables .where(_.typeFullName(".*FastAPI.*")) .l - val appIncludeRouterCalls = - variables.invokingCalls - .nameExact("include_router") - .l - val includedRouters = appIncludeRouterCalls.argument.argumentIndexGte(1).moduleVariables.l - val definitionsOfRouters = includedRouters.definitions.whereNot(_.source.isCall.nameExact("import")).l + val appIncludeRouterCalls = variables.invokingCalls.nameExact("include_router") + val includedRouters = appIncludeRouterCalls.argument.argumentIndexGte(1).moduleVariables + val definitionsOfRouters = includedRouters.definitions.whereNot(_.source.isCall.nameExact("import")) val List(adminRouter, normalRouter, itemsRouter) = definitionsOfRouters.map(x => (x.code, x.method.fullName)).sortBy(_._1).l: @unchecked @@ -1641,4 +1638,28 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) { } } } + + "external method named `import_table`" should { + val cpg = code(""" + |import boto3 + |client = boto3.client("s3") + |response = client.import_table() + |""".stripMargin) + + "have correct methodFullName for `import_table" in { + cpg.call("import_table").l match { + case List(importTable) => + importTable.methodFullName shouldBe "boto3.py:.client..import_table" + case result => fail(s"Expected single call to import_table, but got $result") + } + } + + "provide meaningful typeFullName for `response`" in { + cpg.assignment.target.isIdentifier.name("response").l match { + case List(response) => + response.typeFullName shouldBe "boto3.py:.client..import_table." + case result => fail(s"Expected single assignment to response, but got $result") + } + } + } } diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala similarity index 95% rename from joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala rename to joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala index 6b3facb4d231..ff1526453a45 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala @@ -1,5 +1,6 @@ -package io.joern.pysrc2cpg +package io.joern.pysrc2cpg.testfixtures +import io.joern.pysrc2cpg.Py2Cpg import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.X2Cpg.defaultOverlayCreators import io.shiftleft.codepropertygraph.generated.Cpg diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala similarity index 60% rename from joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala rename to joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala index a9b8a22e450a..177f456fde34 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala @@ -1,26 +1,26 @@ -package io.joern.pysrc2cpg +package io.joern.pysrc2cpg.testfixtures import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.layers.dataflows.* -import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} -import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} -import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.frontendspecific.pysrc2cpg.{ - DynamicTypeHintFullNamePass, - ImportsPass, - PythonImportResolverPass, - PythonInheritanceNamePass, - PythonTypeHintCallLinker, - PythonTypeRecoveryPassGenerator -} +import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.testfixtures.SemanticCpgTestFixture +import io.joern.dataflowengineoss.testfixtures.SemanticTestCpg +import io.joern.pysrc2cpg.Py2CpgOnFileSystem +import io.joern.pysrc2cpg.Py2CpgOnFileSystemConfig +import io.joern.x2cpg.frontendspecific.pysrc2cpg.DynamicTypeHintFullNamePass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.ImportsPass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonImportResolverPass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonInheritanceNamePass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonTypeHintCallLinker +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonTypeRecoveryPassGenerator import io.joern.x2cpg.passes.base.AstLinkerPass import io.joern.x2cpg.passes.callgraph.NaiveCallLinker -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg} +import io.joern.x2cpg.testfixtures.Code2CpgFixture +import io.joern.x2cpg.testfixtures.DefaultTestCpg +import io.joern.x2cpg.testfixtures.LanguageFrontend import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import io.shiftleft.semanticcpg.layers.LayerCreatorContext +import io.shiftleft.semanticcpg.language.ICallResolver +import io.shiftleft.semanticcpg.language.NoResolve trait PythonFrontend extends LanguageFrontend { override val fileSuffix: String = ".py" @@ -48,7 +48,7 @@ class PySrcTestCpg extends DefaultTestCpg with PythonFrontend with SemanticTestC new PythonTypeHintCallLinker(this).createAndApply() new NaiveCallLinker(this).createAndApply() - // Some of passes above create new methods, so, we + // Some of the passes above create new methods, so, we // need to run the ASTLinkerPass one more time new AstLinkerPass(this).createAndApply() applyOssDataFlow() @@ -58,20 +58,20 @@ class PySrcTestCpg extends DefaultTestCpg with PythonFrontend with SemanticTestC class PySrc2CpgFixture( withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), withPostProcessing: Boolean = true ) extends Code2CpgFixture(() => new PySrcTestCpg() .withOssDataflow(withOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) - with SemanticCpgTestFixture(extraFlows) { + with SemanticCpgTestFixture(semantics) { implicit val resolver: ICallResolver = NoResolve protected def flowToResultPairs(path: Path): List[(String, Integer)] = - path.resultPairs().collect { case (firstElement: String, secondElement: Option[Integer]) => + path.resultPairs().collect { case (firstElement: String, secondElement) => (firstElement, secondElement.getOrElse(-1)) } } diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt index 9f2372e82bb6..78f72c6531f7 100644 --- a/joern-cli/frontends/rubysrc2cpg/build.sbt +++ b/joern-cli/frontends/rubysrc2cpg/build.sbt @@ -65,4 +65,4 @@ joernTypeStubsDlTask := { Compile / compile := ((Compile / compile) dependsOn joernTypeStubsDlTask).value Universal / packageName := name.value -Universal / topLevelDirectory := None \ No newline at end of file +Universal / topLevelDirectory := None diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexer.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexer.g4 deleted file mode 100644 index 0aef432c8278..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexer.g4 +++ /dev/null @@ -1,897 +0,0 @@ -lexer grammar DeprecatedRubyLexer; - -// -------------------------------------------------------- -// Auxiliary tokens and features -// -------------------------------------------------------- - -@header { - package io.joern.rubysrc2cpg.deprecated.parser; -} - -tokens { - STRING_INTERPOLATION_END, - REGULAR_EXPRESSION_INTERPOLATION_END, - REGULAR_EXPRESSION_START, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - QUOTED_EXPANDED_REGULAR_EXPRESSION_END, - QUOTED_EXPANDED_STRING_LITERAL_END, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - DELIMITED_STRING_INTERPOLATION_END, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - - // The following tokens are created by `RubyLexerPostProcessor` only. - NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE, - EXPANDED_LITERAL_CHARACTER_SEQUENCE -} - -options { - superClass = DeprecatedRubyLexerBase; -} - -// -------------------------------------------------------- -// Keywords -// -------------------------------------------------------- - -LINE__:'__LINE__'; -ENCODING__: '__ENCODING__'; -FILE__: '__FILE__'; -BEGIN_: 'BEGIN'; -END_: 'END'; -ALIAS: 'alias'; -AND: 'and'; -BEGIN: 'begin'; -BREAK: 'break'; -CASE: 'case'; -CLASS: 'class'; -DEF: 'def'; -IS_DEFINED: 'defined?'; -DO: 'do'; -ELSE: 'else'; -ELSIF: 'elsif'; -END: 'end'; -ENSURE: 'ensure'; -FOR: 'for'; -FALSE: 'false'; -IF: 'if'; -IN: 'in'; -MODULE: 'module'; -NEXT: 'next'; -NIL: 'nil'; -NOT: 'not'; -OR: 'or'; -REDO: 'redo'; -RESCUE: 'rescue'; -RETRY: 'retry'; -RETURN: 'return'; -SELF: 'self'; -SUPER: 'super'; -THEN: 'then'; -TRUE: 'true'; -UNDEF: 'undef'; -UNLESS: 'unless'; -UNTIL: 'until'; -WHEN: 'when'; -WHILE: 'while'; -YIELD: 'yield'; - -fragment KEYWORD - : LINE__ - | ENCODING__ - | FILE__ - | BEGIN_ - | END_ - | ALIAS - | AND - | BEGIN - | BREAK - | CASE - | CLASS - | DEF - | IS_DEFINED - | DO - | ELSE - | ELSIF - | END - | ENSURE - | FOR - | FALSE - | IF - | IN - | MODULE - | NEXT - | NIL - | NOT - | OR - | REDO - | RESCUE - | RETRY - | RETURN - | SELF - | SUPER - | THEN - | TRUE - | UNDEF - | UNLESS - | UNTIL - | WHEN - | WHILE - | YIELD - ; - -// -------------------------------------------------------- -// Punctuators -// -------------------------------------------------------- - -LBRACK: '['; -RBRACK: ']'; -LPAREN: '('; -RPAREN: ')'; -LCURLY: '{'; -RCURLY: '}' - { - if (isEndOfInterpolation()) { - popMode(); - setType(popInterpolationEndTokenType()); - } - } -; -COLON: ':'; -COLON2: '::'; -COMMA: ','; -SEMI: ';'; -DOT: '.'; -DOT2: '..'; -DOT3: '...'; -QMARK: '?'; -EQGT: '=>'; -MINUSGT: '->'; - -fragment PUNCTUATOR - : LBRACK - | RBRACK - | LPAREN - | RPAREN - | LCURLY - | RCURLY - | COLON2 - | COMMA - | SEMI - | DOT2 - | DOT3 - | QMARK - | COLON - | EQGT - ; - -// -------------------------------------------------------- -// Operators -// -------------------------------------------------------- - -EMARK: '!'; -EMARKEQ: '!='; -EMARKTILDE: '!~'; -AMP: '&'; -AMP2: '&&'; -AMPDOT: '&.'; -BAR: '|'; -BAR2: '||'; -EQ: '='; -EQ2: '=='; -EQ3: '==='; -CARET: '^'; -LTEQGT: '<=>'; -EQTILDE: '=~'; -GT: '>'; -GTEQ: '>='; -LT: '<'; -LTEQ: '<='; -LT2: '<<'; -GT2: '>>'; -PLUS: '+'; -MINUS: '-'; -STAR: '*'; -STAR2: '**'; -SLASH: '/' - { - if (isStartOfRegexLiteral()) { - setType(REGULAR_EXPRESSION_START); - pushMode(REGULAR_EXPRESSION_MODE); - } - } -; -PERCENT: '%'; -TILDE: '~'; -// These tokens should only occur after a DEF token, as they are solely used to (re)define unary + and - operators. -// This way we won't emit the wrong token in e.g. `x+@y` (which means + between x and @y) -PLUSAT: '+@' {previousNonWsTokenTypeOrEOF() == DEF}?; -MINUSAT: '-@' {previousNonWsTokenTypeOrEOF() == DEF}?; - -ASSIGNMENT_OPERATOR - : ASSIGNMENT_OPERATOR_NAME '=' - ; - -fragment ASSIGNMENT_OPERATOR_NAME - : AMP - | AMP2 - | BAR - | BAR2 - | CARET - | LT2 - | GT2 - | PLUS - | MINUS - | STAR - | STAR2 - | PERCENT - | SLASH - ; - -fragment OPERATOR_METHOD_NAME - : CARET - | AMP - | BAR - | LTEQGT - | EQ2 - | EQ3 - | EQTILDE - | GT - | GTEQ - | LT - | LTEQ - | LT2 - | GT2 - | PLUS - | MINUS - | STAR - | SLASH - | PERCENT - | STAR2 - | TILDE - | PLUSAT - | MINUSAT - | '[]' - | '[]=' - ; - -// -------------------------------------------------------- -// String literals -// -------------------------------------------------------- - -SINGLE_QUOTED_STRING_LITERAL - : '\'' SINGLE_QUOTED_STRING_CHARACTER*? '\'' - ; - -fragment SINGLE_QUOTED_STRING_CHARACTER - : SINGLE_QUOTED_STRING_NON_ESCAPED_CHARACTER - | SINGLE_QUOTED_ESCAPE_SEQUENCE - ; - -fragment SINGLE_QUOTED_STRING_NON_ESCAPED_CHARACTER - : ~['\\] - ; - -fragment SINGLE_QUOTED_ESCAPE_SEQUENCE - : SINGLE_ESCAPE_CHARACTER_SEQUENCE - | SINGLE_QUOTED_STRING_NON_ESCAPED_CHARACTER_SEQUENCE - ; - -fragment SINGLE_ESCAPE_CHARACTER_SEQUENCE - : '\\' SINGLE_QUOTED_STRING_META_CHARACTER - ; - -fragment SINGLE_QUOTED_STRING_META_CHARACTER - : ['\\] - ; - -fragment SINGLE_QUOTED_STRING_NON_ESCAPED_CHARACTER_SEQUENCE - : '\\' SINGLE_QUOTED_STRING_NON_ESCAPED_CHARACTER - ; - -DOUBLE_QUOTED_STRING_START - : '"' - -> pushMode(DOUBLE_QUOTED_STRING_MODE) - ; - -QUOTED_NON_EXPANDED_STRING_LITERAL_START - : '%q' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_NON_EXPANDED_STRING_LITERAL_END); - _input.consume(); - } - -> pushMode(NON_EXPANDED_DELIMITED_STRING_MODE) - ; - -QUOTED_EXPANDED_STRING_LITERAL_START - : '%Q' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_EXPANDED_STRING_LITERAL_END); - _input.consume(); - pushMode(EXPANDED_DELIMITED_STRING_MODE); - } - // This check exists to prevent issuing a QUOTED_EXPANDED_STRING_LITERAL_START - // in obvious arithmetic expressions, such as `20 %(x+1)`. - // Note, however, that we can't have a perfect test at this stage. For instance, - // in `x = 1; x %(2)`, it's clear that's an arithmetic expression, but we - // will still emit a QUOTED_EXPANDED_STRING_LITERAL_START. - | '%(' {!isNumericTokenType(previousTokenTypeOrEOF())}? - { - pushQuotedDelimiter('('); - pushQuotedEndTokenType(QUOTED_EXPANDED_STRING_LITERAL_END); - pushMode(EXPANDED_DELIMITED_STRING_MODE); - } - ; - -QUOTED_EXPANDED_REGULAR_EXPRESSION_START - : '%r' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_EXPANDED_REGULAR_EXPRESSION_END); - _input.consume(); - } - -> pushMode(EXPANDED_DELIMITED_STRING_MODE) - ; - -QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START - : '%x' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END); - _input.consume(); - } - -> pushMode(EXPANDED_DELIMITED_STRING_MODE) - ; - -// -------------------------------------------------------- -// String (Word) array literals -// -------------------------------------------------------- - -QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START - : '%w' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END); - _input.consume(); - } - -> pushMode(NON_EXPANDED_DELIMITED_ARRAY_MODE) - ; - -QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START - : '%W' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END); - _input.consume(); - } - -> pushMode(EXPANDED_DELIMITED_ARRAY_MODE) - ; - -// -------------------------------------------------------- -// Here doc literals -// -------------------------------------------------------- - -HERE_DOC_IDENTIFIER - : '<<' [-~]? [\t]* IDENTIFIER - ; - -HERE_DOC - : '<<' [-~]? [\t]* IDENTIFIER [a-zA-Z_0-9]* NL ( {!heredocEndAhead(getText())}? . )* [a-zA-Z_] [a-zA-Z_0-9]* - ; - -// -------------------------------------------------------- -// Symbol array literals -// -------------------------------------------------------- - -QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START - : '%i' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END); - _input.consume(); - } - -> pushMode(NON_EXPANDED_DELIMITED_ARRAY_MODE) - ; - -QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START - : '%I' {!Character.isAlphabetic(_input.LA(1))}? - { - pushQuotedDelimiter(_input.LA(1)); - pushQuotedEndTokenType(QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END); - _input.consume(); - } - -> pushMode(EXPANDED_DELIMITED_ARRAY_MODE) - ; - -// -------------------------------------------------------- -// Data section -// -------------------------------------------------------- - -END_OF_PROGRAM_MARKER - : '__END__' {getCharPositionInLine() == 7}? '\r'? '\n' - -> pushMode(DATA_SECTION_MODE), skip - ; - -// -------------------------------------------------------- -// Numeric literals -// -------------------------------------------------------- - -DECIMAL_INTEGER_LITERAL - : UNPREFIXED_DECIMAL_INTEGER_LITERAL - | PREFIXED_DECIMAL_INTEGER_LITERAL - ; - -BINARY_INTEGER_LITERAL - : '0' [bB] BINARY_DIGIT ('_'? BINARY_DIGIT)* - ; - -OCTAL_INTEGER_LITERAL - : '0' [_oO]? OCTAL_DIGIT ('_'? OCTAL_DIGIT)* - ; - -HEXADECIMAL_INTEGER_LITERAL - : '0' [xX] HEXADECIMAL_DIGIT ('_'? HEXADECIMAL_DIGIT)* - ; - -FLOAT_LITERAL_WITHOUT_EXPONENT - : UNPREFIXED_DECIMAL_INTEGER_LITERAL '.' DIGIT_DECIMAL_PART - ; - -FLOAT_LITERAL_WITH_EXPONENT - : SIGNIFICAND_PART EXPONENT_PART - ; - -fragment UNPREFIXED_DECIMAL_INTEGER_LITERAL - : '0' - | DECIMAL_DIGIT_EXCEPT_0 ('_'? DECIMAL_DIGIT)* - ; - -fragment PREFIXED_DECIMAL_INTEGER_LITERAL - : '0' [dD] DIGIT_DECIMAL_PART - ; - -fragment SIGNIFICAND_PART - : FLOAT_LITERAL_WITHOUT_EXPONENT - | UNPREFIXED_DECIMAL_INTEGER_LITERAL - ; - -fragment EXPONENT_PART - : [eE] ('+' | '-')? DIGIT_DECIMAL_PART - ; - -fragment BINARY_DIGIT - : [0-1] - ; - -fragment OCTAL_DIGIT - : [0-7] - ; - -fragment DIGIT_DECIMAL_PART - : DECIMAL_DIGIT ('_'? DECIMAL_DIGIT)* - ; - -fragment DECIMAL_DIGIT - : [0-9] - ; - -fragment DECIMAL_DIGIT_EXCEPT_0 - : [1-9] - ; - -fragment HEXADECIMAL_DIGIT - : DECIMAL_DIGIT - | [a-f] - | [A-F] - ; - -// -------------------------------------------------------- -// Whitespaces -// -------------------------------------------------------- - -NL: LINE_TERMINATOR+; -WS: WHITESPACE+; - -fragment WHITESPACE - : [\u0009] - | [\u000b] - | [\u000c] - | [\u000d] - | [\u0020] - | LINE_TERMINATOR_ESCAPE_SEQUENCE - ; - -fragment LINE_TERMINATOR_ESCAPE_SEQUENCE - : '\\' LINE_TERMINATOR - ; - -fragment LINE_TERMINATOR - : '\r'? '\n' - ; - -// -------------------------------------------------------- -// Symbols -// -------------------------------------------------------- - -SYMBOL_LITERAL - : ':' (SYMBOL_NAME | (CONSTANT_IDENTIFIER | LOCAL_VARIABLE_IDENTIFIER) '=') - // This check exists to prevent issuing a SYMBOL_LITERAL in whitespace-free associations, e.g. - // in `foo(x:y)`, so that `:y` is not a SYMBOL_LITERAL - // or in `{:x=>1}`, so that `:x=` is not a SYMBOL_LITERAL - {previousTokenTypeOrEOF() != LOCAL_VARIABLE_IDENTIFIER && _input.LA(1) != '>'}? - ; - -fragment SYMBOL_NAME - : INSTANCE_VARIABLE_IDENTIFIER - | GLOBAL_VARIABLE_IDENTIFIER - | CLASS_VARIABLE_IDENTIFIER - | CONSTANT_IDENTIFIER - | LOCAL_VARIABLE_IDENTIFIER - | METHOD_ONLY_IDENTIFIER - | OPERATOR_METHOD_NAME - | KEYWORD - // NOTE: Even though we have PLUSAT and MINUSAT in OPERATOR_METHOD_NAME, the former - // are not emitted unless there's a DEF token before them, cf. their predicate. - // Thus, we need to add them explicitly here in order to recognize standalone SYMBOL_LITERAL tokens as well. - | '+@' - | '-@' - ; - -// -------------------------------------------------------- -// Identifiers -// -------------------------------------------------------- - -LOCAL_VARIABLE_IDENTIFIER - : (LOWERCASE_CHARACTER | '_') IDENTIFIER_CHARACTER* - ; - -GLOBAL_VARIABLE_IDENTIFIER - : '$' IDENTIFIER_START_CHARACTER IDENTIFIER_CHARACTER* - | '$' [0-9]+ - ; - -INSTANCE_VARIABLE_IDENTIFIER - : '@' IDENTIFIER_START_CHARACTER IDENTIFIER_CHARACTER* - ; - -CLASS_VARIABLE_IDENTIFIER - : '@@' IDENTIFIER_START_CHARACTER IDENTIFIER_CHARACTER* - ; - -CONSTANT_IDENTIFIER - : UPPERCASE_CHARACTER IDENTIFIER_CHARACTER* - ; - -fragment METHOD_ONLY_IDENTIFIER - : (CONSTANT_IDENTIFIER | LOCAL_VARIABLE_IDENTIFIER) ('!' | '?') - ; - - -// Similarly to PLUSAT/MINUSAT, this should only occur after a DEF token. -// Otherwise, the assignment `x=nil` would be parsed as (ASSIGNMENT_LIKE_METHOD_IDENTIFIER, NIL) -// instead of the more appropriate (LOCAL_VARIABLE_IDENTIFIER, EQ, NIL). -ASSIGNMENT_LIKE_METHOD_IDENTIFIER - : (CONSTANT_IDENTIFIER | LOCAL_VARIABLE_IDENTIFIER) '=' {previousNonWsTokenTypeOrEOF() == DEF}? - ; - -fragment IDENTIFIER_CHARACTER - : IDENTIFIER_START_CHARACTER - | DECIMAL_DIGIT - ; - -fragment IDENTIFIER_START_CHARACTER - : LOWERCASE_CHARACTER - | UPPERCASE_CHARACTER - | '_' - ; - -fragment LOWERCASE_CHARACTER - : [a-z] - ; - -fragment UPPERCASE_CHARACTER - : [A-Z] - ; - -fragment IDENTIFIER - : LOCAL_VARIABLE_IDENTIFIER - | GLOBAL_VARIABLE_IDENTIFIER - | CLASS_VARIABLE_IDENTIFIER - | INSTANCE_VARIABLE_IDENTIFIER - | CONSTANT_IDENTIFIER - | METHOD_ONLY_IDENTIFIER - | ASSIGNMENT_LIKE_METHOD_IDENTIFIER - ; - -// -------------------------------------------------------- -// Comments (are skipped) -// -------------------------------------------------------- - -SINGLE_LINE_COMMENT - : '#' COMMENT_CONTENT? - -> skip; - -MULTI_LINE_COMMENT - : MULTI_LINE_COMMENT_BEGIN_LINE .*? MULTI_LINE_COMMENT_END_LINE - -> skip; - -fragment COMMENT_CONTENT - : (~[\r\n])+ // Meaning (~LINE_TERMINATOR)+ - ; - -fragment MULTI_LINE_COMMENT_BEGIN_LINE - : '=begin' {getCharPositionInLine() == 6}? REST_OF_BEGIN_END_LINE? LINE_TERMINATOR - ; - -fragment MULTI_LINE_COMMENT_END_LINE - : '=end' {getCharPositionInLine() == 4}? REST_OF_BEGIN_END_LINE? (LINE_TERMINATOR | EOF) - ; - -fragment REST_OF_BEGIN_END_LINE - : WHITESPACE+ COMMENT_CONTENT - ; - -// -------------------------------------------------------- -// Unrecognized characters -// -------------------------------------------------------- - -// Any other character shall still be recognized so that the -// recovery mechanism in `io.joern.rubysrc2cpg.astcreation.AntlrParser` -// also handles them. Otherwise, the lexer would complain, not emit -// and the recovery mechanism would not be able to act. - -// Note: this must be the very last rule in this lexer specification, as -// otherwise this token would take precedence over any token defined after. -UNRECOGNIZED - : . - ; - -// -------------------------------------------------------- -// Double quoted string mode -// -------------------------------------------------------- - -mode DOUBLE_QUOTED_STRING_MODE; - -DOUBLE_QUOTED_STRING_END - : '"' - -> popMode - ; - -DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE - : DOUBLE_QUOTED_STRING_CHARACTER+ - ; - -fragment INTERPOLATED_CHARACTER_SEQUENCE_FRAGMENT - : '#' GLOBAL_VARIABLE_IDENTIFIER - | '#' CLASS_VARIABLE_IDENTIFIER - | '#' INSTANCE_VARIABLE_IDENTIFIER - ; - -INTERPOLATED_CHARACTER_SEQUENCE - : INTERPOLATED_CHARACTER_SEQUENCE_FRAGMENT - ; - -STRING_INTERPOLATION_BEGIN - : '#{' - { - pushInterpolationEndTokenType(STRING_INTERPOLATION_END); - pushMode(DEFAULT_MODE); - } - ; - -fragment DOUBLE_QUOTED_STRING_CHARACTER - : ~["#\\] - | '#' {_input.LA(1) != '$' && _input.LA(1) != '@' && _input.LA(1) != '{'}? - | DOUBLE_ESCAPE_SEQUENCE - ; - -fragment DOUBLE_ESCAPE_SEQUENCE - : SIMPLE_ESCAPE_SEQUENCE - | NON_ESCAPED_SEQUENCE - | LINE_TERMINATOR_ESCAPE_SEQUENCE - | OCTAL_ESCAPE_SEQUENCE - | HEXADECIMAL_ESCAPE_SEQUENCE - | CONTROL_ESCAPE_SEQUENCE - ; - -fragment CONTROL_ESCAPE_SEQUENCE - : '\\' ('C-' | 'c') CONTROL_ESCAPED_CHARACTER - ; - -fragment CONTROL_ESCAPED_CHARACTER - : DOUBLE_ESCAPE_SEQUENCE - | '?' - | ~[\\?] - ; - -fragment OCTAL_ESCAPE_SEQUENCE - : '\\' OCTAL_DIGIT OCTAL_DIGIT? OCTAL_DIGIT? - ; - -fragment HEXADECIMAL_ESCAPE_SEQUENCE - : '\\x' HEXADECIMAL_DIGIT HEXADECIMAL_DIGIT? - ; - -fragment NON_ESCAPED_SEQUENCE - : '\\' NON_ESCAPED_DOUBLE_QUOTED_STRING_CHARACTER - ; - -fragment NON_ESCAPED_DOUBLE_QUOTED_STRING_CHARACTER - : ~[\r\nA-Za-z0-9] - ; - -fragment SIMPLE_ESCAPE_SEQUENCE - : '\\' DOUBLE_ESCAPED_CHARACTER - ; - -fragment DOUBLE_ESCAPED_CHARACTER - : [ntrfvaebsu] - ; - -// -------------------------------------------------------- -// Expanded delimited string mode -// -------------------------------------------------------- - -mode EXPANDED_DELIMITED_STRING_MODE; - -DELIMITED_STRING_INTERPOLATION_BEGIN - : '#{' - { - pushInterpolationEndTokenType(DELIMITED_STRING_INTERPOLATION_END); - pushMode(DEFAULT_MODE); - } - ; - -EXPANDED_VARIABLE_CHARACTER_SEQUENCE - : INTERPOLATED_CHARACTER_SEQUENCE_FRAGMENT - ; - -EXPANDED_LITERAL_CHARACTER - : NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE - | NON_ESCAPED_LITERAL_CHARACTER - { - consumeQuotedCharAndMaybePopMode(_input.LA(-1)); - } - ; - -// -------------------------------------------------------- -// Non-expanded delimited string mode -// -------------------------------------------------------- - -mode NON_EXPANDED_DELIMITED_STRING_MODE; - - -fragment NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE - : '\\' NON_ESCAPED_LITERAL_CHARACTER - ; - -fragment NON_ESCAPED_LITERAL_CHARACTER - : ~[\r\n] - | '\n' {_input.LA(1) != '\r'}? - ; - -NON_EXPANDED_LITERAL_CHARACTER - : NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE - | NON_ESCAPED_LITERAL_CHARACTER - { - consumeQuotedCharAndMaybePopMode(_input.LA(-1)); - } - ; - -// -------------------------------------------------------- -// Expanded delimited array mode -// -------------------------------------------------------- - -mode EXPANDED_DELIMITED_ARRAY_MODE; - -DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN - : '#{' - { - pushInterpolationEndTokenType(DELIMITED_ARRAY_ITEM_INTERPOLATION_END); - pushMode(DEFAULT_MODE); - } - ; - -EXPANDED_ARRAY_ITEM_SEPARATOR - : NON_EXPANDED_ARRAY_ITEM_DELIMITER - ; - -EXPANDED_ARRAY_ITEM_CHARACTER - : NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE - | NON_ESCAPED_LITERAL_CHARACTER - { - consumeQuotedCharAndMaybePopMode(_input.LA(-1)); - } - ; - -// -------------------------------------------------------- -// Non-expanded delimited array mode -// -------------------------------------------------------- - -mode NON_EXPANDED_DELIMITED_ARRAY_MODE; - -fragment NON_EXPANDED_ARRAY_ITEM_DELIMITER - : [\u0009] - | [\u000a] - | [\u000b] - | [\u000c] - | [\u000d] - | [\u0020] - | '\\' ('\r'? '\n') - ; - -NON_EXPANDED_ARRAY_ITEM_SEPARATOR - : NON_EXPANDED_ARRAY_ITEM_DELIMITER - ; - -NON_EXPANDED_ARRAY_ITEM_CHARACTER - : NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE - | NON_ESCAPED_LITERAL_CHARACTER - { - consumeQuotedCharAndMaybePopMode(_input.LA(-1)); - } - ; - -// -------------------------------------------------------- -// Regex literal mode -// -------------------------------------------------------- - -mode REGULAR_EXPRESSION_MODE; - -REGULAR_EXPRESSION_END - : '/' REGULAR_EXPRESSION_OPTION* - -> popMode - ; - -REGULAR_EXPRESSION_BODY - : REGULAR_EXPRESSION_CHARACTER+ - ; - -REGULAR_EXPRESSION_INTERPOLATION_BEGIN - : '#{' - { - pushInterpolationEndTokenType(REGULAR_EXPRESSION_INTERPOLATION_END); - pushMode(DEFAULT_MODE); - } - ; - -fragment REGULAR_EXPRESSION_OPTION - : [imxo] - ; - -fragment REGULAR_EXPRESSION_CHARACTER - : ~[/#\\] - | '#' {_input.LA(1) != '$' && _input.LA(1) != '@' && _input.LA(1) != '{'}? - | REGULAR_EXPRESSION_NON_ESCAPED_SEQUENCE - | REGULAR_EXPRESSION_ESCAPE_SEQUENCE - | LINE_TERMINATOR_ESCAPE_SEQUENCE - | INTERPOLATED_CHARACTER_SEQUENCE - ; - -fragment REGULAR_EXPRESSION_NON_ESCAPED_SEQUENCE - : '\\' REGULAR_EXPRESSION_NON_ESCAPED_CHARACTER - ; - -fragment REGULAR_EXPRESSION_NON_ESCAPED_CHARACTER - : ~[\r\n] - | '\n' {_input.LA(1) != '\r'}? - ; - -fragment REGULAR_EXPRESSION_ESCAPE_SEQUENCE - : '\\' '/' - ; - -// -------------------------------------------------------- -// Data section mode -// -------------------------------------------------------- - -mode DATA_SECTION_MODE; - -DATA_SECTION_CONTENT - : .*? EOF - -> popMode, skip - ; diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyParser.g4 deleted file mode 100644 index 6eb263e22a5e..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyParser.g4 +++ /dev/null @@ -1,758 +0,0 @@ -parser grammar DeprecatedRubyParser; - -@header { - package io.joern.rubysrc2cpg.deprecated.parser; -} - -options { - tokenVocab = DeprecatedRubyLexer; -} - -// -------------------------------------------------------- -// Program -// -------------------------------------------------------- - -program - : compoundStatement EOF - ; - -compoundStatement - : (SEMI | NL)* statements? (SEMI | NL)* - ; - -// -------------------------------------------------------- -// Statements -// -------------------------------------------------------- - -statements - : statement ((SEMI | NL)+ statement)* - ; - -statement - : ALIAS NL? definedMethodNameOrSymbol NL? definedMethodNameOrSymbol # aliasStatement - | UNDEF NL? definedMethodNameOrSymbol (COMMA NL? definedMethodNameOrSymbol)* # undefStatement - | BEGIN_ LCURLY compoundStatement RCURLY # beginStatement - | END_ LCURLY compoundStatement RCURLY # endStatement - | statement mod=(IF | UNLESS | WHILE | UNTIL | RESCUE) NL? statement # modifierStatement - | expressionOrCommand # expressionOrCommandStatement - ; - -// -------------------------------------------------------- -// Expressions -// -------------------------------------------------------- - -expressionOrCommand - : expression # expressionExpressionOrCommand - | (EMARK NL?)? invocationWithoutParentheses # invocationExpressionOrCommand - | NOT NL? expressionOrCommand # notExpressionOrCommand - | expressionOrCommand op=(OR | AND) NL? expressionOrCommand # orAndExpressionOrCommand - ; - -expression - : singleLeftHandSide op=(EQ | ASSIGNMENT_OPERATOR) NL? multipleRightHandSide # singleAssignmentExpression - | multipleLeftHandSide EQ NL? multipleRightHandSide # multipleAssignmentExpression - | primary # primaryExpression - | op=(TILDE | PLUS | EMARK) NL? expression # unaryExpression - | expression STAR2 NL? expression # powerExpression - | MINUS NL? expression # unaryMinusExpression - | expression op=(STAR | SLASH | PERCENT) NL? expression # multiplicativeExpression - | expression op=(PLUS | MINUS) NL? expression # additiveExpression - | expression op=(LT2 | GT2) NL? expression # bitwiseShiftExpression - | expression op=AMP NL? expression # bitwiseAndExpression - | expression op=(BAR | CARET) NL? expression # bitwiseOrExpression - | expression op=(GT | GTEQ | LT | LTEQ) NL? expression # relationalExpression - | expression op=(LTEQGT | EQ2 | EQ3 | EMARKEQ | EQTILDE | EMARKTILDE) NL? expression? # equalityExpression - | expression op=AMP2 NL? expression # operatorAndExpression - | expression op=BAR2 NL? expression # operatorOrExpression - | expression op=(DOT2 | DOT3) NL? expression? # rangeExpression - | expression QMARK NL? expression NL? COLON NL? expression # conditionalOperatorExpression - | IS_DEFINED NL? expression # isDefinedExpression - ; - -primary - : classDefinition # classDefinitionPrimary - | moduleDefinition # moduleDefinitionPrimary - | methodDefinition # methodDefinitionPrimary - | procDefinition # procDefinitionPrimary - | yieldWithOptionalArgument # yieldWithOptionalArgumentPrimary - | ifExpression # ifExpressionPrimary - | unlessExpression # unlessExpressionPrimary - | caseExpression # caseExpressionPrimary - | whileExpression # whileExpressionPrimary - | untilExpression # untilExpressionPrimary - | forExpression # forExpressionPrimary - | RETURN argumentsWithParentheses # returnWithParenthesesPrimary - | jumpExpression # jumpExpressionPrimary - | beginExpression # beginExpressionPrimary - | LPAREN compoundStatement RPAREN # groupingExpressionPrimary - | variableReference # variableReferencePrimary - | COLON2 CONSTANT_IDENTIFIER # simpleScopedConstantReferencePrimary - | primary COLON2 CONSTANT_IDENTIFIER # chainedScopedConstantReferencePrimary - | arrayConstructor # arrayConstructorPrimary - | hashConstructor # hashConstructorPrimary - | literal # literalPrimary - | stringExpression # stringExpressionPrimary - | stringInterpolation # stringInterpolationPrimary - | quotedStringExpression # quotedStringExpressionPrimary - | regexInterpolation # regexInterpolationPrimary - | quotedRegexInterpolation # quotedRegexInterpolationPrimary - | IS_DEFINED LPAREN expressionOrCommand RPAREN # isDefinedPrimary - | SUPER argumentsWithParentheses? block? # superExpressionPrimary - | primary LBRACK indexingArguments? RBRACK # indexingExpressionPrimary - | methodOnlyIdentifier # methodOnlyIdentifierPrimary - | methodIdentifier block # invocationWithBlockOnlyPrimary - | methodIdentifier argumentsWithParentheses block? # invocationWithParenthesesPrimary - | primary NL? (DOT | COLON2| AMPDOT) NL? methodName argumentsWithParentheses? block? # chainedInvocationPrimary - | primary COLON2 methodName block? # chainedInvocationWithoutArgumentsPrimary - ; - -// -------------------------------------------------------- -// Assignments -// -------------------------------------------------------- - -singleLeftHandSide - : variableIdentifier # variableIdentifierOnlySingleLeftHandSide - | primary LBRACK arguments? RBRACK # primaryInsideBracketsSingleLeftHandSide - | primary (DOT | COLON2) (LOCAL_VARIABLE_IDENTIFIER | CONSTANT_IDENTIFIER) # xdotySingleLeftHandSide - | COLON2 CONSTANT_IDENTIFIER # scopedConstantAccessSingleLeftHandSide - ; - -multipleLeftHandSide - : (multipleLeftHandSideItem COMMA NL?)+ (multipleLeftHandSideItem | packingLeftHandSide)? # multipleLeftHandSideAndpackingLeftHandSideMultipleLeftHandSide - | packingLeftHandSide # packingLeftHandSideOnlyMultipleLeftHandSide - | groupedLeftHandSide # groupedLeftHandSideOnlyMultipleLeftHandSide - ; - -multipleLeftHandSideItem - : singleLeftHandSide - | groupedLeftHandSide - ; - -packingLeftHandSide - : STAR singleLeftHandSide - ; - -groupedLeftHandSide - : LPAREN multipleLeftHandSide RPAREN - ; - -multipleRightHandSide - : expressionOrCommands (COMMA NL? splattingArgument)? - | splattingArgument - ; - -expressionOrCommands - : expressionOrCommand (COMMA NL? expressionOrCommand)* - ; - -// -------------------------------------------------------- -// Invocation expressions -// -------------------------------------------------------- - -invocationWithoutParentheses - : chainedCommandWithDoBlock # chainedCommandDoBlockInvocationWithoutParentheses - | command # singleCommandOnlyInvocationWithoutParentheses - | RETURN arguments? # returnArgsInvocationWithoutParentheses - | BREAK arguments # breakArgsInvocationWithoutParentheses - | NEXT arguments # nextArgsInvocationWithoutParentheses - ; - -command - : SUPER argumentsWithoutParentheses # superCommand - | YIELD argumentsWithoutParentheses # yieldCommand - | methodIdentifier argumentsWithoutParentheses # simpleMethodCommand - | primary (DOT | COLON2| AMPDOT) NL? methodName argumentsWithoutParentheses # memberAccessCommand - ; - -chainedCommandWithDoBlock - : commandWithDoBlock ((DOT | COLON2) methodName argumentsWithParentheses?)* - ; - -commandWithDoBlock - : SUPER argumentsWithoutParentheses doBlock # argsAndDoBlockCommandWithDoBlock - | methodIdentifier argumentsWithoutParentheses doBlock # argsAndDoBlockAndMethodIdCommandWithDoBlock - | primary (DOT | COLON2) methodName argumentsWithoutParentheses doBlock # primaryMethodArgsDoBlockCommandWithDoBlock - ; - -argumentsWithoutParentheses - : arguments - ; - -arguments - : argument (COMMA NL? argument)* - ; - -argument - : HERE_DOC_IDENTIFIER # hereDocArgument - | blockArgument # blockArgumentArgument - | splattingArgument # splattingArgumentArgument - | association # associationArgument - | expression # expressionArgument - | command # commandArgument - ; - -blockArgument - : AMP expression - ; - -// -------------------------------------------------------- -// Arguments -// -------------------------------------------------------- - -splattingArgument - : STAR expressionOrCommand - | STAR2 expressionOrCommand - ; - -indexingArguments - : expressions (COMMA NL?)? # expressionsOnlyIndexingArguments - | expressions COMMA NL? splattingArgument # expressionsAndSplattingIndexingArguments - | associations (COMMA NL?)? # associationsOnlyIndexingArguments - | splattingArgument # splattingOnlyIndexingArguments - | command # commandOnlyIndexingArguments - ; - -argumentsWithParentheses - : LPAREN NL? RPAREN # blankArgsArgumentsWithParentheses - | LPAREN NL? arguments (COMMA)? NL? RPAREN # argsOnlyArgumentsWithParentheses - | LPAREN NL? expressions COMMA NL? chainedCommandWithDoBlock NL? RPAREN # expressionsAndChainedCommandWithDoBlockArgumentsWithParentheses - | LPAREN NL? chainedCommandWithDoBlock NL? RPAREN # chainedCommandWithDoBlockOnlyArgumentsWithParentheses - ; - -expressions - : expression (COMMA NL? expression)* - ; - -// -------------------------------------------------------- -// Blocks -// -------------------------------------------------------- - -block - : braceBlock # braceBlockBlock - | doBlock # doBlockBlock - ; - -braceBlock - : LCURLY NL? blockParameter? bodyStatement RCURLY - ; - -doBlock - : DO NL? blockParameter? bodyStatement END - ; - -blockParameter - : BAR blockParameters? BAR - ; - -blockParameters - : singleLeftHandSide - | multipleLeftHandSide - ; - -// -------------------------------------------------------- -// Arrays -// -------------------------------------------------------- - -arrayConstructor - : LBRACK NL? indexingArguments? NL? RBRACK # bracketedArrayConstructor - | QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START - nonExpandedArrayElements? - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END # nonExpandedWordArrayConstructor - | QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START - nonExpandedArrayElements? - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END # nonExpandedSymbolArrayConstructor - | QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START - expandedArrayElements? - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END # expandedSymbolArrayConstructor - | QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START - expandedArrayElements? - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END # expandedWordArrayConstructor - ; - - -expandedArrayElements - : EXPANDED_ARRAY_ITEM_SEPARATOR* - expandedArrayElement (EXPANDED_ARRAY_ITEM_SEPARATOR+ expandedArrayElement)* - EXPANDED_ARRAY_ITEM_SEPARATOR* - ; - -expandedArrayElement - : (EXPANDED_ARRAY_ITEM_CHARACTER | delimitedArrayItemInterpolation)+ - ; - -delimitedArrayItemInterpolation - : DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN - compoundStatement - DELIMITED_ARRAY_ITEM_INTERPOLATION_END - ; - -nonExpandedArrayElements - : NON_EXPANDED_ARRAY_ITEM_SEPARATOR* - nonExpandedArrayElement (NON_EXPANDED_ARRAY_ITEM_SEPARATOR+ nonExpandedArrayElement)* - NON_EXPANDED_ARRAY_ITEM_SEPARATOR* - ; - -nonExpandedArrayElement - : NON_EXPANDED_ARRAY_ITEM_CHARACTER+ - ; - -// -------------------------------------------------------- -// Hashes -// -------------------------------------------------------- - -hashConstructor - : LCURLY NL? (hashConstructorElements COMMA?)? NL? RCURLY - ; - -hashConstructorElements - : hashConstructorElement (COMMA NL? hashConstructorElement)* - ; - -hashConstructorElement - : association - | STAR2 expression - ; - -associations - : association (COMMA NL? association)* - ; - -association - : (expression | keyword) (EQGT|COLON) (NL? expression)? - ; - -// -------------------------------------------------------- -// Method definitions -// -------------------------------------------------------- - -methodDefinition - : DEF NL? methodNamePart methodParameterPart bodyStatement END - | DEF NL? methodIdentifier methodParameterPart EQ NL? expression - ; - - -procDefinition - : MINUSGT (LPAREN parameters? RPAREN)? block - ; - -methodNamePart - : definedMethodName # simpleMethodNamePart - | singletonObject NL? (DOT | COLON2) NL? definedMethodName # singletonMethodNamePart - ; - -singletonObject - : variableIdentifier - | pseudoVariableIdentifier - | LPAREN expressionOrCommand RPAREN - ; - -definedMethodName - : methodName - | assignmentLikeMethodIdentifier - ; - -assignmentLikeMethodIdentifier - : ASSIGNMENT_LIKE_METHOD_IDENTIFIER - ; - -methodName - : methodIdentifier - | operatorMethodName - | keyword - ; - -methodIdentifier - : LOCAL_VARIABLE_IDENTIFIER - | CONSTANT_IDENTIFIER - | methodOnlyIdentifier - ; - -methodOnlyIdentifier - : (LOCAL_VARIABLE_IDENTIFIER | CONSTANT_IDENTIFIER | keyword) (EMARK | QMARK) - ; - -methodParameterPart - : LPAREN NL? parameters? NL? RPAREN - | parameters? - ; - -parameters - : parameter (COMMA NL? parameter)* - ; - -parameter - : optionalParameter - | mandatoryParameter - | arrayParameter - | hashParameter - | keywordParameter - | procParameter - ; - -mandatoryParameter - : LOCAL_VARIABLE_IDENTIFIER - ; - -optionalParameter - : LOCAL_VARIABLE_IDENTIFIER EQ NL? expression - ; - -arrayParameter - : STAR LOCAL_VARIABLE_IDENTIFIER? - ; - -hashParameter - : STAR2 LOCAL_VARIABLE_IDENTIFIER? - ; - -keywordParameter - : LOCAL_VARIABLE_IDENTIFIER COLON (NL? expression)? - ; - -procParameter - : AMP LOCAL_VARIABLE_IDENTIFIER? - ; - - -// -------------------------------------------------------- -// Conditional expressions -// -------------------------------------------------------- - -ifExpression - : IF NL? expressionOrCommand thenClause elsifClause* elseClause? END - ; - -thenClause - : (SEMI | NL)+ compoundStatement - | (SEMI | NL)? THEN compoundStatement - ; - -elsifClause - : ELSIF NL? expressionOrCommand thenClause - ; - -elseClause - : ELSE compoundStatement - ; - -unlessExpression - : UNLESS NL? expressionOrCommand thenClause elseClause? END - ; - -caseExpression - : CASE NL? expressionOrCommand? (SEMI | NL)* whenClause+ elseClause? END - ; - -whenClause - : WHEN NL? whenArgument thenClause - ; - -whenArgument - : expressions (COMMA splattingArgument)? - | splattingArgument - ; - -// -------------------------------------------------------- -// Iteration expressions -// -------------------------------------------------------- - -whileExpression - : WHILE NL? expressionOrCommand doClause END - ; - -doClause - : (SEMI | NL)+ compoundStatement - | DO compoundStatement - ; - -untilExpression - : UNTIL NL? expressionOrCommand doClause END - ; - -forExpression - : FOR NL? forVariable IN NL? expressionOrCommand doClause END - ; - -forVariable - : singleLeftHandSide - | multipleLeftHandSide - ; - -// -------------------------------------------------------- -// Begin expression -// -------------------------------------------------------- - -beginExpression - : BEGIN bodyStatement END - ; - -bodyStatement - : compoundStatement rescueClause* elseClause? ensureClause? - ; - -rescueClause - : RESCUE exceptionClass? NL? exceptionVariableAssignment? thenClause - ; - -exceptionClass - : expression - | multipleRightHandSide - ; - -exceptionVariableAssignment - : EQGT singleLeftHandSide - ; - -ensureClause - : ENSURE compoundStatement - ; - -// -------------------------------------------------------- -// Class definitions -// -------------------------------------------------------- - -classDefinition - : CLASS NL? classOrModuleReference (LT NL? expressionOrCommand)? bodyStatement END - | CLASS NL? LT2 NL? expressionOrCommand (SEMI | NL)+ bodyStatement END - ; - -classOrModuleReference - : scopedConstantReference - | CONSTANT_IDENTIFIER - ; - -// -------------------------------------------------------- -// Module definitions -// -------------------------------------------------------- - -moduleDefinition - : MODULE NL? classOrModuleReference bodyStatement END - ; - -// -------------------------------------------------------- -// Yield expressions -// -------------------------------------------------------- - -yieldWithOptionalArgument - : YIELD (LPAREN arguments? RPAREN)? - ; - -// -------------------------------------------------------- -// Jump expressions -// -------------------------------------------------------- - -jumpExpression - : BREAK - | NEXT - | REDO - | RETRY - ; - -// -------------------------------------------------------- -// Variable references -// -------------------------------------------------------- - -variableReference - : variableIdentifier # variableIdentifierVariableReference - | pseudoVariableIdentifier # pseudoVariableIdentifierVariableReference - ; - -variableIdentifier - : LOCAL_VARIABLE_IDENTIFIER - | GLOBAL_VARIABLE_IDENTIFIER - | INSTANCE_VARIABLE_IDENTIFIER - | CLASS_VARIABLE_IDENTIFIER - | CONSTANT_IDENTIFIER - ; - -pseudoVariableIdentifier - : NIL # nilPseudoVariableIdentifier - | TRUE # truePseudoVariableIdentifier - | FALSE # falsePseudoVariableIdentifier - | SELF # selfPseudoVariableIdentifier - | FILE__ # filePseudoVariableIdentifier - | LINE__ # linePseudoVariableIdentifier - | ENCODING__ # encodingPseudoVariableIdentifier - ; - -scopedConstantReference - : COLON2 CONSTANT_IDENTIFIER - | primary COLON2 CONSTANT_IDENTIFIER - ; - -// -------------------------------------------------------- -// Literals -// -------------------------------------------------------- - -literal - : HERE_DOC # hereDocLiteral - | numericLiteral # numericLiteralLiteral - | symbol # symbolLiteral - | REGULAR_EXPRESSION_START REGULAR_EXPRESSION_BODY? REGULAR_EXPRESSION_END # regularExpressionLiteral - ; - -symbol - : SYMBOL_LITERAL - | COLON stringExpression - ; - -// -------------------------------------------------------- -// Strings -// -------------------------------------------------------- - -stringExpression - : simpleString # simpleStringExpression - | stringInterpolation # interpolatedStringExpression - | stringExpression stringExpression+ # concatenatedStringExpression - ; - -quotedStringExpression - : QUOTED_NON_EXPANDED_STRING_LITERAL_START - NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE? - QUOTED_NON_EXPANDED_STRING_LITERAL_END # nonExpandedQuotedStringLiteral - | QUOTED_EXPANDED_STRING_LITERAL_START - (EXPANDED_LITERAL_CHARACTER_SEQUENCE | delimitedStringInterpolation)* - QUOTED_EXPANDED_STRING_LITERAL_END # expandedQuotedStringLiteral - | QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START - (EXPANDED_LITERAL_CHARACTER_SEQUENCE | delimitedStringInterpolation)* - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END # expandedExternalCommandLiteral - ; - -simpleString - : SINGLE_QUOTED_STRING_LITERAL # singleQuotedStringLiteral - | DOUBLE_QUOTED_STRING_START DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE? DOUBLE_QUOTED_STRING_END # doubleQuotedStringLiteral - ; - -delimitedStringInterpolation - : DELIMITED_STRING_INTERPOLATION_BEGIN - compoundStatement - DELIMITED_STRING_INTERPOLATION_END - ; - -stringInterpolation - : DOUBLE_QUOTED_STRING_START - (DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE | interpolatedStringSequence)+ - DOUBLE_QUOTED_STRING_END - ; - -interpolatedStringSequence - : STRING_INTERPOLATION_BEGIN compoundStatement STRING_INTERPOLATION_END - ; - -// -------------------------------------------------------- -// Regex interpolation -// -------------------------------------------------------- - -regexInterpolation - : REGULAR_EXPRESSION_START - (REGULAR_EXPRESSION_BODY | interpolatedRegexSequence)+ - REGULAR_EXPRESSION_END - ; - -interpolatedRegexSequence - : REGULAR_EXPRESSION_INTERPOLATION_BEGIN compoundStatement REGULAR_EXPRESSION_INTERPOLATION_END - ; - -quotedRegexInterpolation - : QUOTED_EXPANDED_REGULAR_EXPRESSION_START - (EXPANDED_LITERAL_CHARACTER_SEQUENCE | delimitedStringInterpolation)* - QUOTED_EXPANDED_REGULAR_EXPRESSION_END - ; - - -// -------------------------------------------------------- -// Numerics -// -------------------------------------------------------- - -numericLiteral - : (PLUS | MINUS)? unsignedNumericLiteral - ; - -unsignedNumericLiteral - : DECIMAL_INTEGER_LITERAL - | BINARY_INTEGER_LITERAL - | OCTAL_INTEGER_LITERAL - | HEXADECIMAL_INTEGER_LITERAL - | FLOAT_LITERAL_WITHOUT_EXPONENT - | FLOAT_LITERAL_WITH_EXPONENT - ; - -// -------------------------------------------------------- -// Helpers -// -------------------------------------------------------- - -definedMethodNameOrSymbol - : definedMethodName - | symbol - ; - -keyword - : LINE__ - | ENCODING__ - | FILE__ - | BEGIN_ - | END_ - | ALIAS - | AND - | BEGIN - | BREAK - | CASE - | CLASS - | DEF - | IS_DEFINED - | DO - | ELSE - | ELSIF - | END - | ENSURE - | FOR - | FALSE - | IF - | IN - | MODULE - | NEXT - | NIL - | NOT - | OR - | REDO - | RESCUE - | RETRY - | RETURN - | SELF - | SUPER - | THEN - | TRUE - | UNDEF - | UNLESS - | UNTIL - | WHEN - | WHILE - | YIELD - ; - -operatorMethodName - : CARET - | AMP - | BAR - | LTEQGT - | EQ2 - | EQ3 - | EQTILDE - | GT - | GTEQ - | LT - | LTEQ - | LT2 - | GT2 - | PLUS - | MINUS - | STAR - | SLASH - | PERCENT - | STAR2 - | TILDE - | PLUSAT - | MINUSAT - | LBRACK RBRACK - | LBRACK RBRACK EQ - ; diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 index 321ad6097bfe..74cc22e06b52 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 @@ -32,96 +32,6 @@ options { superClass = RubyLexerBase; } -// -------------------------------------------------------- -// Keywords -// -------------------------------------------------------- - -LINE__:'__LINE__'; -ENCODING__: '__ENCODING__'; -FILE__: '__FILE__'; -BEGIN_: 'BEGIN'; -END_: 'END'; -ALIAS: 'alias'; -AND: 'and'; -BEGIN: 'begin'; -BREAK: 'break'; -CASE: 'case'; -CLASS: 'class'; -DEF: 'def'; -IS_DEFINED: 'defined?'; -DO: 'do'; -ELSE: 'else'; -ELSIF: 'elsif'; -END: 'end'; -ENSURE: 'ensure'; -FOR: 'for'; -FALSE: 'false'; -IF: 'if'; -IN: 'in'; -MODULE: 'module'; -NEXT: 'next'; -NIL: 'nil'; -NOT: 'not'; -OR: 'or'; -REDO: 'redo'; -RESCUE: 'rescue'; -RETRY: 'retry'; -RETURN: 'return'; -SELF: 'self'; -SUPER: 'super'; -THEN: 'then'; -TRUE: 'true'; -UNDEF: 'undef'; -UNLESS: 'unless'; -UNTIL: 'until'; -WHEN: 'when'; -WHILE: 'while'; -YIELD: 'yield'; - -fragment KEYWORD - : LINE__ - | ENCODING__ - | FILE__ - | BEGIN_ - | END_ - | ALIAS - | AND - | BEGIN - | BREAK - | CASE - | CLASS - | DEF - | IS_DEFINED - | DO - | ELSE - | ELSIF - | END - | ENSURE - | FOR - | FALSE - | IF - | IN - | MODULE - | NEXT - | NIL - | NOT - | OR - | REDO - | RESCUE - | RETRY - | RETURN - | SELF - | SUPER - | THEN - | TRUE - | UNDEF - | UNLESS - | UNTIL - | WHEN - | WHILE - | YIELD - ; - // -------------------------------------------------------- // Punctuators // -------------------------------------------------------- @@ -544,9 +454,98 @@ fragment SYMBOL_NAME // -------------------------------------------------------- // Identifiers // -------------------------------------------------------- - LOCAL_VARIABLE_IDENTIFIER - : (LOWERCASE_CHARACTER | '_') IDENTIFIER_CHARACTER* + : (LOWERCASE_CHARACTER | '_') IDENTIFIER_CHARACTER* { setKeywordTokenType(); } + ; + +// -------------------------------------------------------- +// Keywords +// -------------------------------------------------------- + +LINE__:'__LINE__'; +ENCODING__: '__ENCODING__'; +FILE__: '__FILE__'; +BEGIN_: 'BEGIN'; +END_: 'END'; +ALIAS: 'alias'; +AND: 'and'; +BEGIN: 'begin'; +BREAK: 'break'; +CASE: 'case'; +CLASS: 'class'; +DEF: 'def'; +IS_DEFINED: 'defined?'; +DO: 'do'; +ELSE: 'else'; +ELSIF: 'elsif'; +END: 'end'; +ENSURE: 'ensure'; +FOR: 'for'; +FALSE: 'false'; +IF: 'if'; +IN: 'in'; +MODULE: 'module'; +NEXT: 'next'; +NIL: 'nil'; +NOT: 'not'; +OR: 'or'; +REDO: 'redo'; +RESCUE: 'rescue'; +RETRY: 'retry'; +RETURN: 'return'; +SELF: 'self'; +SUPER: 'super'; +THEN: 'then'; +TRUE: 'true'; +UNDEF: 'undef'; +UNLESS: 'unless'; +UNTIL: 'until'; +WHEN: 'when'; +WHILE: 'while'; +YIELD: 'yield'; + +fragment KEYWORD + : LINE__ + | ENCODING__ + | FILE__ + | BEGIN_ + | END_ + | ALIAS + | AND + | BEGIN + | BREAK + | CASE + | CLASS + | DEF + | IS_DEFINED + | DO + | ELSE + | ELSIF + | END + | ENSURE + | FOR + | FALSE + | IF + | IN + | MODULE + | NEXT + | NIL + | NOT + | OR + | REDO + | RESCUE + | RETRY + | RETURN + | SELF + | SUPER + | THEN + | TRUE + | UNDEF + | UNLESS + | UNTIL + | WHEN + | WHILE + | YIELD ; GLOBAL_VARIABLE_IDENTIFIER diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 index 986106504530..7e2c8fc8b569 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 @@ -83,8 +83,7 @@ multipleLeftHandSideExceptPacking ; packingLeftHandSide - : STAR leftHandSide? - | STAR leftHandSide (COMMA multipleLeftHandSideItem)* + : STAR leftHandSide? (COMMA multipleLeftHandSideItem)* ; groupedLeftHandSide @@ -97,10 +96,9 @@ multipleLeftHandSideItem ; multipleRightHandSide - : operatorExpressionList (COMMA splattingRightHandSide)? - | splattingRightHandSide + : (operatorExpressionList | splattingRightHandSide) (COMMA (operatorExpressionList | splattingRightHandSide))* ; - + splattingRightHandSide : splattingArgument ; @@ -119,6 +117,7 @@ methodName : methodIdentifier | keyword | pseudoVariable + | LBRACK RBRACK ; methodOnlyIdentifier @@ -130,23 +129,31 @@ methodInvocationWithoutParentheses # commandMethodInvocationWithoutParentheses | chainedCommandWithDoBlock ((DOT | COLON2) methodName commandArgumentList)? # chainedMethodInvocationWithoutParentheses - | RETURN primaryValueList + | RETURN primaryValueListWithAssociation # returnMethodInvocationWithoutParentheses | BREAK primaryValueList # breakMethodInvocationWithoutParentheses | NEXT primaryValueList # nextMethodInvocationWithoutParentheses - | YIELD primaryValueList + | YIELD primaryValueListWithAssociation # yieldMethodInvocationWithoutParentheses ; command - : primary NL? (AMPDOT | DOT | COLON2) methodName commandArgument + : operatorExpression QMARK NL* operatorExpression NL* COLON NL* operatorExpression + # commandTernaryOperatorExpression + | primary NL? (AMPDOT | DOT | COLON2) methodName commandArgument # memberAccessCommand - | methodIdentifier commandArgument + | methodIdentifier simpleCommandArgumentList # simpleCommand ; +simpleCommandArgumentList + : associationList + | primaryValueList (COMMA NL* associationList)? + | argumentList + ; + commandArgument : commandArgumentList # commandArgumentCommandArgumentList @@ -168,19 +175,40 @@ commandWithDoBlock | primary (DOT | COLON2) methodName argumentList doBlock ; +bracketedArrayElementList + : bracketedArrayElement (COMMA? NL* bracketedArrayElement)* COMMA? + ; + +bracketedArrayElement + : indexingArgument + | command + | hashLiteral + | splattingArgument + | indexingArgumentList + ; + indexingArgumentList - : command - # commandIndexingArgumentList - | operatorExpressionList COMMA? + : operatorExpressionList COMMA? # operatorExpressionListIndexingArgumentList | operatorExpressionList COMMA splattingArgument # operatorExpressionListWithSplattingArgumentIndexingArgumentList + | indexingArgument (COMMA? NL* indexingArgument)* + #indexingArgumentIndexingArgumentList | associationList COMMA? # associationListIndexingArgumentList - | splattingArgument + | splattingArgument (COMMA NL* splattingArgument)* # splattingArgumentIndexingArgumentList ; +indexingArgument + : symbol + #symbolIndexingArgument + | association + #associationIndexingArgument + | sign=(PLUS | MINUS)? unsignedNumericLiteral + #numericLiteralIndexingArgument + ; + splattingArgument : STAR operatorExpression | STAR2 operatorExpression @@ -189,10 +217,6 @@ splattingArgument operatorExpressionList : operatorExpression (COMMA NL* operatorExpression)* ; - -operatorExpressionList2 - : operatorExpression (COMMA NL* operatorExpression)+ - ; argumentWithParentheses : LPAREN NL* COMMA? NL* RPAREN @@ -208,27 +232,37 @@ argumentWithParentheses argumentList : blockArgument # blockArgumentArgumentList - | splattingArgument (COMMA NL* blockArgument)? - # splattingArgumentArgumentList - | operatorExpressionList (COMMA NL* associationList)? (COMMA NL* splattingArgument)? (COMMA NL* blockArgument)? - # operatorsArgumentList - | associationList (COMMA NL* splattingArgument)? (COMMA NL* blockArgument)? - # associationsArgumentList + | argumentListItem (COMMA NL* argumentListItem)* + # argumentListItemArgumentList + | LBRACK indexingArgumentList? RBRACK + # arrayArgumentList | command # singleCommandArgumentList ; - + +argumentListItem + : splattingArgument + | operatorExpressionList + | associationList + | blockArgument + ; + commandArgumentList : associationList | primaryValueList (COMMA NL* associationList)? - ; + ; primaryValueList : primaryValue (COMMA NL* primaryValue)* ; +primaryValueListWithAssociation + : (primaryValue | association)? (COMMA NL* (primaryValue | association))* + | methodInvocationWithoutParentheses + ; + blockArgument - : AMP operatorExpression + : AMP operatorExpression? ; // -------------------------------------------------------- @@ -268,6 +302,10 @@ primary # primaryValuePrimary ; +hashLiteral + : LCURLY NL* (associationList COMMA?)? NL* RCURLY + ; + primaryValue : // Assignment expressions lhs=variable assignmentOperator NL* rhs=operatorExpression @@ -282,7 +320,7 @@ primaryValue # assignmentWithRescue // Definitions - | CLASS classPath (LT commandOrPrimaryValueClass)? (SEMI | NL) bodyStatement END + | CLASS classPath (LT commandOrPrimaryValueClass)? (SEMI | NL)? bodyStatement END # classDefinition | CLASS LT2 commandOrPrimaryValueClass (SEMI | NL) bodyStatement END # singletonClassDefinition @@ -294,7 +332,7 @@ primaryValue # singletonMethodDefinition | DEF definedMethodName (LPAREN parameterList? RPAREN)? EQ NL* statement # endlessMethodDefinition - | MINUSGT (LPAREN parameterList? RPAREN)? block + | MINUSGT lambdaExpressionParameterList? block # lambdaExpression // Control structures @@ -316,27 +354,12 @@ primaryValue # whileExpression | FOR NL* forVariable IN NL* commandOrPrimaryValue doClause END # forExpression - - // Non-nested calls - | SUPER argumentWithParentheses? block? - # superWithParentheses - | SUPER argumentList? block? - # superWithoutParentheses - | isDefinedKeyword LPAREN expressionOrCommand RPAREN - # isDefinedExpression - | isDefinedKeyword primaryValue - # isDefinedCommand - | methodOnlyIdentifier - # methodCallExpression - | methodIdentifier block - # methodCallWithBlockExpression - | methodIdentifier argumentWithParentheses block? - # methodCallWithParenthesesExpression - | variableReference - # methodCallOrVariableReference - + + | methodCallsWithParentheses + # methodCallWithParentheses + // Literals - | LBRACK NL* indexingArgumentList? NL* RBRACK + | LBRACK NL* bracketedArrayElementList? NL* RBRACK # bracketedArrayLiteral | QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START quotedNonExpandedArrayElementList? QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END # quotedNonExpandedStringArrayLiteral @@ -346,8 +369,8 @@ primaryValue # quotedExpandedStringArrayLiteral | QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START quotedExpandedArrayElementList? QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END # quotedExpandedSymbolArrayLiteral - | LCURLY NL* (associationList COMMA?)? NL* RCURLY - # hashLiteral + | hashLiteral + # primaryValueHashLiteral | sign=(PLUS | MINUS)? unsignedNumericLiteral # numericLiteral | singleQuotedString singleOrDoubleQuotedString* @@ -397,16 +420,45 @@ primaryValue # relationalExpression | primaryValue equalityOperator NL* primaryValue # equalityExpression - | primaryValue andOperator=AMP2 NL* primaryValue + | primaryValue andOperator=AMP2 NL* (primaryValue | RETURN) # logicalAndExpression - | primaryValue orOperator=BAR2 NL* primaryValue + | primaryValue orOperator=BAR2 NL* (primaryValue | RETURN) # logicalOrExpression - | primaryValue rangeOperator NL* primaryValue - # rangeExpression + | primaryValue rangeOperator NL* primaryValue + # boundedRangeExpression + | primaryValue rangeOperator + # endlessRangeExpression + | rangeOperator primaryValue + # beginlessRangeExpression | hereDoc # hereDocs ; +lambdaExpressionParameterList + : LPAREN blockParameterList? RPAREN + | blockParameterList + ; + +// Non-nested calls +methodCallsWithParentheses + : SUPER argumentWithParentheses? block? + # superWithParentheses + | SUPER argumentList? block? + # superWithoutParentheses + | isDefinedKeyword LPAREN expressionOrCommand RPAREN + # isDefinedExpression + | isDefinedKeyword primaryValue + # isDefinedCommand + | methodOnlyIdentifier + # methodCallExpression + | methodIdentifier block + # methodCallWithBlockExpression + | methodIdentifier argumentWithParentheses block? + # methodCallWithParenthesesExpression + | variableReference + # methodCallOrVariableReference + ; + // This is required to make chained calls work. For classes, we cannot move up the `primaryValue` due to the possible // presence of AMPDOT when inheriting (class Foo < Bar::Baz), but the command rule doesn't allow chained calls // in if statements to be created properly, and ends throwing away everything after the first call. Splitting these @@ -443,7 +495,38 @@ doBlock blockParameter : BAR NL* BAR - | BAR NL* parameterList NL* BAR + | BAR NL* blockParameterList NL* BAR + ; + +blockParameterList + : mandatoryOrOptionalOrGroupedParameterList (COMMA NL* arrayParameter)? (COMMA NL* mandatoryOrGroupedParameterList)* (COMMA NL* hashParameter)? (COMMA NL* procParameter)? + | arrayParameter (COMMA NL* mandatoryOrGroupedParameterList)* (COMMA NL* hashParameter)? (COMMA NL* procParameter)? + | hashParameter (COMMA NL* procParameter)? + | procParameter + ; + +mandatoryOrOptionalOrGroupedParameterList + : mandatoryOrOptionalOrGroupedParameter (COMMA NL* mandatoryOrOptionalOrGroupedParameter)* + ; + +mandatoryOrOptionalOrGroupedParameter + : mandatoryParameter + | optionalParameter + | groupedParameterList + ; + +mandatoryOrGroupedParameterList + : mandatoryOrGroupedParameter (COMMA NL* mandatoryOrGroupedParameter)* + ; + +mandatoryOrGroupedParameter + : mandatoryParameter + | groupedParameterList + ; + +groupedParameterList + : LPAREN mandatoryParameter (COMMA NL* arrayParameter)? (COMMA NL* mandatoryParameter)* RPAREN + | LPAREN arrayParameter (COMMA NL* mandatoryParameter)* RPAREN ; thenClause @@ -515,8 +598,8 @@ methodParameterPart ; parameterList - : mandatoryOrOptionalParameterList (COMMA NL* arrayParameter)? (COMMA NL* hashParameter)? (COMMA NL* procParameter)? - | arrayParameter (COMMA NL* hashParameter)? (COMMA NL* procParameter)? + : mandatoryOrOptionalParameterList (COMMA NL* arrayParameter)? (COMMA NL* mandatoryParameterList)? (COMMA NL* hashParameter)? (COMMA NL* procParameter)? + | arrayParameter (COMMA NL* mandatoryParameterList)? (COMMA NL* hashParameter)? (COMMA NL* procParameter)? | hashParameter (COMMA NL* procParameter)? | procParameter ; @@ -524,7 +607,11 @@ parameterList mandatoryOrOptionalParameterList : mandatoryOrOptionalParameter (COMMA NL* mandatoryOrOptionalParameter)* ; - + +mandatoryParameterList + : mandatoryParameter (COMMA NL* mandatoryParameter)* + ; + mandatoryOrOptionalParameter : mandatoryParameter # mandatoryMandatoryOrOptionalParameter @@ -553,7 +640,7 @@ hashParameter ; procParameter - : AMP procParameterName + : AMP procParameterName? ; procParameterName @@ -591,6 +678,9 @@ associationList association : associationKey (EQGT | COLON) NL* operatorExpression + # associationElement + | associationHashArgument + # associationHashArg ; associationKey @@ -598,6 +688,10 @@ associationKey | keyword ; +associationHashArgument + : STAR2 (LOCAL_VARIABLE_IDENTIFIER | methodCallsWithParentheses | (LPAREN methodInvocationWithoutParentheses RPAREN))? + ; + regexpLiteralContent : REGULAR_EXPRESSION_BODY | REGULAR_EXPRESSION_INTERPOLATION_BEGIN compoundStatement REGULAR_EXPRESSION_INTERPOLATION_END @@ -616,25 +710,11 @@ doubleQuotedString : DOUBLE_QUOTED_STRING_START doubleQuotedStringContent* DOUBLE_QUOTED_STRING_END ; -quotedExpandedExternalCommandString - : QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START - quotedExpandedLiteralStringContent* - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END - ; - doubleQuotedStringContent : DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE | STRING_INTERPOLATION_BEGIN compoundStatement STRING_INTERPOLATION_END ; -quotedNonExpandedLiteralString - : QUOTED_NON_EXPANDED_STRING_LITERAL_START NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE? QUOTED_NON_EXPANDED_STRING_LITERAL_END - ; - -quotedExpandedLiteralString - : QUOTED_EXPANDED_STRING_LITERAL_START quotedExpandedLiteralStringContent* QUOTED_EXPANDED_STRING_LITERAL_END - ; - quotedExpandedLiteralStringContent : EXPANDED_LITERAL_CHARACTER_SEQUENCE | DELIMITED_STRING_INTERPOLATION_BEGIN compoundStatement DELIMITED_STRING_INTERPOLATION_END diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala index f1e97e8e61aa..1db133b9c32a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala @@ -1,22 +1,27 @@ package io.joern.rubysrc2cpg import io.joern.rubysrc2cpg.Frontend.* -import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} +import io.joern.x2cpg.DependencyDownloadConfig +import io.joern.x2cpg.X2CpgConfig +import io.joern.x2cpg.X2CpgMain +import io.joern.x2cpg.passes.frontend.TypeRecoveryParserConfig +import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.joern.x2cpg.typestub.TypeStubConfig -import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser final case class Config( antlrCacheMemLimit: Double = 0.6d, - useDeprecatedFrontend: Boolean = false, downloadDependencies: Boolean = false, - useTypeStubs: Boolean = true + useTypeStubs: Boolean = true, + antlrDebug: Boolean = false, + antlrProfiling: Boolean = false ) extends X2CpgConfig[Config] with DependencyDownloadConfig[Config] with TypeRecoveryParserConfig[Config] with TypeStubConfig[Config] { - this.defaultIgnoredFilesRegex = List("spec", "test", "tests").flatMap { directory => + this.defaultIgnoredFilesRegex = List("spec", "test", "tests", "vendor").flatMap { directory => List(s"(^|\\\\)$directory($$|\\\\)".r.unanchored, s"(^|/)$directory($$|/)".r.unanchored) } @@ -24,8 +29,12 @@ final case class Config( copy(antlrCacheMemLimit = value).withInheritedFields(this) } - def withUseDeprecatedFrontend(value: Boolean): Config = { - copy(useDeprecatedFrontend = value).withInheritedFields(this) + def withAntlrDebugging(value: Boolean): Config = { + copy(antlrDebug = value).withInheritedFields(this) + } + + def withAntlrProfiling(value: Boolean): Config = { + copy(antlrProfiling = value).withInheritedFields(this) } override def withDownloadDependencies(value: Boolean): Config = { @@ -54,13 +63,19 @@ private object Frontend { failure(s"$x may result in too many evictions and reduce performance, try a value between 0.3 - 0.8.") case x if x > 0.8 => failure(s"$x may result in too much memory usage and thrashing, try a value between 0.3 - 0.8.") - case x => + case _ => success } .text("sets the heap usage threshold at which the ANTLR DFA cache is cleared during parsing (default 0.6)"), - opt[Unit]("useDeprecatedFrontend") - .action((_, c) => c.withUseDeprecatedFrontend(true)) - .text("uses the original (but deprecated) Ruby frontend (default false)"), + opt[Unit]("antlrDebug") + .hidden() + .action((_, c) => c.withAntlrDebugging(true)), + opt[Unit]("antlrProfile") + .hidden() + .action((_, c) => c.withAntlrProfiling(true)), + opt[Unit]("enable-file-content") + .action((_, c) => c.withDisableFileContent(false)) + .text("Enable file content"), DependencyDownloadConfig.parserOptions, XTypeRecoveryConfig.parserOptionsForParserConfig, TypeStubConfig.parserOptions @@ -68,8 +83,12 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new RubySrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new RubySrc2Cpg()) with FrontendHTTPServer[Config, RubySrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, rubySrc2Cpg: RubySrc2Cpg): Unit = { - rubySrc2Cpg.run(config) + if (config.serverMode) { startup() } + else { rubySrc2Cpg.run(config) } } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 00b61157a7db..7c9301ed293d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -2,22 +2,24 @@ package io.joern.rubysrc2cpg import better.files.File import io.joern.rubysrc2cpg.astcreation.AstCreator +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.StatementList import io.joern.rubysrc2cpg.datastructures.RubyProgramSummary -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.parser.RubyParser +import io.joern.rubysrc2cpg.parser.{RubyNodeCreator, RubyParser} import io.joern.rubysrc2cpg.passes.{ AstCreationPass, ConfigFileCreationPass, DependencyPass, - DependencySummarySolverPass, + DependencySummarySolverPass +} +import io.joern.rubysrc2cpg.utils.DependencyDownloader +import io.joern.x2cpg.X2Cpg.withNewEmptyCpg +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.{ ImplicitRequirePass, ImportsPass, RubyImportResolverPass, - RubyTypeHintCallLinker + RubyTypeHintCallLinker, + RubyTypeRecoveryPassGenerator } -import io.joern.rubysrc2cpg.utils.DependencyDownloader -import io.joern.x2cpg.X2Cpg.withNewEmptyCpg import io.joern.x2cpg.passes.base.AstLinkerPass import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.joern.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass, XTypeRecoveryConfig} @@ -42,22 +44,26 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply() new ConfigFileCreationPass(cpg).createAndApply() new DependencyPass(cpg).createAndApply() - if (config.useDeprecatedFrontend) { - deprecatedCreateCpgAction(cpg, config) - } else { - newCreateCpgAction(cpg, config) - } + createCpgAction(cpg, config) } } - private def newCreateCpgAction(cpg: Cpg, config: Config): Unit = { - Using.resource(new parser.ResourceManagedParser(config.antlrCacheMemLimit)) { parser => + private def createCpgAction(cpg: Cpg, config: Config): Unit = { + Using.resource( + new parser.ResourceManagedParser(config.antlrCacheMemLimit, config.antlrDebug, config.antlrProfiling) + ) { parser => val astCreators = ConcurrentTaskUtil .runUsingThreadPool(RubySrc2Cpg.generateParserTasks(parser, config, cpg.metaData.root.headOption)) .flatMap { case Failure(exception) => logger.warn(s"Could not parse file, skipping - ", exception); None case Success(astCreator) => Option(astCreator) } + .filter(x => { + if x.fileContent.isBlank then logger.info(s"File content empty, skipping - ${x.fileName}") + + !x.fileContent.isBlank + }) + // Pre-parse the AST creators for high level structures val internalProgramSummary = ConcurrentTaskUtil .runUsingThreadPool(astCreators.map(x => () => x.summarize()).iterator) @@ -76,8 +82,6 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { val programSummary = internalProgramSummary ++= dependencySummary AstCreationPass(cpg, astCreators.map(_.withSummary(programSummary))).createAndApply() - if (cpg.dependency.name.contains("zeitwerk")) ImplicitRequirePass(cpg, programSummary).createAndApply() - ImportsPass(cpg).createAndApply() if config.downloadDependencies then { DependencySummarySolverPass(cpg, dependencySummary).createAndApply() } @@ -85,62 +89,15 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { } } - private def deprecatedCreateCpgAction(cpg: Cpg, config: Config): Unit = try { - Using.resource(new deprecated.astcreation.ResourceManagedParser(config.antlrCacheMemLimit)) { parser => - if (config.downloadDependencies && !scala.util.Properties.isWin) { - val tempDir = File.newTemporaryDirectory() - try { - downloadDependency(config.inputPath, tempDir.toString()) - new deprecated.passes.AstPackagePass( - cpg, - tempDir.toString(), - parser, - RubySrc2Cpg.packageTableInfo, - config.inputPath - )(config.schemaValidation).createAndApply() - } finally { - tempDir.delete() - } - } - val parsedFiles = { - val tasks = SourceFiles - .determine( - config.inputPath, - RubySrc2Cpg.RubySourceFileExtensions, - ignoredFilesRegex = Option(config.ignoredFilesRegex), - ignoredFilesPath = Option(config.ignoredFiles) - ) - .map(x => - () => - parser.parse(x) match - case Failure(exception) => - logger.warn(s"Could not parse file: $x, skipping", exception); throw exception - case Success(ast) => x -> ast - ) - .iterator - ConcurrentTaskUtil.runUsingThreadPool(tasks).flatMap(_.toOption) - } - - new io.joern.rubysrc2cpg.deprecated.ParseInternalStructures(parsedFiles, cpg.metaData.root.headOption) - .populatePackageTable() - val astCreationPass = - new deprecated.passes.AstCreationPass(cpg, parsedFiles, RubySrc2Cpg.packageTableInfo, config) - astCreationPass.createAndApply() - } - } finally { - RubySrc2Cpg.packageTableInfo.clear() - } - private def downloadDependency(inputPath: String, tempPath: String): Unit = { if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { - ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { + ExternalCommand.run(Seq("bundle", "config", "set", "--local", "path", tempPath), inputPath).toTry match { case Success(configOutput) => logger.info(s"Gem config successfully done: $configOutput") case Failure(exception) => logger.error(s"Error while configuring Gem Path: ${exception.getMessage}") } - val command = s"bundle install" - ExternalCommand.run(command, inputPath) match { + ExternalCommand.run(Seq("bundle", "install"), inputPath).toTry match { case Success(bundleOutput) => logger.info(s"Dependency installed successfully: $bundleOutput") case Failure(exception) => @@ -152,26 +109,13 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { object RubySrc2Cpg { - // TODO: Global mutable state is bad and should be avoided in the next iteration of the Ruby frontend - val packageTableInfo = new deprecated.utils.PackageTable() private val RubySourceFileExtensions: Set[String] = Set(".rb") def postProcessingPasses(cpg: Cpg, config: Config): List[CpgPassBase] = { - if (config.useDeprecatedFrontend) { - List(new deprecated.passes.RubyImportResolverPass(cpg, packageTableInfo)) - ++ new deprecated.passes.RubyTypeRecoveryPassGenerator(cpg).generate() ++ List( - new deprecated.passes.RubyTypeHintCallLinker(cpg), - new NaiveCallLinker(cpg), - - // Some of passes above create new methods, so, we - // need to run the ASTLinkerPass one more time - new AstLinkerPass(cpg) - ) - } else { - List(new RubyImportResolverPass(cpg)) ++ - new passes.RubyTypeRecoveryPassGenerator(cpg, config = XTypeRecoveryConfig(iterations = 4)) - .generate() ++ List(new RubyTypeHintCallLinker(cpg), new NaiveCallLinker(cpg), new AstLinkerPass(cpg)) - } + val implicitRequirePass = if (cpg.dependency.name.contains("zeitwerk")) ImplicitRequirePass(cpg) :: Nil else Nil + implicitRequirePass ++ List(ImportsPass(cpg), RubyImportResolverPass(cpg)) ++ + new RubyTypeRecoveryPassGenerator(cpg, config = XTypeRecoveryConfig(iterations = 4)) + .generate() ++ List(new RubyTypeHintCallLinker(cpg), new NaiveCallLinker(cpg), new AstLinkerPass(cpg)) } def generateParserTasks( @@ -190,7 +134,16 @@ object RubySrc2Cpg { .map { fileName => () => resourceManagedParser.parse(File(config.inputPath), fileName) match { case Failure(exception) => throw exception - case Success(ctx) => new AstCreator(fileName, ctx, projectRoot)(config.schemaValidation) + case Success(ctx) => + val fileContent = (File(config.inputPath) / fileName).contentAsString + new AstCreator( + fileName, + ctx, + projectRoot, + enableFileContents = !config.disableFileContent, + fileContent = fileContent, + rootNode = Option(new RubyNodeCreator().visit(ctx).asInstanceOf[StatementList]) + )(config.schemaValidation) } } .iterator diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index ea422185d4e1..270e900ef6a3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1,17 +1,16 @@ package io.joern.rubysrc2cpg.astcreation import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* -import io.joern.rubysrc2cpg.datastructures.{BlockScope, NamespaceScope, RubyProgramSummary, RubyScope, RubyStubbedType} +import io.joern.rubysrc2cpg.datastructures.{BlockScope, NamespaceScope, RubyProgramSummary, RubyScope} import io.joern.rubysrc2cpg.parser.{RubyNodeCreator, RubyParser} import io.joern.rubysrc2cpg.passes.Defines -import io.joern.x2cpg.utils.NodeBuilders.{newBindingNode, newModifierNode} +import io.joern.rubysrc2cpg.utils.FreshNameGenerator +import io.joern.x2cpg.utils.NodeBuilders.{newModifierNode, newThisParameterNode} import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{DiffGraphBuilder, ModifierTypes} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder import java.util.regex.Matcher @@ -19,16 +18,23 @@ class AstCreator( val fileName: String, protected val programCtx: RubyParser.ProgramContext, protected val projectRoot: Option[String] = None, - protected val programSummary: RubyProgramSummary = RubyProgramSummary() + protected val programSummary: RubyProgramSummary = RubyProgramSummary(), + val enableFileContents: Boolean = false, + val fileContent: String = "", + val rootNode: Option[RubyExpression] = None )(implicit withSchemaValidation: ValidationMode) extends AstCreatorBase(fileName) with AstCreatorHelper with AstForStatementsCreator with AstForExpressionsCreator + with AstForControlStructuresCreator with AstForFunctionsCreator with AstForTypesCreator with AstSummaryVisitor - with AstNodeBuilder[RubyNode, AstCreator] { + with AstNodeBuilder[RubyExpression, AstCreator] { + + val tmpGen: FreshNameGenerator[String] = FreshNameGenerator(i => s"") + val procParamGen: FreshNameGenerator[Left[String, Nothing]] = FreshNameGenerator(i => Left(s"")) /* Used to track variable names and their LOCAL nodes. */ @@ -36,24 +42,32 @@ class AstCreator( protected val logger: Logger = LoggerFactory.getLogger(getClass) + protected var fileNode: Option[NewFile] = None + protected var parseLevel: AstParseLevel = AstParseLevel.FULL_AST + override protected def offset(node: RubyExpression): Option[(Int, Int)] = node.offset + protected val relativeFileName: String = projectRoot .map(fileName.stripPrefix) .map(_.stripPrefix(java.io.File.separator)) .getOrElse(fileName) - private def internalLineAndColNum: Option[Integer] = Option(1) + private def internalLineAndColNum: Option[Int] = Option(1) /** The relative file name, in a unix path delimited format. */ private def relativeUnixStyleFileName = relativeFileName.replaceAll(Matcher.quoteReplacement(java.io.File.separator), "/") - override def createAst(): BatchedUpdate.DiffGraphBuilder = { - val rootNode = new RubyNodeCreator().visit(programCtx).asInstanceOf[StatementList] - val ast = astForRubyFile(rootNode) + override def createAst(): DiffGraphBuilder = { + val astRootNode = rootNode.match { + case Some(node) => node.asInstanceOf[StatementList] + case None => new RubyNodeCreator(tmpGen, procParamGen).visit(programCtx).asInstanceOf[StatementList] + } + + val ast = astForRubyFile(astRootNode) Ast.storeInDiffGraph(ast, diffGraph) diffGraph } @@ -63,7 +77,9 @@ class AstCreator( * allowing for a straightforward representation of out-of-method statements. */ protected def astForRubyFile(rootStatements: StatementList): Ast = { - val fileNode = NewFile().name(relativeFileName) + fileNode = + if enableFileContents then Option(NewFile().name(relativeFileName).content(fileContent)) + else Option(NewFile().name(relativeFileName)) val fullName = s"$relativeUnixStyleFileName:${NamespaceTraversal.globalNamespaceName}" val namespaceBlock = NewNamespaceBlock() .filename(relativeFileName) @@ -74,13 +90,15 @@ class AstCreator( val rubyFakeMethodAst = astInFakeMethod(rootStatements) scope.popScope() - Ast(fileNode).withChild(Ast(namespaceBlock).withChild(rubyFakeMethodAst)) + Ast(fileNode.get).withChild(Ast(namespaceBlock).withChild(rubyFakeMethodAst)) } private def astInFakeMethod(rootNode: StatementList): Ast = { - val name = Defines.Program - val fullName = computeMethodFullName(name) - val code = rootNode.text + val name = Defines.Main + // From the
method onwards, we do not embed the namespace name in the full names + val fullName = + s"${scope.surroundingScopeFullName.head.stripSuffix(NamespaceTraversal.globalNamespaceName)}$name" + val code = rootNode.text val methodNode_ = methodNode( node = rootNode, name = name, @@ -89,6 +107,15 @@ class AstCreator( signature = None, fileName = relativeFileName ) + val thisParameterNode = newThisParameterNode( + name = Defines.Self, + code = Defines.Self, + typeFullName = Defines.Any, + line = methodNode_.lineNumber, + column = methodNode_.columnNumber + ) + val thisParameterAst = Ast(thisParameterNode) + scope.addToScope(Defines.Self, thisParameterNode) val methodReturn = methodReturnNode(rootNode, Defines.Any) scope.newProgramScope @@ -102,7 +129,7 @@ class AstCreator( scope.popScope() methodAst( methodNode_, - Seq.empty, + thisParameterAst :: Nil, bodyAst, methodReturn, newModifierNode(ModifierTypes.MODULE) :: newModifierNode(ModifierTypes.VIRTUAL) :: Nil diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index d519c4e1deb8..8358a539f803 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -5,7 +5,7 @@ import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{ InstanceFieldIdentifier, MemberAccess, RubyFieldIdentifier, - RubyNode + RubyExpression } import io.joern.rubysrc2cpg.datastructures.{BlockScope, FieldDecl} import io.joern.rubysrc2cpg.passes.Defines @@ -15,16 +15,39 @@ import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, Operators} +import scala.collection.mutable + trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - protected def computeClassFullName(name: String): String = s"${scope.surroundingScopeFullName.head}.$name" - protected def computeMethodFullName(name: String): String = s"${scope.surroundingScopeFullName.head}:$name" + private val usedFullNames = mutable.Set.empty[String] + + /** Ensures a unique full name is assigned based on the current scope. + * @param name + * the name of the entity. + * @param counter + * an optional counter, used to create unique instances in the case of redefinitions. + * @return + * a unique full name. + */ + protected def computeFullName(name: String, counter: Option[Int] = None): String = { + val candidate = counter match { + case Some(cnt) => s"${scope.surroundingScopeFullName.head}.$name$cnt" + case None => s"${scope.surroundingScopeFullName.head}.$name" + } + if (usedFullNames.contains(candidate)) { + computeFullName(name, counter.map(_ + 1).orElse(Option(0))) + } else { + usedFullNames.add(candidate) + candidate + } + } + + override def column(node: RubyExpression): Option[Int] = node.column + override def columnEnd(node: RubyExpression): Option[Int] = node.columnEnd + override def line(node: RubyExpression): Option[Int] = node.line + override def lineEnd(node: RubyExpression): Option[Int] = node.lineEnd - override def column(node: RubyNode): Option[Int] = node.column - override def columnEnd(node: RubyNode): Option[Int] = node.columnEnd - override def line(node: RubyNode): Option[Int] = node.line - override def lineEnd(node: RubyNode): Option[Int] = node.lineEnd - override def code(node: RubyNode): String = shortenCode(node.text) + override def code(node: RubyExpression): String = shortenCode(node.text) protected def isBuiltin(x: String): Boolean = kernelFunctions.contains(x) protected def prefixAsKernelDefined(x: String): String = s"$kernelPrefix$pathSep$x" @@ -32,7 +55,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def isBundledClass(x: String): Boolean = GlobalTypes.bundledClasses.contains(x) protected def pathSep = "." - private def astForFieldInstance(name: String, node: RubyNode & RubyFieldIdentifier): Ast = { + private def astForFieldInstance(name: String, node: RubyExpression & RubyFieldIdentifier): Ast = { val identName = node match { case _: InstanceFieldIdentifier => Defines.Self case _: ClassFieldIdentifier => scope.surroundingTypeFullName.map(_.split("[.]").last).getOrElse(Defines.Any) @@ -47,7 +70,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As ) } - protected def handleVariableOccurrence(node: RubyNode): Ast = { + protected def handleVariableOccurrence(node: RubyExpression): Ast = { val name = code(node) val identifier = identifierNode(node, name, name, Defines.Any) val typeRef = scope.tryResolveTypeReference(name) @@ -92,12 +115,19 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As astForAssignment(Ast(lhs), Ast(rhs), lineNumber, columnNumber) } - protected def astForAssignment(lhs: Ast, rhs: Ast, lineNumber: Option[Int], columnNumber: Option[Int]): Ast = { - val code = Seq(lhs, rhs).flatMap(_.root).collect { case x: ExpressionNew => x.code }.mkString(" = ") + protected def astForAssignment( + lhs: Ast, + rhs: Ast, + lineNumber: Option[Int], + columnNumber: Option[Int], + code: Option[String] = None + ): Ast = { + val _code = + code.getOrElse(Seq(lhs, rhs).flatMap(_.root).collect { case x: ExpressionNew => x.code }.mkString(" = ")) val assignment = NewCall() .name(Operators.assignment) .methodFullName(Operators.assignment) - .code(code) + .code(_code) .dispatchType(DispatchTypes.STATIC_DISPATCH) .lineNumber(lineNumber) .columnNumber(columnNumber) @@ -147,8 +177,8 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As "&" -> Operators.and, "|" -> Operators.or, "^" -> Operators.xor, - "<<" -> Operators.shiftLeft, - ">>" -> Operators.logicalShiftRight +// "<<" -> Operators.shiftLeft, Note: Generally Ruby abstracts this as an append operator based on the LHS + ">>" -> Operators.logicalShiftRight ) protected val AssignmentOperatorNames: Map[String, String] = Map( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForControlStructuresCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForControlStructuresCreator.scala new file mode 100644 index 000000000000..4abb68005d5e --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForControlStructuresCreator.scala @@ -0,0 +1,272 @@ +package io.joern.rubysrc2cpg.astcreation + +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{ + BinaryExpression, + BreakExpression, + CaseExpression, + ControlFlowStatement, + DoWhileExpression, + ElseClause, + ForExpression, + IfExpression, + MemberCall, + NextExpression, + RescueExpression, + ReturnExpression, + RubyExpression, + SimpleIdentifier, + SingleAssignment, + SplattingRubyNode, + StatementList, + UnaryExpression, + Unknown, + UnlessExpression, + UntilExpression, + WhenClause, + WhileExpression +} +import io.joern.rubysrc2cpg.passes.Defines +import io.joern.x2cpg.{Ast, ValidationMode} +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{ + NewBlock, + NewFieldIdentifier, + NewIdentifier, + NewLiteral, + NewLocal +} + +trait AstForControlStructuresCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => + + protected def astForControlStructureExpression(node: ControlFlowStatement): Ast = node match { + case node: WhileExpression => astForWhileStatement(node) + case node: DoWhileExpression => astForDoWhileStatement(node) + case node: UntilExpression => astForUntilStatement(node) + case node: CaseExpression => blockAst(NewBlock(), astsForCaseExpression(node).toList) + case node: IfExpression => astForIfExpression(node) + case node: UnlessExpression => astForUnlessStatement(node) + case node: ForExpression => astForForExpression(node) + case node: RescueExpression => astForRescueExpression(node) + case node: NextExpression => astForNextExpression(node) + case node: BreakExpression => astForBreakExpression(node) + } + + private def astForWhileStatement(node: WhileExpression): Ast = { + val conditionAst = astForExpression(node.condition) + val bodyAsts = astsForStatement(node.body) + whileAst(Some(conditionAst), bodyAsts, Option(code(node)), line(node), column(node)) + } + + private def astForDoWhileStatement(node: DoWhileExpression): Ast = { + val conditionAst = astForExpression(node.condition) + val bodyAsts = astsForStatement(node.body) + doWhileAst(Some(conditionAst), bodyAsts, Option(code(node)), line(node), column(node)) + } + + // `until T do B` is lowered as `while !T do B` + private def astForUntilStatement(node: UntilExpression): Ast = { + val notCondition = astForExpression(UnaryExpression("!", node.condition)(node.condition.span)) + val bodyAsts = astsForStatement(node.body) + whileAst(Some(notCondition), bodyAsts, Option(code(node)), line(node), column(node)) + } + + // Recursively lowers into a ternary conditional call + private def astForIfExpression(node: IfExpression): Ast = { + def builder(node: IfExpression, conditionAst: Ast, thenAst: Ast, elseAsts: List[Ast]): Ast = { + // We want to make sure there's always an «else» clause in a ternary operator. + // The default value is a `nil` literal. + val elseAsts_ = if (elseAsts.isEmpty) { + List(astForNilBlock) + } else { + elseAsts + } + + val call = callNode(node, code(node), Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH) + callAst(call, conditionAst :: thenAst :: elseAsts_) + } + + foldIfExpression(builder)(node) + } + + // `unless T do B` is lowered as `if !T then B` + private def astForUnlessStatement(node: UnlessExpression): Ast = { + val notConditionAst = astForExpression(UnaryExpression("!", node.condition)(node.condition.span)) + val thenAst = node.trueBranch match + case stmtList: StatementList => astForStatementList(stmtList) + case _ => astForStatementList(StatementList(List(node.trueBranch))(node.trueBranch.span)) + val elseAsts = node.falseBranch.map(astForElseClause).toList + val ifNode = controlStructureNode(node, ControlStructureTypes.IF, code(node)) + controlStructureAst(ifNode, Some(notConditionAst), thenAst :: elseAsts) + } + + protected def astForElseClause(node: RubyExpression): Ast = { + node match + case elseNode: ElseClause => + elseNode.thenClause match + case stmtList: StatementList => astForStatementList(stmtList) + case node => + logger.warn(s"Expecting statement list in ${code(node)} ($relativeFileName), skipping") + astForUnknown(node) + case elseNode => + logger.warn(s"Expecting else clause in ${code(elseNode)} ($relativeFileName), skipping") + astForUnknown(elseNode) + } + + private def astForForExpression(node: ForExpression): Ast = { + val forEachNode = controlStructureNode(node, ControlStructureTypes.FOR, code(node)) + + def collectionAst = astForExpression(node.iterableVariable) + val collectionNode = node.iterableVariable + + val iterIdentifier = + identifierNode( + node = node.forVariable, + name = node.forVariable.span.text, + code = node.forVariable.span.text, + typeFullName = Defines.Any + ) + val iterVarLocal = NewLocal().name(node.forVariable.span.text).code(node.forVariable.span.text) + scope.addToScope(node.forVariable.span.text, iterVarLocal) + + val idxName = "_idx_" + val idxLocal = NewLocal().name(idxName).code(idxName).typeFullName(Defines.getBuiltInType(Defines.Integer)) + val idxIdenAtAssign = identifierNode( + node = collectionNode, + name = idxName, + code = idxName, + typeFullName = Defines.getBuiltInType(Defines.Integer) + ) + + val idxAssignment = + callNode(node, s"$idxName = 0", Operators.assignment, Operators.assignment, DispatchTypes.STATIC_DISPATCH) + val idxAssignmentArgs = + List(Ast(idxIdenAtAssign), Ast(NewLiteral().code("0").typeFullName(Defines.getBuiltInType(Defines.Integer)))) + val idxAssignmentAst = callAst(idxAssignment, idxAssignmentArgs) + + val idxIdAtCond = idxIdenAtAssign.copy + val collectionCountAccess = callNode( + node, + s"${node.iterableVariable.span.text}.length", + Operators.fieldAccess, + Operators.fieldAccess, + DispatchTypes.STATIC_DISPATCH + ) + val fieldAccessAst = callAst( + collectionCountAccess, + collectionAst :: Ast(NewFieldIdentifier().canonicalName("length").code("length")) :: Nil + ) + + val idxLt = callNode( + node, + s"$idxName < ${node.iterableVariable.span.text}.length", + Operators.lessThan, + Operators.lessThan, + DispatchTypes.STATIC_DISPATCH + ) + val idxLtArgs = List(Ast(idxIdAtCond), fieldAccessAst) + val ltCallCond = callAst(idxLt, idxLtArgs) + + val idxIdAtCollAccess = idxIdenAtAssign.copy + val collectionIdxAccess = callNode( + node, + s"${node.iterableVariable.span.text}[$idxName++]", + Operators.indexAccess, + Operators.indexAccess, + DispatchTypes.STATIC_DISPATCH + ) + val postIncrAst = callAst( + callNode(node, s"$idxName++", Operators.postIncrement, Operators.postIncrement, DispatchTypes.STATIC_DISPATCH), + Ast(idxIdAtCollAccess) :: Nil + ) + + val indexAccessAst = callAst(collectionIdxAccess, collectionAst :: postIncrAst :: Nil) + val iteratorAssignmentNode = callNode( + node, + s"${node.forVariable.span.text} = ${node.iterableVariable.span.text}[$idxName++]", + Operators.assignment, + Operators.assignment, + DispatchTypes.STATIC_DISPATCH + ) + val iteratorAssignmentArgs = List(Ast(iterIdentifier), indexAccessAst) + val iteratorAssignmentAst = callAst(iteratorAssignmentNode, iteratorAssignmentArgs) + val doBodyAst = astsForStatement(node.doBlock) + + val locals = Ast(idxLocal) + .withRefEdge(idxIdenAtAssign, idxLocal) + .withRefEdge(idxIdAtCond, idxLocal) + .withRefEdge(idxIdAtCollAccess, idxLocal) :: Ast(iterVarLocal).withRefEdge(iterIdentifier, iterVarLocal) :: Nil + + val conditionAsts = ltCallCond :: Nil + val initAsts = idxAssignmentAst :: Nil + val updateAsts = iteratorAssignmentAst :: Nil + + forAst( + forNode = forEachNode, + locals = locals, + initAsts = initAsts, + conditionAsts = conditionAsts, + updateAsts = updateAsts, + bodyAsts = doBodyAst + ) + } + + protected def astsForCaseExpression(node: CaseExpression): Seq[Ast] = { + // TODO: Clean up the below + def goCase(expr: Option[SimpleIdentifier]): List[RubyExpression] = { + val elseThenClause: Option[RubyExpression] = node.elseClause.map(_.asInstanceOf[ElseClause].thenClause) + val whenClauses = node.whenClauses.map(_.asInstanceOf[WhenClause]) + val ifElseChain = whenClauses.foldRight[Option[RubyExpression]](elseThenClause) { + (whenClause: WhenClause, restClause: Option[RubyExpression]) => + // We translate multiple match expressions into an or expression. + // + // A single match expression is compared using `.===` to the case target expression if it is present + // otherwise it is treated as a conditional. + // + // There may be a splat as the last match expression, + // `case y when *x then c end` or + // `case when *x then c end` + // which is translated to `x.include? y` and `x.any?` conditions respectively + + val conditions = whenClause.matchExpressions.map { mExpr => + expr.map(e => BinaryExpression(mExpr, "===", e)(mExpr.span)).getOrElse(mExpr) + } ++ whenClause.matchSplatExpression.iterator.flatMap { + case splat @ SplattingRubyNode(exprList) => + expr + .map { e => + List(MemberCall(exprList, ".", "include?", List(e))(splat.span)) + } + .getOrElse { + List(MemberCall(exprList, ".", "any?", List())(splat.span)) + } + case e => + logger.warn(s"Unrecognised RubyNode (${e.getClass}) in case match splat expression") + List(Unknown()(e.span)) + } + // There is always at least one match expression or a splat + // will become an unknown in condition at the end + val condition = conditions.init.foldRight(conditions.last) { (cond, condAcc) => + BinaryExpression(cond, "||", condAcc)(whenClause.span) + } + val conditional = IfExpression( + condition, + whenClause.thenClause.asStatementList, + List(), + restClause.map { els => ElseClause(els.asStatementList)(els.span) } + )(node.span) + Some(conditional) + } + ifElseChain.iterator.toList + } + def generatedNode: StatementList = node.expression + .map { e => + val tmp = SimpleIdentifier(None)(e.span.spanStart(this.tmpGen.fresh)) + StatementList( + List(SingleAssignment(tmp, "=", e)(e.span)) ++ + goCase(Some(tmp)) + )(node.span) + } + .getOrElse(StatementList(goCase(None))(node.span)) + astsForStatement(generatedNode) + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 8c1000f388f3..75675fe41dcc 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -5,7 +5,6 @@ import io.joern.rubysrc2cpg.datastructures.BlockScope import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.GlobalTypes import io.joern.rubysrc2cpg.passes.Defines.{RubyOperators, getBuiltInType} -import io.joern.rubysrc2cpg.utils.FreshNameGenerator import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ @@ -17,46 +16,51 @@ import io.shiftleft.codepropertygraph.generated.{ PropertyNames } -trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - val tmpGen: FreshNameGenerator[String] = FreshNameGenerator(i => s"") - - protected def astForExpression(node: RubyNode): Ast = node match - case node: StaticLiteral => astForStaticLiteral(node) - case node: HereDocNode => astForHereDoc(node) - case node: DynamicLiteral => astForDynamicLiteral(node) - case node: UnaryExpression => astForUnary(node) - case node: BinaryExpression => astForBinary(node) - case node: MemberAccess => astForMemberAccess(node) - case node: MemberCall => astForMemberCall(node) - case node: ObjectInstantiation => astForObjectInstantiation(node) - case node: IndexAccess => astForIndexAccess(node) - case node: SingleAssignment => astForSingleAssignment(node) - case node: AttributeAssignment => astForAttributeAssignment(node) - case node: TypeIdentifier => astForTypeIdentifier(node) - case node: RubyIdentifier => astForSimpleIdentifier(node) - case node: SimpleCall => astForSimpleCall(node) - case node: RequireCall => astForRequireCall(node) - case node: IncludeCall => astForIncludeCall(node) - case node: YieldExpr => astForYield(node) - case node: RangeExpression => astForRange(node) - case node: ArrayLiteral => astForArrayLiteral(node) - case node: HashLiteral => astForHashLiteral(node) - case node: Association => astForAssociation(node) - case node: IfExpression => astForIfExpression(node) - case node: UnlessExpression => astForUnlessExpression(node) - case node: RescueExpression => astForRescueExpression(node) - case node: CaseExpression => blockAst(NewBlock(), astsForCaseExpression(node).toList) - case node: MandatoryParameter => astForMandatoryParameter(node) - case node: SplattingRubyNode => astForSplattingRubyNode(node) - case node: AnonymousTypeDeclaration => astForAnonymousTypeDeclaration(node) - case node: ProcOrLambdaExpr => astForProcOrLambdaExpr(node) - case node: RubyCallWithBlock[_] => astForCallWithBlock(node) - case node: SelfIdentifier => astForSelfIdentifier(node) - case node: BreakStatement => astForBreakStatement(node) - case node: StatementList => astForStatementList(node) - case node: DummyNode => Ast(node.node) - case node: Unknown => astForUnknown(node) +import scala.collection.mutable + +trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { + this: AstCreator => + + /** For tracking aliased calls that occur on the LHS of a member access or call. + */ + protected val baseAstCache = mutable.Map.empty[RubyExpression, String] + + protected def astForExpression(node: RubyExpression): Ast = node match + case node: ControlFlowStatement => astForControlStructureExpression(node) + case node: StaticLiteral => astForStaticLiteral(node) + case node: HereDocNode => astForHereDoc(node) + case node: DynamicLiteral => astForDynamicLiteral(node) + case node: UnaryExpression => astForUnary(node) + case node: BinaryExpression => astForBinary(node) + case node: MemberAccess => astForMemberAccess(node) + case node: MemberCall => astForMemberCall(node) + case node: ObjectInstantiation => astForObjectInstantiation(node) + case node: IndexAccess => astForIndexAccess(node) + case node: SingleAssignment => astForSingleAssignment(node) + case node: AttributeAssignment => astForAttributeAssignment(node) + case node: TypeIdentifier => astForTypeIdentifier(node) + case node: RubyIdentifier => astForSimpleIdentifier(node) + case node: SimpleCall => astForSimpleCall(node) + case node: RequireCall => astForRequireCall(node) + case node: IncludeCall => astForIncludeCall(node) + case node: RaiseCall => astForRaiseCall(node) + case node: YieldExpr => astForYield(node) + case node: RangeExpression => astForRange(node) + case node: ArrayLiteral => astForArrayLiteral(node) + case node: HashLiteral => astForHashLiteral(node) + case node: Association => astForAssociation(node) + case node: MandatoryParameter => astForMandatoryParameter(node) + case node: SplattingRubyNode => astForSplattingRubyNode(node) + case node: AnonymousTypeDeclaration => astForAnonymousTypeDeclaration(node) + case node: ProcOrLambdaExpr => astForProcOrLambdaExpr(node) + case node: SingletonObjectMethodDeclaration => astForSingletonObjectMethodDeclaration(node) + case node: RubyCallWithBlock[_] => astForCallWithBlock(node) + case node: SelfIdentifier => astForSelfIdentifier(node) + case node: StatementList => astForStatementList(node) + case node: ReturnExpression => astForReturnExpression(node) + case node: AccessModifier => astForSimpleIdentifier(node.toSimpleIdentifier) + case node: DummyNode => Ast(node.node) + case node: Unknown => astForUnknown(node) case x => logger.warn(s"Unhandled expression of type ${x.getClass.getSimpleName}") astForUnknown(node) @@ -71,7 +75,8 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { // Helper for nil literals to put in empty clauses protected def astForNilLiteral: Ast = Ast(NewLiteral().code("nil").typeFullName(getBuiltInType(Defines.NilClass))) - protected def astForNilBlock: Ast = blockAst(NewBlock(), List(astForNilLiteral)) + + protected def astForNilBlock: Ast = blockAst(NewBlock(), List(astForNilLiteral)) protected def astForDynamicLiteral(node: DynamicLiteral): Ast = { val fmtValueAsts = node.expressions.map { @@ -151,19 +156,19 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { /** Attempts to extract a type from the base of a member call. */ - protected def typeFromCallTarget(baseNode: RubyNode): Option[String] = { - scope.lookupVariable(baseNode.text) match { - // fixme: This should be under type recovery logic - case Some(decl: NewLocal) if decl.typeFullName != Defines.Any => Option(decl.typeFullName) - case Some(decl: NewMethodParameterIn) if decl.typeFullName != Defines.Any => Option(decl.typeFullName) - case Some(decl: NewLocal) if decl.dynamicTypeHintFullName.nonEmpty => decl.dynamicTypeHintFullName.headOption - case Some(decl: NewMethodParameterIn) if decl.dynamicTypeHintFullName.nonEmpty => - decl.dynamicTypeHintFullName.headOption + protected def typeFromCallTarget(baseNode: RubyExpression): Option[String] = { + baseNode match { + case literal: LiteralExpr => Option(literal.typeFullName) case _ => - astForExpression(baseNode).nodes - .flatMap(_.properties.get(PropertyNames.TYPE_FULL_NAME).map(_.toString)) - .filterNot(_ == XDefines.Any) - .headOption + scope.lookupVariable(baseNode.text) match { + // fixme: This should be under type recovery logic + case Some(decl: NewLocal) if decl.typeFullName != Defines.Any => Option(decl.typeFullName) + case Some(decl: NewMethodParameterIn) if decl.typeFullName != Defines.Any => Option(decl.typeFullName) + case Some(decl: NewLocal) if decl.dynamicTypeHintFullName.nonEmpty => decl.dynamicTypeHintFullName.headOption + case Some(decl: NewMethodParameterIn) if decl.dynamicTypeHintFullName.nonEmpty => + decl.dynamicTypeHintFullName.headOption + case _ => None + } } } @@ -171,38 +176,50 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { Ast(typeRefNode(node, code(node), node.typeFullName)) } - protected def astForMemberCall(node: MemberCall): Ast = { + protected def astForMemberCall(node: MemberCall, isStatic: Boolean = false): Ast = { def createMemberCall(n: MemberCall): Ast = { - val baseAst = astForExpression(n.target) // this wil be something like self.Foo - val receiverAst = astForExpression(MemberAccess(n.target, ".", n.methodName)(n.span)) + val receiverAst = astForFieldAccess(MemberAccess(n.target, ".", n.methodName)(n.span), stripLeadingAt = true) + val (baseAst, baseCode) = astForMemberAccessTarget(n.target) val builtinType = n.target match { case MemberAccess(_: SelfIdentifier, _, memberName) if isBundledClass(memberName) => Option(prefixAsBundledType(memberName)) case x: TypeIdentifier if x.isBuiltin => Option(x.typeFullName) case _ => None } - val (receiverFullName, methodFullName) = receiverAst.nodes + val methodFullName = receiverAst.nodes .collectFirst { - case _ if builtinType.isDefined => builtinType.get -> s"${builtinType.get}:${n.methodName}" - case x: NewMethodRef => x.methodFullName -> x.methodFullName + case _ if builtinType.isDefined => s"${builtinType.get}.${n.methodName}" + case x: NewMethodRef => x.methodFullName case _ => (n.target match { case ma: MemberAccess => scope.tryResolveTypeReference(ma.memberName).map(_.name) case _ => typeFromCallTarget(n.target) - }).map(x => x -> s"$x:${n.methodName}") - .getOrElse(XDefines.Any -> XDefines.DynamicCallUnknownFullName) + }).map(x => s"$x.${n.methodName}") + .getOrElse(XDefines.DynamicCallUnknownFullName) } - .getOrElse(XDefines.Any -> XDefines.DynamicCallUnknownFullName) + .getOrElse(XDefines.DynamicCallUnknownFullName) val argumentAsts = n.arguments.map(astForMethodCallArgument) - val dispatchType = DispatchTypes.DYNAMIC_DISPATCH + val dispatchType = if (isStatic) DispatchTypes.STATIC_DISPATCH else DispatchTypes.DYNAMIC_DISPATCH - val call = callNode(n, code(n), n.methodName, XDefines.DynamicCallUnknownFullName, dispatchType) + val callCode = if (baseCode.contains(" target case x: SimpleIdentifier => scope.getSurroundingType(x.text).map(_.fullName) match { @@ -210,20 +227,97 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val typeName = surroundingType.split('.').last TypeIdentifier(s"$surroundingType")(x.span.spanStart(typeName)) case None if scope.lookupVariable(x.text).isDefined => x - case None => MemberAccess(SelfIdentifier()(x.span.spanStart(Defines.Self)), ".", x.text)(x.span) + case None if x.text.charAt(0).isUpper => // calls have lower-case first character + MemberAccess(SelfIdentifier()(x.span.spanStart(Defines.Self)), ".", x.text)(x.span) + case None => MemberCall(SelfIdentifier()(x.span.spanStart(Defines.Self)), ".", x.text, Nil)(x.span) } - case x @ MemberAccess(ma, op, memberName) => x.copy(target = determineMemberAccessBase(ma))(x.span) - case _ => target + case x @ MemberAccess(ma, _, _) => x.copy(target = determineMemberAccessBase(ma))(x.span) + case _ => target } node.target match { + case _: LiteralExpr => + createMemberCall(node) case x: SimpleIdentifier if isBundledClass(x.text) => createMemberCall(node.copy(target = TypeIdentifier(prefixAsBundledType(x.text))(x.span))(node.span)) case x: SimpleIdentifier => createMemberCall(node.copy(target = determineMemberAccessBase(x))(node.span)) case memAccess: MemberAccess => createMemberCall(node.copy(target = determineMemberAccessBase(memAccess))(node.span)) - case x => createMemberCall(node) + case _ => createMemberCall(node) + } + } + + protected def astForFieldAccess(node: MemberAccess, stripLeadingAt: Boolean = false): Ast = { + val (memberName, memberCode) = node.target match { + case _ if node.memberName == Defines.Initialize => Defines.Initialize -> Defines.Initialize + case _ if stripLeadingAt => node.memberName -> node.memberName.stripPrefix("@") + case _: TypeIdentifier => node.memberName -> node.memberName + case _ if !node.memberName.startsWith("@") && node.memberName.headOption.exists(_.isLower) => + s"@${node.memberName}" -> node.memberName + case _ => node.memberName -> node.memberName + } + + val fieldIdentifierAst = Ast(fieldIdentifierNode(node, memberName, memberCode)) + val (targetAst, _code) = astForMemberAccessTarget(node.target) + val code = s"$_code${node.op}$memberCode" + val memberType = typeFromCallTarget(node.target) + .flatMap(scope.tryResolveTypeReference) + .map(_.fields) + .getOrElse(List.empty) + .collectFirst { + case x if x.name == memberName => + scope.tryResolveTypeReference(x.typeName).map(_.name).getOrElse(Defines.Any) + } + .orElse(Option(Defines.Any)) + val fieldAccess = callNode( + node, + code, + Operators.fieldAccess, + Operators.fieldAccess, + DispatchTypes.STATIC_DISPATCH, + signature = None, + typeFullName = Option(Defines.Any) + ).possibleTypes(IndexedSeq(memberType.get)) + callAst(fieldAccess, Seq(targetAst, fieldIdentifierAst)) + } + + private def astForMemberAccessTarget(target: RubyExpression): (Ast, String) = { + target match { + case simpleLhs: (LiteralExpr | SimpleIdentifier | SelfIdentifier | TypeIdentifier) => + astForExpression(simpleLhs) -> code(target) + case target: MemberAccess => handleTmpGen(target, astForFieldAccess(target, stripLeadingAt = true)) + case target => handleTmpGen(target, astForExpression(target)) + } + } + + private def handleTmpGen(target: RubyExpression, rhs: Ast): (Ast, String) = { + // Check cache + val createAssignmentToTmp = !baseAstCache.contains(target) + val tmpName = baseAstCache + .updateWith(target) { + case Some(tmpName) => + // TODO: Type ref nodes are automatically committed on creation, so if we have found a suitable cached AST, + // we want to clean this creation up. + Option(tmpName) + case None => + val tmpName = this.tmpGen.fresh + val tmpGenLocal = NewLocal().name(tmpName).code(tmpName).typeFullName(Defines.Any) + scope.addToScope(tmpName, tmpGenLocal) match { + case BlockScope(block) => diffGraph.addEdge(block, tmpGenLocal, EdgeTypes.AST) + case _ => + } + Option(tmpName) + } + .get + val tmpIden = NewIdentifier().name(tmpName).code(tmpName).typeFullName(Defines.Any) + val tmpIdenAst = + scope.lookupVariable(tmpName).map(x => Ast(tmpIden).withRefEdge(tmpIden, x)).getOrElse(Ast(tmpIden)) + val code = s"$tmpName = ${target.text}" + if (createAssignmentToTmp) { + astForAssignment(tmpIdenAst, rhs, target.line, target.column, Option(code)) -> s"($code)" + } else { + tmpIdenAst -> s"($code)" } } @@ -253,17 +347,42 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } } - protected def astForObjectInstantiation(node: RubyNode & ObjectInstantiation): Ast = { - val className = node.target.text - val callName = "new" - val methodName = Defines.Initialize + /* `foo() do end` is lowered as a METHOD node shaped like so: + * ``` + * = def 0() + * + * end + * foo(, ) + * ``` + */ + protected def astForCallWithBlock[C <: RubyCall](node: RubyExpression & RubyCallWithBlock[C]): Ast = { + val Seq(typeRef, _) = astForDoBlock(node.block): @unchecked + val typeRefDummyNode = typeRef.root.map(DummyNode(_)(node.span)).toList + + // Create call with argument referencing the MethodRef + val callWithLambdaArg = node.withoutBlock match { + case x: SimpleCall => astForSimpleCall(x.copy(arguments = x.arguments ++ typeRefDummyNode)(x.span)) + case x: MemberCall => astForMemberCall(x.copy(arguments = x.arguments ++ typeRefDummyNode)(x.span)) + case x => + logger.warn(s"Unhandled call-with-block type ${code(x)}, creating anonymous method structures only") + Ast() + } + + callWithLambdaArg + } + + protected def astForObjectInstantiation(node: RubyExpression & ObjectInstantiation): Ast = { /* We short-cut the call edge from `new` call to `initialize` method, however we keep the modelling of the receiver as referring to the singleton class. */ - val (receiverTypeFullName, fullName) = scope.tryResolveTypeReference(className) match { - case Some(typeMetaData) => s"${typeMetaData.name}" -> s"${typeMetaData.name}:$methodName" - case None => XDefines.Any -> XDefines.DynamicCallUnknownFullName + val (receiverTypeFullName, fullName) = node.target match { + case x: (SimpleIdentifier | MemberAccess) => + scope.tryResolveTypeReference(x.text) match { + case Some(typeMetaData) => s"${typeMetaData.name}" -> s"${typeMetaData.name}.${Defines.Initialize}" + case None => XDefines.Any -> XDefines.DynamicCallUnknownFullName + } + case _ => XDefines.Any -> XDefines.DynamicCallUnknownFullName } /* Similarly to some other frontends, we lower the constructor into two operations, e.g., @@ -273,9 +392,9 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) - val tmpName = tmpGen.fresh + val tmpName = this.tmpGen.fresh val tmpTypeHint = receiverTypeFullName.stripSuffix("") - val tmp = SimpleIdentifier(Option(className))(node.span.spanStart(tmpName)) + val tmp = SimpleIdentifier(None)(node.span.spanStart(tmpName)) val tmpLocal = NewLocal().name(tmpName).code(tmpName).dynamicTypeHintFullName(Seq(tmpTypeHint)) scope.addToScope(tmpName, tmpLocal) @@ -286,12 +405,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } // Assign tmp to - val receiverAst = Ast(identifierNode(node, className, className, receiverTypeFullName)) - val allocCall = callNode(node, code(node), Operators.alloc, Operators.alloc, DispatchTypes.STATIC_DISPATCH) - val allocAst = callAst(allocCall, Seq.empty, Option(receiverAst)) + val allocCall = callNode(node, code(node), Operators.alloc, Operators.alloc, DispatchTypes.STATIC_DISPATCH) + val allocAst = callAst(allocCall, Seq.empty) val assignmentCall = callNode( node, - s"${tmp.text} = ${code(node)}", + s"${tmp.text} = ${code(node.target)}.${Defines.Initialize}", Operators.assignment, Operators.assignment, DispatchTypes.STATIC_DISPATCH @@ -302,12 +420,21 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val argumentAsts = node match { case x: SimpleObjectInstantiation => x.arguments.map(astForMethodCallArgument) case x: ObjectInstantiationWithBlock => - val Seq(_, methodRef) = astForDoBlock(x.block): @unchecked - x.arguments.map(astForMethodCallArgument) :+ methodRef + val Seq(typeRef, _) = astForDoBlock(x.block): @unchecked + x.arguments.map(astForMethodCallArgument) :+ typeRef } - val constructorCall = callNode(node, code(node), callName, fullName, DispatchTypes.DYNAMIC_DISPATCH) - val constructorCallAst = callAst(constructorCall, argumentAsts, Option(tmpIdentifier)) + val constructorCall = + callNode( + node, + code(node), + Defines.Initialize, + XDefines.DynamicCallUnknownFullName, + DispatchTypes.DYNAMIC_DISPATCH + ) + if fullName != XDefines.DynamicCallUnknownFullName then constructorCall.dynamicTypeHintFullName(Seq(fullName)) + val constructorRecv = astForExpression(MemberAccess(node.target, ".", Defines.Initialize)(node.span)) + val constructorCallAst = callAst(constructorCall, argumentAsts, Option(tmpIdentifier), Option(constructorRecv)) val retIdentifierAst = tmpIdentifier scope.popScope() @@ -327,7 +454,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { astForUnknown(node) case Some(op) => node.rhs match { - case cfNode: ControlFlowExpression => + case cfNode: ControlFlowStatement => def elseAssignNil(span: TextSpan) = Option { ElseClause( StatementList( @@ -340,12 +467,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { )(span.spanStart(s"else\n\t${node.lhs.span.text} ${node.op} nil\nend")) } - def transform(e: RubyNode & ControlFlowExpression): RubyNode = + def transform(e: RubyExpression & ControlFlowStatement): RubyExpression = transformLastRubyNodeInControlFlowExpressionBody( e, x => reassign(node.lhs, node.op, x, transform), elseAssignNil ) + astForExpression(transform(cfNode)) case _ => // The if the LHS defines a new variable, put the local variable into scope @@ -358,8 +486,17 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case _ => } astForExpression(node.lhs) + case SplattingRubyNode(nameNode: SimpleIdentifier) if scope.lookupVariable(code(nameNode)).isEmpty => + val name = code(nameNode) + val local = localNode(nameNode, name, name, Defines.Any) + scope.addToScope(name, local) match { + case BlockScope(block) => diffGraph.addEdge(block, local, EdgeTypes.AST) + case _ => + } + astForExpression(node.lhs) case _ => astForExpression(node.lhs) } + val rhsAst = astForExpression(node.rhs) // If this is a simple object instantiation assignment, we can give the LHS variable a type hint @@ -386,21 +523,21 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } private def reassign( - lhs: RubyNode, + lhs: RubyExpression, op: String, - rhs: RubyNode, - transform: (RubyNode & ControlFlowExpression) => RubyNode - ): RubyNode = { - def stmtListAssigningLastExpression(stmts: List[RubyNode]): List[RubyNode] = stmts match { - case (head: ControlFlowClause) :: Nil => clauseAssigningLastExpression(head) :: Nil - case (head: ControlFlowExpression) :: Nil => transform(head) :: Nil + rhs: RubyExpression, + transform: (RubyExpression & ControlFlowStatement) => RubyExpression + ): RubyExpression = { + def stmtListAssigningLastExpression(stmts: List[RubyExpression]): List[RubyExpression] = stmts match { + case (head: ControlFlowClause) :: Nil => clauseAssigningLastExpression(head) :: Nil + case (head: ControlFlowStatement) :: Nil => transform(head) :: Nil case head :: Nil => SingleAssignment(lhs, op, head)(rhs.span.spanStart(s"${lhs.span.text} $op ${head.span.text}")) :: Nil case Nil => List.empty case head :: tail => head :: stmtListAssigningLastExpression(tail) } - def clauseAssigningLastExpression(x: RubyNode & ControlFlowClause): RubyNode = x match { + def clauseAssigningLastExpression(x: RubyExpression & ControlFlowClause): RubyExpression = x match { case RescueClause(exceptionClassList, assignment, thenClause) => RescueClause(exceptionClassList, assignment, reassign(lhs, op, thenClause, transform))(x.span) case EnsureClause(thenClause) => EnsureClause(reassign(lhs, op, thenClause, transform))(x.span) @@ -412,22 +549,30 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } rhs match { - case StatementList(statements) => StatementList(stmtListAssigningLastExpression(statements))(rhs.span) - case clause: ControlFlowClause => clauseAssigningLastExpression(clause) - case expr: ControlFlowExpression => transform(expr) + case StatementList(statements) => StatementList(stmtListAssigningLastExpression(statements))(rhs.span) + case clause: ControlFlowClause => clauseAssigningLastExpression(clause) + case expr: ControlFlowStatement => transform(expr) case _ => SingleAssignment(lhs, op, rhs)(rhs.span.spanStart(s"${lhs.span.text} $op ${rhs.span.text}")) } } - // `x.y = 1` is lowered as `x.y=(1)`, i.e. as calling `y=` on `x` with argument `1` + // `x.y = 1` is approximated as `x.y = 1`, i.e. as calling `x.y =` assignment with argument `1` + // This has the benefit of avoiding unnecessary call resolution protected def astForAttributeAssignment(node: AttributeAssignment): Ast = { - val call = SimpleCall(node, List(node.rhs))(node.span) - val memberAccess = MemberAccess(node.target, ".", s"${node.attributeName}=")(node.span) - astForMemberCallWithoutBlock(call, memberAccess) + val memberAccess = MemberAccess(node.target, ".", s"@${node.attributeName}")( + node.span.spanStart(s"${node.target.text}.${node.attributeName}") + ) + + val assignmentOp = AssignmentOperatorNames(node.assignmentOperator) + + val lhsAst = astForFieldAccess(memberAccess, stripLeadingAt = true) + val rhsAst = astForExpression(node.rhs) + val call = callNode(node, code(node), assignmentOp, assignmentOp, DispatchTypes.STATIC_DISPATCH) + callAst(call, Seq(lhsAst, rhsAst)) } - protected def astForSimpleIdentifier(node: RubyNode & RubyIdentifier): Ast = { + protected def astForSimpleIdentifier(node: RubyExpression & RubyIdentifier): Ast = { val name = code(node) if (isBundledClass(name)) { val typeFullName = prefixAsBundledType(name) @@ -445,7 +590,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } } - protected def astForMandatoryParameter(node: RubyNode): Ast = handleVariableOccurrence(node) + protected def astForMandatoryParameter(node: RubyExpression): Ast = handleVariableOccurrence(node) protected def astForSimpleCall(node: SimpleCall): Ast = { node.target match @@ -462,7 +607,16 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case _ => None } pathOpt.foreach(path => scope.addRequire(projectRoot.get, fileName, path, node.isRelative, node.isWildCard)) - astForSimpleCall(node.asSimpleCall) + + val callName = node.target.text + val requireCallNode = NewCall() + .name(node.target.text) + .code(code(node)) + .methodFullName(getBuiltInType(callName)) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .typeFullName(Defines.Any) + val arguments = astForExpression(node.argument) :: Nil + callAst(requireCallNode, arguments) } protected def astForIncludeCall(node: IncludeCall): Ast = { @@ -472,24 +626,28 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { astForSimpleCall(node.asSimpleCall) } - /** A yield in Ruby could either return the result of the block, or simply call the block, depending on runtime - * conditions. Thus we embed this in a conditional expression where the condition itself is some non-deterministic - * placeholder. + protected def astForRaiseCall(node: RaiseCall): Ast = { + val throwControlStruct = controlStructureNode(node, ControlStructureTypes.THROW, code(node)) + val args = node.arguments.map(astForExpression) + Ast(throwControlStruct).withChildren(args) + } + + /** A yield in Ruby calls an explicit (or implicit) proc parameter and returns its value. This can be lowered as + * block.call(), which is effectively how one invokes a proc parameter in any case. */ protected def astForYield(node: YieldExpr): Ast = { scope.useProcParam match { case Some(param) => - val call = astForExpression( - SimpleCall(SimpleIdentifier()(node.span.spanStart(param)), node.arguments)(node.span.spanStart(param)) - ) - val ret = returnAst(returnNode(node, code(node))) - val cond = astForExpression( - SimpleCall(SimpleIdentifier()(node.span.spanStart(tmpGen.fresh)), List())(node.span.spanStart("")) - ) - callAst( - callNode(node, code(node), Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH), - List(cond, call, ret) - ) + // We do not know if we necessarily have an explicit proc param here, or if we need to create a new one + if (scope.lookupVariable(param).isEmpty) { + scope.anonProcParam.map { param => + val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + astForParameter(paramNode, -1) + } + } + val loweredCall = + MemberCall(SimpleIdentifier()(node.span.spanStart(param)), ".", "call", node.arguments)(node.span) + astForExpression(loweredCall) case None => logger.warn(s"Yield expression outside of method scope: ${code(node)} ($relativeFileName), skipping") astForUnknown(node) @@ -505,39 +663,29 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } protected def astForArrayLiteral(node: ArrayLiteral): Ast = { - if (node.isDynamic) { - logger.warn(s"Interpolated array literals are not supported yet: ${code(node)} ($relativeFileName), skipping") - astForUnknown(node) - } else { - val arguments = if (node.text.startsWith("%")) { - val argumentsType = - if (node.isStringArray) getBuiltInType(Defines.String) - else getBuiltInType(Defines.Symbol) - node.elements.map { - case element @ StaticLiteral(_) => StaticLiteral(argumentsType)(element.span) - case element => element - } - } else { - node.elements + val arguments = if (node.text.startsWith("%")) { + val argumentsType = + if (node.isStringArray) getBuiltInType(Defines.String) + else getBuiltInType(Defines.Symbol) + node.elements.map { + case element @ StaticLiteral(_) => StaticLiteral(argumentsType)(element.span) + case element @ DynamicLiteral(_, expressions) => DynamicLiteral(argumentsType, expressions)(element.span) + case element => element } - val argumentAsts = arguments.map(astForExpression) - - val call = - callNode( - node, - code(node), - Operators.arrayInitializer, - Operators.arrayInitializer, - DispatchTypes.STATIC_DISPATCH - ) - callAst(call, argumentAsts) + } else { + node.elements } + val argumentAsts = arguments.map(astForExpression) + + val call = + callNode(node, code(node), Operators.arrayInitializer, Operators.arrayInitializer, DispatchTypes.STATIC_DISPATCH) + callAst(call, argumentAsts) } protected def astForHashLiteral(node: HashLiteral): Ast = { - val tmp = tmpGen.fresh + val tmp = this.tmpGen.fresh - def tmpAst(tmpNode: Option[RubyNode] = None) = astForSimpleIdentifier( + def tmpAst(tmpNode: Option[RubyExpression] = None) = astForSimpleIdentifier( SimpleIdentifier()(tmpNode.map(_.span).getOrElse(node.span).spanStart(tmp)) ) @@ -548,7 +696,8 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { val argumentAsts = node.elements.flatMap(elem => elem match - case associationNode: Association => astForAssociationHash(associationNode, tmp) + case associationNode: Association => astForAssociationHash(associationNode, tmp) + case splattingRubyNode: SplattingRubyNode => astForSplattingRubyNode(splattingRubyNode) :: Nil case node => logger.warn(s"Could not represent element: ${code(node)} ($relativeFileName), skipping") astForUnknown(node) :: Nil @@ -573,6 +722,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { protected def astForAssociationHash(node: Association, tmp: String): List[Ast] = { node.key match { + case mod: AccessModifier => + // Modifiers aren't allowed here, will be shadowed by a simple identifier + astForAssociationHash(node.copy(key = mod.toSimpleIdentifier)(node.span), tmp) + case iden: SimpleIdentifier => + // An identifier here will always be interpreted as a symbol + val sym = StaticLiteral(getBuiltInType(Defines.Symbol))(iden.span.spanStart(s":${iden.text}")) + astForAssociationHash(node.copy(key = sym)(node.span), tmp) case rangeExpr: RangeExpression => val expandedList = generateStaticLiteralsForRange(rangeExpr).map { x => astForSingleKeyValue(x, node.value, tmp) @@ -583,7 +739,6 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } else { astForSingleKeyValue(node.key, node.value, tmp) :: Nil } - case _ => astForSingleKeyValue(node.key, node.value, tmp) :: Nil } } @@ -595,7 +750,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case (s"${GlobalTypes.`kernelPrefix`}.Integer", s"${GlobalTypes.`kernelPrefix`}.Integer") => generateRange(lb.span.text.toInt, ub.span.text.toInt, node.rangeOperator.exclusive) .map(x => - StaticLiteral(lb.typeFullName)(TextSpan(lb.line, lb.column, lb.lineEnd, lb.columnEnd, x.toString)) + StaticLiteral(lb.typeFullName)(TextSpan(lb.line, lb.column, lb.lineEnd, lb.columnEnd, None, x.toString)) ) .toList case (s"${GlobalTypes.`kernelPrefix`}.String", s"${GlobalTypes.`kernelPrefix`}.String") => @@ -612,7 +767,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { generateRange(lbVal(0).toInt, ubVal(0).toInt, node.rangeOperator.exclusive) .map(x => StaticLiteral(lb.typeFullName)( - TextSpan(lb.line, lb.column, lb.lineEnd, lb.columnEnd, s"\'${x.toChar.toString}\'") + TextSpan(lb.line, lb.column, lb.lineEnd, lb.columnEnd, None, s"\'${x.toChar.toString}\'") ) ) .toList @@ -637,13 +792,22 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { callAst(call, Seq(key, value)) } - protected def astForSingleKeyValue(keyNode: RubyNode, valueNode: RubyNode, tmp: String): Ast = { + protected def astForSingleKeyValue(keyNode: RubyExpression, valueNode: RubyExpression, tmp: String): Ast = { astForExpression( SingleAssignment( IndexAccess( - SimpleIdentifier()(TextSpan(keyNode.line, keyNode.column, keyNode.lineEnd, keyNode.columnEnd, tmp)), + SimpleIdentifier()(TextSpan(keyNode.line, keyNode.column, keyNode.lineEnd, keyNode.columnEnd, None, tmp)), List(keyNode) - )(TextSpan(keyNode.line, keyNode.column, keyNode.lineEnd, keyNode.columnEnd, s"$tmp[${keyNode.span.text}]")), + )( + TextSpan( + keyNode.line, + keyNode.column, + keyNode.lineEnd, + keyNode.columnEnd, + None, + s"$tmp[${keyNode.span.text}]" + ) + ), "=", valueNode )( @@ -652,34 +816,13 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { keyNode.column, keyNode.lineEnd, keyNode.columnEnd, + None, s"$tmp[${keyNode.span.text}] = ${valueNode.span.text}" ) ) ) } - // Recursively lowers into a ternary conditional call - protected def astForIfExpression(node: IfExpression): Ast = { - def builder(node: IfExpression, conditionAst: Ast, thenAst: Ast, elseAsts: List[Ast]): Ast = { - // We want to make sure there's always an «else» clause in a ternary operator. - // The default value is a `nil` literal. - val elseAsts_ = if (elseAsts.isEmpty) { - List(astForNilBlock) - } else { - elseAsts - } - - val call = callNode(node, code(node), Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH) - callAst(call, conditionAst :: thenAst :: elseAsts_) - } - foldIfExpression(builder)(node) - } - - protected def astForUnlessExpression(node: UnlessExpression): Ast = { - val notConditionAst = UnaryExpression("!", node.condition)(node.condition.span) - astForExpression(IfExpression(notConditionAst, node.trueBranch, List(), node.falseBranch)(node.span)) - } - protected def astForRescueExpression(node: RescueExpression): Ast = { val tryAst = astForStatementList(node.body.asStatementList) val rescueAsts = node.rescueClauses @@ -696,27 +839,35 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { case x: NewMethodParameterIn => Ast(x.dynamicTypeHintFullName(classes)) } .toList - astForStatementList(x.thenClause.asStatementList).withChildren(variables) + val rescueNode = controlStructureNode(x.thenClause.asStatementList, ControlStructureTypes.CATCH, "catch") + Ast(rescueNode).withChild(astForStatementList(x.thenClause.asStatementList).withChildren(variables)) } - val elseAst = node.elseClause.map { x => astForStatementList(x.thenClause.asStatementList) } - val ensureAst = node.ensureClause.map { x => astForStatementList(x.thenClause.asStatementList) } - tryCatchAstWithOrder( - NewControlStructure() - .controlStructureType(ControlStructureTypes.TRY) - .code(code(node)), - tryAst, - rescueAsts ++ elseAst.toSeq, - ensureAst - ) + val elseAst = node.elseClause.map { x => + val astForClause = controlStructureNode(x.thenClause.asStatementList, ControlStructureTypes.ELSE, "else") + Ast(astForClause).withChild(astForStatementList(x.thenClause.asStatementList)) + } + + val ensureAst = node.ensureClause.map { x => + val astForEnsureClause = + controlStructureNode(x.thenClause.asStatementList, ControlStructureTypes.FINALLY, "finally") + Ast(astForEnsureClause).withChild(astForStatementList(x.thenClause.asStatementList)) + } + + val tryNode = controlStructureNode(node.body.asStatementList, ControlStructureTypes.TRY, "try") + tryCatchAst(tryNode, tryAst, rescueAsts ++ elseAst, ensureAst) } private def astForSelfIdentifier(node: SelfIdentifier): Ast = { val thisIdentifier = identifierNode(node, Defines.Self, code(node), scope.surroundingTypeFullName.getOrElse(Defines.Any)) - Ast(thisIdentifier) + + scope + .lookupVariable(Defines.Self) + .map(selfParam => Ast(thisIdentifier).withRefEdge(thisIdentifier, selfParam)) + .getOrElse(Ast(thisIdentifier)) } - protected def astForUnknown(node: RubyNode): Ast = { + protected def astForUnknown(node: RubyExpression): Ast = { val className = node.getClass.getSimpleName val text = code(node) logger.warn(s"Could not represent expression: $text ($className) ($relativeFileName), skipping") @@ -724,16 +875,11 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } private def astForMemberCallWithoutBlock(node: SimpleCall, memberAccess: MemberAccess): Ast = { - val receiverAst = astForFieldAccess(memberAccess) - val methodName = memberAccess.memberName - // TODO: Type recovery should potentially resolve this - val methodFullName = typeFromCallTarget(memberAccess.target) - .map(x => s"$x:$methodName") - .getOrElse(XDefines.DynamicCallUnknownFullName) - val argumentAsts = node.arguments.map(astForMethodCallArgument) - val call = - callNode(node, code(node), methodName, XDefines.DynamicCallUnknownFullName, DispatchTypes.DYNAMIC_DISPATCH) - .possibleTypes(IndexedSeq(methodFullName)) + val receiverAst = astForFieldAccess(memberAccess) + val methodName = memberAccess.memberName + val methodFullName = XDefines.DynamicCallUnknownFullName + val argumentAsts = node.arguments.map(astForMethodCallArgument) + val call = callNode(node, code(node), methodName, methodFullName, DispatchTypes.DYNAMIC_DISPATCH) callAst(call, argumentAsts, Some(receiverAst)) } @@ -753,7 +899,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { ) // Check if this is a method invocation of a member imported into scope match { case Some(m) => - scope.typeForMethod(m).map(t => t.name -> s"${t.name}:${m.name}").getOrElse(defaultResult) + scope.typeForMethod(m).map(t => t.name -> s"${t.name}.${m.name}").getOrElse(defaultResult) case None => defaultResult } @@ -766,28 +912,47 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { if methodFullName != methodFullNameHint then call.possibleTypes(IndexedSeq(methodFullNameHint)) - val receiverAst = astForExpression( - MemberAccess(SelfIdentifier()(node.span.spanStart(Defines.Self)), ".", call.name)(node.span) + val receiverAst = astForFieldAccess( + MemberAccess(SelfIdentifier()(node.span.spanStart(Defines.Self)), ".", call.name)(node.span), + stripLeadingAt = true ) - val baseAst = Ast(identifierNode(node, Defines.Self, Defines.Self, receiverType)) + val selfIdentifier = identifierNode(node, Defines.Self, Defines.Self, receiverType) + val baseAst = scope + .lookupVariable(Defines.Self) + .map(selfParam => Ast(selfIdentifier).withRefEdge(selfIdentifier, selfParam)) + .getOrElse(Ast(selfIdentifier)) callAst(call, argumentAst, Option(baseAst), Option(receiverAst)) } private def astForProcOrLambdaExpr(node: ProcOrLambdaExpr): Ast = { - val Seq(_, methodRef) = astForDoBlock(node.block): @unchecked - methodRef + val Seq(typeRef, _) = astForDoBlock(node.block): @unchecked + typeRef } - private def astForMethodCallArgument(node: RubyNode): Ast = { + private def astForSingletonObjectMethodDeclaration(node: SingletonObjectMethodDeclaration): Ast = { + val methodAstsWithRefs = astForMethodDeclaration(node, isSingletonObjectMethod = true) + + // Set span contents + methodAstsWithRefs.flatMap(_.nodes).foreach { + case m: NewMethodRef => DummyNode(m.copy)(node.body.span.spanStart(m.code)) + case _ => + } + + val Seq(typeRef, _) = methodAstsWithRefs + + typeRef + } + + private def astForMethodCallArgument(node: RubyExpression): Ast = { node match // Associations in method calls are keyword arguments case assoc: Association => astForKeywordArgument(assoc) case block: RubyBlock => - val Seq(methodDecl, typeDecl, _, methodRef) = astForDoBlock(block) + val Seq(methodDecl, typeDecl, typeRef, _) = astForDoBlock(block) Ast.storeInDiffGraph(methodDecl, diffGraph) Ast.storeInDiffGraph(typeDecl, diffGraph) - methodRef + typeRef case selfMethod: SingletonMethodDeclaration => // Last element is the method declaration, the prefix methods would be `foo = def foo (...)` pointers in other // contexts, but this would be empty as a method call argument @@ -799,59 +964,43 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { } Ast.storeInDiffGraph(methodDeclAst, diffGraph) scope.surroundingScopeFullName - .map(s => Ast(methodRefNode(node, selfMethod.span.text, s"$s:${selfMethod.methodName}", Defines.Any))) + .map(s => Ast(methodRefNode(node, selfMethod.span.text, s"$s.${selfMethod.methodName}", Defines.Any))) .getOrElse(Ast()) case _ => astForExpression(node) } private def astForKeywordArgument(assoc: Association): Ast = { + + def setArgumentName(argumentAst: Ast, name: String): Ast = { + argumentAst.root.collectFirst { case x: ExpressionNew => + x.argumentName_=(Option(name)) + x.argumentIndex_=(-1) + } + argumentAst + } + val value = astForExpression(assoc.value) - assoc.key match - case keyIdentifier: SimpleIdentifier => - value.root.collectFirst { case x: ExpressionNew => - x.argumentName_=(Option(keyIdentifier.text)) - x.argumentIndex_=(-1) - } - value - case _: StaticLiteral => astForExpression(assoc) + assoc.key match { + case keyIdentifier: SimpleIdentifier => setArgumentName(value, keyIdentifier.text) + case symbol @ StaticLiteral(typ) if typ == getBuiltInType(Defines.Symbol) => + setArgumentName(value, symbol.text.stripPrefix(":")) + case _: (LiteralExpr | RubyCall) => astForExpression(assoc) case x => logger.warn(s"Not explicitly handled argument association key of type ${x.getClass.getSimpleName}") astForExpression(assoc) - } - - protected def astForFieldAccess(node: MemberAccess): Ast = { - val fieldIdentifierAst = Ast(fieldIdentifierNode(node, node.memberName, node.memberName)) - val targetAst = astForExpression(node.target) - val code = s"${node.target.text}${node.op}${node.memberName}" - val memberType = typeFromCallTarget(node.target) - .flatMap(scope.tryResolveTypeReference) - .map(_.fields) - .getOrElse(List.empty) - .collectFirst { - case x if x.name == node.memberName => - scope.tryResolveTypeReference(x.typeName).map(_.name).getOrElse(Defines.Any) - } - .orElse(Option(Defines.Any)) - val fieldAccess = callNode( - node, - code, - Operators.fieldAccess, - Operators.fieldAccess, - DispatchTypes.STATIC_DISPATCH, - signature = None, - typeFullName = Option(Defines.Any) - ).possibleTypes(IndexedSeq(memberType.get)) - callAst(fieldAccess, Seq(targetAst, fieldIdentifierAst)) + } } protected def astForSplattingRubyNode(node: SplattingRubyNode): Ast = { val splattingCall = callNode(node, code(node), RubyOperators.splat, RubyOperators.splat, DispatchTypes.STATIC_DISPATCH) - val argumentAst = astsForStatement(node.name) + val argumentAst = astsForStatement(node.target) callAst(splattingCall, argumentAst) } - private def getBinaryOperatorName(op: String): Option[String] = BinaryOperatorNames.get(op) - private def getUnaryOperatorName(op: String): Option[String] = UnaryOperatorNames.get(op) + private def getBinaryOperatorName(op: String): Option[String] = BinaryOperatorNames.get(op) + + private def getUnaryOperatorName(op: String): Option[String] = UnaryOperatorNames.get(op) + private def getAssignmentOperatorName(op: String): Option[String] = AssignmentOperatorNames.get(op) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index 4bdc09643f52..a8601327a0f2 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -26,7 +26,10 @@ import scala.collection.mutable trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - val procParamGen = FreshNameGenerator(i => Left(s"")) + /** As expressions may be discarded, we cannot store closure ASTs in the diffgraph at the point of creation. So we + * assume every reference to this map means that the closure AST was successfully propagated. + */ + protected val closureToRefs = mutable.Map.empty[RubyExpression, Seq[NewNode]] /** Creates method declaration related structures. * @param node @@ -36,12 +39,22 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th * @return * a method declaration with additional refs and types if specified. */ - protected def astForMethodDeclaration(node: MethodDeclaration, isClosure: Boolean = false): Seq[Ast] = { + protected def astForMethodDeclaration( + node: RubyExpression & ProcedureDeclaration, + isClosure: Boolean = false, + isSingletonObjectMethod: Boolean = false + ): Seq[Ast] = { val isInTypeDecl = scope.surroundingAstLabel.contains(NodeTypes.TYPE_DECL) val isConstructor = (node.methodName == Defines.Initialize) && isInTypeDecl val methodName = node.methodName + // TODO: body could be a try - val fullName = computeMethodFullName(methodName) + + val fullName = node match { + case x: SingletonObjectMethodDeclaration => computeFullName(s"class<<${x.baseClass.span.text}.$methodName") + case _ => computeFullName(methodName) + } + val method = methodNode( node = node, name = methodName, @@ -55,27 +68,32 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val isSurroundedByProgramScope = scope.isSurroundedByProgramScope if (isConstructor) scope.pushNewScope(ConstructorScope(fullName)) - else scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) - - val thisParameterAst = Ast( - newThisParameterNode( - code = Defines.Self, - typeFullName = scope.surroundingTypeFullName.getOrElse(Defines.Any), - line = method.lineNumber, - column = method.columnNumber - ) + else scope.pushNewScope(MethodScope(fullName, this.procParamGen.fresh)) + + val thisParameterNode = newThisParameterNode( + name = Defines.Self, + code = Defines.Self, + typeFullName = scope.surroundingTypeFullName.getOrElse(Defines.Any), + line = method.lineNumber, + column = method.columnNumber ) + val thisParameterAst = Ast(thisParameterNode) + scope.addToScope(Defines.Self, thisParameterNode) val parameterAsts = thisParameterAst :: astForParameters(node.parameters) val optionalStatementList = statementListForOptionalParams(node.parameters) val methodReturn = methodReturnNode(node, Defines.Any) - val refs = - List(typeRefNode(node, methodName, fullName), methodRefNode(node, methodName, fullName, fullName)).map(Ast.apply) + val refs = { + val typeRef = + if isClosure then typeRefNode(node, s"$methodName&Proc", s"$fullName&Proc") + else typeRefNode(node, methodName, fullName) + List(typeRef, methodRefNode(node, methodName, fullName, fullName)).map(Ast.apply) + } // Consider which variables are captured from the outer scope - val stmtBlockAst = if (isClosure) { + val stmtBlockAst = if (isClosure || isSingletonObjectMethod) { val baseStmtBlockAst = astForMethodBody(node.body, optionalStatementList) transformAsClosureBody(refs, baseStmtBlockAst) } else { @@ -90,11 +108,11 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } // For yield statements where there isn't an explicit proc parameter - val anonProcParam = scope.anonProcParam.map { param => - val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + val anonProcParam = scope.procParamName.map { p => val nextIndex = - parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) - astForParameter(paramNode, nextIndex) + parameterAsts.flatMap(_.root).lastOption.map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) + + Ast(p.index(nextIndex)) } scope.popScope() @@ -104,29 +122,43 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th scope.surroundingAstLabel.foreach(typeDeclNode_.astParentType(_)) scope.surroundingScopeFullName.foreach(typeDeclNode_.astParentFullName(_)) createMethodTypeBindings(method, typeDeclNode_) - if isClosure then - Ast(typeDeclNode_) - .withChild(Ast(newModifierNode(ModifierTypes.LAMBDA))) - .withChild( - // This member refers back to itself, as itself is the type decl bound to the respective method - Ast(NewMember().name("call").code("call").dynamicTypeHintFullName(Seq(fullName)).typeFullName(Defines.Any)) - ) + if isClosure then Ast(typeDeclNode_).withChild(Ast(newModifierNode(ModifierTypes.LAMBDA))) else Ast(typeDeclNode_) } - val modifiers = mutable.Buffer(ModifierTypes.VIRTUAL) + // Due to lambdas being invoked by `call()`, this additional type ref holding that member is created. + val lambdaTypeDeclAst = if isClosure then { + val typeDeclNode_ = typeDeclNode(node, s"$methodName&Proc", s"$fullName&Proc", relativeFileName, code(node)) + scope.surroundingAstLabel.foreach(typeDeclNode_.astParentType(_)) + scope.surroundingScopeFullName.foreach(typeDeclNode_.astParentFullName(_)) + Ast(typeDeclNode_) + .withChild( + // This member refers back to itself, as itself is the type decl bound to the respective method + Ast(NewMember().name("call").code("call").dynamicTypeHintFullName(Seq(fullName)).typeFullName(Defines.Any)) + ) + } else Ast() + + val accessModifier = + // Initialize is guaranteed `private` by the Ruby interpreter (we include our method here) + if (methodName == Defines.Initialize || methodName == Defines.TypeDeclBody) ModifierTypes.PRIVATE + //
functions are private functions on the Object class + else if (isSurroundedByProgramScope) ModifierTypes.PRIVATE + // Else, use whatever modifier has been user-defined (or is default for current scope) + else currentAccessModifier + val modifiers = mutable.Buffer(ModifierTypes.VIRTUAL, accessModifier) if (isClosure) modifiers.addOne(ModifierTypes.LAMBDA) if (isConstructor) modifiers.addOne(ModifierTypes.CONSTRUCTOR) val prefixMemberAst = - if isClosure || isSurroundedByProgramScope then Ast() // program scope members are set elsewhere + if isClosure || isSingletonObjectMethod || isSurroundedByProgramScope then + Ast() // program scope members are set elsewhere else { // Singleton constructors that initialize @@ fields should have their members linked under the singleton class val methodMember = scope.surroundingTypeFullName match { case Some(astParentTfn) => memberForMethod(method, Option(NodeTypes.TYPE_DECL), Option(astParentTfn)) case None => memberForMethod(method, scope.surroundingAstLabel, scope.surroundingScopeFullName) } - Ast(memberForMethod(method, scope.surroundingAstLabel, scope.surroundingScopeFullName)) + Ast(memberForMethod(method, Option(NodeTypes.TYPE_DECL), scope.surroundingScopeFullName)) } // For closures, we also want the method/type refs for upstream use val methodAst_ = { @@ -141,19 +173,23 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } // Each of these ASTs are linked via AstLinker as per the astParent* properties - (prefixMemberAst :: methodAst_ :: methodTypeDeclAst :: Nil).foreach(Ast.storeInDiffGraph(_, diffGraph)) + (prefixMemberAst :: methodAst_ :: methodTypeDeclAst :: lambdaTypeDeclAst :: Nil) + .foreach(Ast.storeInDiffGraph(_, diffGraph)) // In the case of a closure, we expect this method to return a method ref, otherwise, we bind a pointer to a // method ref, e.g. self.foo = def foo(...) - if isClosure then refs else createMethodRefPointer(method) :: Nil + if isClosure || isSingletonObjectMethod then refs else createMethodRefPointer(method) :: Nil } private def transformAsClosureBody(refs: List[Ast], baseStmtBlockAst: Ast) = { // Determine which locals are captured val capturedLocalNodes = baseStmtBlockAst.nodes - .collect { case x: NewIdentifier => x } + .collect { case x: NewIdentifier if x.name != Defines.Self => x } // Self identifiers are handled separately .distinctBy(_.name) - .flatMap(i => scope.lookupVariable(i.name)) + .map(i => scope.lookupVariableInOuterScope(i.name)) + .filter(_.nonEmpty) + .flatten .toSet + val capturedIdentifiers = baseStmtBlockAst.nodes.collect { case i: NewIdentifier if capturedLocalNodes.map(_.name).contains(i.name) => i } @@ -163,30 +199,42 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th case _ => false }) - val methodRefOption = refs.flatMap(_.nodes).collectFirst { case x: NewMethodRef => x } + val typeRefOption = refs.flatMap(_.nodes).collectFirst { case x: NewTypeRef => x } + val astChildren = mutable.Buffer.empty[NewNode] + val refEdges = mutable.Buffer.empty[(NewNode, NewNode)] + val captureEdges = mutable.Buffer.empty[(NewNode, NewNode)] capturedLocalNodes .collect { case local: NewLocal => - val closureBindingId = scope.surroundingScopeFullName.map(x => s"$x:${local.name}") + val closureBindingId = scope.variableScopeFullName(local.name).map(x => s"$x.${local.name}") (local, local.name, local.code, closureBindingId) case param: NewMethodParameterIn => - val closureBindingId = scope.surroundingScopeFullName.map(x => s"$x:${param.name}") + val closureBindingId = scope.variableScopeFullName(param.name).map(x => s"$x.${param.name}") (param, param.name, param.code, closureBindingId) } - .collect { case (decl, name, code, Some(closureBindingId)) => - val local = newLocalNode(name, code, Option(closureBindingId)) - val closureBinding = newClosureBindingNode(closureBindingId, name, EvaluationStrategies.BY_REFERENCE) + .collect { case (capturedLocal, name, code, Some(closureBindingId)) => + val capturingLocal = + newLocalNode(name = name, typeFullName = Defines.Any, closureBindingId = Option(closureBindingId)) + + val closureBinding = newClosureBindingNode( + closureBindingId = closureBindingId, + originalName = name, + evaluationStrategy = EvaluationStrategies.BY_REFERENCE + ) // Create new local node for lambda, with corresponding REF edges to identifiers and closure binding - capturedBlockAst.withChild(Ast(local)) - capturedIdentifiers.filter(_.name == name).foreach(i => capturedBlockAst.withRefEdge(i, local)) - diffGraph.addEdge(closureBinding, decl, EdgeTypes.REF) + val _refEdges = + capturedIdentifiers.filter(_.name == name).map(i => i -> capturingLocal) :+ (closureBinding, capturedLocal) - methodRefOption.foreach(methodRef => diffGraph.addEdge(methodRef, closureBinding, EdgeTypes.CAPTURE)) + astChildren.addOne(capturingLocal) + refEdges.addAll(_refEdges.toList) + captureEdges.addAll(typeRefOption.map(typeRef => typeRef -> closureBinding).toList) } - capturedBlockAst + val astWithAstChildren = astChildren.foldLeft(capturedBlockAst) { case (ast, child) => ast.withChild(Ast(child)) } + val astWithRefEdges = refEdges.foldLeft(astWithAstChildren) { case (ast, (src, dst)) => ast.withRefEdge(src, dst) } + captureEdges.foldLeft(astWithRefEdges) { case (ast, (src, dst)) => ast.withCaptureEdge(src, dst) } } /** Creates the bindings between the method and its types. This is useful for resolving function pointers and imports. @@ -198,7 +246,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } // TODO: remaining cases - protected def astForParameter(node: RubyNode, index: Int): Ast = { + protected def astForParameter(node: RubyExpression, index: Int): Ast = { node match { case node: (MandatoryParameter | OptionalParameter) => val parameterIn = parameterInNode( @@ -222,9 +270,8 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th evaluationStrategy = EvaluationStrategies.BY_REFERENCE, typeFullName = None ) - scope.addToScope(node.name, parameterIn) - scope.setProcParam(node.name) - Ast(parameterIn) + scope.setProcParam(node.name, parameterIn) + Ast() // The proc parameter is retrieved later under method AST creation case node: CollectionParameter => val typeFullName = node match { case ArrayParameter(_) => prefixAsKernelDefined("Array") @@ -241,6 +288,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th ) scope.addToScope(node.name, parameterIn) Ast(parameterIn) + case node: GroupedParameter => + val parameterIn = parameterInNode( + node = node.tmpParam, + name = node.name, + code = code(node.tmpParam), + index = index, + isVariadic = false, + evaluationStrategy = EvaluationStrategies.BY_REFERENCE, + typeFullName = None + ) + scope.addToScope(node.name, parameterIn) + Ast(parameterIn) case node => logger.warn( s"${node.getClass.getSimpleName} parameters are not supported yet: ${code(node)} ($relativeFileName), skipping" @@ -249,11 +308,11 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } - private def generateTextSpan(node: RubyNode, text: String): TextSpan = { - TextSpan(node.span.line, node.span.column, node.span.lineEnd, node.span.columnEnd, text) + private def generateTextSpan(node: RubyExpression, text: String): TextSpan = { + TextSpan(node.span.line, node.span.column, node.span.lineEnd, node.span.columnEnd, node.span.offset, text) } - protected def statementForOptionalParam(node: OptionalParameter): RubyNode = { + protected def statementForOptionalParam(node: OptionalParameter): RubyExpression = { val defaultExprNode = node.defaultExpression IfExpression( @@ -301,7 +360,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th protected def astForSingletonMethodDeclaration(node: SingletonMethodDeclaration): Seq[Ast] = { node.target match { case targetNode: SingletonMethodIdentifier => - val fullName = computeMethodFullName(node.methodName) + val fullName = computeFullName(node.methodName) val (astParentType, astParentFullName, thisParamCode, addEdge) = targetNode match { case _: SelfIdentifier => @@ -321,7 +380,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } - scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) + scope.pushNewScope(MethodScope(fullName, this.procParamGen.fresh)) val method = methodNode( node = node, name = node.methodName, @@ -343,25 +402,30 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th createMethodTypeBindings(method, methodTypeDecl_) - val thisParameterAst = Ast( - newThisParameterNode( - name = Defines.Self, - code = thisParamCode, - typeFullName = astParentFullName.getOrElse(Defines.Any), - line = method.lineNumber, - column = method.columnNumber - ) + val thisNodeTypeFullName = astParentFullName match { + case Some(fn) => s"$fn" + case None => Defines.Any + } + + val thisNode = newThisParameterNode( + name = Defines.Self, + code = thisParamCode, + typeFullName = thisNodeTypeFullName, + line = method.lineNumber, + column = method.columnNumber ) + val thisParameterAst = Ast(thisNode) + scope.addToScope(Defines.Self, thisNode) - val parameterAsts = astForParameters(node.parameters) + val parameterAsts = thisParameterAst :: astForParameters(node.parameters) val optionalStatementList = statementListForOptionalParams(node.parameters) val stmtBlockAst = astForMethodBody(node.body, optionalStatementList) - val anonProcParam = scope.anonProcParam.map { param => - val paramNode = ProcParameter(param)(node.span.spanStart(s"&$param")) + val anonProcParam = scope.procParamName.map { p => val nextIndex = - parameterAsts.lastOption.flatMap(_.root).map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(1) - astForParameter(paramNode, nextIndex) + parameterAsts.flatMap(_.root).lastOption.map { case m: NewMethodParameterIn => m.index + 1 }.getOrElse(0) + + Ast(p.index(nextIndex)) } scope.popScope() @@ -373,7 +437,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th val _methodAst = methodAst( method, - (thisParameterAst +: parameterAsts) ++ anonProcParam, + parameterAsts ++ anonProcParam, stmtBlockAst, methodReturnNode(node, Defines.Any), newModifierNode(ModifierTypes.VIRTUAL) :: Nil @@ -417,7 +481,11 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th .methodFullName(Operators.fieldAccess) .dispatchType(DispatchTypes.STATIC_DISPATCH) .typeFullName(Defines.Any) - callAst(fieldAccess, Seq(Ast(self), Ast(fi))) + val selfAst = scope + .lookupVariable(Defines.Self) + .map(selfParam => Ast(self).withRefEdge(self, selfParam)) + .getOrElse(Ast(self)) + callAst(fieldAccess, Seq(selfAst, Ast(fi))) } astForAssignment(methodRefIdent, methodRefNode, method.lineNumber, method.columnNumber) @@ -426,24 +494,24 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } - private def astForParameters(parameters: List[RubyNode]): List[Ast] = { + private def astForParameters(parameters: List[RubyExpression]): List[Ast] = { parameters.zipWithIndex.map { case (parameterNode, index) => astForParameter(parameterNode, index + 1) } } - private def statementListForOptionalParams(params: List[RubyNode]): StatementList = { + private def statementListForOptionalParams(params: List[RubyExpression]): StatementList = { StatementList( params .collect { case x: OptionalParameter => x } .map(statementForOptionalParam) - )(TextSpan(None, None, None, None, "")) + )(TextSpan(None, None, None, None, None, "")) } private def astForMethodBody( - body: RubyNode, + body: RubyExpression, optionalStatementList: StatementList, returnLastExpression: Boolean = true ): Ast = { @@ -472,7 +540,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } - private def astForConstructorMethodBody(body: RubyNode, optionalStatementList: StatementList): Ast = { + private def astForConstructorMethodBody(body: RubyExpression, optionalStatementList: StatementList): Ast = { if (this.parseLevel == AstParseLevel.SIGNATURES) { Ast() } else { @@ -490,4 +558,18 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th } } + private val accessModifierStack: mutable.Stack[String] = mutable.Stack.empty + + protected def currentAccessModifier: String = { + accessModifierStack.headOption.getOrElse(ModifierTypes.PUBLIC) + } + + protected def pushAccessModifier(name: String): Unit = { + accessModifierStack.push(name) + } + + protected def popAccessModifier(): Unit = { + if (accessModifierStack.nonEmpty) accessModifierStack.pop() + } + } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 68504e6cd561..c1d571775f74 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -1,61 +1,55 @@ package io.joern.rubysrc2cpg.astcreation -import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{RubyStatement, *} import io.joern.rubysrc2cpg.datastructures.BlockScope import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.codepropertygraph.generated.nodes.{NewControlStructure, NewMethod, NewMethodRef, NewTypeDecl} +import io.shiftleft.codepropertygraph.generated.nodes.NewControlStructure +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, ModifierTypes} trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - protected def astsForStatement(node: RubyNode): Seq[Ast] = node match - case node: WhileExpression => astForWhileStatement(node) :: Nil - case node: DoWhileExpression => astForDoWhileStatement(node) :: Nil - case node: UntilExpression => astForUntilStatement(node) :: Nil - case node: IfExpression => astForIfStatement(node) :: Nil - case node: UnlessExpression => astForUnlessStatement(node) :: Nil - case node: ForExpression => astForForExpression(node) :: Nil - case node: CaseExpression => astsForCaseExpression(node) - case node: StatementList => astForStatementList(node) :: Nil - case node: SimpleCallWithBlock => astForCallWithBlock(node) :: Nil - case node: MemberCallWithBlock => astForCallWithBlock(node) :: Nil - case node: ReturnExpression => astForReturnStatement(node) :: Nil - case node: AnonymousTypeDeclaration => astForAnonymousTypeDeclaration(node) :: Nil - case node: TypeDeclaration => astForClassDeclaration(node) - case node: FieldsDeclaration => astsForFieldDeclarations(node) - case node: MethodDeclaration => astForMethodDeclaration(node) - case node: SingletonMethodDeclaration => astForSingletonMethodDeclaration(node) - case node: MultipleAssignment => node.assignments.map(astForExpression) - case node: BreakStatement => astForBreakStatement(node) :: Nil - case _ => astForExpression(node) :: Nil - - private def astForWhileStatement(node: WhileExpression): Ast = { - val conditionAst = astForExpression(node.condition) - val bodyAsts = astsForStatement(node.body) - whileAst(Some(conditionAst), bodyAsts, Option(code(node)), line(node), column(node)) - } - - private def astForDoWhileStatement(node: DoWhileExpression): Ast = { - val conditionAst = astForExpression(node.condition) - val bodyAsts = astsForStatement(node.body) - doWhileAst(Some(conditionAst), bodyAsts, Option(code(node)), line(node), column(node)) - } - - // `until T do B` is lowered as `while !T do B` - private def astForUntilStatement(node: UntilExpression): Ast = { - val notCondition = astForExpression(UnaryExpression("!", node.condition)(node.condition.span)) - val bodyAsts = astsForStatement(node.body) - whileAst(Some(notCondition), bodyAsts, Option(code(node)), line(node), column(node)) + protected def astsForStatement(node: RubyExpression): Seq[Ast] = { + baseAstCache.clear() // A safe approximation on where to reset the cache + node match { + case node: IfExpression => astForIfStatement(node) + case node: CaseExpression => astsForCaseExpression(node) + case node: StatementList => astForStatementList(node) :: Nil + case node: ReturnExpression => astForReturnExpression(node) :: Nil + case node: AnonymousTypeDeclaration => astForAnonymousTypeDeclaration(node) :: Nil + case node: TypeDeclaration => astForClassDeclaration(node) + case node: FieldsDeclaration => astsForFieldDeclarations(node) + case node: AccessModifier => registerAccessModifier(node) + case node: MethodDeclaration => astForMethodDeclaration(node) + case node: SingletonMethodDeclaration => astForSingletonMethodDeclaration(node) + case node: MultipleAssignment => node.assignments.map(astForExpression) + case node: BreakExpression => astForBreakExpression(node) :: Nil + case node: SingletonStatementList => astForSingletonStatementList(node) + case _ => astForExpression(node) :: Nil + } } - private def astForIfStatement(node: IfExpression): Ast = { + private def astForIfStatement(node: IfExpression): Seq[Ast] = { def builder(node: IfExpression, conditionAst: Ast, thenAst: Ast, elseAsts: List[Ast]): Ast = { val ifNode = controlStructureNode(node, ControlStructureTypes.IF, code(node)) controlStructureAst(ifNode, Some(conditionAst), thenAst :: elseAsts) } - foldIfExpression(builder)(node) + + foldIfExpression(builder)(node) :: Nil + } + + /** Registers the currently set access modifier for the current type (until it is reset later). + */ + private def registerAccessModifier(node: AccessModifier): Seq[Ast] = { + val modifier = node match { + case PrivateModifier() => ModifierTypes.PRIVATE + case ProtectedModifier() => ModifierTypes.PROTECTED + case PublicModifier() => ModifierTypes.PUBLIC + } + popAccessModifier() // pop off the current modifier in scope + pushAccessModifier(modifier) // push new one on + Nil } // Rewrites a nested `if T_1 then E_1 elsif T_2 then E_2 elsif ... elsif T_n then E_n else E_{n+1}` @@ -67,11 +61,11 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t builder(node, conditionAst, thenAst, elseAsts) } - private def astForThenClause(node: RubyNode): Ast = astForStatementList(node.asStatementList) + private def astForThenClause(node: RubyExpression): Ast = astForStatementList(node.asStatementList) private def astsForElseClauses( - elsIfClauses: List[RubyNode], - elseClause: Option[RubyNode], + elsIfClauses: List[RubyExpression], + elseClause: Option[RubyExpression], astForIf: IfExpression => Ast ): List[Ast] = { elsIfClauses match @@ -88,96 +82,6 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t Nil } - private def astForElseClause(node: RubyNode): Ast = { - node match - case elseNode: ElseClause => - elseNode.thenClause match - case stmtList: StatementList => astForStatementList(stmtList) - case node => - logger.warn(s"Expecting statement list in ${code(node)} ($relativeFileName), skipping") - astForUnknown(node) - case elseNode => - logger.warn(s"Expecting else clause in ${code(elseNode)} ($relativeFileName), skipping") - astForUnknown(elseNode) - } - - // `unless T do B` is lowered as `if !T then B` - private def astForUnlessStatement(node: UnlessExpression): Ast = { - val notConditionAst = astForExpression(UnaryExpression("!", node.condition)(node.condition.span)) - val thenAst = node.trueBranch match - case stmtList: StatementList => astForStatementList(stmtList) - case _ => astForStatementList(StatementList(List(node.trueBranch))(node.trueBranch.span)) - val elseAsts = node.falseBranch.map(astForElseClause).toList - val ifNode = controlStructureNode(node, ControlStructureTypes.IF, code(node)) - controlStructureAst(ifNode, Some(notConditionAst), thenAst :: elseAsts) - } - - private def astForForExpression(node: ForExpression): Ast = { - val forEachNode = controlStructureNode(node, ControlStructureTypes.FOR, code(node)) - val doBodyAst = astsForStatement(node.doBlock) - val iteratorNode = astForExpression(node.forVariable) - val iterableNode = astForExpression(node.iterableVariable) - Ast(forEachNode).withChild(iteratorNode).withChild(iterableNode).withChildren(doBodyAst) - } - - protected def astsForCaseExpression(node: CaseExpression): Seq[Ast] = { - def goCase(expr: Option[SimpleIdentifier]): List[RubyNode] = { - val elseThenClause: Option[RubyNode] = node.elseClause.map(_.asInstanceOf[ElseClause].thenClause) - val whenClauses = node.whenClauses.map(_.asInstanceOf[WhenClause]) - val ifElseChain = whenClauses.foldRight[Option[RubyNode]](elseThenClause) { - (whenClause: WhenClause, restClause: Option[RubyNode]) => - // We translate multiple match expressions into an or expression. - // - // A single match expression is compared using `.===` to the case target expression if it is present - // otherwise it is treated as a conditional. - // - // There may be a splat as the last match expression, - // `case y when *x then c end` or - // `case when *x then c end` - // which is translated to `x.include? y` and `x.any?` conditions respectively - - val conditions = whenClause.matchExpressions.map { mExpr => - expr.map(e => BinaryExpression(mExpr, "===", e)(mExpr.span)).getOrElse(mExpr) - } ++ (whenClause.matchSplatExpression.iterator.flatMap { - case splat @ SplattingRubyNode(exprList) => - expr - .map { e => - List(MemberCall(exprList, ".", "include?", List(e))(splat.span)) - } - .getOrElse { - List(MemberCall(exprList, ".", "any?", List())(splat.span)) - } - case e => - logger.warn(s"Unrecognised RubyNode (${e.getClass}) in case match splat expression") - List(Unknown()(e.span)) - }) - // There is always at least one match expression or a splat - // a splat will become an unknown in condition at the end - val condition = conditions.init.foldRight(conditions.last) { (cond, condAcc) => - BinaryExpression(cond, "||", condAcc)(whenClause.span) - } - val conditional = IfExpression( - condition, - whenClause.thenClause.asStatementList, - List(), - restClause.map { els => ElseClause(els.asStatementList)(els.span) } - )(node.span) - Some(conditional) - } - ifElseChain.iterator.toList - } - def generatedNode: StatementList = node.expression - .map { e => - val tmp = SimpleIdentifier(None)(e.span.spanStart(tmpGen.fresh)) - StatementList( - List(SingleAssignment(tmp, "=", e)(e.span)) ++ - goCase(Some(tmp)) - )(node.span) - } - .getOrElse(StatementList(goCase(None))(node.span)) - astsForStatement(generatedNode) - } - protected def astForStatementList(node: StatementList): Ast = { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) @@ -186,56 +90,38 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t blockAst(block, statementAsts) } - /* `foo() do end` is lowered as a METHOD node shaped like so: - * ``` - * = def 0() - * - * end - * foo(, ) - * ``` - */ - protected def astForCallWithBlock[C <: RubyCall](node: RubyNode & RubyCallWithBlock[C]): Ast = { - val Seq(_, methodRefAst) = astForDoBlock(node.block): @unchecked - val methodRefDummyNode = methodRefAst.root.map(DummyNode(_)(node.span)).toList - - // Create call with argument referencing the MethodRef - val callWithLambdaArg = node.withoutBlock match { - case x: SimpleCall => astForSimpleCall(x.copy(arguments = x.arguments ++ methodRefDummyNode)(x.span)) - case x: MemberCall => astForMemberCall(x.copy(arguments = x.arguments ++ methodRefDummyNode)(x.span)) - case x => - logger.warn(s"Unhandled call-with-block type ${code(x)}, creating anonymous method structures only") - Ast() - } - - callWithLambdaArg - } - - protected def astForDoBlock(block: Block & RubyNode): Seq[Ast] = { - // Create closure structures: [MethodDecl, TypeRef, MethodRef] - val methodName = nextClosureName() - - val methodAstsWithRefs = block.body match { - case x: Block => - astForMethodDeclaration(x.toMethodDeclaration(methodName, Option(block.parameters)), isClosure = true) - case _ => - astForMethodDeclaration(block.toMethodDeclaration(methodName, Option(block.parameters)), isClosure = true) - } - - // Set span contents - methodAstsWithRefs.flatMap(_.nodes).foreach { - case m: NewMethodRef => DummyNode(m.copy)(block.span.spanStart(m.code)) - case _ => + protected def astForDoBlock(block: Block & RubyExpression): Seq[Ast] = { + if (closureToRefs.contains(block)) { + closureToRefs(block).map(x => Ast(x.copy)) + } else { + val methodName = nextClosureName() + // Create closure structures: [TypeRef, MethodRef] + val methodRefAsts = block.body match { + case x: Block => + astForMethodDeclaration(x.toMethodDeclaration(methodName, Option(block.parameters)), isClosure = true) + case _ => + astForMethodDeclaration(block.toMethodDeclaration(methodName, Option(block.parameters)), isClosure = true) + } + closureToRefs.put(block, methodRefAsts.flatMap(_.root)) + methodRefAsts } - - methodAstsWithRefs } - protected def astForReturnStatement(node: ReturnExpression): Ast = { + protected def astForReturnExpression(node: ReturnExpression): Ast = { val argumentAsts = node.expressions.map(astForExpression) val returnNode_ = returnNode(node, code(node)) returnAst(returnNode_, argumentAsts) } + protected def astForNextExpression(node: NextExpression): Ast = { + val nextNode = NewControlStructure() + .controlStructureType(ControlStructureTypes.CONTINUE) + .lineNumber(line(node)) + .columnNumber(column(node)) + .code(code(node)) + Ast(nextNode) + } + protected def astForStatementListReturningLastExpression(node: StatementList): Ast = { val block = blockNode(node) scope.pushNewScope(BlockScope(block)) @@ -250,7 +136,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t blockAst(block, stmtAsts) } - private def astsForImplicitReturnStatement(node: RubyNode): Seq[Ast] = { + private def astsForImplicitReturnStatement(node: RubyExpression): Seq[Ast] = { def elseReturnNil(span: TextSpan) = Option { ElseClause( StatementList( @@ -262,27 +148,33 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } node match - case expr: ControlFlowExpression => - def transform(e: RubyNode & ControlFlowExpression): RubyNode = + case expr: ControlFlowStatement => + def transform(e: RubyExpression & ControlFlowStatement): RubyExpression = transformLastRubyNodeInControlFlowExpressionBody(e, returnLastNode(_, transform), elseReturnNil) astsForStatement(transform(expr)) case node: MemberCallWithBlock => returnAstForRubyCall(node) case node: SimpleCallWithBlock => returnAstForRubyCall(node) case _: (LiteralExpr | BinaryExpression | UnaryExpression | SimpleIdentifier | SelfIdentifier | IndexAccess | Association | YieldExpr | RubyCall | RubyFieldIdentifier | HereDocNode | Unknown) => - astForReturnStatement(ReturnExpression(List(node))(node.span)) :: Nil + astForReturnExpression(ReturnExpression(List(node))(node.span)) :: Nil case node: SingleAssignment => - astForSingleAssignment(node) :: List(astForReturnStatement(ReturnExpression(List(node.lhs))(node.span))) + astForSingleAssignment(node) :: List(astForReturnExpression(ReturnExpression(List(node.lhs))(node.span))) + case node: DefaultMultipleAssignment => + astsForStatement(node) ++ astsForImplicitReturnStatement(ArrayLiteral(node.assignments.map(_.lhs))(node.span)) + case node: GroupedParameterDesugaring => + // If the desugaring is the last expression, then we should return nil + val nilReturnSpan = node.span.spanStart("return nil") + val nilReturnLiteral = StaticLiteral(Defines.NilClass)(nilReturnSpan) + astsForStatement(node) ++ astsForImplicitReturnStatement(nilReturnLiteral) case node: AttributeAssignment => List( astForAttributeAssignment(node), astForReturnFieldAccess(MemberAccess(node.target, node.op, node.attributeName)(node.span)) ) case node: MemberAccess => astForReturnMemberCall(node) :: Nil - case ret: ReturnExpression => astForReturnStatement(ret) :: Nil - case node: MethodDeclaration => - (astForMethodDeclaration(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList - case _: BreakStatement => astsForStatement(node).toList + case ret: ReturnExpression => astForReturnExpression(ret) :: Nil + case node: (MethodDeclaration | SingletonMethodDeclaration) => + (astsForStatement(node) :+ astForReturnMethodDeclarationSymbolName(node)).toList case node => logger.warn( s"Implicit return here not supported yet: ${node.text} (${node.getClass.getSimpleName}), only generating statement" @@ -290,7 +182,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t astsForStatement(node).toList } - private def returnAstForRubyCall[C <: RubyCall](node: RubyNode & RubyCallWithBlock[C]): Seq[Ast] = { + private def returnAstForRubyCall[C <: RubyCall](node: RubyExpression & RubyCallWithBlock[C]): Seq[Ast] = { val callAst = astForCallWithBlock(node) returnAst(returnNode(node, code(node)), List(callAst)) :: Nil } @@ -301,7 +193,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t // The evaluation of a MethodDeclaration returns its name in symbol form. // E.g. `def f = 0` ===> `:f` - private def astForReturnMethodDeclarationSymbolName(node: MethodDeclaration): Ast = { + private def astForReturnMethodDeclarationSymbolName(node: RubyExpression & ProcedureDeclaration): Ast = { val literalNode_ = literalNode(node, s":${node.methodName}", getBuiltInType(Defines.Symbol)) val returnNode_ = returnNode(node, literalNode_.code) returnAst(returnNode_, Seq(Ast(literalNode_))) @@ -311,11 +203,7 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t returnAst(returnNode(node, code(node)), List(astForMemberAccess(node))) } - private def astForReturnMemberCall(node: MemberCall): Ast = { - returnAst(returnNode(node, code(node)), List(astForMemberCall(node))) - } - - protected def astForBreakStatement(node: BreakStatement): Ast = { + protected def astForBreakExpression(node: BreakExpression): Ast = { val _node = NewControlStructure() .controlStructureType(ControlStructureTypes.BREAK) .lineNumber(line(node)) @@ -324,6 +212,10 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t Ast(_node) } + protected def astForSingletonStatementList(list: SingletonStatementList): Seq[Ast] = { + list.statements.map(astForExpression) + } + /** Wraps the last RubyNode with a ReturnExpression. * @param x * the node to wrap a return around. If a StatementList is given, then the ReturnExpression will wrap around the @@ -331,17 +223,20 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t * @return * the RubyNode with an explicit expression */ - private def returnLastNode(x: RubyNode, transform: (RubyNode & ControlFlowExpression) => RubyNode): RubyNode = { - def statementListReturningLastExpression(stmts: List[RubyNode]): List[RubyNode] = stmts match { - case (head: ControlFlowClause) :: Nil => clauseReturningLastExpression(head) :: Nil - case (head: ControlFlowExpression) :: Nil => transform(head) :: Nil - case (head: ReturnExpression) :: Nil => head :: Nil - case head :: Nil => ReturnExpression(head :: Nil)(head.span) :: Nil - case Nil => List.empty - case head :: tail => head :: statementListReturningLastExpression(tail) + private def returnLastNode( + x: RubyExpression, + transform: (RubyExpression & ControlFlowStatement) => RubyExpression + ): RubyExpression = { + def statementListReturningLastExpression(stmts: List[RubyExpression]): List[RubyExpression] = stmts match { + case (head: ControlFlowClause) :: Nil => clauseReturningLastExpression(head) :: Nil + case (head: ControlFlowStatement) :: Nil => transform(head) :: Nil + case (head: ReturnExpression) :: Nil => head :: Nil + case head :: Nil => ReturnExpression(head :: Nil)(head.span) :: Nil + case Nil => List.empty + case head :: tail => head :: statementListReturningLastExpression(tail) } - def clauseReturningLastExpression(x: RubyNode & ControlFlowClause): RubyNode = x match { + def clauseReturningLastExpression(x: RubyExpression & ControlFlowClause): RubyExpression = x match { case RescueClause(exceptionClassList, assignment, thenClause) => RescueClause(exceptionClassList, assignment, returnLastNode(thenClause, transform))(x.span) case EnsureClause(thenClause) => EnsureClause(returnLastNode(thenClause, transform))(x.span) @@ -352,12 +247,11 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t } x match { - case StatementList(statements) => StatementList(statementListReturningLastExpression(statements))(x.span) - case clause: ControlFlowClause => clauseReturningLastExpression(clause) - case node: ControlFlowExpression => transform(node) - case node: BreakStatement => node - case node: ReturnExpression => node - case _ => ReturnExpression(x :: Nil)(x.span) + case StatementList(statements) => StatementList(statementListReturningLastExpression(statements))(x.span) + case clause: ControlFlowClause => clauseReturningLastExpression(clause) + case node: ControlFlowStatement => transform(node) + case node: ReturnExpression => node + case _ => ReturnExpression(x :: Nil)(x.span) } } @@ -369,10 +263,10 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t * RubyNode with transform function applied */ protected def transformLastRubyNodeInControlFlowExpressionBody( - node: RubyNode & ControlFlowExpression, - transform: RubyNode => RubyNode, + node: RubyExpression & ControlFlowStatement, + transform: RubyExpression => RubyExpression, defaultElseBranch: TextSpan => Option[ElseClause] - ): RubyNode = { + ): RubyExpression = { node match { case RescueExpression(body, rescueClauses, elseClause, ensureClause) => // Ensure never returns a value, only the main body, rescue & else clauses @@ -406,6 +300,8 @@ trait AstForStatementsCreator(implicit withSchemaValidation: ValidationMode) { t whenClauses.map(transform), elseClause.map(transform).orElse(defaultElseBranch(node.span)) )(node.span) + case next: NextExpression => next + case break: BreakExpression => break } } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 4b827f02db83..bca905f42548 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -1,13 +1,14 @@ package io.joern.rubysrc2cpg.astcreation import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* -import io.joern.rubysrc2cpg.datastructures.{BlockScope, MethodScope, ModuleScope, TypeScope} +import io.joern.rubysrc2cpg.datastructures.{BlockScope, MethodScope, ModuleScope, NamespaceScope, TypeScope} import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, ValidationMode} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ DispatchTypes, + EdgeTypes, EvaluationStrategies, ModifierTypes, NodeTypes, @@ -19,7 +20,7 @@ import scala.collection.mutable trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - protected def astForClassDeclaration(node: RubyNode & TypeDeclaration): Seq[Ast] = { + protected def astForClassDeclaration(node: RubyExpression & TypeDeclaration): Seq[Ast] = { node.name match case name: SimpleIdentifier => astForSimpleNamedClassDeclaration(node, name) case name => @@ -27,46 +28,90 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: astForUnknown(node) :: Nil } - private def getBaseClassName(node: RubyNode): Option[String] = { + private def getBaseClassName(node: RubyExpression): String = { node match case simpleIdentifier: SimpleIdentifier => - val name = simpleIdentifier.text - scope.lookupVariable(name) match { - case Some(_) => Option(name) // in the case of singleton classes, we want to keep the variable name - case None => scope.tryResolveTypeReference(name).map(_.name).orElse(Option(name)) - } + simpleIdentifier.text case _: SelfIdentifier => - scope.surroundingTypeFullName + Defines.Self case qualifiedBaseClass: MemberAccess => - scope - .tryResolveTypeReference(qualifiedBaseClass.toString) - .map(_.name) - .orElse(Option(qualifiedBaseClass.toString)) + qualifiedBaseClass.text.replace("::", ".") + case qualifiedBaseClass: MemberCall => + qualifiedBaseClass.text.replace("::", ".") case x => logger.warn( - s"Base class names of type ${x.getClass} are not supported yet: ${code(node)} ($relativeFileName), skipping" + s"Base class names of type ${x.getClass} are not supported yet: ${code(node)} ($relativeFileName), returning string as-is" ) - None + x.text } private def astForSimpleNamedClassDeclaration( - node: RubyNode & TypeDeclaration, + node: RubyExpression & TypeDeclaration, nameIdentifier: SimpleIdentifier ): Seq[Ast] = { - val className = nameIdentifier.text - val inheritsFrom = node.baseClass.flatMap(getBaseClassName).toList - val classFullName = computeClassFullName(className) - val typeDecl = typeDeclNode( - node = node, - name = className, - fullName = classFullName, - filename = relativeFileName, - code = code(node), - inherits = inheritsFrom, - alias = None - ) - scope.surroundingAstLabel.foreach(typeDecl.astParentType(_)) - scope.surroundingScopeFullName.foreach(typeDecl.astParentFullName(_)) + val className = nameIdentifier.text + val inheritsFrom = node.baseClass.map(getBaseClassName).toList + pushAccessModifier(ModifierTypes.PUBLIC) + + /** Pushes new NamespaceScope onto scope stack and populates AST_PARENT_FULL_NAME and AST_PARENT_TYPE for TypeDecls + * that are declared in a namespace + * @param typeDecl + * \- TypeDecl node + * @param astParentFullName + * \- Fullname of AstParent + * @return + * typeDecl node with updated fields + */ + def populateAstParentValues(typeDecl: NewTypeDecl, astParentFullName: String): NewTypeDecl = { + val namespaceBlockFullName = s"${scope.surroundingScopeFullName.getOrElse("")}.$astParentFullName" + scope.pushNewScope(NamespaceScope(namespaceBlockFullName)) + + val namespaceBlock = + NewNamespaceBlock().name(astParentFullName).fullName(astParentFullName).filename(relativeFileName) + + diffGraph.addNode(namespaceBlock) + + fileNode.foreach(diffGraph.addEdge(_, namespaceBlock, EdgeTypes.AST)) + + typeDecl.astParentFullName(astParentFullName) + typeDecl.astParentType(NodeTypes.NAMESPACE_BLOCK) + + typeDecl.fullName(computeFullName(className)) + typeDecl + } + + val (typeDecl, classFullName, shouldPopAdditionalScope) = node match { + case x: NamespaceDeclaration if x.namespaceParts.isDefined => + val className = nameIdentifier.text + val typeDeclTemp = typeDeclNode( + node = node, + name = className, + fullName = Defines.Any, + filename = relativeFileName, + code = code(node), + inherits = inheritsFrom, + alias = None + ) + populateAstParentValues(typeDeclTemp, x.namespaceParts.get.mkString(".")) + val classFullName = typeDeclTemp.fullName + + (typeDeclTemp, classFullName, true) + case _ => + val classFullName = computeFullName(className) + val typeDeclTemp = typeDeclNode( + node = node, + name = className, + fullName = classFullName, + filename = relativeFileName, + code = code(node), + inherits = inheritsFrom, + alias = None + ) + scope.surroundingAstLabel.foreach(typeDeclTemp.astParentType(_)) + scope.surroundingScopeFullName.foreach(typeDeclTemp.astParentFullName(_)) + (typeDeclTemp, classFullName, false) + } + /* In Ruby, there are semantic differences between the ordinary class and singleton class (think "meta" class in Python). Similar to how Java allows both static and dynamic methods/fields/etc. within the same type declaration, @@ -98,17 +143,18 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: val classBody = node.body.asInstanceOf[StatementList] // for now (bodyStatement is a superset of stmtList) - def handleDefaultConstructor(bodyAsts: Seq[Ast]): Seq[Ast] = bodyAsts match { - case bodyAsts if scope.shouldGenerateDefaultConstructor && this.parseLevel == AstParseLevel.FULL_AST => + val classBodyAsts = { + val bodyAsts = classBody.statements.flatMap(astsForStatement) + if (scope.shouldGenerateDefaultConstructor && this.parseLevel == AstParseLevel.FULL_AST) { val bodyStart = classBody.span.spanStart() val initBody = StatementList(List())(bodyStart) val methodDecl = astForMethodDeclaration(MethodDeclaration(Defines.Initialize, List(), initBody)(bodyStart)) methodDecl ++ bodyAsts - case bodyAsts => bodyAsts + } else { + bodyAsts + } } - val classBodyAsts = handleDefaultConstructor(classBody.statements.flatMap(astsForStatement)) - val fields = node match { case classDecl: ClassDeclaration => classDecl.fields case moduleDecl: ModuleDeclaration => moduleDecl.fields @@ -146,14 +192,25 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: .withChildren(fieldSingletonMemberNodes.map(_._2)) val bodyMemberCallAst = node.bodyMemberCall match { - case Some(bodyMemberCall) => astForMemberCall(bodyMemberCall) + case Some(bodyMemberCall) => astForTypeDeclBodyCall(bodyMemberCall, classFullName) case None => Ast() } (typeDeclAst :: singletonTypeDeclAst :: Nil).foreach(Ast.storeInDiffGraph(_, diffGraph)) + + if shouldPopAdditionalScope then scope.popScope() + popAccessModifier() prefixAst :: bodyMemberCallAst :: Nil } + private def astForTypeDeclBodyCall(node: TypeDeclBodyCall, typeFullName: String): Ast = { + val callAst = astForMemberCall(node.toMemberCall, isStatic = true) + callAst.nodes.collectFirst { + case c: NewCall if c.name == Defines.TypeDeclBody => c.methodFullName(s"$typeFullName.${Defines.TypeDeclBody}") + } + callAst + } + private def createTypeRefPointer(typeDecl: NewTypeDecl): Ast = { if (scope.isSurroundedByProgramScope) { // We aim to preserve whether it's a `class` or `module` in the `code` property @@ -179,7 +236,11 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: .methodFullName(Operators.fieldAccess) .dispatchType(DispatchTypes.STATIC_DISPATCH) .typeFullName(Defines.Any) - callAst(fieldAccess, Seq(Ast(self), Ast(fi))) + val selfAst = scope + .lookupVariable(Defines.Self) + .map(selfParam => Ast(self).withRefEdge(self, selfParam)) + .getOrElse(Ast(self)) + callAst(fieldAccess, Seq(selfAst, Ast(fi))) } astForAssignment(typeRefIdent, typeRefNode, typeDecl.lineNumber, typeDecl.columnNumber) } else { @@ -191,84 +252,49 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: node.fieldNames.flatMap(astsForSingleFieldDeclaration(node, _)) } - private def astsForSingleFieldDeclaration(node: FieldsDeclaration, nameNode: RubyNode): Seq[Ast] = { + private def astsForSingleFieldDeclaration(node: FieldsDeclaration, nameNode: RubyExpression): Seq[Ast] = { nameNode match case nameAsSymbol: StaticLiteral if nameAsSymbol.isSymbol => val fieldName = nameAsSymbol.innerText.prepended('@') val memberNode_ = memberNode(nameAsSymbol, fieldName, code(node), Defines.Any) val memberAst = Ast(memberNode_) - val getterAst = Option.when(node.hasGetter)(astForGetterMethod(node, fieldName)) - val setterAst = Option.when(node.hasSetter)(astForSetterMethod(node, fieldName)) - Seq(memberAst) ++ getterAst.toList ++ setterAst.toList + val getterAst = Option.when(node.hasGetter)(astForGetterMethod(node, fieldName)).getOrElse(Nil) + val setterAst = Option.when(node.hasSetter)(astForSetterMethod(node, fieldName)).getOrElse(Nil) + Seq(memberAst) ++ getterAst ++ setterAst case _ => logger.warn(s"Unsupported field declaration: ${nameNode.text}, skipping") Seq() } // creates a `def () { return }` METHOD, for = @. - private def astForGetterMethod(node: FieldsDeclaration, fieldName: String): Ast = { - val name = fieldName.drop(1) - val fullName = computeMethodFullName(name) - val method = methodNode( - node = node, - name = name, - fullName = fullName, - code = s"def $name (...)", - signature = None, - fileName = relativeFileName, - astParentType = scope.surroundingAstLabel, - astParentFullName = scope.surroundingScopeFullName - ) - scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) - val block_ = blockNode(node) - scope.pushNewScope(BlockScope(block_)) - // TODO: Should it be `return this.@abc`? - val returnAst_ = { - val returnNode_ = returnNode(node, s"return $fieldName") - val fieldNameIdentifier = identifierNode(node, fieldName, fieldName, Defines.Any) - returnAst(returnNode_, Seq(Ast(fieldNameIdentifier))) - } - - val methodBody = blockAst(block_, List(returnAst_)) - scope.popScope() - scope.popScope() - methodAst(method, Seq(), methodBody, methodReturnNode(node, Defines.Any)) + private def astForGetterMethod(node: FieldsDeclaration, fieldName: String): Seq[Ast] = { + val name = fieldName.drop(1) + val code = s"def $name (...)" + val methodDecl = MethodDeclaration( + name, + Nil, + StatementList(InstanceFieldIdentifier()(node.span.spanStart(fieldName)) :: Nil)( + node.span.spanStart(s"return $fieldName") + ) + )(node.span.spanStart(code)) + astForMethodDeclaration(methodDecl) } // creates a `def =(x) { = x }` METHOD, for = @ - private def astForSetterMethod(node: FieldsDeclaration, fieldName: String): Ast = { - val name = fieldName.drop(1) + "=" - val fullName = computeMethodFullName(name) - val method = methodNode( - node = node, - name = name, - fullName = fullName, - code = s"def $name (...)", - signature = None, - fileName = relativeFileName, - astParentType = scope.surroundingAstLabel, - astParentFullName = scope.surroundingScopeFullName - ) - scope.pushNewScope(MethodScope(fullName, procParamGen.fresh)) - val parameter = parameterInNode(node, "x", "x", 1, false, EvaluationStrategies.BY_REFERENCE) - val methodBody = { - val block_ = blockNode(node) - scope.pushNewScope(BlockScope(block_)) - val lhs = identifierNode(node, fieldName, fieldName, Defines.Any) - val rhs = identifierNode(node, parameter.name, parameter.name, Defines.Any) - val assignmentCall = callNode( - node, - s"${lhs.code} = ${rhs.code}", - Operators.assignment, - Operators.assignment, - DispatchTypes.STATIC_DISPATCH - ) - val assignmentAst = callAst(assignmentCall, Seq(Ast(lhs), Ast(rhs))) - scope.popScope() - blockAst(blockNode(node), List(assignmentAst)) - } - scope.popScope() - methodAst(method, Seq(Ast(parameter)), methodBody, methodReturnNode(node, Defines.Any)) + private def astForSetterMethod(node: FieldsDeclaration, fieldName: String): Seq[Ast] = { + val name = fieldName.drop(1) + "=" + val code = s"def $name (...)" + val assignment = SingleAssignment( + InstanceFieldIdentifier()(node.span.spanStart(fieldName)), + "=", + SimpleIdentifier()(node.span.spanStart("x")) + )(node.span.spanStart(s"$fieldName = x")) + val methodDecl = MethodDeclaration( + name, + MandatoryParameter("x")(node.span.spanStart("x")) :: Nil, + StatementList(assignment :: Nil)(node.span.spanStart(s"return $fieldName")) + )(node.span.spanStart(code)) + astForMethodDeclaration(methodDecl) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala index a1aceea6f561..9f7396f9a5c9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstSummaryVisitor.scala @@ -1,7 +1,7 @@ package io.joern.rubysrc2cpg.astcreation -import better.files.File -import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.StatementList +import flatgraph.DiffGraphApplier +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{RubyExpression, StatementList} import io.joern.rubysrc2cpg.datastructures.{RubyField, RubyMethod, RubyProgramSummary, RubyStubbedType, RubyType} import io.joern.rubysrc2cpg.parser.RubyNodeCreator import io.joern.rubysrc2cpg.passes.Defines @@ -12,7 +12,6 @@ import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Local, Member, Method, TypeDecl} import io.shiftleft.semanticcpg.language.* -import overflowdb.{BatchedUpdate, Config} import java.io.File as JavaFile import java.util.regex.Matcher @@ -23,13 +22,18 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A def summarize(asExternal: Boolean = false): RubyProgramSummary = { this.parseLevel = AstParseLevel.SIGNATURES + Using.resource(Cpg.empty) { cpg => // Build and store compilation unit AST - val rootNode = new RubyNodeCreator().visit(programCtx).asInstanceOf[StatementList] - val ast = astForRubyFile(rootNode) + val rootNode = this.rootNode match { + case Some(rootNode) => rootNode.asInstanceOf[StatementList] + case None => new RubyNodeCreator().visit(programCtx).asInstanceOf[StatementList] + } + + val ast = astForRubyFile(rootNode) Ast.storeInDiffGraph(ast, diffGraph) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) - CpgLoader.createIndexes(cpg) + DiffGraphApplier.applyDiff(cpg.graph, diffGraph) + // Link basic AST elements AstLinkerPass(cpg).createAndApply() // Summarize findings @@ -38,7 +42,7 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A } def withSummary(newSummary: RubyProgramSummary): AstCreator = { - AstCreator(fileName, programCtx, projectRoot, newSummary) + AstCreator(fileName, programCtx, projectRoot, newSummary, enableFileContents, fileContent, rootNode) } private def summarize(cpg: Cpg, asExternal: Boolean): RubyProgramSummary = { @@ -116,7 +120,7 @@ trait AstSummaryVisitor(implicit withSchemaValidation: ValidationMode) { this: A }.toSet // Map module types val typeEntries = namespace.method.collectFirst { - case m: Method if m.name == Defines.Program => + case m: Method if m.name == Defines.Main => val childrenTypes = m.astChildren.collectAll[TypeDecl].l val fullName = if childrenTypes.nonEmpty && asExternal then buildFullName(childrenTypes.head) else s"${m.fullName}" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala index 01a57a1f6414..231b91910b91 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala @@ -3,7 +3,7 @@ package io.joern.rubysrc2cpg.astcreation import io.joern.rubysrc2cpg.passes.{Defines, GlobalTypes} import io.shiftleft.codepropertygraph.generated.nodes.NewNode -import scala.annotation.tailrec +import java.util.Objects object RubyIntermediateAst { @@ -12,12 +12,15 @@ object RubyIntermediateAst { column: Option[Int], lineEnd: Option[Int], columnEnd: Option[Int], + offset: Option[(Int, Int)], text: String ) { - def spanStart(newText: String = ""): TextSpan = TextSpan(line, column, line, column, newText) + def spanStart(newText: String = ""): TextSpan = TextSpan(line, column, line, column, offset, newText) } - sealed class RubyNode(val span: TextSpan) { + /** Most-if-not-all constructs in Ruby evaluate to some value, so we name the base class `RubyExpression`. + */ + sealed class RubyExpression(val span: TextSpan) { def line: Option[Int] = span.line def column: Option[Int] = span.column @@ -26,19 +29,47 @@ object RubyIntermediateAst { def columnEnd: Option[Int] = span.columnEnd + def offset: Option[(Int, Int)] = span.offset + def text: String = span.text + + override def hashCode(): Int = Objects.hash(span) + + override def equals(obj: Any): Boolean = { + obj match { + case o: RubyExpression => o.span == span + case _ => false + } + } } - implicit class RubyNodeHelper(node: RubyNode) { + /** Ruby statements evaluate to some value (and thus are expressions), but also perform some operation, e.g., + * assignments, method definitions, etc. + */ + sealed trait RubyStatement extends RubyExpression + + implicit class RubyExpressionHelper(node: RubyExpression) { def asStatementList: StatementList = node match { case stmtList: StatementList => stmtList case _ => StatementList(List(node))(node.span) } } - final case class Unknown()(span: TextSpan) extends RubyNode(span) + final case class Unknown()(span: TextSpan) extends RubyExpression(span) - final case class StatementList(statements: List[RubyNode])(span: TextSpan) extends RubyNode(span) { + final case class StatementList(statements: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with RubyStatement { + override def text: String = statements.size match + case 0 | 1 => span.text + case _ => "(...)" + + def size: Int = statements.size + } + + final case class SingletonStatementList(statements: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with RubyStatement { override def text: String = statements.size match case 0 | 1 => span.text case _ => "(...)" @@ -48,106 +79,157 @@ object RubyIntermediateAst { sealed trait AllowedTypeDeclarationChild - sealed trait TypeDeclaration extends AllowedTypeDeclarationChild { - def name: RubyNode - def baseClass: Option[RubyNode] - def body: RubyNode - def bodyMemberCall: Option[MemberCall] + sealed trait TypeDeclaration extends AllowedTypeDeclarationChild with RubyStatement { + def name: RubyExpression + def baseClass: Option[RubyExpression] + def body: RubyExpression + def bodyMemberCall: Option[TypeDeclBodyCall] + } + + sealed trait NamespaceDeclaration extends RubyStatement { + def namespaceParts: Option[List[String]] } final case class ModuleDeclaration( - name: RubyNode, - body: RubyNode, - fields: List[RubyNode & RubyFieldIdentifier], - bodyMemberCall: Option[MemberCall] + name: RubyExpression, + body: RubyExpression, + fields: List[RubyExpression & RubyFieldIdentifier], + bodyMemberCall: Option[TypeDeclBodyCall], + namespaceParts: Option[List[String]] )(span: TextSpan) - extends RubyNode(span) - with TypeDeclaration { - def baseClass: Option[RubyNode] = None + extends RubyExpression(span) + with TypeDeclaration + with NamespaceDeclaration { + def baseClass: Option[RubyExpression] = None } final case class ClassDeclaration( - name: RubyNode, - baseClass: Option[RubyNode], - body: RubyNode, - fields: List[RubyNode & RubyFieldIdentifier], - bodyMemberCall: Option[MemberCall] + name: RubyExpression, + baseClass: Option[RubyExpression], + body: RubyExpression, + fields: List[RubyExpression & RubyFieldIdentifier], + bodyMemberCall: Option[TypeDeclBodyCall], + namespaceParts: Option[List[String]] )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with TypeDeclaration + with NamespaceDeclaration - sealed trait AnonymousTypeDeclaration extends RubyNode with TypeDeclaration + sealed trait AnonymousTypeDeclaration extends RubyExpression with TypeDeclaration final case class AnonymousClassDeclaration( - name: RubyNode, - baseClass: Option[RubyNode], - body: RubyNode, - bodyMemberCall: Option[MemberCall] = None + name: RubyExpression, + baseClass: Option[RubyExpression], + body: RubyExpression, + bodyMemberCall: Option[TypeDeclBodyCall] = None )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with AnonymousTypeDeclaration final case class SingletonClassDeclaration( - name: RubyNode, - baseClass: Option[RubyNode], - body: RubyNode, - bodyMemberCall: Option[MemberCall] = None + name: RubyExpression, + baseClass: Option[RubyExpression], + body: RubyExpression, + bodyMemberCall: Option[TypeDeclBodyCall] = None )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with AnonymousTypeDeclaration - final case class FieldsDeclaration(fieldNames: List[RubyNode])(span: TextSpan) - extends RubyNode(span) + final case class FieldsDeclaration(fieldNames: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) with AllowedTypeDeclarationChild { def hasGetter: Boolean = text.startsWith("attr_reader") || text.startsWith("attr_accessor") def hasSetter: Boolean = text.startsWith("attr_writer") || text.startsWith("attr_accessor") } - final case class MethodDeclaration(methodName: String, parameters: List[RubyNode], body: RubyNode)(span: TextSpan) - extends RubyNode(span) + sealed trait ProcedureDeclaration extends RubyStatement { + def methodName: String + def parameters: List[RubyExpression] + def body: RubyExpression + } + + final case class MethodDeclaration(methodName: String, parameters: List[RubyExpression], body: RubyExpression)( + span: TextSpan + ) extends RubyExpression(span) + with ProcedureDeclaration with AllowedTypeDeclarationChild final case class SingletonMethodDeclaration( - target: RubyNode, + target: RubyExpression, methodName: String, - parameters: List[RubyNode], - body: RubyNode + parameters: List[RubyExpression], + body: RubyExpression )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) + with ProcedureDeclaration with AllowedTypeDeclarationChild + final case class SingletonObjectMethodDeclaration( + methodName: String, + parameters: List[RubyExpression], + body: RubyExpression, + baseClass: RubyExpression + )(span: TextSpan) + extends RubyExpression(span) + with ProcedureDeclaration + sealed trait MethodParameter { def name: String } - final case class MandatoryParameter(name: String)(span: TextSpan) extends RubyNode(span) with MethodParameter + final case class MandatoryParameter(name: String)(span: TextSpan) extends RubyExpression(span) with MethodParameter - final case class OptionalParameter(name: String, defaultExpression: RubyNode)(span: TextSpan) - extends RubyNode(span) + final case class OptionalParameter(name: String, defaultExpression: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with MethodParameter + + final case class GroupedParameter( + name: String, + tmpParam: RubyExpression, + multipleAssignment: GroupedParameterDesugaring + )(span: TextSpan) + extends RubyExpression(span) with MethodParameter sealed trait CollectionParameter extends MethodParameter - final case class ArrayParameter(name: String)(span: TextSpan) extends RubyNode(span) with CollectionParameter + final case class ArrayParameter(name: String)(span: TextSpan) extends RubyExpression(span) with CollectionParameter + + final case class HashParameter(name: String)(span: TextSpan) extends RubyExpression(span) with CollectionParameter - final case class HashParameter(name: String)(span: TextSpan) extends RubyNode(span) with CollectionParameter + final case class ProcParameter(name: String)(span: TextSpan) extends RubyExpression(span) with MethodParameter - final case class ProcParameter(name: String)(span: TextSpan) extends RubyNode(span) with MethodParameter + final case class SingleAssignment(lhs: RubyExpression, op: String, rhs: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with RubyStatement - final case class SingleAssignment(lhs: RubyNode, op: String, rhs: RubyNode)(span: TextSpan) extends RubyNode(span) + trait MultipleAssignment extends RubyStatement { + def assignments: List[SingleAssignment] + } - final case class MultipleAssignment(assignments: List[SingleAssignment])(span: TextSpan) extends RubyNode(span) + final case class DefaultMultipleAssignment(assignments: List[SingleAssignment])(span: TextSpan) + extends RubyExpression(span) + with MultipleAssignment - final case class SplattingRubyNode(name: RubyNode)(span: TextSpan) extends RubyNode(span) + final case class GroupedParameterDesugaring(assignments: List[SingleAssignment])(span: TextSpan) + extends RubyExpression(span) + with MultipleAssignment - final case class AttributeAssignment(target: RubyNode, op: String, attributeName: String, rhs: RubyNode)( - span: TextSpan - ) extends RubyNode(span) + final case class SplattingRubyNode(target: RubyExpression)(span: TextSpan) extends RubyExpression(span) + + final case class AttributeAssignment( + target: RubyExpression, + op: String, + attributeName: String, + assignmentOperator: String, + rhs: RubyExpression + )(span: TextSpan) + extends RubyExpression(span) - /** Any structure that conditionally modifies the control flow of the program. + /** Any structure that conditionally modifies the control flow of the program. These also behave as statements. */ - sealed trait ControlFlowExpression + sealed trait ControlFlowStatement extends RubyStatement /** A control structure's clause, which may contain an additional control structures. */ @@ -164,81 +246,99 @@ object RubyIntermediateAst { sealed trait SingletonMethodIdentifier final case class RescueExpression( - body: RubyNode, + body: RubyExpression, rescueClauses: List[RescueClause], elseClause: Option[ElseClause], ensureClause: Option[EnsureClause] )(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + extends RubyExpression(span) + with ControlFlowStatement final case class RescueClause( - exceptionClassList: Option[RubyNode], - variables: Option[RubyNode], - thenClause: RubyNode + exceptionClassList: Option[RubyExpression], + variables: Option[RubyExpression], + thenClause: RubyExpression )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with ControlFlowClause - final case class EnsureClause(thenClause: RubyNode)(span: TextSpan) extends RubyNode(span) with ControlFlowClause + final case class EnsureClause(thenClause: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with ControlFlowClause - final case class WhileExpression(condition: RubyNode, body: RubyNode)(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + final case class WhileExpression(condition: RubyExpression, body: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with ControlFlowStatement - final case class DoWhileExpression(condition: RubyNode, body: RubyNode)(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + final case class DoWhileExpression(condition: RubyExpression, body: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with ControlFlowStatement - final case class UntilExpression(condition: RubyNode, body: RubyNode)(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + final case class UntilExpression(condition: RubyExpression, body: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with ControlFlowStatement final case class IfExpression( - condition: RubyNode, - thenClause: RubyNode, - elsifClauses: List[RubyNode], - elseClause: Option[RubyNode] + condition: RubyExpression, + thenClause: RubyExpression, + elsifClauses: List[RubyExpression], + elseClause: Option[RubyExpression] )(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + extends RubyExpression(span) + with ControlFlowStatement + with RubyStatement - final case class ElsIfClause(condition: RubyNode, thenClause: RubyNode)(span: TextSpan) - extends RubyNode(span) + final case class ElsIfClause(condition: RubyExpression, thenClause: RubyExpression)(span: TextSpan) + extends RubyExpression(span) with ControlFlowClause - final case class ElseClause(thenClause: RubyNode)(span: TextSpan) extends RubyNode(span) with ControlFlowClause + final case class ElseClause(thenClause: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with ControlFlowClause - final case class UnlessExpression(condition: RubyNode, trueBranch: RubyNode, falseBranch: Option[RubyNode])( - span: TextSpan - ) extends RubyNode(span) - with ControlFlowExpression + final case class UnlessExpression( + condition: RubyExpression, + trueBranch: RubyExpression, + falseBranch: Option[RubyExpression] + )(span: TextSpan) + extends RubyExpression(span) + with ControlFlowStatement - final case class ForExpression(forVariable: RubyNode, iterableVariable: RubyNode, doBlock: RubyNode)(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + final case class ForExpression( + forVariable: RubyExpression, + iterableVariable: RubyExpression, + doBlock: RubyExpression + )(span: TextSpan) + extends RubyExpression(span) + with ControlFlowStatement final case class CaseExpression( - expression: Option[RubyNode], - whenClauses: List[RubyNode], - elseClause: Option[RubyNode] + expression: Option[RubyExpression], + whenClauses: List[RubyExpression], + elseClause: Option[RubyExpression] )(span: TextSpan) - extends RubyNode(span) - with ControlFlowExpression + extends RubyExpression(span) + with ControlFlowStatement final case class WhenClause( - matchExpressions: List[RubyNode], - matchSplatExpression: Option[RubyNode], - thenClause: RubyNode + matchExpressions: List[RubyExpression], + matchSplatExpression: Option[RubyExpression], + thenClause: RubyExpression )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with ControlFlowClause - final case class ReturnExpression(expressions: List[RubyNode])(span: TextSpan) extends RubyNode(span) + final case class NextExpression()(span: TextSpan) extends RubyExpression(span) with ControlFlowStatement + + final case class BreakExpression()(span: TextSpan) extends RubyExpression(span) with ControlFlowStatement + + final case class ReturnExpression(expressions: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with RubyStatement /** Represents an unqualified identifier e.g. `X`, `x`, `@@x`, `$x`, `$<`, etc. */ final case class SimpleIdentifier(typeFullName: Option[String] = None)(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with RubyIdentifier with SingletonMethodIdentifier { override def toString: String = s"SimpleIdentifier(${span.text}, $typeFullName)" @@ -246,18 +346,20 @@ object RubyIntermediateAst { /** Represents a type reference successfully determined, e.g. module A; end; A */ - final case class TypeIdentifier(typeFullName: String)(span: TextSpan) extends RubyNode(span) with RubyIdentifier { + final case class TypeIdentifier(typeFullName: String)(span: TextSpan) + extends RubyExpression(span) + with RubyIdentifier { def isBuiltin: Boolean = typeFullName.startsWith(GlobalTypes.builtinPrefix) override def toString: String = s"TypeIdentifier(${span.text}, $typeFullName)" } /** Represents a InstanceFieldIdentifier e.g `@x` */ - final case class InstanceFieldIdentifier()(span: TextSpan) extends RubyNode(span) with RubyFieldIdentifier + final case class InstanceFieldIdentifier()(span: TextSpan) extends RubyExpression(span) with RubyFieldIdentifier /** Represents a ClassFieldIdentifier e.g `@@x` */ - final case class ClassFieldIdentifier()(span: TextSpan) extends RubyNode(span) with RubyFieldIdentifier + final case class ClassFieldIdentifier()(span: TextSpan) extends RubyExpression(span) with RubyFieldIdentifier - final case class SelfIdentifier()(span: TextSpan) extends RubyNode(span) with SingletonMethodIdentifier + final case class SelfIdentifier()(span: TextSpan) extends RubyExpression(span) with SingletonMethodIdentifier /** Represents some kind of literal expression. */ @@ -266,7 +368,7 @@ object RubyIntermediateAst { } /** Represents a non-interpolated literal. */ - final case class StaticLiteral(typeFullName: String)(span: TextSpan) extends RubyNode(span) with LiteralExpr { + final case class StaticLiteral(typeFullName: String)(span: TextSpan) extends RubyExpression(span) with LiteralExpr { def isSymbol: Boolean = text.startsWith(":") def isString: Boolean = text.startsWith("\"") || text.startsWith("'") @@ -282,17 +384,22 @@ object RubyIntermediateAst { } } - final case class DynamicLiteral(typeFullName: String, expressions: List[RubyNode])(span: TextSpan) - extends RubyNode(span) + final case class DynamicLiteral(typeFullName: String, expressions: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) with LiteralExpr - final case class RangeExpression(lowerBound: RubyNode, upperBound: RubyNode, rangeOperator: RangeOperator)( - span: TextSpan - ) extends RubyNode(span) + final case class RangeExpression( + lowerBound: RubyExpression, + upperBound: RubyExpression, + rangeOperator: RangeOperator + )(span: TextSpan) + extends RubyExpression(span) - final case class RangeOperator(exclusive: Boolean)(span: TextSpan) extends RubyNode(span) + final case class RangeOperator(exclusive: Boolean)(span: TextSpan) extends RubyExpression(span) - final case class ArrayLiteral(elements: List[RubyNode])(span: TextSpan) extends RubyNode(span) with LiteralExpr { + final case class ArrayLiteral(elements: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with LiteralExpr { def isSymbolArray: Boolean = text.take(2).toLowerCase.startsWith("%i") def isStringArray: Boolean = text.take(2).toLowerCase.startsWith("%w") @@ -304,48 +411,70 @@ object RubyIntermediateAst { def typeFullName: String = Defines.getBuiltInType(Defines.Array) } - final case class HashLiteral(elements: List[RubyNode])(span: TextSpan) extends RubyNode(span) with LiteralExpr { + final case class HashLiteral(elements: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with LiteralExpr { def typeFullName: String = Defines.getBuiltInType(Defines.Hash) } - final case class Association(key: RubyNode, value: RubyNode)(span: TextSpan) extends RubyNode(span) + final case class Association(key: RubyExpression, value: RubyExpression)(span: TextSpan) extends RubyExpression(span) /** Represents a call. */ sealed trait RubyCall { - def target: RubyNode - def arguments: List[RubyNode] + def target: RubyExpression + def arguments: List[RubyExpression] } /** Represents traditional calls, e.g. `foo`, `foo x, y`, `foo(x,y)` */ - final case class SimpleCall(target: RubyNode, arguments: List[RubyNode])(span: TextSpan) - extends RubyNode(span) + final case class SimpleCall(target: RubyExpression, arguments: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) with RubyCall final case class RequireCall( - target: RubyNode, - argument: RubyNode, + target: RubyExpression, + argument: RubyExpression, isRelative: Boolean = false, isWildCard: Boolean = false )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with RubyCall { - def arguments: List[RubyNode] = List(argument) - def asSimpleCall: SimpleCall = SimpleCall(target, arguments)(span) + def arguments: List[RubyExpression] = List(argument) + def asSimpleCall: SimpleCall = SimpleCall(target, arguments)(span) } - final case class IncludeCall(target: RubyNode, argument: RubyNode)(span: TextSpan) - extends RubyNode(span) + final case class IncludeCall(target: RubyExpression, argument: RubyExpression)(span: TextSpan) + extends RubyExpression(span) with RubyCall { - def arguments: List[RubyNode] = List(argument) - def asSimpleCall: SimpleCall = SimpleCall(target, arguments)(span) + def arguments: List[RubyExpression] = List(argument) + def asSimpleCall: SimpleCall = SimpleCall(target, arguments)(span) + } + + final case class RaiseCall(target: RubyExpression, arguments: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) + with RubyCall + + sealed trait AccessModifier extends AllowedTypeDeclarationChild { + def toSimpleIdentifier: SimpleIdentifier + } + + final case class PublicModifier()(span: TextSpan) extends RubyExpression(span) with AccessModifier { + override def toSimpleIdentifier: SimpleIdentifier = SimpleIdentifier(None)(span) + } + + final case class PrivateModifier()(span: TextSpan) extends RubyExpression(span) with AccessModifier { + override def toSimpleIdentifier: SimpleIdentifier = SimpleIdentifier(None)(span) + } + + final case class ProtectedModifier()(span: TextSpan) extends RubyExpression(span) with AccessModifier { + override def toSimpleIdentifier: SimpleIdentifier = SimpleIdentifier(None)(span) } /** Represents standalone `proc { ... }` or `lambda { ... }` expressions */ - final case class ProcOrLambdaExpr(block: Block)(span: TextSpan) extends RubyNode(span) + final case class ProcOrLambdaExpr(block: Block)(span: TextSpan) extends RubyExpression(span) - final case class YieldExpr(arguments: List[RubyNode])(span: TextSpan) extends RubyNode(span) + final case class YieldExpr(arguments: List[RubyExpression])(span: TextSpan) extends RubyExpression(span) /** Represents a call with a block argument. */ @@ -353,38 +482,53 @@ object RubyIntermediateAst { def block: Block - def withoutBlock: RubyNode & C + def withoutBlock: RubyExpression & C } - final case class SimpleCallWithBlock(target: RubyNode, arguments: List[RubyNode], block: Block)(span: TextSpan) - extends RubyNode(span) + final case class SimpleCallWithBlock(target: RubyExpression, arguments: List[RubyExpression], block: Block)( + span: TextSpan + ) extends RubyExpression(span) with RubyCallWithBlock[SimpleCall] { def withoutBlock: SimpleCall = SimpleCall(target, arguments)(span) } /** Represents member calls, e.g. `x.y(z,w)` */ - final case class MemberCall(target: RubyNode, op: String, methodName: String, arguments: List[RubyNode])( + final case class MemberCall(target: RubyExpression, op: String, methodName: String, arguments: List[RubyExpression])( span: TextSpan - ) extends RubyNode(span) + ) extends RubyExpression(span) with RubyCall + /** Special class for `` calls of type decls. + */ + final case class TypeDeclBodyCall(target: RubyExpression, typeName: String)(span: TextSpan) + extends RubyExpression(span) + with RubyCall { + + def toMemberCall: MemberCall = MemberCall(target, op, Defines.TypeDeclBody, arguments)(span) + + def arguments: List[RubyExpression] = Nil + + def op: String = "::" + } + final case class MemberCallWithBlock( - target: RubyNode, + target: RubyExpression, op: String, methodName: String, - arguments: List[RubyNode], + arguments: List[RubyExpression], block: Block )(span: TextSpan) - extends RubyNode(span) + extends RubyExpression(span) with RubyCallWithBlock[MemberCall] { def withoutBlock: MemberCall = MemberCall(target, op, methodName, arguments)(span) } /** Represents index accesses, e.g. `x[0]`, `self.x.y[1, 2]` */ - final case class IndexAccess(target: RubyNode, indices: List[RubyNode])(span: TextSpan) extends RubyNode(span) + final case class IndexAccess(target: RubyExpression, indices: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) - final case class MemberAccess(target: RubyNode, op: String, memberName: String)(span: TextSpan) - extends RubyNode(span) { + final case class MemberAccess(target: RubyExpression, op: String, memberName: String)(span: TextSpan) + extends RubyExpression(span) { override def toString: String = s"${target.text}.$memberName" } @@ -392,39 +536,41 @@ object RubyIntermediateAst { */ sealed trait ObjectInstantiation extends RubyCall - final case class SimpleObjectInstantiation(target: RubyNode, arguments: List[RubyNode])(span: TextSpan) - extends RubyNode(span) + final case class SimpleObjectInstantiation(target: RubyExpression, arguments: List[RubyExpression])(span: TextSpan) + extends RubyExpression(span) with ObjectInstantiation - final case class ObjectInstantiationWithBlock(target: RubyNode, arguments: List[RubyNode], block: Block)( + final case class ObjectInstantiationWithBlock(target: RubyExpression, arguments: List[RubyExpression], block: Block)( span: TextSpan - ) extends RubyNode(span) + ) extends RubyExpression(span) with ObjectInstantiation with RubyCallWithBlock[SimpleObjectInstantiation] { def withoutBlock: SimpleObjectInstantiation = SimpleObjectInstantiation(target, arguments)(span) } /** Represents a `do` or `{ .. }` (braces) block. */ - final case class Block(parameters: List[RubyNode], body: RubyNode)(span: TextSpan) extends RubyNode(span) { - - def toMethodDeclaration(name: String, parameters: Option[List[RubyNode]]): MethodDeclaration = parameters match { - case Some(givenParameters) => MethodDeclaration(name, givenParameters, body)(span) - case None => MethodDeclaration(name, this.parameters, body)(span) - } - + final case class Block(parameters: List[RubyExpression], body: RubyExpression)(span: TextSpan) + extends RubyExpression(span) + with RubyStatement { + + def toMethodDeclaration(name: String, parameters: Option[List[RubyExpression]]): MethodDeclaration = + parameters match { + case Some(givenParameters) => MethodDeclaration(name, givenParameters, body)(span) + case None => MethodDeclaration(name, this.parameters, body)(span) + } } /** A dummy class for wrapping around `NewNode` and allowing it to integrate with RubyNode classes. */ - final case class DummyNode(node: NewNode)(span: TextSpan) extends RubyNode(span) + final case class DummyNode(node: NewNode)(span: TextSpan) extends RubyExpression(span) - final case class UnaryExpression(op: String, expression: RubyNode)(span: TextSpan) extends RubyNode(span) + final case class UnaryExpression(op: String, expression: RubyExpression)(span: TextSpan) extends RubyExpression(span) - final case class BinaryExpression(lhs: RubyNode, op: String, rhs: RubyNode)(span: TextSpan) extends RubyNode(span) + final case class BinaryExpression(lhs: RubyExpression, op: String, rhs: RubyExpression)(span: TextSpan) + extends RubyExpression(span) - final case class HereDocNode(content: String)(span: TextSpan) extends RubyNode(span) + final case class HereDocNode(content: String)(span: TextSpan) extends RubyExpression(span) - final case class AliasStatement(oldName: String, newName: String)(span: TextSpan) extends RubyNode(span) + final case class AliasStatement(oldName: String, newName: String)(span: TextSpan) extends RubyExpression(span) - final case class BreakStatement()(span: TextSpan) extends RubyNode(span) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala index 2b98bce044aa..5a43cc4a139d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyScope.scala @@ -4,10 +4,10 @@ import better.files.File import io.joern.rubysrc2cpg.passes.GlobalTypes import io.joern.rubysrc2cpg.passes.GlobalTypes.builtinPrefix import io.joern.x2cpg.Defines -import io.joern.rubysrc2cpg.passes.Defines as RDefines -import io.joern.x2cpg.datastructures.* +import io.joern.x2cpg.datastructures.{TypedScopeElement, *} import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{DeclarationNew, NewLocal, NewMethodParameterIn} +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import java.io.File as JFile import scala.collection.mutable @@ -47,7 +47,8 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) /** @return * using the stack, will initialize a new module scope object. */ - def newProgramScope: Option[ProgramScope] = surroundingScopeFullName.map(ProgramScope.apply) + def newProgramScope: Option[ProgramScope] = + surroundingScopeFullName.map(_.stripSuffix(NamespaceTraversal.globalNamespaceName)).map(ProgramScope.apply) /** @return * true if the top of the stack is the program/module. @@ -124,6 +125,13 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) } } + def lookupVariableInOuterScope(identifier: String): List[DeclarationNew] = { + stack.drop(1).collect { + case scopeElement if scopeElement.variables.contains(identifier) => + scopeElement.variables(identifier) + } + } + def addRequire( projectRoot: String, currentFilePath: String, @@ -224,15 +232,26 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) (ScopeElement(MethodScope(fullName, param, true), variables), param.fold(x => x, x => x)) } - /** Get the name of the implicit or explict proc param */ + /** Get the name of the implicit or explicit proc param */ def anonProcParam: Option[String] = stack.collectFirst { case ScopeElement(MethodScope(_, Left(param), true), _) => param } - /** Set the name of explict proc param */ - def setProcParam(param: String): Unit = updateSurrounding { + /** Set the name of explicit proc param */ + def setProcParam(param: String, paramNode: NewMethodParameterIn): Unit = updateSurrounding { case ScopeElement(MethodScope(fullName, _, _), variables) => - (ScopeElement(MethodScope(fullName, Right(param)), variables), ()) + (ScopeElement(MethodScope(fullName, Right(param), true), variables ++ Map(paramNode.name -> paramNode)), ()) + } + + /** If a proc param is used, provides the node to add to the AST. + */ + def procParamName: Option[NewMethodParameterIn] = { + stack + .collectFirst { + case ScopeElement(MethodScope(_, Left(param), true), _) => param + case ScopeElement(MethodScope(_, Right(param), true), _) => param + } + .flatMap(lookupVariable(_).collect { case p: NewMethodParameterIn => p }) } def surroundingTypeFullName: Option[String] = stack.collectFirst { case ScopeElement(x: TypeLikeScope, _) => @@ -332,7 +351,7 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) case None if GlobalTypes.kernelFunctions.contains(normalizedTypeName) => Option(RubyType(s"${GlobalTypes.kernelPrefix}.$normalizedTypeName", List.empty, List.empty)) case None if GlobalTypes.bundledClasses.contains(normalizedTypeName) => - Option(RubyType(s"<${GlobalTypes.builtinPrefix}.$normalizedTypeName>", List.empty, List.empty)) + Option(RubyType(s"${GlobalTypes.builtinPrefix}.$normalizedTypeName", List.empty, List.empty)) case None => None case x => x @@ -340,4 +359,22 @@ class RubyScope(summary: RubyProgramSummary, projectRoot: Option[String]) } } + /** @param identifier + * the name of the variable. + * @return + * the full name of the variable's scope, if available. + */ + def variableScopeFullName(identifier: String): Option[String] = { + stack + .collectFirst { + case scopeElement if scopeElement.variables.contains(identifier) => + scopeElement + } + .map { + case ScopeElement(x: NamespaceLikeScope, _) => x.fullName + case ScopeElement(x: TypeLikeScope, _) => x.fullName + case ScopeElement(x: MethodLikeScope, _) => x.fullName + } + } + } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala index f7661770e69c..bc78eb9593d6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/ScopeElement.scala @@ -1,6 +1,6 @@ package io.joern.rubysrc2cpg.datastructures -import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{RubyFieldIdentifier, RubyNode} +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{RubyFieldIdentifier, RubyExpression} import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.datastructures.{NamespaceLikeScope, TypedScopeElement} import io.shiftleft.codepropertygraph.generated.nodes.NewBlock @@ -16,7 +16,7 @@ case class FieldDecl( typeFullName: String, isStatic: Boolean, isInitialized: Boolean, - node: RubyNode & RubyFieldIdentifier + node: RubyExpression & RubyFieldIdentifier ) extends TypedScopeElement /** A type-like scope with a full name. @@ -35,7 +35,7 @@ trait TypeLikeScope extends TypedScopeElement { * the relative file name. */ case class ProgramScope(fileName: String) extends TypeLikeScope { - override def fullName: String = s"$fileName:${Defines.Program}" + override def fullName: String = s"$fileName${Defines.Main}" } /** A Ruby module/abstract class. diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/ParseInternalStructures.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/ParseInternalStructures.scala deleted file mode 100644 index 6f810a9dd14e..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/ParseInternalStructures.scala +++ /dev/null @@ -1,156 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated - -import io.joern.rubysrc2cpg.RubySrc2Cpg -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.utils.PackageTable -import org.antlr.v4.runtime.ParserRuleContext -import org.antlr.v4.runtime.misc.Interval -import org.slf4j.LoggerFactory - -import java.io.File as JFile -import scala.collection.mutable -import scala.jdk.CollectionConverters.* -import scala.util.{Failure, Try} - -class ParseInternalStructures( - parsedFiles: List[(String, DeprecatedRubyParser.ProgramContext)], - projectRoot: Option[String] = None -) { - - private val logger = LoggerFactory.getLogger(getClass) - - def populatePackageTable(): Unit = { - parsedFiles.foreach { case (fileName, programCtx) => - Try { - val relativeFilename: String = - projectRoot.map(fileName.stripPrefix).map(_.stripPrefix(JFile.separator)).getOrElse(fileName) - implicit val classStack: mutable.Stack[String] = mutable.Stack[String]() - parseForStructures(relativeFilename, programCtx) - } match { - case Failure(exception) => - logger.warn(s"Exception encountered while scanning for internal structures in file '$fileName'", exception) - case _ => // do nothing - } - } - } - - private def parseForStructures(relativeFilename: String, programCtx: ProgramContext)(implicit - classStack: mutable.Stack[String] - ): Unit = { - val name = ":program" - val fullName = s"$relativeFilename:$name" - classStack.push(fullName) - if ( - programCtx.compoundStatement() != null && - programCtx.compoundStatement().statements() != null - ) { - programCtx.compoundStatement().statements().statement().asScala.foreach(parseStatement) - } - classStack.pop() - } - - private def parseStatement(ctx: StatementContext)(implicit classStack: mutable.Stack[String]): Unit = ctx match { - case ctx: ExpressionOrCommandStatementContext => parseExpressionOrCommand(ctx.expressionOrCommand()) - case _ => - } - - private def parseExpressionOrCommand( - ctx: ExpressionOrCommandContext - )(implicit classStack: mutable.Stack[String]): Unit = ctx match { - case ctx: ExpressionExpressionOrCommandContext => parseExpressionContext(ctx.expression()) - case _ => - } - - private def parseExpressionContext(ctx: ExpressionContext)(implicit classStack: mutable.Stack[String]): Unit = - ctx match { - case ctx: PrimaryExpressionContext => parsePrimaryContext(ctx.primary()) - case _ => - } - - private def parsePrimaryContext(ctx: PrimaryContext)(implicit classStack: mutable.Stack[String]): Unit = ctx match { - case ctx: MethodDefinitionPrimaryContext => parseMethodDefinitionContext(ctx.methodDefinition()) - case ctx: ModuleDefinitionPrimaryContext => parseModuleDefinitionContext(ctx.moduleDefinition()) - case ctx: ClassDefinitionPrimaryContext => parseClassDefinition(ctx.classDefinition()) - case _ => - } - - private def parseModuleDefinitionContext( - moduleDefinitionContext: ModuleDefinitionContext - )(implicit classStack: mutable.Stack[String]): Unit = { - val className = moduleDefinitionContext.classOrModuleReference().CONSTANT_IDENTIFIER().getText - classStack.push(className) - parseClassBody(moduleDefinitionContext.bodyStatement()) - } - - private def parseClassDefinition( - classDef: ClassDefinitionContext - )(implicit classStack: mutable.Stack[String]): Unit = { - Option(classDef).foreach { ctx => - Option(ctx.classOrModuleReference()).map(_.CONSTANT_IDENTIFIER().getText).foreach { className => - classStack.push(className) - parseClassBody(ctx.bodyStatement()) - } - } - } - - private def parseClassBody(ctx: BodyStatementContext)(implicit classStack: mutable.Stack[String]): Unit = { - Option(ctx).map(_.compoundStatement()).map(_.statements()).foreach(_.statement().asScala.foreach(parseStatement)) - } - - private def parseMethodDefinitionContext( - ctx: MethodDefinitionContext - )(implicit classStack: mutable.Stack[String]): Unit = { - val maybeMethodName = Option(ctx.methodNamePart()) match - case Some(ctxMethodNamePart) => - readMethodNamePart(ctxMethodNamePart) - case None => - readMethodIdentifier(ctx.methodIdentifier()) - - maybeMethodName.foreach { methodName => - val classType = if (classStack.isEmpty) "Standalone" else classStack.top - val classPath = classStack.reverse.toList.mkString(".") - RubySrc2Cpg.packageTableInfo.addPackageMethod(PackageTable.InternalModule, methodName, classPath, classType) - } - } - - private def readMethodNamePart(ctx: MethodNamePartContext): Option[String] = { - ctx match - case context: SimpleMethodNamePartContext => - Option(context.definedMethodName().methodName()) match - case Some(methodNameCtx) => Try(methodNameCtx.methodIdentifier().getText).toOption - case None => None - case context: SingletonMethodNamePartContext => - Option(context.definedMethodName().methodName()) match - case Some(methodNameCtx) => Try(methodNameCtx.methodIdentifier().getText).toOption - case None => None - case _ => None - } - - private def readMethodIdentifier(ctx: MethodIdentifierContext): Option[String] = { - if (ctx.methodOnlyIdentifier() != null) { - readMethodOnlyIdentifier(ctx.methodOnlyIdentifier()) - } else if (ctx.LOCAL_VARIABLE_IDENTIFIER() != null) { - Option(ctx.LOCAL_VARIABLE_IDENTIFIER().getSymbol.getText) - } else { - None - } - } - - private def readMethodOnlyIdentifier(ctx: MethodOnlyIdentifierContext): Option[String] = { - if (ctx.LOCAL_VARIABLE_IDENTIFIER() != null || ctx.CONSTANT_IDENTIFIER() != null) { - text(ctx) - } else { - None - } - } - - private def text(ctx: ParserRuleContext): Option[String] = Try { - val a = ctx.getStart.getStartIndex - val b = ctx.getStop.getStopIndex - val intv = new Interval(a, b) - val input = ctx.getStart.getInputStream - input.getText(intv) - }.toOption - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AntlrParser.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AntlrParser.scala deleted file mode 100644 index 89ce67e1f3e8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AntlrParser.scala +++ /dev/null @@ -1,71 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.{ - DeprecatedRubyLexer, - DeprecatedRubyLexerPostProcessor, - DeprecatedRubyParser -} -import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.atn.ATN -import org.antlr.v4.runtime.dfa.DFA -import org.slf4j.LoggerFactory - -import scala.util.Try - -/** A consumable wrapper for the RubyParser class used to parse the given file and be disposed thereafter. - * @param filename - * the file path to the file to be parsed. - */ -class AntlrParser(filename: String) { - - private val charStream = CharStreams.fromFileName(filename) - private val lexer = new DeprecatedRubyLexer(charStream) - private val tokenStream = new CommonTokenStream(DeprecatedRubyLexerPostProcessor(lexer)) - val parser: DeprecatedRubyParser = new DeprecatedRubyParser(tokenStream) - - def parse(): Try[DeprecatedRubyParser.ProgramContext] = Try(parser.program()) -} - -/** A re-usable parser object that clears the ANTLR DFA-cache if it determines that the memory usage is becoming large. - * Once this parser is closed, the whole cache is evicted. - * - * This is done in this way since clearing the cache after each file is inefficient, since the cache must be re-built - * every time, but the cache can become unnecessarily large at times. The cache also does not evict itself at the end - * of parsing. - * - * @param clearLimit - * the percentage of used heap to clear the DFA-cache on. - */ -class ResourceManagedParser(clearLimit: Double) extends AutoCloseable { - - private val logger = LoggerFactory.getLogger(getClass) - private val runtime = Runtime.getRuntime - private var maybeDecisionToDFA: Option[Array[DFA]] = None - private var maybeAtn: Option[ATN] = None - - def parse(filename: String): Try[DeprecatedRubyParser.ProgramContext] = { - val antlrParser = AntlrParser(filename) - val interp = antlrParser.parser.getInterpreter - // We need to grab a live instance in order to get the static variables as they are protected from static access - maybeDecisionToDFA = Option(interp.decisionToDFA) - maybeAtn = Option(interp.atn) - val usedMemory = runtime.freeMemory.toDouble / runtime.totalMemory.toDouble - if (usedMemory >= clearLimit) { - logger.info(s"Runtime memory consumption at $usedMemory, clearing ANTLR DFA cache") - clearDFA() - } - antlrParser.parse() - } - - /** Clears the shared DFA cache. - */ - private def clearDFA(): Unit = if (maybeDecisionToDFA.isDefined && maybeAtn.isDefined) { - val decisionToDFA = maybeDecisionToDFA.get - val atn = maybeAtn.get - for (d <- decisionToDFA.indices) { - decisionToDFA(d) = new DFA(atn.getDecisionState(d), d) - } - } - - override def close(): Unit = clearDFA() -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala deleted file mode 100644 index 347319eca95f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreator.scala +++ /dev/null @@ -1,760 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.deprecated.utils.PackageContext -import io.joern.x2cpg.Ast.storeInDiffGraph -import io.joern.x2cpg.Defines.DynamicCallUnknownFullName -import io.joern.x2cpg.X2Cpg.stripQuotes -import io.joern.x2cpg.datastructures.Global -import io.joern.x2cpg.utils.NodeBuilders.newModifierNode -import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode, Defines as XDefines} -import io.shiftleft.codepropertygraph.generated.* -import io.shiftleft.codepropertygraph.generated.nodes.* -import org.antlr.v4.runtime.ParserRuleContext -import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate - -import java.io.File as JFile -import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} -import scala.collection.immutable.Seq -import scala.collection.mutable -import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters.* -import scala.util.{Failure, Success} - -class AstCreator( - filename: String, - programCtx: DeprecatedRubyParser.ProgramContext, - protected val packageContext: PackageContext, - projectRoot: Option[String] = None -)(implicit withSchemaValidation: ValidationMode) - extends AstCreatorBase(filename) - with AstNodeBuilder[ParserRuleContext, AstCreator] - with AstForPrimitivesCreator - with AstForStatementsCreator(filename) - with AstForFunctionsCreator - with AstForExpressionsCreator - with AstForDeclarationsCreator - with AstForTypesCreator - with AstForControlStructuresCreator - with AstCreatorHelper - with AstForHereDocsCreator { - - protected val scope: RubyScope = new RubyScope() - - private val logger = LoggerFactory.getLogger(this.getClass) - - protected val classStack: mutable.Stack[String] = mutable.Stack[String]() - - protected val packageStack: mutable.Stack[String] = mutable.Stack[String]() - - protected val pathSep = "." - - protected val relativeFilename: String = - projectRoot.map(filename.stripPrefix).map(_.stripPrefix(JFile.separator)).getOrElse(filename) - - // The below are for adding implicit return nodes to methods - - // This is true if the last statement of a method is being processed. The last statement could be a if-else as well - protected val processingLastMethodStatement: AtomicBoolean = AtomicBoolean(false) - // a monotonically increasing block id unique within this file - protected val blockIdCounter: AtomicInteger = AtomicInteger(1) - // block id of the block currently being processed - protected val currentBlockId: AtomicInteger = AtomicInteger(0) - /* - * This is a hash of parent block id ---> child block id. If there are multiple children, any one child can be present. - * The value of this entry for a block is read AFTER its last statement has been processed. Absence of the the block - * in this hash implies this is a leaf block. - */ - protected val blockChildHash: mutable.Map[Int, Int] = mutable.HashMap[Int, Int]() - - private val builtInCallNames = mutable.HashSet[String]() - // Hashmap to store used variable names, to avoid duplicates in case of un-named variables - protected val usedVariableNames = mutable.HashMap.empty[String, Int] - - override def createAst(): BatchedUpdate.DiffGraphBuilder = createAstForProgramCtx(programCtx) - - private def createAstForProgramCtx(programCtx: DeprecatedRubyParser.ProgramContext) = { - val name = ":program" - val fullName = s"$relativeFilename:$name" - val programMethod = - methodNode( - programCtx, - name, - name, - fullName, - None, - relativeFilename, - Option(NodeTypes.TYPE_DECL), - Option(fullName) - ) - - classStack.push(fullName) - scope.pushNewScope(programMethod) - - val statementAsts = - if ( - programCtx.compoundStatement() != null && - programCtx.compoundStatement().statements() != null - ) { - astForStatements(programCtx.compoundStatement().statements(), false, false) ++ blockMethods - } else { - logger.error(s"File $filename has no compound statement. Needs to be examined") - List[Ast](Ast()) - } - - val methodRetNode = methodReturnNode(programCtx, Defines.Any) - - // For all the builtIn's encountered create assignment ast, minus user-defined methods with the same name - val lineColNum = 1 - val builtInMethodAst = builtInCallNames - .filterNot(methodNameToMethod.contains) - .map { builtInCallName => - val identifierNode = NewIdentifier() - .code(builtInCallName) - .name(builtInCallName) - .lineNumber(lineColNum) - .columnNumber(lineColNum) - .typeFullName(Defines.Any) - scope.addToScope(builtInCallName, identifierNode) - val typeRefNode = NewTypeRef() - .code(prefixAsBuiltin(builtInCallName)) - .typeFullName(prefixAsBuiltin(builtInCallName)) - .lineNumber(lineColNum) - .columnNumber(lineColNum) - astForAssignment(identifierNode, typeRefNode, Some(lineColNum), Some(lineColNum)) - } - .toList - - val methodRefAssignmentAsts = methodNameToMethod.values - .filterNot(_.astParentType == NodeTypes.TYPE_DECL) - .map { methodNode => - // Create a methodRefNode and assign it to the identifier version of the method, which will help in type propagation to resolve calls - val methodRefNode = NewMethodRef() - .code("def " + methodNode.name + "(...)") - .methodFullName(methodNode.fullName) - .typeFullName(methodNode.fullName) - .lineNumber(lineColNum) - .columnNumber(lineColNum) - - val methodNameIdentifier = NewIdentifier() - .code(methodNode.name) - .name(methodNode.name) - .typeFullName(Defines.Any) - .lineNumber(lineColNum) - .columnNumber(lineColNum) - scope.addToScope(methodNode.name, methodNameIdentifier) - val methodRefAssignmentAst = - astForAssignment(methodNameIdentifier, methodRefNode, methodNode.lineNumber, methodNode.columnNumber) - methodRefAssignmentAst - } - .toList - - val typeRefAssignmentAst = typeDeclNameToTypeDecl.values.map { typeDeclNode => - - val typeRefNode = NewTypeRef() - .code("class " + typeDeclNode.name + "(...)") - .typeFullName(typeDeclNode.fullName) - .lineNumber(typeDeclNode.lineNumber) - .columnNumber(typeDeclNode.columnNumber) - - val typeDeclNameIdentifier = NewIdentifier() - .code(typeDeclNode.name) - .name(typeDeclNode.name) - .typeFullName(Defines.Any) - .lineNumber(lineColNum) - .columnNumber(lineColNum) - scope.addToScope(typeDeclNode.name, typeDeclNameIdentifier) - val typeRefAssignmentAst = - astForAssignment(typeDeclNameIdentifier, typeRefNode, typeDeclNode.lineNumber, typeDeclNode.columnNumber) - typeRefAssignmentAst - } - - val methodDefInArgumentAsts = methodDefInArgument.toList - val locals = scope.createAndLinkLocalNodes(diffGraph).map(Ast.apply) - val programAst = - methodAst( - programMethod, - Seq.empty[Ast], - blockAst( - blockNode(programCtx), - locals ++ builtInMethodAst ++ methodRefAssignmentAsts ++ typeRefAssignmentAst ++ methodDefInArgumentAsts ++ statementAsts.toList - ), - methodRetNode, - newModifierNode(ModifierTypes.MODULE) :: Nil - ) - - scope.popScope() - - val fileNode = NewFile().name(relativeFilename).order(1) - val namespaceBlock = globalNamespaceBlock() - val ast = Ast(fileNode).withChild(Ast(namespaceBlock).withChild(programAst)) - - classStack.popAll() - - storeInDiffGraph(ast, diffGraph) - diffGraph - } - - def astForPrimaryContext(ctx: PrimaryContext): Seq[Ast] = ctx match { - case ctx: ClassDefinitionPrimaryContext if ctx.hasClassDefinition => astForClassDeclaration(ctx) - case ctx: ClassDefinitionPrimaryContext => astForClassExpression(ctx) - case ctx: ModuleDefinitionPrimaryContext => astForModuleDefinitionPrimaryContext(ctx) - case ctx: MethodDefinitionPrimaryContext => astForMethodDefinitionContext(ctx.methodDefinition()) - case ctx: ProcDefinitionPrimaryContext => astForProcDefinitionContext(ctx.procDefinition()) - case ctx: YieldWithOptionalArgumentPrimaryContext => - Seq(astForYieldCall(ctx, Option(ctx.yieldWithOptionalArgument().arguments()))) - case ctx: IfExpressionPrimaryContext => Seq(astForIfExpression(ctx.ifExpression())) - case ctx: UnlessExpressionPrimaryContext => Seq(astForUnlessExpression(ctx.unlessExpression())) - case ctx: CaseExpressionPrimaryContext => astForCaseExpressionPrimaryContext(ctx) - case ctx: WhileExpressionPrimaryContext => Seq(astForWhileExpression(ctx.whileExpression())) - case ctx: UntilExpressionPrimaryContext => Seq(astForUntilExpression(ctx.untilExpression())) - case ctx: ForExpressionPrimaryContext => Seq(astForForExpression(ctx.forExpression())) - case ctx: ReturnWithParenthesesPrimaryContext => - Seq(returnAst(returnNode(ctx, code(ctx)), astForArgumentsWithParenthesesContext(ctx.argumentsWithParentheses()))) - case ctx: JumpExpressionPrimaryContext => astForJumpExpressionPrimaryContext(ctx) - case ctx: BeginExpressionPrimaryContext => astForBeginExpressionPrimaryContext(ctx) - case ctx: GroupingExpressionPrimaryContext => astForCompoundStatement(ctx.compoundStatement(), false, false) - case ctx: VariableReferencePrimaryContext => Seq(astForVariableReference(ctx.variableReference())) - case ctx: SimpleScopedConstantReferencePrimaryContext => - astForSimpleScopedConstantReferencePrimaryContext(ctx) - case ctx: ChainedScopedConstantReferencePrimaryContext => - astForChainedScopedConstantReferencePrimaryContext(ctx) - case ctx: ArrayConstructorPrimaryContext => astForArrayLiteral(ctx.arrayConstructor()) - case ctx: HashConstructorPrimaryContext => astForHashConstructorPrimaryContext(ctx) - case ctx: LiteralPrimaryContext => astForLiteralPrimaryExpression(ctx) - case ctx: StringExpressionPrimaryContext => astForStringExpression(ctx.stringExpression) - case ctx: QuotedStringExpressionPrimaryContext => astForQuotedStringExpression(ctx.quotedStringExpression) - case ctx: RegexInterpolationPrimaryContext => - astForRegexInterpolationPrimaryContext(ctx.regexInterpolation) - case ctx: QuotedRegexInterpolationPrimaryContext => astForQuotedRegexInterpolation(ctx.quotedRegexInterpolation) - case ctx: IsDefinedPrimaryContext => Seq(astForIsDefinedPrimaryExpression(ctx)) - case ctx: SuperExpressionPrimaryContext => Seq(astForSuperExpression(ctx)) - case ctx: IndexingExpressionPrimaryContext => astForIndexingExpressionPrimaryContext(ctx) - case ctx: MethodOnlyIdentifierPrimaryContext => astForMethodOnlyIdentifier(ctx.methodOnlyIdentifier()) - case ctx: InvocationWithBlockOnlyPrimaryContext => astForInvocationWithBlockOnlyPrimaryContext(ctx) - case ctx: InvocationWithParenthesesPrimaryContext => astForInvocationWithParenthesesPrimaryContext(ctx) - case ctx: ChainedInvocationPrimaryContext => astForChainedInvocationPrimaryContext(ctx) - case ctx: ChainedInvocationWithoutArgumentsPrimaryContext => - astForChainedInvocationWithoutArgumentsPrimaryContext(ctx) - case _ => - logger.error(s"astForPrimaryContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - def astForExpressionContext(ctx: ExpressionContext): Seq[Ast] = ctx match { - case ctx: PrimaryExpressionContext => astForPrimaryContext(ctx.primary()) - case ctx: UnaryExpressionContext => Seq(astForUnaryExpression(ctx)) - case ctx: PowerExpressionContext => Seq(astForPowerExpression(ctx)) - case ctx: UnaryMinusExpressionContext => Seq(astForUnaryMinusExpression(ctx)) - case ctx: MultiplicativeExpressionContext => Seq(astForMultiplicativeExpression(ctx)) - case ctx: AdditiveExpressionContext => Seq(astForAdditiveExpression(ctx)) - case ctx: BitwiseShiftExpressionContext => Seq(astForBitwiseShiftExpression(ctx)) - case ctx: BitwiseAndExpressionContext => Seq(astForBitwiseAndExpression(ctx)) - case ctx: BitwiseOrExpressionContext => Seq(astForBitwiseOrExpression(ctx)) - case ctx: RelationalExpressionContext => Seq(astForRelationalExpression(ctx)) - case ctx: EqualityExpressionContext => Seq(astForEqualityExpression(ctx)) - case ctx: OperatorAndExpressionContext => Seq(astForAndExpression(ctx)) - case ctx: OperatorOrExpressionContext => Seq(astForOrExpression(ctx)) - case ctx: RangeExpressionContext => astForRangeExpressionContext(ctx) - case ctx: ConditionalOperatorExpressionContext => Seq(astForTernaryConditionalOperator(ctx)) - case ctx: SingleAssignmentExpressionContext => astForSingleAssignmentExpressionContext(ctx) - case ctx: MultipleAssignmentExpressionContext => astForMultipleAssignmentExpressionContext(ctx) - case ctx: IsDefinedExpressionContext => Seq(astForIsDefinedExpression(ctx)) - case _ => - logger.error(s"astForExpressionContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - protected def astForIndexingArgumentsContext(ctx: IndexingArgumentsContext): Seq[Ast] = ctx match { - case ctx: DeprecatedRubyParser.CommandOnlyIndexingArgumentsContext => - astForCommand(ctx.command()) - case ctx: DeprecatedRubyParser.ExpressionsOnlyIndexingArgumentsContext => - ctx - .expressions() - .expression() - .asScala - .flatMap(astForExpressionContext) - .toSeq - case ctx: DeprecatedRubyParser.ExpressionsAndSplattingIndexingArgumentsContext => - val expAsts = ctx - .expressions() - .expression() - .asScala - .flatMap(astForExpressionContext) - .toSeq - val splatAsts = astForExpressionOrCommand(ctx.splattingArgument().expressionOrCommand()) - val callNode = createOpCall(ctx.COMMA, Operators.arrayInitializer, code(ctx)) - Seq(callAst(callNode, expAsts ++ splatAsts)) - case ctx: AssociationsOnlyIndexingArgumentsContext => - astForAssociationsContext(ctx.associations()) - case ctx: DeprecatedRubyParser.SplattingOnlyIndexingArgumentsContext => - astForExpressionOrCommand(ctx.splattingArgument().expressionOrCommand()) - case _ => - logger.error(s"astForIndexingArgumentsContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - private def astForBeginExpressionPrimaryContext(ctx: BeginExpressionPrimaryContext): Seq[Ast] = - astForBodyStatementContext(ctx.beginExpression().bodyStatement()) - - private def astForChainedInvocationPrimaryContext(ctx: ChainedInvocationPrimaryContext): Seq[Ast] = { - val hasBlockStmt = ctx.block() != null - val primaryAst = astForPrimaryContext(ctx.primary()) - val methodNameAst = - if (!hasBlockStmt && code(ctx.methodName()) == "new") astForCallToConstructor(ctx.methodName(), primaryAst) - else astForMethodNameContext(ctx.methodName()) - - val terminalNode = if (ctx.COLON2() != null) { - ctx.COLON2() - } else if (ctx.DOT() != null) { - ctx.DOT() - } else { - ctx.AMPDOT() - } - - val argsAst = if (ctx.argumentsWithParentheses() != null) { - astForArgumentsWithParenthesesContext(ctx.argumentsWithParentheses()) - } else { - Seq() - } - - if (hasBlockStmt) { - val blockName = methodNameAst.head.nodes.head - .asInstanceOf[NewCall] - .name - val blockMethodName = blockName + terminalNode.getSymbol.getLine - val blockMethodAsts = - astForBlockFunction( - ctxStmt = ctx.block().compoundStatement.statements(), - ctxParam = ctx.block().blockParameter, - blockMethodName, - line(ctx).head, - column(ctx).head, - lineEnd(ctx).head, - columnEnd(ctx).head - ) - val blockMethodNode = - blockMethodAsts.head.nodes.head - .asInstanceOf[NewMethod] - - blockMethods.addOne(blockMethodAsts.head) - - val callNode = NewCall() - .name(blockName) - .methodFullName(blockMethodNode.fullName) - .typeFullName(Defines.Any) - .code(blockMethodNode.code) - .lineNumber(blockMethodNode.lineNumber) - .columnNumber(blockMethodNode.columnNumber) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - - val methodRefNode = NewMethodRef() - .methodFullName(blockMethodNode.fullName) - .typeFullName(Defines.Any) - .code(blockMethodNode.code) - .lineNumber(blockMethodNode.lineNumber) - .columnNumber(blockMethodNode.columnNumber) - - Seq(callAst(callNode, argsAst ++ Seq(Ast(methodRefNode)), primaryAst.headOption)) - } else { - val callNode = methodNameAst.head.nodes - .filter(node => node.isInstanceOf[NewCall]) - .head - .asInstanceOf[NewCall] - - if (callNode.name == "call" && ctx.primary().isInstanceOf[ProcDefinitionPrimaryContext]) { - // this is a proc.call - val baseCallNode = primaryAst.head.nodes.head.asInstanceOf[NewCall] - Seq(callAst(baseCallNode, argsAst)) - } else { - callNode - .code(text(ctx)) - .lineNumber(terminalNode.lineNumber) - .columnNumber(terminalNode.columnNumber) - - primaryAst.headOption.flatMap(_.root) match { - case Some(methodNode: NewMethod) => - val methodRefNode = NewMethodRef() - .code("def " + methodNode.name + "(...)") - .methodFullName(methodNode.fullName) - .typeFullName(Defines.Any) - blockMethods.addOne(primaryAst.head) - Seq(callAst(callNode, Seq(Ast(methodRefNode)) ++ argsAst)) - case _ => - Seq(callAst(callNode, argsAst, primaryAst.headOption)) - } - } - } - } - - private def astForCallToConstructor(ctx: MethodNameContext, receiverAst: Seq[Ast]): Seq[Ast] = { - val receiverTypeName = receiverAst.flatMap(_.root).collectFirst { case x: NewIdentifier => x } match - case Some(receiverNode) if receiverNode.typeFullName != Defines.Any => - receiverNode.typeFullName - case Some(receiverNode) if typeDeclNameToTypeDecl.contains(receiverNode.name) => - typeDeclNameToTypeDecl(receiverNode.name).fullName - case _ => Defines.Any - - val name = XDefines.ConstructorMethodName - val (methodFullName, typeFullName) = - if (receiverTypeName != Defines.Any) - (Seq(receiverTypeName, XDefines.ConstructorMethodName).mkString(pathSep), receiverTypeName) - else (XDefines.DynamicCallUnknownFullName, Defines.Any) - - val constructorCall = - callNode(ctx, code(ctx), name, methodFullName, DispatchTypes.STATIC_DISPATCH, None, Option(typeFullName)) - Seq(Ast(constructorCall)) - } - - def astForChainedInvocationWithoutArgumentsPrimaryContext( - ctx: ChainedInvocationWithoutArgumentsPrimaryContext - ): Seq[Ast] = { - val methodNameAst = astForMethodNameContext(ctx.methodName()) - val baseAst = astForPrimaryContext(ctx.primary()) - - val blocksAst = if (ctx.block() != null) { - Seq(astForBlock(ctx.block())) - } else { - Seq() - } - val callNode = methodNameAst.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] - callNode - .code(text(ctx)) - .lineNumber(ctx.COLON2().getSymbol().getLine()) - .columnNumber(ctx.COLON2().getSymbol().getCharPositionInLine()) - Seq(callAst(callNode, baseAst ++ blocksAst)) - } - - private def astForChainedScopedConstantReferencePrimaryContext( - ctx: ChainedScopedConstantReferencePrimaryContext - ): Seq[Ast] = { - val primaryAst = astForPrimaryContext(ctx.primary()) - val localVar = ctx.CONSTANT_IDENTIFIER() - val varSymbol = localVar.getSymbol - val node = createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, List(Defines.Any)) - val constAst = Ast(node) - - val operatorName = getOperatorName(ctx.COLON2().getSymbol) - val callNode = createOpCall(ctx.COLON2, operatorName, code(ctx)) - Seq(callAst(callNode, primaryAst ++ Seq(constAst))) - } - - private def astForGroupedLeftHandSideContext(ctx: GroupedLeftHandSideContext): Seq[Ast] = { - astForMultipleLeftHandSideContext(ctx.multipleLeftHandSide()) - } - - private def astForPackingLeftHandSideContext(ctx: PackingLeftHandSideContext): Seq[Ast] = { - astForSingleLeftHandSideContext(ctx.singleLeftHandSide()) - } - - def astForMultipleLeftHandSideContext(ctx: MultipleLeftHandSideContext): Seq[Ast] = ctx match { - case ctx: MultipleLeftHandSideAndpackingLeftHandSideMultipleLeftHandSideContext => - val multipleLHSAsts = ctx.multipleLeftHandSideItem.asScala.flatMap { item => - if (item.singleLeftHandSide != null) { - astForSingleLeftHandSideContext(item.singleLeftHandSide()) - } else { - astForGroupedLeftHandSideContext(item.groupedLeftHandSide()) - } - }.toList - - val paramAsts = - if (ctx.packingLeftHandSide() != null) { - val packingLHSAst = astForPackingLeftHandSideContext(ctx.packingLeftHandSide()) - multipleLHSAsts ++ packingLHSAst - } else { - multipleLHSAsts - } - - paramAsts - - case ctx: PackingLeftHandSideOnlyMultipleLeftHandSideContext => - astForPackingLeftHandSideContext(ctx.packingLeftHandSide()) - case ctx: GroupedLeftHandSideOnlyMultipleLeftHandSideContext => - astForGroupedLeftHandSideContext(ctx.groupedLeftHandSide()) - case _ => - logger.error(s"astForMultipleLeftHandSideContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - // TODO: Clean-up and take into account other hash elements - private def astForHashConstructorPrimaryContext(ctx: HashConstructorPrimaryContext): Seq[Ast] = { - if (ctx.hashConstructor().hashConstructorElements() == null) return Seq(Ast()) - val hashCtorElemCtxs = ctx.hashConstructor().hashConstructorElements().hashConstructorElement().asScala - val associationCtxs = hashCtorElemCtxs.filter(_.association() != null).map(_.association()).toSeq - val expressionCtxs = hashCtorElemCtxs.filter(_.expression() != null).map(_.expression()).toSeq - expressionCtxs.flatMap(astForExpressionContext) ++ associationCtxs.flatMap(astForAssociationContext) - } - - def astForInvocationExpressionOrCommandContext(ctx: InvocationExpressionOrCommandContext): Seq[Ast] = { - if (ctx.EMARK() != null) { - val invocWOParenAsts = astForInvocationWithoutParenthesesContext(ctx.invocationWithoutParentheses()) - val operatorName = getOperatorName(ctx.EMARK().getSymbol) - val callNode = createOpCall(ctx.EMARK, operatorName, code(ctx)) - Seq(callAst(callNode, invocWOParenAsts)) - } else { - astForInvocationWithoutParenthesesContext(ctx.invocationWithoutParentheses()) - } - } - - private def astForInvocationWithoutParenthesesContext(ctx: InvocationWithoutParenthesesContext): Seq[Ast] = - ctx match { - case ctx: SingleCommandOnlyInvocationWithoutParenthesesContext => astForCommand(ctx.command()) - case ctx: ChainedCommandDoBlockInvocationWithoutParenthesesContext => - astForChainedCommandWithDoBlockContext(ctx.chainedCommandWithDoBlock()) - case ctx: ReturnArgsInvocationWithoutParenthesesContext => - val retNode = NewReturn() - .code(text(ctx)) - .lineNumber(ctx.RETURN().getSymbol.getLine) - .columnNumber(ctx.RETURN().getSymbol.getCharPositionInLine) - val argAst = Option(ctx.arguments).map(astForArguments).getOrElse(Seq()) - Seq(returnAst(retNode, argAst)) - case ctx: BreakArgsInvocationWithoutParenthesesContext => - astForBreakArgsInvocation(ctx) - case ctx: NextArgsInvocationWithoutParenthesesContext => - astForNextArgsInvocation(ctx) - case _ => - logger.error( - s"astForInvocationWithoutParenthesesContext() $relativeFilename, ${text(ctx)} All contexts mismatched." - ) - Seq(Ast()) - } - - private def astForInvocationWithBlockOnlyPrimaryContext(ctx: InvocationWithBlockOnlyPrimaryContext): Seq[Ast] = { - val methodIdAst = astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - val blockName = methodIdAst.head.nodes.head - .asInstanceOf[NewCall] - .name - - val isYieldMethod = if (blockName.endsWith(YIELD_SUFFIX)) { - val lookupMethodName = blockName.take(blockName.length - YIELD_SUFFIX.length) - methodNamesWithYield.contains(lookupMethodName) - } else { - false - } - - if (isYieldMethod) { - /* - * This is a yield block. Create a fake method out of it. The yield call will be a call to the yield block - */ - astForBlockFunction( - ctx.block().compoundStatement.statements(), - ctx.block().blockParameter, - blockName, - line(ctx).head, - lineEnd(ctx).head, - column(ctx).head, - columnEnd(ctx).head - ) - } else { - val blockAst = Seq(astForBlock(ctx.block())) - // this is expected to be a call node - val callNode = methodIdAst.head.nodes.head.asInstanceOf[NewCall] - Seq(callAst(callNode, blockAst)) - } - } - - private def astForInvocationWithParenthesesPrimaryContext(ctx: InvocationWithParenthesesPrimaryContext): Seq[Ast] = { - val methodIdAst = astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - val parenAst = astForArgumentsWithParenthesesContext(ctx.argumentsWithParentheses()) - val callNode = methodIdAst.head.nodes.filter(_.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] - callNode.name(resolveAlias(callNode.name)) - - if (ctx.block() != null) { - val isYieldMethod = if (callNode.name.endsWith(YIELD_SUFFIX)) { - val lookupMethodName = callNode.name.take(callNode.name.length - YIELD_SUFFIX.length) - methodNamesWithYield.contains(lookupMethodName) - } else { - false - } - if (isYieldMethod) { - val methAst = astForBlock(ctx.block(), Some(callNode.name)) - blockMethods.addOne(methAst) - Seq(callAst(callNode, parenAst)) - } else { - val blockAst = Seq(astForBlock(ctx.block())) - Seq(callAst(callNode, parenAst ++ blockAst)) - } - } else - Seq(callAst(callNode, parenAst)) - } - - def astForCallNode(ctx: ParserRuleContext, code: String, isYieldBlock: Boolean = false): Ast = { - val name = if (isYieldBlock) { - s"${resolveAlias(text(ctx))}$YIELD_SUFFIX" - } else { - val calleeName = resolveAlias(text(ctx)) - // Add the call name to the global builtIn callNames set - if (isBuiltin(calleeName)) builtInCallNames.add(calleeName) - calleeName - } - - callAst(callNode(ctx, code, name, DynamicCallUnknownFullName, DispatchTypes.STATIC_DISPATCH)) - } - - private def astForMethodOnlyIdentifier(ctx: MethodOnlyIdentifierContext): Seq[Ast] = { - if (ctx.LOCAL_VARIABLE_IDENTIFIER() != null) { - Seq(astForCallNode(ctx, code(ctx))) - } else if (ctx.CONSTANT_IDENTIFIER() != null) { - Seq(astForCallNode(ctx, code(ctx))) - } else if (ctx.keyword() != null) { - Seq(astForCallNode(ctx, code(ctx.keyword()))) - } else { - Seq(Ast()) - } - } - - def astForMethodIdentifierContext(ctx: MethodIdentifierContext, code: String): Seq[Ast] = { - // the local/const identifiers are definitely method names - if (ctx.methodOnlyIdentifier() != null) { - astForMethodOnlyIdentifier(ctx.methodOnlyIdentifier()) - } else if (ctx.LOCAL_VARIABLE_IDENTIFIER() != null) { - val localVar = ctx.LOCAL_VARIABLE_IDENTIFIER() - val varSymbol = localVar.getSymbol - Seq(astForCallNode(ctx, code, methodNamesWithYield.contains(varSymbol.getText))) - } else if (ctx.CONSTANT_IDENTIFIER() != null) { - Seq(astForCallNode(ctx, code)) - } else { - Seq.empty - } - } - - def astForRescueClauseContext(ctx: RescueClauseContext): Ast = { - val asts = ListBuffer.empty[Ast] - - if (ctx.exceptionClass() != null) { - val exceptionClass = ctx.exceptionClass() - - if (exceptionClass.expression() != null) { - asts.addAll(astForExpressionContext(exceptionClass.expression())) - } else { - asts.addAll(astForMultipleRightHandSideContext(exceptionClass.multipleRightHandSide())) - } - } - - if (ctx.exceptionVariableAssignment() != null) { - asts.addAll(astForSingleLeftHandSideContext(ctx.exceptionVariableAssignment().singleLeftHandSide())) - } - - asts.addAll(astForCompoundStatement(ctx.thenClause().compoundStatement(), false)) - blockAst(blockNode(ctx), asts.toList) - } - - private def astForSimpleScopedConstantReferencePrimaryContext( - ctx: SimpleScopedConstantReferencePrimaryContext - ): Seq[Ast] = { - val localVar = ctx.CONSTANT_IDENTIFIER - val varSymbol = localVar.getSymbol - val node = createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, List(Defines.Any)) - - val operatorName = getOperatorName(ctx.COLON2.getSymbol) - val callNode = createOpCall(ctx.COLON2, operatorName, code(ctx)) - - Seq(callAst(callNode, Seq(Ast(node)))) - } - - private def astForCommandWithDoBlockContext(ctx: CommandWithDoBlockContext): Seq[Ast] = ctx match { - case ctx: ArgsAndDoBlockCommandWithDoBlockContext => - val argsAsts = astForArguments(ctx.argumentsWithoutParentheses().arguments()) - val doBlockAst = Seq(astForDoBlock(ctx.doBlock())) - argsAsts ++ doBlockAst - case ctx: DeprecatedRubyParser.ArgsAndDoBlockAndMethodIdCommandWithDoBlockContext => - val methodIdAsts = astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - methodIdAsts.headOption.flatMap(_.root) match - case Some(methodIdRoot: NewCall) if methodIdRoot.name == "define_method" => - ctx.argumentsWithoutParentheses.arguments.argument.asScala.headOption - .map { methodArg => - // TODO: methodArg will name the method, but this could be an identifier or even a string concatenation - // which is not assumed below - val methodName = stripQuotes(methodArg.getText) - Seq(astForDoBlock(ctx.doBlock(), Option(methodName))) - } - .getOrElse(Seq.empty) - case _ => - val argsAsts = astForArguments(ctx.argumentsWithoutParentheses().arguments()) - val doBlockAsts = Seq(astForDoBlock(ctx.doBlock())) - methodIdAsts ++ argsAsts ++ doBlockAsts - case ctx: DeprecatedRubyParser.PrimaryMethodArgsDoBlockCommandWithDoBlockContext => - val argsAsts = astForArguments(ctx.argumentsWithoutParentheses().arguments()) - val doBlockAsts = Seq(astForDoBlock(ctx.doBlock())) - val methodNameAsts = astForMethodNameContext(ctx.methodName()) - val primaryAsts = astForPrimaryContext(ctx.primary()) - primaryAsts ++ methodNameAsts ++ argsAsts ++ doBlockAsts - case _ => - logger.error(s"astForCommandWithDoBlockContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - private def astForChainedCommandWithDoBlockContext(ctx: ChainedCommandWithDoBlockContext): Seq[Ast] = { - val cmdAsts = astForCommandWithDoBlockContext(ctx.commandWithDoBlock) - val mNameAsts = ctx.methodName.asScala.flatMap(astForMethodNameContext).toSeq - val apAsts = ctx - .argumentsWithParentheses() - .asScala - .flatMap(astForArgumentsWithParenthesesContext) - .toSeq - cmdAsts ++ mNameAsts ++ apAsts - } - - protected def astForArgumentsWithParenthesesContext(ctx: ArgumentsWithParenthesesContext): Seq[Ast] = ctx match { - case _: BlankArgsArgumentsWithParenthesesContext => Seq.empty - case ctx: ArgsOnlyArgumentsWithParenthesesContext => astForArguments(ctx.arguments) - case ctx: ExpressionsAndChainedCommandWithDoBlockArgumentsWithParenthesesContext => - val expAsts = ctx.expressions.expression.asScala - .flatMap(astForExpressionContext) - .toSeq - val ccDoBlock = astForChainedCommandWithDoBlockContext(ctx.chainedCommandWithDoBlock) - expAsts ++ ccDoBlock - case ctx: ChainedCommandWithDoBlockOnlyArgumentsWithParenthesesContext => - astForChainedCommandWithDoBlockContext(ctx.chainedCommandWithDoBlock) - case _ => - logger.error(s"astForArgumentsWithParenthesesContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - private def astForBlockParametersContext(ctx: BlockParametersContext): Seq[Ast] = - if (ctx.singleLeftHandSide != null) { - astForSingleLeftHandSideContext(ctx.singleLeftHandSide) - } else if (ctx.multipleLeftHandSide != null) { - astForMultipleLeftHandSideContext(ctx.multipleLeftHandSide) - } else { - Seq.empty - } - - protected def astForBlockParameterContext(ctx: BlockParameterContext): Seq[Ast] = - if (ctx.blockParameters != null) { - astForBlockParametersContext(ctx.blockParameters) - } else { - Seq.empty - } - - def astForAssociationContext(ctx: AssociationContext): Seq[Ast] = { - val terminalNode = Option(ctx.COLON).getOrElse(ctx.EQGT) - val operatorText = getOperatorName(terminalNode.getSymbol) - val expressions = ctx.expression.asScala - - val callArgs = - Option(ctx.keyword) match { - case Some(ctxKeyword) => - val expr1Ast = astForCallNode(ctx, code(ctxKeyword)) - val expr2Asts = astForExpressionContext(expressions.head) - Seq(expr1Ast) ++ expr2Asts - case None => - val expr1Asts = astForExpressionContext(expressions.head) - val expr2Asts = expressions.lift(1).flatMap(astForExpressionContext) - expr1Asts ++ expr2Asts - } - - val callNode = createOpCall(terminalNode, operatorText, code(ctx)) - Seq(callAst(callNode, callArgs)) - } - - private def astForAssociationsContext(ctx: AssociationsContext): Seq[Ast] = { - ctx.association.asScala - .flatMap(astForAssociationContext) - .toSeq - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreatorHelper.scala deleted file mode 100644 index d8ad2c3c609f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstCreatorHelper.scala +++ /dev/null @@ -1,361 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines as RubyDefines -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} -import org.antlr.v4.runtime.misc.Interval -import org.antlr.v4.runtime.tree.TerminalNode -import org.antlr.v4.runtime.{ParserRuleContext, Token} - -import scala.collection.mutable -import scala.util.Try - -trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - import io.joern.rubysrc2cpg.deprecated.astcreation.GlobalTypes.* - - protected def line(ctx: ParserRuleContext): Option[Int] = - Try(ctx.getStart.getLine).toOption - - protected def column(ctx: ParserRuleContext): Option[Int] = - Try(ctx.getStart.getCharPositionInLine).toOption - - protected def lineEnd(ctx: ParserRuleContext): Option[Int] = - Try(ctx.getStop.getLine).toOption - - protected def columnEnd(ctx: ParserRuleContext): Option[Int] = - Try(ctx.getStop.getCharPositionInLine).toOption - - override def code(node: ParserRuleContext): String = shortenCode(text(node)) - - protected def text(ctx: ParserRuleContext): String = Try { - val a = ctx.getStart.getStartIndex - val b = ctx.getStop.getStopIndex - val intv = new Interval(a, b) - val input = ctx.getStart.getInputStream - input.getText(intv) - }.getOrElse("") - - protected def isBuiltin(x: String): Boolean = builtinFunctions.contains(x) - - protected def prefixAsBuiltin(x: String): String = s"$builtinPrefix$pathSep$x" - - protected def methodsWithName(name: String): List[String] = { - packageContext.packageTable.getMethodFullNameUsingName(methodName = name) - } - - private def methodTableToCallNode( - methodFullName: String, - name: String, - code: String, - typeFullName: String, - dynamicTypeHints: Seq[String] = Seq(), - ctx: Option[ParserRuleContext] = None - ): NewCall = { - callNode(ctx.orNull, code, name, methodFullName, DispatchTypes.DYNAMIC_DISPATCH, None, Option(typeFullName)) - .dynamicTypeHintFullName(dynamicTypeHints) - } - - /** Checks that the name is not `this` and that the method has been referred to more than just an initial assignment - * to METHOD_REF. - * - * @param name - * the identifier name. - * @return - * true if this appears to be more like a method call than an identifier. - */ - private def isMethodCall(name: String): Boolean = { - name != "this" && scope.numVariableReferences(name) == 0 - } - - protected def createIdentifierWithScope( - ctx: ParserRuleContext, - name: String, - code: String, - typeFullName: String, - dynamicTypeHints: Seq[String] = Seq(), - definitelyIdentifier: Boolean = false - ): NewNode = { - methodsWithName(name) match - case method :: _ if !definitelyIdentifier && isMethodCall(name) => - methodTableToCallNode(method, name, code, typeFullName, dynamicTypeHints, Option(ctx)) - case _ => - val newNode = identifierNode(ctx, name, code, typeFullName, dynamicTypeHints) - scope.addToScope(name, newNode) - newNode - } - - protected def createIdentifierWithScope( - name: String, - code: String, - typeFullName: String, - dynamicTypeHints: Seq[String], - lineNumber: Option[Int], - columnNumber: Option[Int], - definitelyIdentifier: Boolean - ): NewNode = { - methodsWithName(name) match - case method :: _ if !definitelyIdentifier && isMethodCall(name) => - methodTableToCallNode(method, name, code, typeFullName, dynamicTypeHints, None) - case _ => - val newNode = NewIdentifier() - .name(name) - .code(code) - .typeFullName(typeFullName) - .dynamicTypeHintFullName(dynamicTypeHints) - .lineNumber(lineNumber) - .columnNumber(columnNumber) - scope.addToScope(name, newNode) - newNode - } - - protected def createOpCall( - node: TerminalNode, - operation: String, - code: String, - typeFullName: String = RubyDefines.Any - ): NewCall = { - NewCall() - .name(operation) - .methodFullName(operation) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .lineNumber(node.lineNumber) - .columnNumber(node.columnNumber) - .typeFullName(typeFullName) - .code(code) - } - - protected def createLiteralNode( - code: String, - typeFullName: String, - dynamicTypeHints: Seq[String] = Seq.empty, - lineNumber: Option[Int] = None, - columnNumber: Option[Int] = None - ): NewLiteral = { - val newLiteral = NewLiteral() - .code(code) - .typeFullName(typeFullName) - .dynamicTypeHintFullName(dynamicTypeHints) - lineNumber.foreach(newLiteral.lineNumber(_)) - columnNumber.foreach(newLiteral.columnNumber(_)) - newLiteral - } - - protected def astForAssignment( - lhs: NewNode, - rhs: NewNode, - lineNumber: Option[Int] = None, - colNumber: Option[Int] = None - ): Ast = { - val code = Seq(lhs, rhs).collect { case x: AstNodeNew => x.code }.mkString(" = ") - val assignment = NewCall() - .name(Operators.assignment) - .methodFullName(Operators.assignment) - .code(code) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .lineNumber(lineNumber) - .columnNumber(colNumber) - - callAst(assignment, Seq(Ast(lhs), Ast(rhs))) - } - - protected def createThisIdentifier( - ctx: ParserRuleContext, - typeFullName: String = RubyDefines.Any, - dynamicTypeHints: List[String] = List.empty - ): NewIdentifier = - createIdentifierWithScope(ctx, "this", "this", typeFullName, dynamicTypeHints, true).asInstanceOf[NewIdentifier] - - protected def newFieldIdentifier(ctx: ParserRuleContext): NewFieldIdentifier = { - val c = code(ctx) - val name = c.replaceAll("@", "") - NewFieldIdentifier() - .code(c) - .canonicalName(name) - .lineNumber(ctx.start.getLine) - .columnNumber(ctx.start.getCharPositionInLine) - } - - protected def astForFieldAccess(ctx: ParserRuleContext, baseNode: NewNode): Ast = { - val fieldAccess = - callNode(ctx, code(ctx), Operators.fieldAccess, Operators.fieldAccess, DispatchTypes.STATIC_DISPATCH) - val fieldIdentifier = newFieldIdentifier(ctx) - val astChildren = Seq(baseNode, fieldIdentifier) - callAst(fieldAccess, astChildren.map(Ast.apply)) - } - - protected def createMethodParameterIn( - name: String, - lineNumber: Option[Int] = None, - colNumber: Option[Int] = None, - typeFullName: String = RubyDefines.Any, - order: Int = -1, - index: Int = -1 - ): NewMethodParameterIn = { - NewMethodParameterIn() - .name(name) - .code(name) - .lineNumber(lineNumber) - .typeFullName(typeFullName) - .columnNumber(colNumber) - .order(order) - .index(index) - } - - protected def getUnusedVariableNames( - usedVariableNames: mutable.HashMap[String, Int], - variableName: String - ): String = { - val counter = usedVariableNames.get(variableName).map(_ + 1).getOrElse(0) - val currentVariableName = s"${variableName}_$counter" - usedVariableNames.put(variableName, counter) - currentVariableName - } - - protected def astForControlStructure( - parserTypeName: String, - node: TerminalNode, - controlStructureType: String, - code: String - ): Ast = - Ast( - NewControlStructure() - .parserTypeName(parserTypeName) - .controlStructureType(controlStructureType) - .code(code) - .lineNumber(node.lineNumber) - .columnNumber(node.columnNumber) - ) - - protected def returnNode(node: TerminalNode, code: String): NewReturn = - NewReturn() - .lineNumber(node.lineNumber) - .columnNumber(node.columnNumber) - .code(code) - - protected def getOperatorName(token: Token): String = token.getType match { - case ASSIGNMENT_OPERATOR => Operators.assignment - case DOT2 => Operators.range - case DOT3 => Operators.range - case EMARK => Operators.not - case EQ => Operators.assignment - case COLON2 => RubyOperators.scopeResolution - case DOT => Operators.fieldAccess - case EQGT => RubyOperators.keyValueAssociation - case COLON => RubyOperators.activeRecordAssociation - case _ => RubyOperators.none - } - - implicit class TerminalNodeExt(n: TerminalNode) { - - def lineNumber: Int = n.getSymbol.getLine - - def columnNumber: Int = n.getSymbol.getCharPositionInLine - - } - -} - -object RubyOperators { - val none = ".none" - val patternMatch = ".patternMatch" - val notPatternMatch = ".notPatternMatch" - val scopeResolution = ".scopeResolution" - val defined = ".defined" - val keyValueAssociation = ".keyValueAssociation" - val activeRecordAssociation = ".activeRecordAssociation" - val undef = ".undef" - val superKeyword = ".super" - val stringConcatenation = ".stringConcatenation" - val formattedString = ".formatString" - val formattedValue = ".formatValue" -} - -object GlobalTypes { - val builtinPrefix = "__builtin" - /* Sources: - * https://ruby-doc.org/docs/ruby-doc-bundle/Manual/man-1.4/function.html - * https://ruby-doc.org/3.2.2/Kernel.html - * - * We comment-out methods that require an explicit "receiver" (target of member access.) - */ - val builtinFunctions = Set( - "Array", - "Complex", - "Float", - "Hash", - "Integer", - "Rational", - "String", - "__callee__", - "__dir__", - "__method__", - "abort", - "at_exit", - "autoload", - "autoload?", - "binding", - "block_given?", - "callcc", - "caller", - "caller_locations", - "catch", - "chomp", - "chomp!", - "chop", - "chop!", - // "class", - // "clone", - "eval", - "exec", - "exit", - "exit!", - "fail", - "fork", - "format", - // "frozen?", - "gets", - "global_variables", - "gsub", - "gsub!", - "iterator?", - "lambda", - "load", - "local_variables", - "loop", - "open", - "p", - "print", - "printf", - "proc", - "putc", - "puts", - "raise", - "rand", - "readline", - "readlines", - "require", - "require_relative", - "select", - "set_trace_func", - "sleep", - "spawn", - "sprintf", - "srand", - "sub", - "sub!", - "syscall", - "system", - "tap", - "test", - // "then", - "throw", - "trace_var", - // "trap", - "untrace_var", - "warn" - // "yield_self", - ) -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForControlStructuresCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForControlStructuresCreator.scala deleted file mode 100644 index 89424fad8c9b..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForControlStructuresCreator.scala +++ /dev/null @@ -1,135 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.codepropertygraph.generated.nodes.NewControlStructure - -import scala.jdk.CollectionConverters.* -trait AstForControlStructuresCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - private def astForWhenArgumentContext(ctx: WhenArgumentContext): Seq[Ast] = { - val expAsts = - ctx.expressions.expression.asScala - .flatMap(astForExpressionContext) - .toList - - if (ctx.splattingArgument != null) { - expAsts ++ astForExpressionOrCommand(ctx.splattingArgument().expressionOrCommand()) - } else { - expAsts - } - } - - protected def astForCaseExpressionPrimaryContext(ctx: CaseExpressionPrimaryContext): Seq[Ast] = { - val codeString = s"case ${Option(ctx.caseExpression().expressionOrCommand).map(code).getOrElse("")}".stripTrailing() - val switchNode = controlStructureNode(ctx, ControlStructureTypes.SWITCH, codeString) - val conditionAst = Option(ctx.caseExpression().expressionOrCommand()).toList - .flatMap(astForExpressionOrCommand) - .headOption - - val whenThenAstsList = ctx - .caseExpression() - .whenClause() - .asScala - .flatMap(wh => { - val whenNode = - jumpTargetNode(wh, "case", s"case ${code(wh)}", Option(wh.getClass.getSimpleName)) - - val whenACondAsts = astForWhenArgumentContext(wh.whenArgument()) - val thenAsts = astForCompoundStatement( - wh.thenClause().compoundStatement(), - isMethodBody = true, - canConsiderAsLeaf = false - ) ++ Seq(Ast(NewControlStructure().controlStructureType(ControlStructureTypes.BREAK))) - Seq(Ast(whenNode)) ++ whenACondAsts ++ thenAsts - }) - .toList - - val stmtAsts = whenThenAstsList ++ (Option(ctx.caseExpression().elseClause()) match - case Some(elseClause) => - Ast( - // name = "default" for behaviour determined by CfgCreator.cfgForJumpTarget - jumpTargetNode(elseClause, "default", "else", Option(elseClause.getClass.getSimpleName)) - ) +: astForCompoundStatement(elseClause.compoundStatement(), isMethodBody = true, canConsiderAsLeaf = false) - case None => Seq.empty[Ast] - ) - val block = blockNode(ctx.caseExpression()) - Seq(controlStructureAst(switchNode, conditionAst, Seq(Ast(block).withChildren(stmtAsts)))) - } - - protected def astForNextArgsInvocation(ctx: NextArgsInvocationWithoutParenthesesContext): Seq[Ast] = { - /* - * While this is a `CONTINUE` for now, if we detect that this is the LHS of an `IF` then this becomes a `RETURN` - */ - Seq( - astForControlStructure( - ctx.getClass.getSimpleName, - ctx.NEXT(), - ControlStructureTypes.CONTINUE, - Defines.ModifierNext - ).withChildren(astForArguments(ctx.arguments())) - ) - } - - protected def astForBreakArgsInvocation(ctx: BreakArgsInvocationWithoutParenthesesContext): Seq[Ast] = { - Option(ctx.arguments()) match { - case Some(args) => - /* - * This is break with args inside a block. The argument passed to break will be returned by the bloc - * Model this as a return since this is effectively a return - */ - val retNode = returnNode(ctx.BREAK(), code(ctx)) - val argAst = astForArguments(args) - Seq(returnAst(retNode, argAst)) - case None => - Seq( - astForControlStructure(ctx.getClass.getSimpleName, ctx.BREAK(), ControlStructureTypes.BREAK, code(ctx)) - .withChildren(astForArguments(ctx.arguments)) - ) - } - } - - protected def astForJumpExpressionPrimaryContext(ctx: JumpExpressionPrimaryContext): Seq[Ast] = { - val parserTypeName = ctx.getClass.getSimpleName - val controlStructureAst = ctx.jumpExpression() match - case expr if expr.BREAK() != null => - astForControlStructure(parserTypeName, expr.BREAK(), ControlStructureTypes.BREAK, code(ctx)) - case expr if expr.NEXT() != null => - astForControlStructure(parserTypeName, expr.NEXT(), ControlStructureTypes.CONTINUE, Defines.ModifierNext) - case expr if expr.REDO() != null => - astForControlStructure(parserTypeName, expr.REDO(), ControlStructureTypes.CONTINUE, Defines.ModifierRedo) - case expr if expr.RETRY() != null => - astForControlStructure(parserTypeName, expr.RETRY(), ControlStructureTypes.CONTINUE, Defines.ModifierRetry) - case _ => - Ast() - Seq(controlStructureAst) - } - - protected def astForRescueClause(ctx: BodyStatementContext): Ast = { - val compoundStatementAsts = astForCompoundStatement(ctx.compoundStatement) - val elseClauseAsts = Option(ctx.elseClause) match - case Some(ctx) => astForCompoundStatement(ctx.compoundStatement) - case None => Seq.empty - - /* - * TODO Conversion of last statement to return AST is needed here - * This can be done after the data flow engine issue with return from a try block is fixed - */ - val tryBodyAsts = compoundStatementAsts ++ elseClauseAsts - val tryBodyAst = blockAst(blockNode(ctx), tryBodyAsts.toList) - - val finallyAst = Option(ctx.ensureClause) match - case Some(ctx) => astForCompoundStatement(ctx.compoundStatement).headOption - case None => None - - val catchAsts = ctx.rescueClause.asScala - .map(astForRescueClauseContext) - .toSeq - - val tryNode = controlStructureNode(ctx, ControlStructureTypes.TRY, "try") - tryCatchAstWithOrder(tryNode, tryBodyAst, catchAsts, finallyAst) - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForDeclarationsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForDeclarationsCreator.scala deleted file mode 100644 index f49b17a5495f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForDeclarationsCreator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.x2cpg.Ast -import io.shiftleft.codepropertygraph.generated.nodes.{NewJumpTarget, NewLiteral} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators} -import org.antlr.v4.runtime.ParserRuleContext -import org.slf4j.LoggerFactory - -import scala.jdk.CollectionConverters.CollectionHasAsScala - -trait AstForDeclarationsCreator { this: AstCreator => - - private val logger = LoggerFactory.getLogger(this.getClass) - - protected def astForArguments(ctx: ArgumentsContext): Seq[Ast] = { - ctx.argument().asScala.flatMap(astForArgument).toSeq - } - - protected def astForArgument(ctx: ArgumentContext): Seq[Ast] = { - ctx match { - case ctx: BlockArgumentArgumentContext => astForExpressionContext(ctx.blockArgument.expression) - case ctx: SplattingArgumentArgumentContext => astForExpressionOrCommand(ctx.splattingArgument.expressionOrCommand) - case ctx: ExpressionArgumentContext => astForExpressionContext(ctx.expression) - case ctx: AssociationArgumentContext => astForAssociationContext(ctx.association) - case ctx: CommandArgumentContext => astForCommand(ctx.command) - case ctx: HereDocArgumentContext => astForHereDocArgument(ctx) - case _ => - logger.error(s"astForArgument() $relativeFilename, ${ctx.getText} All contexts mismatched.") - Seq() - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForExpressionsCreator.scala deleted file mode 100644 index 0897863c28a9..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForExpressionsCreator.scala +++ /dev/null @@ -1,512 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.deprecated.passes.Defines.* -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, NewCall, NewIdentifier} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators} -import org.antlr.v4.runtime.ParserRuleContext -import org.slf4j.LoggerFactory - -import scala.collection.immutable.Set -import scala.jdk.CollectionConverters.CollectionHasAsScala - -trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - private val logger = LoggerFactory.getLogger(this.getClass) - protected var lastModifier: Option[String] = None - - protected def astForPowerExpression(ctx: PowerExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.exponentiation, ctx.expression().asScala) - - protected def astForOrExpression(ctx: OperatorOrExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.or, ctx.expression().asScala) - - protected def astForAndExpression(ctx: OperatorAndExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.and, ctx.expression().asScala) - - protected def astForUnaryExpression(ctx: UnaryExpressionContext): Ast = ctx.op.getType match { - case TILDE => astForBinaryOperatorExpression(ctx, Operators.not, Seq(ctx.expression())) - case PLUS => astForBinaryOperatorExpression(ctx, Operators.plus, Seq(ctx.expression())) - case EMARK => astForBinaryOperatorExpression(ctx, Operators.not, Seq(ctx.expression())) - } - - protected def astForUnaryMinusExpression(ctx: UnaryMinusExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.minus, Seq(ctx.expression())) - - protected def astForAdditiveExpression(ctx: AdditiveExpressionContext): Ast = ctx.op.getType match { - case PLUS => astForBinaryOperatorExpression(ctx, Operators.addition, ctx.expression().asScala) - case MINUS => astForBinaryOperatorExpression(ctx, Operators.subtraction, ctx.expression().asScala) - } - - protected def astForMultiplicativeExpression(ctx: MultiplicativeExpressionContext): Ast = ctx.op.getType match { - case STAR => astForMultiplicativeStarExpression(ctx) - case SLASH => astForMultiplicativeSlashExpression(ctx) - case PERCENT => astForMultiplicativePercentExpression(ctx) - } - - protected def astForMultiplicativeStarExpression(ctx: MultiplicativeExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.multiplication, ctx.expression().asScala) - - protected def astForMultiplicativeSlashExpression(ctx: MultiplicativeExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.division, ctx.expression().asScala) - - protected def astForMultiplicativePercentExpression(ctx: MultiplicativeExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.modulo, ctx.expression().asScala) - - protected def astForEqualityExpression(ctx: EqualityExpressionContext): Ast = ctx.op.getType match { - case LTEQGT => astForBinaryOperatorExpression(ctx, Operators.compare, ctx.expression().asScala) - case EQ2 => astForBinaryOperatorExpression(ctx, Operators.equals, ctx.expression().asScala) - case EQ3 => astForBinaryOperatorExpression(ctx, Operators.is, ctx.expression().asScala) - case EMARKEQ => astForBinaryOperatorExpression(ctx, Operators.notEquals, ctx.expression().asScala) - case EQTILDE => astForBinaryOperatorExpression(ctx, RubyOperators.patternMatch, ctx.expression().asScala) - case EMARKTILDE => astForBinaryOperatorExpression(ctx, RubyOperators.notPatternMatch, ctx.expression().asScala) - } - - protected def astForRelationalExpression(ctx: RelationalExpressionContext): Ast = ctx.op.getType match { - case GT => astForBinaryOperatorExpression(ctx, Operators.greaterThan, ctx.expression().asScala) - case GTEQ => astForBinaryOperatorExpression(ctx, Operators.greaterEqualsThan, ctx.expression().asScala) - case LT => astForBinaryOperatorExpression(ctx, Operators.lessThan, ctx.expression().asScala) - case LTEQ => astForBinaryOperatorExpression(ctx, Operators.lessEqualsThan, ctx.expression().asScala) - } - - protected def astForBitwiseOrExpression(ctx: BitwiseOrExpressionContext): Ast = ctx.op.getType match { - case BAR => astForBinaryOperatorExpression(ctx, Operators.logicalOr, ctx.expression().asScala) - case CARET => astForBinaryOperatorExpression(ctx, Operators.logicalOr, ctx.expression().asScala) - } - - protected def astForBitwiseAndExpression(ctx: BitwiseAndExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, Operators.logicalAnd, ctx.expression().asScala) - - protected def astForBitwiseShiftExpression(ctx: BitwiseShiftExpressionContext): Ast = ctx.op.getType match { - case LT2 => astForBinaryOperatorExpression(ctx, Operators.shiftLeft, ctx.expression().asScala) - case GT2 => astForBinaryOperatorExpression(ctx, Operators.logicalShiftRight, ctx.expression().asScala) - } - - private def astForBinaryOperatorExpression( - ctx: ParserRuleContext, - name: String, - arguments: Iterable[ExpressionContext] - ): Ast = { - val argsAst = arguments.flatMap(astForExpressionContext) - val call = callNode(ctx, code(ctx), name, name, DispatchTypes.STATIC_DISPATCH) - callAst(call, argsAst.toList) - } - - protected def astForIsDefinedExpression(ctx: IsDefinedExpressionContext): Ast = - astForBinaryOperatorExpression(ctx, RubyOperators.defined, Seq(ctx.expression())) - - // TODO: Maybe merge (in DeprecatedRubyParser.g4) isDefinedExpression with isDefinedPrimaryExpression? - protected def astForIsDefinedPrimaryExpression(ctx: IsDefinedPrimaryContext): Ast = { - val argsAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val call = callNode(ctx, code(ctx), RubyOperators.defined, RubyOperators.defined, DispatchTypes.STATIC_DISPATCH) - callAst(call, argsAst.toList) - } - - protected def astForLiteralPrimaryExpression(ctx: LiteralPrimaryContext): Seq[Ast] = ctx.literal() match { - case ctx: NumericLiteralLiteralContext => Seq(astForNumericLiteral(ctx.numericLiteral())) - case ctx: SymbolLiteralContext => astForSymbol(ctx.symbol()) - case ctx: RegularExpressionLiteralContext => Seq(astForRegularExpressionLiteral(ctx)) - case ctx: HereDocLiteralContext => Seq(astForHereDocLiteral(ctx)) - case _ => - logger.error(s"astForLiteralPrimaryExpression() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq() - } - - private def astForSymbol(ctx: SymbolContext): Seq[Ast] = { - if ( - ctx.stringExpression() != null && ctx.stringExpression().children.get(0).isInstanceOf[StringInterpolationContext] - ) { - val node = callNode( - ctx, - code(ctx), - RubyOperators.formattedString, - RubyOperators.formattedString, - DispatchTypes.STATIC_DISPATCH, - None, - Option(Defines.Any) - ) - astForStringExpression(ctx.stringExpression()) ++ Seq(Ast(node)) - } else { - Seq(astForSymbolLiteral(ctx)) - } - } - - protected def astForMultipleRightHandSideContext(ctx: MultipleRightHandSideContext): Seq[Ast] = - if (ctx == null) { - Seq.empty - } else { - val expCmd = ctx.expressionOrCommands() - val exprAsts = Option(expCmd) match - case Some(expCmd) => - expCmd.expressionOrCommand().asScala.flatMap(astForExpressionOrCommand).toSeq - case None => - Seq.empty - - if (ctx.splattingArgument != null) { - val splattingAsts = astForExpressionOrCommand(ctx.splattingArgument.expressionOrCommand) - exprAsts ++ splattingAsts - } else { - exprAsts - } - } - - protected def astForSingleLeftHandSideContext(ctx: SingleLeftHandSideContext): Seq[Ast] = ctx match { - case ctx: VariableIdentifierOnlySingleLeftHandSideContext => - Seq(astForVariableIdentifierHelper(ctx.variableIdentifier, true)) - case ctx: PrimaryInsideBracketsSingleLeftHandSideContext => - val primaryAsts = astForPrimaryContext(ctx.primary) - val argsAsts = astForArguments(ctx.arguments) - val indexAccessCall = createOpCall(ctx.LBRACK, Operators.indexAccess, code(ctx)) - Seq(callAst(indexAccessCall, primaryAsts ++ argsAsts)) - case ctx: XdotySingleLeftHandSideContext => - // TODO handle obj.foo=arg being interpreted as obj.foo(arg) here. - val xAsts = astForPrimaryContext(ctx.primary) - - Seq(ctx.LOCAL_VARIABLE_IDENTIFIER, ctx.CONSTANT_IDENTIFIER) - .flatMap(Option(_)) - .headOption match - case Some(localVar) => - val name = localVar.getSymbol.getText - val node = createIdentifierWithScope(ctx, name, name, Defines.Any, List(Defines.Any), true) - val yAst = Ast(node) - - val callNode = createOpCall(localVar, Operators.fieldAccess, code(ctx)) - Seq(callAst(callNode, xAsts ++ Seq(yAst))) - case None => - Seq.empty - case ctx: ScopedConstantAccessSingleLeftHandSideContext => - val localVar = ctx.CONSTANT_IDENTIFIER - val varSymbol = localVar.getSymbol - val node = - createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, List(Defines.Any), true) - Seq(Ast(node)) - case _ => - logger.error(s"astForSingleLeftHandSideContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq.empty - } - - protected def astForSingleAssignmentExpressionContext(ctx: SingleAssignmentExpressionContext): Seq[Ast] = { - val rightAst = astForMultipleRightHandSideContext(ctx.multipleRightHandSide) - val leftAst = astForSingleLeftHandSideContext(ctx.singleLeftHandSide) - - val operatorName = getOperatorName(ctx.op) - val opCallNode = - callNode(ctx, code(ctx), operatorName, operatorName, DispatchTypes.STATIC_DISPATCH, None, Option(Defines.Any)) - .lineNumber(ctx.op.getLine) - .columnNumber(ctx.op.getCharPositionInLine) - if (leftAst.size == 1 && rightAst.size > 1) { - /* - * This is multiple RHS packed into a single LHS. That is, packing left hand side. - * This is as good as multiple RHS packed into an array and put into a single LHS - */ - val packedRHS = getPackedRHS(rightAst, wrapInBrackets = true) - Seq(callAst(opCallNode, leftAst ++ packedRHS)) - } else { - Seq(callAst(opCallNode, leftAst ++ rightAst)) - } - } - - protected def astForMultipleAssignmentExpressionContext(ctx: MultipleAssignmentExpressionContext): Seq[Ast] = { - val rhsAsts = astForMultipleRightHandSideContext(ctx.multipleRightHandSide()) - val lhsAsts = astForMultipleLeftHandSideContext(ctx.multipleLeftHandSide()) - val operatorName = getOperatorName(ctx.EQ.getSymbol) - - /* - * This is multiple LHS and multiple RHS - *Since we have multiple LHS and RHS elements here, we will now create synthetic assignment - * call nodes to model how ruby assigns values from RHS elements to LHS elements. We create - * tuples for each assignment and then pass them to the assignment calls nodes - */ - val assigns = - if (lhsAsts.size < rhsAsts.size) { - /* The rightmost AST in the LHS is a packed variable. - * Pack the extra ASTs and the rightmost AST in the RHS in one array like the if() part - */ - val diff = rhsAsts.size - lhsAsts.size - val packedRHS = getPackedRHS(rhsAsts.takeRight(diff + 1)).headOption.to(Seq) - val alignedAsts = lhsAsts.take(lhsAsts.size - 1) zip rhsAsts.take(lhsAsts.size - 1) - val packedAsts = lhsAsts.takeRight(1) zip packedRHS - alignedAsts ++ packedAsts - } else { - lhsAsts.zip(rhsAsts) - } - - assigns.map { case (lhsAst, rhsAst) => - val lhsCode = lhsAst.nodes.collectFirst { case x: AstNodeNew => x.code }.getOrElse("") - val rhsCode = rhsAst.nodes.collectFirst { case x: AstNodeNew => x.code }.getOrElse("") - val code = s"$lhsCode = $rhsCode" - val syntheticCallNode = createOpCall(ctx.EQ, operatorName, code) - - callAst(syntheticCallNode, Seq(lhsAst, rhsAst)) - } - } - - protected def astForIndexingExpressionPrimaryContext(ctx: IndexingExpressionPrimaryContext): Seq[Ast] = { - val lhsExpressionAst = astForPrimaryContext(ctx.primary()) - val rhsExpressionAst = Option(ctx.indexingArguments).map(astForIndexingArgumentsContext).getOrElse(Seq()) - - val operator = lhsExpressionAst.flatMap(_.nodes).collectFirst { case x: NewIdentifier => x } match - case Some(node) if node.name == "Array" => Operators.arrayInitializer - case _ => Operators.indexAccess - - val callNode = createOpCall(ctx.LBRACK, operator, code(ctx)) - Seq(callAst(callNode, lhsExpressionAst ++ rhsExpressionAst)) - - } - - private def getPackedRHS(astsToConcat: Seq[Ast], wrapInBrackets: Boolean = false) = { - val code = astsToConcat - .flatMap(_.nodes) - .collect { case x: AstNodeNew => x.code } - .mkString(", ") - - val callNode = NewCall() - .name(Operators.arrayInitializer) - .methodFullName(Operators.arrayInitializer) - .typeFullName(Defines.Any) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .code(if (wrapInBrackets) s"[$code]" else code) - Seq(callAst(callNode, astsToConcat)) - } - - def astForStringInterpolationContext(ctx: InterpolatedStringExpressionContext): Seq[Ast] = { - val varAsts = ctx.stringInterpolation.interpolatedStringSequence.asScala - .flatMap(inter => - Seq( - Ast( - callNode( - ctx, - code(inter), - RubyOperators.formattedValue, - RubyOperators.formattedValue, - DispatchTypes.STATIC_DISPATCH, - None, - Option(Defines.Any) - ) - ) - ) ++ - astForStatements(inter.compoundStatement.statements, false, false) - ) - .toSeq - - val literalAsts = ctx - .stringInterpolation() - .DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE() - .asScala - .map(substr => - Ast( - createLiteralNode( - substr.getText, - Defines.String, - List(Defines.String), - Option(substr.lineNumber), - Option(substr.columnNumber) - ) - ) - ) - .toSeq - varAsts ++ literalAsts - } - - // TODO: Return Ast instead of Seq[Ast] - protected def astForStringExpression(ctx: StringExpressionContext): Seq[Ast] = ctx match { - case ctx: SimpleStringExpressionContext => Seq(astForSimpleString(ctx.simpleString)) - case ctx: InterpolatedStringExpressionContext => astForStringInterpolationContext(ctx) - case ctx: ConcatenatedStringExpressionContext => Seq(astForConcatenatedStringExpressions(ctx)) - } - - // Regex interpolation has been modeled just as a set of statements, that suffices to track dataflows - protected def astForRegexInterpolationPrimaryContext(ctx: RegexInterpolationContext): Seq[Ast] = { - val varAsts = ctx - .interpolatedRegexSequence() - .asScala - .flatMap(inter => { - astForStatements(inter.compoundStatement().statements(), false, false) - }) - .toSeq - varAsts - } - - protected def astForSimpleString(ctx: SimpleStringContext): Ast = ctx match { - case ctx: SingleQuotedStringLiteralContext => astForSingleQuotedStringLiteral(ctx) - case ctx: DoubleQuotedStringLiteralContext => astForDoubleQuotedStringLiteral(ctx) - } - - protected def astForConcatenatedStringExpressions(ctx: ConcatenatedStringExpressionContext): Ast = { - val stringExpressionAsts = ctx.stringExpression().asScala.flatMap(astForStringExpression) - val callNode_ = callNode( - ctx, - code(ctx), - RubyOperators.stringConcatenation, - RubyOperators.stringConcatenation, - DispatchTypes.STATIC_DISPATCH - ) - callAst(callNode_, stringExpressionAsts.toSeq) - } - - protected def astForTernaryConditionalOperator(ctx: ConditionalOperatorExpressionContext): Ast = { - val testAst = astForExpressionContext(ctx.expression(0)) - val thenAst = astForExpressionContext(ctx.expression(1)) - val elseAst = astForExpressionContext(ctx.expression(2)) - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - controlStructureAst(ifNode, testAst.headOption, thenAst ++ elseAst) - } - - def astForRangeExpressionContext(ctx: RangeExpressionContext): Seq[Ast] = - Seq(astForBinaryOperatorExpression(ctx, Operators.range, ctx.expression().asScala)) - - protected def astForSuperExpression(ctx: SuperExpressionPrimaryContext): Ast = { - val argsAst = Option(ctx.argumentsWithParentheses()) match - case Some(ctxArgs) => astForArgumentsWithParenthesesContext(ctxArgs) - case None => Seq() - astForSuperCall(ctx, argsAst) - } - - // TODO: Handle the optional block. - // NOTE: `super` is quite complicated semantically speaking. We'll need - // to revisit how to represent them. - protected def astForSuperCall(ctx: ParserRuleContext, arguments: Seq[Ast]): Ast = { - val call = - callNode(ctx, code(ctx), RubyOperators.superKeyword, RubyOperators.superKeyword, DispatchTypes.STATIC_DISPATCH) - callAst(call, arguments.toList) - } - - protected def astForYieldCall(ctx: ParserRuleContext, argumentsCtx: Option[ArgumentsContext]): Ast = { - val args = argumentsCtx.map(astForArguments).getOrElse(Seq()) - val call = callNode(ctx, code(ctx), UNRESOLVED_YIELD, UNRESOLVED_YIELD, DispatchTypes.STATIC_DISPATCH) - callAst(call, args) - } - - protected def astForUntilExpression(ctx: UntilExpressionContext): Ast = { - val testAst = astForExpressionOrCommand(ctx.expressionOrCommand()).headOption - val bodyAst = astForCompoundStatement(ctx.doClause().compoundStatement()) - // TODO: testAst should be negated if it's going to be modelled as a while stmt. - whileAst(testAst, bodyAst, Some(text(ctx)), line(ctx), column(ctx)) - } - - protected def astForForExpression(ctx: ForExpressionContext): Ast = { - val forVarAst = astForForVariableContext(ctx.forVariable()) - val forExprAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val forBodyAst = astForCompoundStatement(ctx.doClause().compoundStatement()) - // TODO: for X in Y is not properly modelled by while Y - val forRootAst = whileAst(forExprAst.headOption, forBodyAst, Some(text(ctx)), line(ctx), column(ctx)) - forVarAst.headOption.map(forRootAst.withChild).getOrElse(forRootAst) - } - - private def astForForVariableContext(ctx: ForVariableContext): Seq[Ast] = { - if (ctx.singleLeftHandSide() != null) { - astForSingleLeftHandSideContext(ctx.singleLeftHandSide()) - } else if (ctx.multipleLeftHandSide() != null) { - astForMultipleLeftHandSideContext(ctx.multipleLeftHandSide()) - } else { - Seq(Ast()) - } - } - - protected def astForWhileExpression(ctx: WhileExpressionContext): Ast = { - val testAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val bodyAst = astForCompoundStatement(ctx.doClause().compoundStatement()) - whileAst(testAst.headOption, bodyAst, Some(text(ctx)), line(ctx), column(ctx)) - } - - protected def astForIfExpression(ctx: IfExpressionContext): Ast = { - val testAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val thenAst = astForCompoundStatement(ctx.thenClause().compoundStatement()) - val elsifAsts = Option(ctx.elsifClause).map(_.asScala).getOrElse(Seq()).map(astForElsifClause) - val elseAst = Option(ctx.elseClause()).map(ctx => astForCompoundStatement(ctx.compoundStatement())).getOrElse(Seq()) - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - controlStructureAst(ifNode, testAst.headOption) - .withChildren(thenAst) - .withChildren(elsifAsts.toSeq) - .withChildren(elseAst) - } - - private def astForElsifClause(ctx: ElsifClauseContext): Ast = { - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - val testAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val bodyAst = astForCompoundStatement(ctx.thenClause().compoundStatement()) - controlStructureAst(ifNode, testAst.headOption, bodyAst) - } - - protected def astForVariableReference(ctx: VariableReferenceContext): Ast = ctx match { - case ctx: VariableIdentifierVariableReferenceContext => astForVariableIdentifierHelper(ctx.variableIdentifier()) - case ctx: PseudoVariableIdentifierVariableReferenceContext => - astForPseudoVariableIdentifier(ctx.pseudoVariableIdentifier()) - } - - private def astForPseudoVariableIdentifier(ctx: PseudoVariableIdentifierContext): Ast = ctx match { - case ctx: NilPseudoVariableIdentifierContext => astForNilLiteral(ctx) - case ctx: TruePseudoVariableIdentifierContext => astForTrueLiteral(ctx) - case ctx: FalsePseudoVariableIdentifierContext => astForFalseLiteral(ctx) - case ctx: SelfPseudoVariableIdentifierContext => astForSelfPseudoIdentifier(ctx) - case ctx: FilePseudoVariableIdentifierContext => astForFilePseudoIdentifier(ctx) - case ctx: LinePseudoVariableIdentifierContext => astForLinePseudoIdentifier(ctx) - case ctx: EncodingPseudoVariableIdentifierContext => astForEncodingPseudoIdentifier(ctx) - } - - protected def astForVariableIdentifierHelper( - ctx: VariableIdentifierContext, - definitelyIdentifier: Boolean = false - ): Ast = { - /* - * Preferences - * 1. If definitelyIdentifier is SET, create a identifier node - * 2. If an identifier with the variable name exists within the scope, create a identifier node - * 3. If a method with the variable name exists, create a method node - * 4. Otherwise default to identifier node creation since there is no reason (point 2) to create a call node - */ - - val variableName = code(ctx) - val isSelfFieldAccess = variableName.startsWith("@") - if (isSelfFieldAccess) { - // Very basic field detection - fieldReferences.updateWith(classStack.top) { - case Some(xs) => Option(xs ++ Set(ctx)) - case None => Option(Set(ctx)) - } - val thisNode = createThisIdentifier(ctx) - astForFieldAccess(ctx, thisNode) - } else if (definitelyIdentifier || scope.lookupVariable(variableName).isDefined) { - val node = createIdentifierWithScope(ctx, variableName, variableName, Defines.Any, List(), definitelyIdentifier) - Ast(node) - } else if (methodNameToMethod.contains(variableName)) { - astForCallNode(ctx, variableName) - } else if (ModifierTypes.ALL.contains(variableName.toUpperCase)) { - lastModifier = Option(variableName.toUpperCase) - Ast() - } else if (ctx.GLOBAL_VARIABLE_IDENTIFIER() != null) { - val globalVar = ctx.GLOBAL_VARIABLE_IDENTIFIER().getText - Ast(createIdentifierWithScope(ctx, globalVar, globalVar, Defines.String, List())) - } else { - val node = createIdentifierWithScope(ctx, variableName, variableName, Defines.Any, List()) - Ast(node) - } - } - - protected def astForUnlessExpression(ctx: UnlessExpressionContext): Ast = { - val testAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val thenAst = astForCompoundStatement(ctx.thenClause().compoundStatement()) - val elseAst = - Option(ctx.elseClause()).map(_.compoundStatement()).map(st => astForCompoundStatement(st)).getOrElse(Seq()) - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - controlStructureAst(ifNode, testAst.headOption, thenAst ++ elseAst) - } - - protected def astForQuotedStringExpression(ctx: QuotedStringExpressionContext): Seq[Ast] = ctx match - case ctx: NonExpandedQuotedStringLiteralContext => Seq(astForNonExpandedQuotedString(ctx)) - case _ => - logger.error(s"Translation for ${text(ctx)} not implemented yet") - Seq() - - private def astForNonExpandedQuotedString(ctx: NonExpandedQuotedStringLiteralContext): Ast = { - Ast(literalNode(ctx, code(ctx), getBuiltInType(Defines.String))) - } - - // TODO: handle interpolation - protected def astForQuotedRegexInterpolation(ctx: QuotedRegexInterpolationContext): Seq[Ast] = { - Seq(Ast(literalNode(ctx, code(ctx), Defines.Regexp))) - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForFunctionsCreator.scala deleted file mode 100644 index 080dfed33e42..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForFunctionsCreator.scala +++ /dev/null @@ -1,417 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.deprecated.utils.PackageContext -import io.joern.x2cpg.utils.NodeBuilders.newModifierNode -import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes} -import org.antlr.v4.runtime.ParserRuleContext -import org.antlr.v4.runtime.tree.TerminalNode -import org.slf4j.LoggerFactory - -import scala.collection.mutable -import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters.* - -trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { - this: AstCreator => - - private val logger = LoggerFactory.getLogger(getClass) - - /* - *Fake methods created from yield blocks and their yield calls will have this suffix in their names - */ - protected val YIELD_SUFFIX = "_yield" - - /* - * This is used to mark call nodes created due to yield calls. This is set in their names at creation. - * The appropriate name wrt the names of their actual methods is set later in them. - */ - protected val UNRESOLVED_YIELD = "unresolved_yield" - - /* - * Stack of variable identifiers incorrectly identified as method identifiers - * Each AST contains exactly one call or identifier node - */ - protected val methodNameAsIdentifierStack: mutable.Stack[Ast] = mutable.Stack.empty - protected val methodAliases: mutable.HashMap[String, String] = mutable.HashMap.empty - protected val methodNameToMethod: mutable.HashMap[String, NewMethod] = mutable.HashMap.empty - protected val methodDefInArgument: ListBuffer[Ast] = ListBuffer.empty - protected val methodNamesWithYield: mutable.HashSet[String] = mutable.HashSet.empty - protected val blockMethods: ListBuffer[Ast] = ListBuffer.empty - - /** @return - * the method name if found as an alias, or the given name if not found. - */ - protected def resolveAlias(name: String): String = { - methodAliases.getOrElse(name, name) - } - - protected def astForMethodDefinitionContext(ctx: MethodDefinitionContext): Seq[Ast] = { - val astMethodName = Option(ctx.methodNamePart()) match - case Some(ctxMethodNamePart) => - astForMethodNamePartContext(ctxMethodNamePart) - case None => - astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - val callNode = astMethodName.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] - - // Create thisParameter if this is an instance method - // TODO may need to revisit to make this more robust - - val (methodName, methodFullName) = if (callNode.name == Defines.Initialize) { - (XDefines.ConstructorMethodName, classStack.reverse :+ XDefines.ConstructorMethodName mkString pathSep) - } else { - (callNode.name, classStack.reverse :+ callNode.name mkString pathSep) - } - val newMethodNode = methodNode(ctx, methodName, code(ctx), methodFullName, None, relativeFilename) - .columnNumber(callNode.columnNumber) - .lineNumber(callNode.lineNumber) - - scope.pushNewScope(newMethodNode) - - val astMethodParamSeq = ctx.methodNamePart() match { - case _: SimpleMethodNamePartContext if !classStack.top.endsWith(":program") => - val thisParameterNode = createMethodParameterIn( - "this", - typeFullName = callNode.methodFullName, - lineNumber = callNode.lineNumber, - colNumber = callNode.columnNumber, - index = 0, - order = 0 - ) - Seq(Ast(thisParameterNode)) ++ astForMethodParameterPartContext(ctx.methodParameterPart()) - case _ => astForMethodParameterPartContext(ctx.methodParameterPart()) - } - - Option(ctx.END()).foreach(endNode => newMethodNode.lineNumberEnd(endNode.getSymbol.getLine)) - - callNode.methodFullName(methodFullName) - - val classType = if (classStack.isEmpty) "Standalone" else classStack.top - val classPath = classStack.reverse.toList.mkString(pathSep) - packageContext.packageTable.addPackageMethod(packageContext.moduleName, callNode.name, classPath, classType) - - val astBody = Option(ctx.bodyStatement()) match { - case Some(ctxBodyStmt) => astForBodyStatementContext(ctxBodyStmt, true) - case None => - val expAst = astForExpressionContext(ctx.expression()) - Seq(lastStmtAsReturnAst(ctx, expAst.head, Option(text(ctx.expression())))) - } - - // process yield calls. - astBody - .flatMap(_.nodes.collect { case x: NewCall => x }.filter(_.name == UNRESOLVED_YIELD)) - .foreach { yieldCallNode => - val name = newMethodNode.name - val methodFullName = classStack.reverse :+ callNode.name mkString pathSep - yieldCallNode.name(name + YIELD_SUFFIX) - yieldCallNode.methodFullName(methodFullName + YIELD_SUFFIX) - methodNamesWithYield.add(newMethodNode.name) - /* - * These are calls to the yield block of this method. - * Add this method to the list of yield blocks. - * The add() is idempotent and so adding the same method multiple times makes no difference. - * It just needs to be added at this place so that it gets added iff it has a yield block - */ - } - - val methodRetNode = NewMethodReturn().typeFullName(Defines.Any) - - val modifierNode = lastModifier match { - case Some(modifier) => NewModifier().modifierType(modifier).code(modifier) - case None => NewModifier().modifierType(ModifierTypes.PUBLIC).code(ModifierTypes.PUBLIC) - } - /* - * public/private/protected modifiers are in a separate statement - * TODO find out how they should be used. Need to do this iff it adds any value - */ - if (methodName != XDefines.ConstructorMethodName) { - methodNameToMethod.put(newMethodNode.name, newMethodNode) - } - - /* Before creating ast, we traverse the method params and identifiers and link them*/ - val identifiers = - astBody.flatMap(ast => ast.nodes.filter(_.isInstanceOf[NewIdentifier])).asInstanceOf[Seq[NewIdentifier]] - - val params = astMethodParamSeq - .flatMap(_.nodes.collect { case x: NewMethodParameterIn => x }) - .toList - val locals = scope.createAndLinkLocalNodes(diffGraph, params.map(_.name).toSet) - - params.foreach { param => - identifiers.filter(_.name == param.name).foreach { identifier => - diffGraph.addEdge(identifier, param, EdgeTypes.REF) - } - } - scope.popScope() - - Seq( - methodAst( - newMethodNode, - astMethodParamSeq, - blockAst(blockNode(ctx), locals.map(Ast.apply) ++ astBody.toList), - methodRetNode, - Seq[NewModifier](modifierNode) - ) - ) - } - - private def astForOperatorMethodNameContext(ctx: OperatorMethodNameContext): Seq[Ast] = { - /* - * This is for operator overloading for the class - */ - val name = code(ctx) - val methodFullName = classStack.reverse :+ name mkString pathSep - - val node = callNode(ctx, code(ctx), name, methodFullName, DispatchTypes.STATIC_DISPATCH, None, Option(Defines.Any)) - ctx.children.asScala - .collectFirst { case x: TerminalNode => x } - .foreach(x => node.lineNumber(x.lineNumber).columnNumber(x.columnNumber)) - Seq(callAst(node)) - } - - protected def astForMethodNameContext(ctx: MethodNameContext): Seq[Ast] = { - if (ctx.methodIdentifier() != null) { - astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - } else if (ctx.operatorMethodName() != null) { - astForOperatorMethodNameContext(ctx.operatorMethodName) - } else if (ctx.keyword() != null) { - val node = - callNode(ctx, code(ctx), code(ctx), code(ctx), DispatchTypes.STATIC_DISPATCH, None, Option(Defines.Any)) - ctx.children.asScala - .collectFirst { case x: TerminalNode => x } - .foreach(x => - node.lineNumber(x.lineNumber).columnNumber(x.columnNumber).name(x.getText).methodFullName(x.getText) - ) - Seq(callAst(node)) - } else { - Seq.empty - } - } - - private def astForSingletonMethodNamePartContext(ctx: SingletonMethodNamePartContext): Seq[Ast] = { - val definedMethodNameAst = astForDefinedMethodNameContext(ctx.definedMethodName()) - val singletonObjAst = astForSingletonObjectContext(ctx.singletonObject()) - definedMethodNameAst ++ singletonObjAst - } - - private def astForSingletonObjectContext(ctx: SingletonObjectContext): Seq[Ast] = { - if (ctx.variableIdentifier() != null) { - Seq(astForVariableIdentifierHelper(ctx.variableIdentifier(), true)) - } else if (ctx.pseudoVariableIdentifier() != null) { - Seq(Ast()) - } else if (ctx.expressionOrCommand() != null) { - astForExpressionOrCommand(ctx.expressionOrCommand()) - } else { - Seq.empty - } - } - - private def astForParametersContext(ctx: ParametersContext): Seq[Ast] = { - if (ctx == null) return Seq() - - // the parameterTupleList holds the parameter terminal node and is the parameter a variadic parameter - val parameterTupleList = ctx.parameter().asScala.map { - case procCtx if procCtx.procParameter() != null => - (Option(procCtx.procParameter().LOCAL_VARIABLE_IDENTIFIER()), false) - case optCtx if optCtx.optionalParameter() != null => - (Option(optCtx.optionalParameter().LOCAL_VARIABLE_IDENTIFIER()), false) - case manCtx if manCtx.mandatoryParameter() != null => - (Option(manCtx.mandatoryParameter().LOCAL_VARIABLE_IDENTIFIER()), false) - case arrCtx if arrCtx.arrayParameter() != null => - (Option(arrCtx.arrayParameter().LOCAL_VARIABLE_IDENTIFIER()), arrCtx.arrayParameter().STAR() != null) - case keywordCtx if keywordCtx.keywordParameter() != null => - (Option(keywordCtx.keywordParameter().LOCAL_VARIABLE_IDENTIFIER()), false) - case _ => (None, false) - } - - parameterTupleList.zipWithIndex.map { case (paraTuple, paraIndex) => - paraTuple match - case (Some(paraValue), isVariadic) => - val varSymbol = paraValue.getSymbol - createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, Seq[String](Defines.Any)) - Ast( - createMethodParameterIn( - varSymbol.getText, - lineNumber = Some(varSymbol.getLine), - colNumber = Some(varSymbol.getCharPositionInLine), - order = paraIndex + 1, - index = paraIndex + 1 - ).isVariadic(isVariadic) - ) - case _ => - Ast( - createMethodParameterIn( - getUnusedVariableNames(usedVariableNames, Defines.TempParameter), - order = paraIndex + 1, - index = paraIndex + 1 - ) - ) - }.toList - } - - // TODO: Rewrite for simplicity and take into account more than parameter names. - private def astForMethodParameterPartContext(ctx: MethodParameterPartContext): Seq[Ast] = { - if (ctx == null || ctx.parameters == null) Seq.empty - else astForParametersContext(ctx.parameters) - } - - private def astForDefinedMethodNameContext(ctx: DefinedMethodNameContext): Seq[Ast] = { - Option(ctx.methodName()) match - case Some(methodNameCtx) => astForMethodNameContext(methodNameCtx) - case None => astForAssignmentLikeMethodIdentifierContext(ctx.assignmentLikeMethodIdentifier()) - } - - private def astForAssignmentLikeMethodIdentifierContext(ctx: AssignmentLikeMethodIdentifierContext): Seq[Ast] = { - Seq( - callAst( - callNode(ctx, code(ctx), code(ctx), code(ctx), DispatchTypes.STATIC_DISPATCH, Some(""), Some(Defines.Any)) - ) - ) - } - - private def astForMethodNamePartContext(ctx: MethodNamePartContext): Seq[Ast] = ctx match { - case ctx: SimpleMethodNamePartContext => astForSimpleMethodNamePartContext(ctx) - case ctx: SingletonMethodNamePartContext => astForSingletonMethodNamePartContext(ctx) - case _ => - logger.error(s"astForMethodNamePartContext() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - private def astForSimpleMethodNamePartContext(ctx: SimpleMethodNamePartContext): Seq[Ast] = - astForDefinedMethodNameContext(ctx.definedMethodName) - - protected def methodForClosureStyleFn(ctx: ParserRuleContext): NewMethod = { - val procMethodName = s"proc_${blockIdCounter.getAndAdd(1)}" - val methodFullName = classStack.reverse :+ procMethodName mkString pathSep - methodNode(ctx, procMethodName, code(ctx), methodFullName, None, relativeFilename) - } - - protected def astForProcDefinitionContext(ctx: ProcDefinitionContext): Seq[Ast] = { - /* - * Model a proc as a method - */ - // Note: For parameters in the Proc definition, an implicit parameter which goes by the name of `this` is added to the cpg - val newMethodNode = methodForClosureStyleFn(ctx) - - scope.pushNewScope(newMethodNode) - - val astMethodParam = astForParametersContext(ctx.parameters()) - val paramNames = astMethodParam.flatMap(_.nodes).collect { case x: NewMethodParameterIn => x.name }.toSet - val astBody = astForCompoundStatement(ctx.block.compoundStatement, true) - val locals = scope.createAndLinkLocalNodes(diffGraph, paramNames).map(Ast.apply) - - val methodRetNode = NewMethodReturn() - .typeFullName(Defines.Any) - - val modifiers = newModifierNode(ModifierTypes.PUBLIC) :: newModifierNode(ModifierTypes.LAMBDA) :: Nil - - val methAst = methodAst( - newMethodNode, - astMethodParam, - blockAst(blockNode(ctx), locals ++ astBody.toList), - methodRetNode, - modifiers - ) - blockMethods.addOne(methAst) - - val callArgs = astMethodParam - .flatMap(_.root) - .collect { case x: NewMethodParameterIn => x } - .map(param => Ast(createIdentifierWithScope(ctx, param.name, param.code, Defines.Any, Seq(), true))) - - val procCallNode = - callNode( - ctx, - code(ctx), - newMethodNode.name, - newMethodNode.fullName, - DispatchTypes.STATIC_DISPATCH, - None, - Option(Defines.Any) - ) - - scope.popScope() - - Seq(callAst(procCallNode, callArgs)) - } - - def astForDefinedMethodNameOrSymbolContext(ctx: DefinedMethodNameOrSymbolContext): Seq[Ast] = - if (ctx == null) { - Seq.empty - } else { - if (ctx.definedMethodName() != null) { - astForDefinedMethodNameContext(ctx.definedMethodName()) - } else { - Seq(astForSymbolLiteral(ctx.symbol())) - } - } - - protected def astForBlockFunction( - ctxStmt: StatementsContext, - ctxParam: Option[BlockParameterContext], - blockMethodName: String, - lineStart: Int, - lineEnd: Int, - colStart: Int, - colEnd: Int - ): Seq[Ast] = { - /* - * Model a block as a method - */ - val methodFullName = classStack.reverse :+ blockMethodName mkString pathSep - val newMethodNode = methodNode(ctxStmt, blockMethodName, code(ctxStmt), methodFullName, None, relativeFilename) - .lineNumber(lineStart) - .lineNumberEnd(lineEnd) - .columnNumber(colStart) - .columnNumberEnd(colEnd) - - scope.pushNewScope(newMethodNode) - val astMethodParam = ctxParam.map(astForBlockParameterContext).getOrElse(Seq()) - - val publicModifier = NewModifier().modifierType(ModifierTypes.PUBLIC) - val paramSeq = astMethodParam.flatMap(_.root).map { - /* In majority of cases, node will be an identifier */ - case identifierNode: NewIdentifier => - val param = NewMethodParameterIn() - .name(identifierNode.name) - .code(identifierNode.code) - .typeFullName(identifierNode.typeFullName) - .lineNumber(identifierNode.lineNumber) - .columnNumber(identifierNode.columnNumber) - .dynamicTypeHintFullName(identifierNode.dynamicTypeHintFullName) - Ast(param) - case _: NewCall => - /* TODO: Occasionally, we might encounter a _ call in cases like "do |_, x|" where we should handle this? - * But for now, we just return an empty AST. Keeping this match explicitly here so we come back */ - Ast() - case _ => - Ast() - } - val paramNames = (astMethodParam ++ paramSeq) - .flatMap(_.root) - .collect { - case x: NewMethodParameterIn => x.name - case x: NewIdentifier => x.name - } - .toSet - val astBody = astForStatements(ctxStmt, true) - val locals = scope.createAndLinkLocalNodes(diffGraph, paramNames).map(Ast.apply) - val methodRetNode = NewMethodReturn().typeFullName(Defines.Any) - - scope.popScope() - - Seq( - methodAst( - newMethodNode, - paramSeq, - blockAst(blockNode(ctxStmt), locals ++ astBody.toList), - methodRetNode, - Seq(publicModifier) - ) - ) - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForHereDocsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForHereDocsCreator.scala deleted file mode 100644 index 64cb8726d8cb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForHereDocsCreator.scala +++ /dev/null @@ -1,74 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.{HereDocArgumentContext, HereDocLiteralContext} -import io.joern.rubysrc2cpg.deprecated.parser.HereDocHandling -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier, NewLiteral} - -import scala.collection.immutable.Seq -import scala.collection.mutable - -trait AstForHereDocsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - private val hereDocTokens = mutable.Stack[(String, NewLiteral)]() - - protected def astForHereDocLiteral(ctx: HereDocLiteralContext): Ast = { - val delimiter = HereDocHandling.getHereDocDelimiter(ctx.HERE_DOC().getText).getOrElse("") - val hereDoc = ctx.HERE_DOC().getText.replaceFirst("<<[~-]", "") - val hereDocTxt = hereDoc.stripPrefix(delimiter).stripSuffix(delimiter).strip() - val literal = NewLiteral() - .code(hereDocTxt) - .typeFullName(Defines.String) - .lineNumber(line(ctx)) - .columnNumber(column(ctx)) - Ast(literal) - } - - protected def astForHereDocArgument(ctx: HereDocArgumentContext): Seq[Ast] = - HereDocHandling.getHereDocDelimiter(ctx.HERE_DOC_IDENTIFIER().getText) match - case Some(delimiter) => - val literal = NewLiteral() - .code("") // build code from the upcoming statements - .typeFullName(Defines.String) - .lineNumber(line(ctx)) - .columnNumber(column(ctx)) - hereDocTokens.push((delimiter, literal)) - Seq(Ast(literal)) - case None => Seq.empty - - /** Will determine, if we have recently met a here doc initializer, if this statement should be converted to a here - * doc literal or returned as-is. - * @param stmt - * the statement AST. - * @return - * the statement AST or nothing if this is determined to be a here doc body. - */ - protected def scanStmtForHereDoc(stmt: Seq[Ast]): Seq[Ast] = { - if (stmt.nonEmpty && hereDocTokens.nonEmpty) { - val (delimiter, literalNode) = hereDocTokens.head - val stmtAst = stmt.head - val atHereDocInitializer = stmt.flatMap(_.nodes).exists { - case x: NewLiteral => hereDocTokens.exists(_._2 == x) - case _ => false - } - if (atHereDocInitializer) { - // We are at the start of the here doc, do nothing - stmt - } else { - // We are in the middle of the here doc, convert statements to here doc body + look out for delimiter - val txt = stmtAst.root match - case Some(x: NewCall) => x.code - case Some(x: NewIdentifier) => x.code - case _ => "" - - if (txt == delimiter) hereDocTokens.pop() - else literalNode.code(s"${literalNode.code}\n$txt".trim) - Seq.empty[Ast] - } - } else { - stmt - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForPrimitivesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForPrimitivesCreator.scala deleted file mode 100644 index 2821d4d8658e..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForPrimitivesCreator.scala +++ /dev/null @@ -1,100 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.deprecated.passes.Defines.getBuiltInType -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.NewCall -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import org.antlr.v4.runtime.ParserRuleContext - -import scala.jdk.CollectionConverters.CollectionHasAsScala - -trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - protected def astForNilLiteral(ctx: DeprecatedRubyParser.NilPseudoVariableIdentifierContext): Ast = - Ast(literalNode(ctx, code(ctx), Defines.NilClass)) - - protected def astForTrueLiteral(ctx: DeprecatedRubyParser.TruePseudoVariableIdentifierContext): Ast = - Ast(literalNode(ctx, code(ctx), Defines.TrueClass)) - - protected def astForFalseLiteral(ctx: DeprecatedRubyParser.FalsePseudoVariableIdentifierContext): Ast = - Ast(literalNode(ctx, code(ctx), Defines.FalseClass)) - - protected def astForSelfPseudoIdentifier(ctx: DeprecatedRubyParser.SelfPseudoVariableIdentifierContext): Ast = - Ast(createIdentifierWithScope(ctx, code(ctx), code(ctx), Defines.Object)) - - protected def astForFilePseudoIdentifier(ctx: DeprecatedRubyParser.FilePseudoVariableIdentifierContext): Ast = - Ast(createIdentifierWithScope(ctx, code(ctx), code(ctx), getBuiltInType(Defines.String))) - - protected def astForLinePseudoIdentifier(ctx: DeprecatedRubyParser.LinePseudoVariableIdentifierContext): Ast = - Ast(createIdentifierWithScope(ctx, code(ctx), code(ctx), getBuiltInType(Defines.Integer))) - - protected def astForEncodingPseudoIdentifier(ctx: DeprecatedRubyParser.EncodingPseudoVariableIdentifierContext): Ast = - Ast(createIdentifierWithScope(ctx, code(ctx), code(ctx), Defines.Encoding)) - - protected def astForNumericLiteral(ctx: DeprecatedRubyParser.NumericLiteralContext): Ast = { - val numericTypeName = - if (isFloatLiteral(ctx.unsignedNumericLiteral)) getBuiltInType(Defines.Float) else getBuiltInType(Defines.Integer) - Ast(literalNode(ctx, code(ctx), numericTypeName)) - } - - protected def astForSymbolLiteral(ctx: DeprecatedRubyParser.SymbolContext): Ast = - Ast(literalNode(ctx, code(ctx), Defines.Symbol)) - - protected def astForSingleQuotedStringLiteral(ctx: DeprecatedRubyParser.SingleQuotedStringLiteralContext): Ast = - Ast(literalNode(ctx, code(ctx), getBuiltInType(Defines.String))) - - protected def astForDoubleQuotedStringLiteral(ctx: DeprecatedRubyParser.DoubleQuotedStringLiteralContext): Ast = - Ast(literalNode(ctx, code(ctx), getBuiltInType(Defines.String))) - - protected def astForRegularExpressionLiteral(ctx: DeprecatedRubyParser.RegularExpressionLiteralContext): Ast = - Ast(literalNode(ctx, code(ctx), Defines.Regexp)) - - private def isFloatLiteral(ctx: DeprecatedRubyParser.UnsignedNumericLiteralContext): Boolean = - Option(ctx.FLOAT_LITERAL_WITH_EXPONENT).isDefined || Option(ctx.FLOAT_LITERAL_WITHOUT_EXPONENT).isDefined - - // TODO: Return Ast instead of Seq[Ast] - protected def astForArrayLiteral(ctx: ArrayConstructorContext): Seq[Ast] = ctx match - case ctx: BracketedArrayConstructorContext => astForBracketedArrayConstructor(ctx) - case ctx: NonExpandedWordArrayConstructorContext => astForNonExpandedWordArrayConstructor(ctx) - case ctx: NonExpandedSymbolArrayConstructorContext => astForNonExpandedSymbolArrayConstructor(ctx) - - private def astForBracketedArrayConstructor(ctx: BracketedArrayConstructorContext): Seq[Ast] = { - Option(ctx.indexingArguments) - .map(astForIndexingArgumentsContext) - .getOrElse(Seq(astForEmptyArrayInitializer(ctx))) - } - - private def astForEmptyArrayInitializer(ctx: ParserRuleContext): Ast = { - Ast(callNode(ctx, code(ctx), Operators.arrayInitializer, Operators.arrayInitializer, DispatchTypes.STATIC_DISPATCH)) - } - - private def astForNonExpandedWordArrayConstructor(ctx: NonExpandedWordArrayConstructorContext): Seq[Ast] = { - Option(ctx.nonExpandedArrayElements) - .map(astForNonExpandedArrayElements(_, astForNonExpandedWordArrayElement)) - .getOrElse(Seq(astForEmptyArrayInitializer(ctx))) - } - - private def astForNonExpandedWordArrayElement(ctx: NonExpandedArrayElementContext): Ast = { - Ast(literalNode(ctx, code(ctx), Defines.String, List(Defines.String))) - } - - private def astForNonExpandedSymbolArrayConstructor(ctx: NonExpandedSymbolArrayConstructorContext): Seq[Ast] = { - Option(ctx.nonExpandedArrayElements) - .map(astForNonExpandedArrayElements(_, astForNonExpandedSymbolArrayElement)) - .getOrElse(Seq(astForEmptyArrayInitializer(ctx))) - } - - private def astForNonExpandedArrayElements( - ctx: NonExpandedArrayElementsContext, - astForNonExpandedArrayElement: NonExpandedArrayElementContext => Ast - ): Seq[Ast] = { - ctx.nonExpandedArrayElement.asScala.map(astForNonExpandedArrayElement).toSeq - } - - private def astForNonExpandedSymbolArrayElement(ctx: NonExpandedArrayElementContext): Ast = { - Ast(literalNode(ctx, code(ctx), Defines.Symbol)) - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForStatementsCreator.scala deleted file mode 100644 index 925dc9e19720..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForStatementsCreator.scala +++ /dev/null @@ -1,426 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import better.files.File -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.x2cpg.Defines.DynamicCallUnknownFullName -import io.joern.x2cpg.Imports.createImportNodeAndLink -import io.joern.x2cpg.X2Cpg.stripQuotes -import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} -import org.antlr.v4.runtime.ParserRuleContext -import org.slf4j.LoggerFactory - -import scala.jdk.CollectionConverters.CollectionHasAsScala - -trait AstForStatementsCreator(filename: String)(implicit withSchemaValidation: ValidationMode) { - this: AstCreator => - - private val logger = LoggerFactory.getLogger(this.getClass) - private val prefixMethods = Set( - "attr_reader", - "attr_writer", - "attr_accessor", - "remove_method", - "public_class_method", - "private_class_method", - "private", - "protected", - "module_function" - ) - - private def astForAliasStatement(ctx: AliasStatementContext): Ast = { - val aliasName = ctx.definedMethodNameOrSymbol(0).getText.substring(1) - val methodName = ctx.definedMethodNameOrSymbol(1).getText.substring(1) - methodAliases.addOne(aliasName, methodName) - Ast() - } - - private def astForUndefStatement(ctx: UndefStatementContext): Ast = { - val undefNames = ctx.definedMethodNameOrSymbol().asScala.flatMap(astForDefinedMethodNameOrSymbolContext).toSeq - val call = callNode(ctx, code(ctx), RubyOperators.undef, RubyOperators.undef, DispatchTypes.STATIC_DISPATCH) - callAst(call, undefNames) - } - - private def astForBeginStatement(ctx: BeginStatementContext): Ast = { - val stmts = Option(ctx.compoundStatement).map(astForCompoundStatement(_)).getOrElse(Seq()) - val blockNode = NewBlock().typeFullName(Defines.Any) - blockAst(blockNode, stmts.toList) - } - - private def astForEndStatement(ctx: EndStatementContext): Ast = { - val stmts = Option(ctx.compoundStatement).map(astForCompoundStatement(_)).getOrElse(Seq()) - val blockNode = NewBlock().typeFullName(Defines.Any) - blockAst(blockNode, stmts.toList) - } - - private def astForModifierStatement(ctx: ModifierStatementContext): Ast = ctx.mod.getType match { - case IF => astForIfModifierStatement(ctx) - case UNLESS => astForUnlessModifierStatement(ctx) - case WHILE => astForWhileModifierStatement(ctx) - case UNTIL => astForUntilModifierStatement(ctx) - case RESCUE => astForRescueModifierStatement(ctx) - } - - private def astForIfModifierStatement(ctx: ModifierStatementContext): Ast = { - val lhs = astForStatement(ctx.statement(0)) - val rhs = astForStatement(ctx.statement(1)).headOption - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - lhs.headOption.flatMap(_.root) match - // If the LHS is a `next` command with a return value, then this if statement is its condition and it becomes a - // `return` - case Some(x: NewControlStructure) if x.code == Defines.ModifierNext && lhs.head.nodes.size > 1 => - val retNode = NewReturn().code(Defines.ModifierNext).lineNumber(x.lineNumber).columnNumber(x.columnNumber) - controlStructureAst(ifNode, rhs, Seq(lhs.head.subTreeCopy(x, replacementNode = Option(retNode)))) - case _ => controlStructureAst(ifNode, rhs, lhs) - } - - private def astForUnlessModifierStatement(ctx: ModifierStatementContext): Ast = { - val lhs = astForStatement(ctx.statement(0)) - val rhs = astForStatement(ctx.statement(1)) - val ifNode = controlStructureNode(ctx, ControlStructureTypes.IF, code(ctx)) - controlStructureAst(ifNode, lhs.headOption, rhs) - } - - private def astForWhileModifierStatement(ctx: ModifierStatementContext): Ast = { - val lhs = astForStatement(ctx.statement(0)) - val rhs = astForStatement(ctx.statement(1)) - whileAst(rhs.headOption, lhs, Some(text(ctx))) - } - - private def astForUntilModifierStatement(ctx: ModifierStatementContext): Ast = { - val lhs = astForStatement(ctx.statement(0)) - val rhs = astForStatement(ctx.statement(1)) - whileAst(rhs.headOption, lhs, Some(text(ctx))) - } - - private def astForRescueModifierStatement(ctx: ModifierStatementContext): Ast = { - val lhs = astForStatement(ctx.statement(0)) - val rhs = astForStatement(ctx.statement(1)) - val throwNode = controlStructureNode(ctx, ControlStructureTypes.THROW, code(ctx)) - controlStructureAst(throwNode, rhs.headOption, lhs) - } - - /** If the last statement is a return, this is returned. If not, then a return node is created. - */ - protected def lastStmtAsReturnAst(ctx: ParserRuleContext, lastStmtAst: Ast, maybeCode: Option[String] = None): Ast = - lastStmtAst.root.collectFirst { case x: NewReturn => x } match - case Some(_) => lastStmtAst - case None => - val code = maybeCode.getOrElse(text(ctx)) - val retNode = returnNode(ctx, code) - lastStmtAst.root match - case Some(method: NewMethod) => returnAst(retNode, Seq(Ast(methodToMethodRef(ctx, method)))) - case _ => returnAst(retNode, Seq(lastStmtAst)) - - protected def astForBodyStatementContext(ctx: BodyStatementContext, isMethodBody: Boolean = false): Seq[Ast] = { - if (ctx.rescueClause.size > 0) Seq(astForRescueClause(ctx)) - else astForCompoundStatement(ctx.compoundStatement(), isMethodBody) - } - - protected def astForCompoundStatement( - ctx: CompoundStatementContext, - isMethodBody: Boolean = false, - canConsiderAsLeaf: Boolean = true - ): Seq[Ast] = { - val stmtAsts = Option(ctx) - .map(_.statements()) - .map(astForStatements(_, isMethodBody, canConsiderAsLeaf)) - .getOrElse(Seq.empty) - if (isMethodBody) { - stmtAsts - } else { - Seq(blockAst(blockNode(ctx), stmtAsts.toList)) - } - } - - /* - * Each statement set can be considered a block. The blocks of a method can be considered to form a hierarchy. - * We can consider the blocks structure as a n-way tree. Leaf blocks are blocks that have no more sub blocks i.e children in the - * hierarchy. The last statement of the block of the method which is the top level/root block i.e. method body should be - * converted into a implicit return. However, if the last statement is a if-else it has sub-blocks/child blocks and the last statement of each leaf block in it - * will have to be converted to a implicit return, unless it is already a implicit return. - * Some sub-blocks are exempt from their last statements being converted to returns. Examples are blocks that are arguments to functions like string interpolation. - * - * isMethodBody => The statement set is the top level block in the method. i.e. the root block - * canConsiderAsLeaf => The statement set can be considered a leaf block. This is set to false by the caller when it is a statement - * set as a part of an expression. Eg. argument in string interpolation. We do not want to construct return nodes out of - * string interpolation arguments. These are exempt blocks for implicit returns. - * blockChildHash => Hash of a block id to any child. Absence of a block in this after all its statements have been processed implies - * that the block is a leaf - * blockIdCounter => A simple counter used to assign an unique id to each block. - */ - protected def astForStatements( - ctx: StatementsContext, - isMethodBody: Boolean = false, - canConsiderAsLeaf: Boolean = true - ): Seq[Ast] = { - - def astsForStmtCtx(stCtx: StatementContext, stmtCount: Int, stmtCounter: Int): Seq[Ast] = { - if (isMethodBody) processingLastMethodStatement.lazySet(stmtCounter == stmtCount) - val stAsts = astForStatement(stCtx) - if (stAsts.nonEmpty && canConsiderAsLeaf && processingLastMethodStatement.get) { - blockChildHash.get(currentBlockId.get) match { - case Some(_) => - // this is a non-leaf block - stAsts - case None => - // this is a leaf block - processingLastMethodStatement.lazySet(!(isMethodBody && stmtCounter == stmtCount)) - Seq(lastStmtAsReturnAst(stCtx, stAsts.head, Option(text(stCtx)))) - } - } else { - stAsts - } - } - - Option(ctx) - .map { ctx => - val stmtCount = ctx.statement.size - val parentBlockId = currentBlockId.get - if (canConsiderAsLeaf) blockChildHash.update(parentBlockId, currentBlockId.get) - currentBlockId.lazySet(blockIdCounter.addAndGet(1)) - - val stmtAsts = Option(ctx) - .map(_.statement) - .map(_.asScala) - .getOrElse(Seq.empty) - .zipWithIndex - .flatMap { case (stmtCtx, idx) => astsForStmtCtx(stmtCtx, stmtCount, idx + 1) } - .toSeq - currentBlockId.lazySet(parentBlockId) - stmtAsts - } - .getOrElse(Seq.empty) - } - - // TODO: return Ast instead of Seq[Ast]. - private def astForStatement(ctx: StatementContext): Seq[Ast] = scanStmtForHereDoc(ctx match { - case ctx: AliasStatementContext => Seq(astForAliasStatement(ctx)) - case ctx: UndefStatementContext => Seq(astForUndefStatement(ctx)) - case ctx: BeginStatementContext => Seq(astForBeginStatement(ctx)) - case ctx: EndStatementContext => Seq(astForEndStatement(ctx)) - case ctx: ModifierStatementContext => Seq(astForModifierStatement(ctx)) - case ctx: ExpressionOrCommandStatementContext => astForExpressionOrCommand(ctx.expressionOrCommand()) - }) - - // TODO: return Ast instead of Seq[Ast] - protected def astForExpressionOrCommand(ctx: ExpressionOrCommandContext): Seq[Ast] = ctx match { - case ctx: InvocationExpressionOrCommandContext => astForInvocationExpressionOrCommandContext(ctx) - case ctx: NotExpressionOrCommandContext => Seq(astForNotKeywordExpressionOrCommand(ctx)) - case ctx: OrAndExpressionOrCommandContext => Seq(astForOrAndExpressionOrCommand(ctx)) - case ctx: ExpressionExpressionOrCommandContext => astForExpressionContext(ctx.expression()) - case _ => - logger.error(s"astForExpressionOrCommand() $relativeFilename, ${text(ctx)} All contexts mismatched.") - Seq(Ast()) - } - - private def astForNotKeywordExpressionOrCommand(ctx: NotExpressionOrCommandContext): Ast = { - val exprOrCommandAst = astForExpressionOrCommand(ctx.expressionOrCommand()) - val call = callNode(ctx, code(ctx), Operators.not, Operators.not, DispatchTypes.STATIC_DISPATCH) - callAst(call, exprOrCommandAst) - } - - private def astForOrAndExpressionOrCommand(ctx: OrAndExpressionOrCommandContext): Ast = ctx.op.getType match { - case OR => astForOrExpressionOrCommand(ctx) - case AND => astForAndExpressionOrCommand(ctx) - } - - private def astForOrExpressionOrCommand(ctx: OrAndExpressionOrCommandContext): Ast = { - val argsAst = ctx.expressionOrCommand().asScala.flatMap(astForExpressionOrCommand) - val call = callNode(ctx, code(ctx), Operators.or, Operators.or, DispatchTypes.STATIC_DISPATCH) - callAst(call, argsAst.toList) - } - - private def astForAndExpressionOrCommand(ctx: OrAndExpressionOrCommandContext): Ast = { - val argsAst = ctx.expressionOrCommand().asScala.flatMap(astForExpressionOrCommand) - val call = callNode(ctx, code(ctx), Operators.and, Operators.and, DispatchTypes.STATIC_DISPATCH) - callAst(call, argsAst.toList) - } - - private def astForSuperCommand(ctx: SuperCommandContext): Ast = - astForSuperCall(ctx, astForArguments(ctx.argumentsWithoutParentheses().arguments())) - - private def astForYieldCommand(ctx: YieldCommandContext): Ast = - astForYieldCall(ctx, Option(ctx.argumentsWithoutParentheses().arguments())) - - private def astForSimpleMethodCommand(ctx: SimpleMethodCommandContext): Seq[Ast] = { - val methodIdentifierAsts = astForMethodIdentifierContext(ctx.methodIdentifier(), code(ctx)) - methodIdentifierAsts.headOption.foreach(methodNameAsIdentifierStack.push) - val argsAsts = astForArguments(ctx.argumentsWithoutParentheses().arguments()) - - /* get args without the method def in it */ - val argAstsWithoutMethods = argsAsts.filterNot(_.root.exists(_.isInstanceOf[NewMethod])) - - /* isolate methods from the original args and create identifier ASTs from it */ - val methodDefAsts = argsAsts.filter(_.root.exists(_.isInstanceOf[NewMethod])) - val methodToIdentifierAsts = methodDefAsts.flatMap { - _.nodes.collectFirst { case methodNode: NewMethod => - Ast( - createIdentifierWithScope( - methodNode.name, - methodNode.name, - Defines.Any, - Seq.empty, - methodNode.lineNumber, - methodNode.columnNumber, - definitelyIdentifier = true - ) - ) - } - } - - /* TODO: we add the isolated method defs later on to the parent instead */ - methodDefInArgument.addAll(methodDefAsts) - - val callNodes = methodIdentifierAsts.head.nodes.collect { case x: NewCall => x } - if (callNodes.size == 1) { - val callNode = callNodes.head - if (callNode.name == "require" || callNode.name == "load") { - resolveRequireOrLoadPath(argsAsts, callNode) - } else if (callNode.name == "require_relative") { - resolveRelativePath(filename, argsAsts, callNode) - } else if (prefixMethods.contains(callNode.name)) { - /* we remove the method definition AST from argument and add its corresponding identifier form */ - Seq(callAst(callNode, argAstsWithoutMethods ++ methodToIdentifierAsts)) - } else { - Seq(callAst(callNode, argsAsts)) - } - } else { - argsAsts - } - } - - private def astForMemberAccessCommand(ctx: MemberAccessCommandContext): Seq[Ast] = { - astForMethodNameContext(ctx.methodName).headOption - .flatMap(_.root) - .collectFirst { case x: NewCall => resolveAlias(x.name) } - .map(methodName => - callNode( - ctx, - code(ctx), - methodName, - DynamicCallUnknownFullName, - DispatchTypes.STATIC_DISPATCH, - None, - Option(Defines.Any) - ) - ) match - case Some(newCall) => - val primaryAst = astForPrimaryContext(ctx.primary()) - val argsAst = astForArguments(ctx.argumentsWithoutParentheses().arguments) - primaryAst.headOption - .flatMap(_.root) - .collectFirst { case x: NewMethod => x } - .map { methodNode => - val methodRefNode = methodToMethodRef(ctx, methodNode) - blockMethods.addOne(primaryAst.head) - Seq(callAst(newCall, Seq(Ast(methodRefNode)) ++ argsAst)) - } - .getOrElse(Seq(callAst(newCall, argsAst, primaryAst.headOption))) - case None => Seq.empty - } - - private def methodToMethodRef(ctx: ParserRuleContext, methodNode: NewMethod): NewMethodRef = - methodRefNode(ctx, s"def ${methodNode.name}(...)", methodNode.fullName, Defines.Any) - - protected def astForCommand(ctx: CommandContext): Seq[Ast] = ctx match { - case ctx: YieldCommandContext => Seq(astForYieldCommand(ctx)) - case ctx: SuperCommandContext => Seq(astForSuperCommand(ctx)) - case ctx: SimpleMethodCommandContext => astForSimpleMethodCommand(ctx) - case ctx: MemberAccessCommandContext => astForMemberAccessCommand(ctx) - } - - private def resolveRequireOrLoadPath(argsAst: Seq[Ast], callNode: NewCall): Seq[Ast] = { - val importedNode = argsAst.headOption.map(_.nodes.collect { case x: NewLiteral => x }).getOrElse(Seq.empty) - if (importedNode.size == 1) { - val node = importedNode.head - val pathValue = stripQuotes(node.code) - val result = pathValue match { - case path if File(path).exists => - path - case path if File(s"$path.rb").exists => - s"$path.rb" - case _ => - pathValue - } - packageStack.append(result) - val importNode = createImportNodeAndLink(result, pathValue, Some(callNode), diffGraph) - Seq(callAst(callNode, argsAst), Ast(importNode)) - } else { - Seq(callAst(callNode, argsAst)) - } - } - - protected def resolveRelativePath(currentFile: String, argsAst: Seq[Ast], callNode: NewCall): Seq[Ast] = { - val importedNode = argsAst.head.nodes.collect { case x: NewLiteral => x } - if (importedNode.size == 1) { - val node = importedNode.head - val pathValue = stripQuotes(node.code) - val updatedPath = if (pathValue.endsWith(".rb")) pathValue else s"$pathValue.rb" - - val currentDirectory = File(currentFile).parent - val file = File(currentDirectory, updatedPath) - packageStack.append(file.pathAsString) - val importNode = createImportNodeAndLink(updatedPath, pathValue, Some(callNode), diffGraph) - Seq(callAst(callNode, argsAst), Ast(importNode)) - } else { - Seq(callAst(callNode, argsAst)) - } - } - - protected def astForBlock(ctx: BlockContext, blockMethodName: Option[String] = None): Ast = ctx match - case ctx: DoBlockBlockContext => astForDoBlock(ctx.doBlock(), blockMethodName) - case ctx: BraceBlockBlockContext => astForBraceBlock(ctx.braceBlock(), blockMethodName) - - private def astForBlockHelper( - ctx: ParserRuleContext, - blockParamCtx: Option[BlockParameterContext], - compoundStmtCtx: CompoundStatementContext, - blockMethodName: Option[String] = None - ) = { - blockMethodName match { - case Some(blockMethodName) => - astForBlockFunction( - compoundStmtCtx.statements(), - blockParamCtx, - blockMethodName, - line(compoundStmtCtx).head, - lineEnd(compoundStmtCtx).head, - column(compoundStmtCtx).head, - columnEnd(compoundStmtCtx).head - ).head - case None => - val blockNode_ = blockNode(ctx, code(ctx), Defines.Any) - val blockBodyAst = astForCompoundStatement(compoundStmtCtx) - val blockParamAst = blockParamCtx.flatMap(astForBlockParameterContext) - blockAst(blockNode_, blockBodyAst.toList ++ blockParamAst) - } - } - - protected def astForDoBlock(ctx: DoBlockContext, blockMethodName: Option[String] = None): Ast = { - astForBlockHelper(ctx, Option(ctx.blockParameter), ctx.bodyStatement().compoundStatement(), blockMethodName) - } - - private def astForBraceBlock(ctx: BraceBlockContext, blockMethodName: Option[String] = None): Ast = { - astForBlockHelper(ctx, Option(ctx.blockParameter), ctx.bodyStatement().compoundStatement(), blockMethodName) - } - - // TODO: This class shouldn't be required and will eventually be phased out. - protected implicit class BlockContextExt(val ctx: BlockContext) { - def compoundStatement: CompoundStatementContext = { - fold(_.bodyStatement.compoundStatement, _.bodyStatement.compoundStatement) - } - - def blockParameter: Option[BlockParameterContext] = { - fold(ctx => Option(ctx.blockParameter()), ctx => Option(ctx.blockParameter())) - } - - private def fold[A](f: DoBlockContext => A, g: BraceBlockContext => A): A = ctx match { - case ctx: DoBlockBlockContext => f(ctx.doBlock()) - case ctx: BraceBlockBlockContext => g(ctx.braceBlock()) - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForTypesCreator.scala deleted file mode 100644 index 5ff0df57e826..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/AstForTypesCreator.scala +++ /dev/null @@ -1,270 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.x2cpg.utils.* -import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes, Operators, nodes} -import org.antlr.v4.runtime.ParserRuleContext - -import scala.collection.mutable - -trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => - - // Maps field references of known types - protected val fieldReferences: mutable.HashMap[String, Set[ParserRuleContext]] = mutable.HashMap.empty - protected val typeDeclNameToTypeDecl: mutable.HashMap[String, NewTypeDecl] = mutable.HashMap.empty - - def astForClassDeclaration(ctx: ClassDefinitionPrimaryContext): Seq[Ast] = { - val className = ctx.className.getOrElse(Defines.Any) - if (className != Defines.Any) { - classStack.push(className) - val fullName = classStack.reverse.mkString(pathSep) - - val bodyAst = astForClassBody(ctx.classDefinition().bodyStatement()).map { ast => - ast.root.foreach { - case node: NewMethod => - node - .astParentType(NodeTypes.TYPE_DECL) - .astParentFullName(fullName) - case _ => - } - ast - } - - if (classStack.nonEmpty) { - classStack.pop() - } - - val typeDecl = typeDeclNode(ctx, className, fullName, relativeFilename, code(ctx).takeWhile(_ != '\n')) - - // create constructor if not explicitly defined - val hasConstructor = - bodyAst.flatMap(_.root).collect { case x: NewMethod => x.name }.contains(XDefines.ConstructorMethodName) - val defaultConstructor = - if (!hasConstructor) - createDefaultConstructor(ctx, typeDecl, bodyAst.flatMap(_.nodes).collect { case x: NewMember => x }) - else Seq.empty - - typeDeclNameToTypeDecl.put(className, typeDecl) - Seq(Ast(typeDecl).withChildren(defaultConstructor ++ bodyAst)) - } else { - Seq.empty - } - } - - /** If no constructor is explicitly defined, will create a default one. - */ - private def createDefaultConstructor( - ctx: ClassDefinitionPrimaryContext, - typeDecl: NewTypeDecl, - fields: Seq[NewMember] - ): Seq[Ast] = { - val name = XDefines.ConstructorMethodName - val code = Seq(typeDecl.name, name).mkString(pathSep) - val fullName = Seq(typeDecl.fullName, name).mkString(pathSep) - - val constructorNode = - methodNode(ctx, name, code, fullName, None, relativeFilename, Option(typeDecl.label), Option(typeDecl.fullName)) - val thisParam = createMethodParameterIn("this", None, None, typeDecl.fullName) - val params = - thisParam +: fields.map(m => createMethodParameterIn(m.name, None, None, m.typeFullName)) - val assignments = fields.map { m => - val thisNode = createThisIdentifier(ctx) - val lhs = astForFieldAccess(ctx, thisNode) - val paramIdentifier = identifierNode(ctx, m.name, m.name, m.typeFullName) - val refParam = params.find(_.name == m.name).get - astForAssignment(lhs.root.get, paramIdentifier) - .withRefEdge(thisNode, thisParam) - .withRefEdge(paramIdentifier, refParam) - }.toList - val body = blockAst(blockNode(ctx), assignments) - val methodReturn = methodReturnNode(ctx, typeDecl.fullName) - - Seq(methodAst(constructorNode, params.map(Ast.apply(_)), body, methodReturn)) - } - - def astForClassExpression(ctx: ClassDefinitionPrimaryContext): Seq[Ast] = { - // TODO test for this is pending due to lack of understanding to generate an example - val astExprOfCommand = astForExpressionOrCommand(ctx.classDefinition().expressionOrCommand()) - val astBodyStatement = astForBodyStatementContext(ctx.classDefinition().bodyStatement()) - val blockNode = NewBlock() - .code(text(ctx)) - val bodyBlockAst = blockAst(blockNode, astBodyStatement.toList) - astExprOfCommand ++ Seq(bodyBlockAst) - } - - def astForModuleDefinitionPrimaryContext(ctx: ModuleDefinitionPrimaryContext): Seq[Ast] = { - val className = ctx.moduleDefinition().classOrModuleReference().classOrModuleName - - if (className != Defines.Any) { - classStack.push(className) - - val fullName = classStack.reverse.mkString(pathSep) - val namespaceBlock = NewNamespaceBlock() - .name(className) - .fullName(fullName) - .filename(relativeFilename) - - val moduleBodyAst = astInFakeMethod(className, fullName, relativeFilename, ctx) - classStack.pop() - Seq(Ast(namespaceBlock).withChildren(moduleBodyAst)) - } else { - Seq.empty - } - - } - - private def astInFakeMethod( - name: String, - fullName: String, - path: String, - ctx: ModuleDefinitionPrimaryContext - ): Seq[Ast] = { - - val fakeGlobalTypeDecl = NewTypeDecl() - .name(name) - .fullName(fullName) - - val bodyAst = astForClassBody(ctx.moduleDefinition().bodyStatement()) - Seq(Ast(fakeGlobalTypeDecl).withChildren(bodyAst)) - } - - private def getClassNameScopedConstantReferenceContext(ctx: ScopedConstantReferenceContext): String = { - val classTerminalNode = ctx.CONSTANT_IDENTIFIER() - - if (ctx.primary() != null) { - val primaryAst = astForPrimaryContext(ctx.primary()) - val moduleNameNode = primaryAst.head.nodes - .filter(node => node.isInstanceOf[NewIdentifier]) - .head - .asInstanceOf[NewIdentifier] - val moduleName = moduleNameNode.name - moduleName + "." + classTerminalNode.getText - } else { - classTerminalNode.getText - } - } - - def membersFromStatementAsts(ast: Ast): Seq[Ast] = - ast.nodes - .collect { case i: NewIdentifier if i.name.startsWith("@") || i.name.isAllUpperCase => i } - .map { i => - val code = ast.root.collect { case c: NewCall => c.code }.getOrElse(i.name) - val modifierType = i.name match - case x if x.startsWith("@@") => ModifierTypes.STATIC - case x if x.isAllUpperCase => ModifierTypes.FINAL - case _ => ModifierTypes.VIRTUAL - val modifierAst = Ast(NewModifier().modifierType(modifierType)) - Ast( - NewMember() - .code(code) - .name(i.name.replaceAll("@", "")) - .typeFullName(i.typeFullName) - .lineNumber(i.lineNumber) - .columnNumber(i.columnNumber) - ).withChild(modifierAst) - } - .toSeq - - /** Handles body statements differently from [[astForBodyStatementContext]] by noting that method definitions should - * be on the root level and assignments where the LHS starts with @@ should be treated as fields. - */ - private def astForClassBody(ctx: BodyStatementContext): Seq[Ast] = { - val rootStatements = - Option(ctx).map(_.compoundStatement()).map(_.statements()).map(astForStatements(_)).getOrElse(Seq()) - retrieveAndGenerateClassChildren(ctx, rootStatements) - } - - /** As class bodies are not treated much differently to other procedure bodies, we need to retrieve certain components - * that would result in the creation of interprocedural constructs. - * - * TODO: This is pretty hacky and the parser could benefit from more specific tokens - */ - private def retrieveAndGenerateClassChildren(classCtx: BodyStatementContext, rootStatements: Seq[Ast]): Seq[Ast] = { - val (memberLikeStmts, blockStmts) = rootStatements - .flatMap { ast => - ast.root match - case Some(_: NewMethod) => Seq(ast) - case Some(x: NewCall) if x.name == Operators.assignment => Seq(ast) ++ membersFromStatementAsts(ast) - case _ => Seq(ast) - } - .partition(_.root match - case Some(_: NewMethod) => true - case Some(_: NewMember) => true - case _ => false - ) - - val methodStmts = memberLikeStmts.filter(_.root.exists(_.isInstanceOf[NewMethod])) - val memberNodes = memberLikeStmts.flatMap(_.root).collect { case m: NewMember => m } - - val uniqueMemberReferences = - (memberNodes ++ fieldReferences.getOrElse(classStack.top, Set.empty).groupBy(_.getText).map { case (code, ctxs) => - NewMember() - .name(code.replaceAll("@", "")) - .code(code) - .typeFullName(Defines.Any) - }).toList.distinctBy(_.name).map { m => - val modifierType = m.name match - case x if x.startsWith("@@") => ModifierTypes.STATIC - case _ => ModifierTypes.VIRTUAL - val modifierAst = Ast(NewModifier().modifierType(modifierType)) - Ast(m).withChild(modifierAst) - } - - // Create class initialization method to host all field initializers - val classInitMethodAst = if (blockStmts.nonEmpty) { - val classInitFullName = (classStack.reverse :+ XDefines.StaticInitMethodName).mkString(pathSep) - val classInitMethod = methodNode( - classCtx, - XDefines.StaticInitMethodName, - XDefines.StaticInitMethodName, - classInitFullName, - None, - relativeFilename, - Option(NodeTypes.TYPE_DECL), - Option(classStack.reverse.mkString(pathSep)) - ) - val classInitBody = blockAst(blockNode(classCtx), blockStmts.toList) - Seq(methodAst(classInitMethod, Seq.empty, classInitBody, methodReturnNode(classCtx, Defines.Any))) - } else { - Seq.empty - } - - classInitMethodAst ++ uniqueMemberReferences ++ methodStmts - } - - implicit class ClassDefinitionPrimaryContextExt(val ctx: ClassDefinitionPrimaryContext) { - - def hasClassDefinition: Boolean = Option(ctx.classDefinition()).isDefined - - def className: Option[String] = - Option(ctx.classDefinition().classOrModuleReference()) match { - case Some(classOrModuleReferenceCtx) => - Option(classOrModuleReferenceCtx) - .map(_.classOrModuleName) - case None => - // TODO the below is just to avoid crashes. This needs to be implemented properly - None - } - } - - implicit class ClassOrModuleReferenceContextExt(val ctx: ClassOrModuleReferenceContext) { - - def hasScopedConstantReference: Boolean = Option(ctx.scopedConstantReference()).isDefined - - def classOrModuleName: String = - Option(ctx) match { - case Some(ct) => - if (ct.hasScopedConstantReference) - getClassNameScopedConstantReferenceContext(ct.scopedConstantReference()) - else - Option(ct.CONSTANT_IDENTIFIER()).map(_.getText) match { - case Some(className) => className - case None => Defines.Any - } - case None => Defines.Any - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala deleted file mode 100644 index 773c656bb44b..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/astcreation/RubyScope.scala +++ /dev/null @@ -1,110 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.astcreation - -import io.joern.x2cpg.datastructures.Scope -import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.codepropertygraph.generated.nodes.{DeclarationNew, NewIdentifier, NewLocal, NewNode} -import overflowdb.BatchedUpdate - -import scala.collection.mutable - -/** Extends the Scope class to help scope variables and create locals. - * - * TODO: Extend this to similarly link parameter nodes (especially `this` node) for consistency. - */ -class RubyScope extends Scope[String, NewIdentifier, NewNode] { - - private type VarMap = Map[String, VarGroup] - private type ScopeNodeType = NewNode - - /** Groups a local node with its referencing identifiers. - */ - private case class VarGroup(local: NewLocal, ids: List[NewIdentifier]) - - /** Links a scope to its variable groupings. - */ - private val scopeToVarMap = mutable.HashMap.empty[ScopeNodeType, VarMap] - - override def addToScope(identifier: String, variable: NewIdentifier): NewNode = { - val scopeNode = super.addToScope(identifier, variable) - stack.headOption.foreach(head => scopeToVarMap.appendIdentifierToVarGroup(head.scopeNode, variable)) - scopeNode - } - - override def popScope(): Option[NewNode] = { - stack.headOption.map(_.scopeNode).foreach(scopeToVarMap.remove) - super.popScope() - } - - /** Will generate local nodes for this scope's variables, excluding those that reference parameters. - * @param paramNames - * the names of parameters. - */ - def createAndLinkLocalNodes( - diffGraph: BatchedUpdate.DiffGraphBuilder, - paramNames: Set[String] = Set.empty - ): List[DeclarationNew] = stack.headOption match - case Some(top) => scopeToVarMap.buildVariableGroupings(top.scopeNode, paramNames ++ Set("this"), diffGraph) - case None => List.empty[DeclarationNew] - - /** @param identifier - * the identifier to count - * @return - * the number of times the given identifier occurs in the immediate scope. - */ - def numVariableReferences(identifier: String): Int = { - stack.map(_.scopeNode).flatMap(scopeToVarMap.get).flatMap(_.get(identifier)).map(_.ids.size).headOption.getOrElse(0) - } - - private implicit class IdentifierExt(node: NewIdentifier) { - - /** Creates a new VarGroup and corresponding NewLocal for the given identifier. - */ - def toNewVarGroup: VarGroup = { - val newLocal = NewLocal() - .name(node.name) - .code(node.name) - .lineNumber(node.lineNumber) - .columnNumber(node.columnNumber) - .typeFullName(node.typeFullName) - VarGroup(newLocal, List(node)) - } - - } - - private implicit class ScopeExt(scopeMap: mutable.Map[ScopeNodeType, VarMap]) { - - /** Registers the identifier to its corresponding variable grouping in the given scope. - */ - def appendIdentifierToVarGroup(key: ScopeNodeType, identifier: NewIdentifier): Unit = - scopeMap.updateWith(key) { - case Some(varMap: VarMap) => - Some(varMap.updatedWith(identifier.name) { - case Some(varGroup: VarGroup) => Some(varGroup.copy(ids = varGroup.ids :+ identifier)) - case None => Some(identifier.toNewVarGroup) - }) - case None => - Some(Map(identifier.name -> identifier.toNewVarGroup)) - } - - /** Will persist the variable groupings that do not represent parameter nodes and link them with REF edges. - * @return - * the list of persisted local nodes. - */ - def buildVariableGroupings( - key: ScopeNodeType, - paramNames: Set[String], - diffGraph: BatchedUpdate.DiffGraphBuilder - ): List[DeclarationNew] = - scopeMap.get(key) match - case Some(varMap) => - varMap.values - .filterNot { case VarGroup(local, _) => paramNames.contains(local.name) } - .map { case VarGroup(local, ids) => - ids.foreach(id => diffGraph.addEdge(id, local, EdgeTypes.REF)) - local - } - .toList - case None => List.empty[DeclarationNew] - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerBase.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerBase.scala deleted file mode 100644 index f48315811f43..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerBase.scala +++ /dev/null @@ -1,49 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyLexer.* -import org.antlr.v4.runtime.Recognizer.EOF -import org.antlr.v4.runtime.{CharStream, Lexer, Token} - -/** Aggregates auxiliary features to DeprecatedRubyLexer in a single place. */ -abstract class DeprecatedRubyLexerBase(input: CharStream) - extends Lexer(input) - with RegexLiteralHandling - with InterpolationHandling - with QuotedLiteralHandling - with HereDocHandling { - - /** The previously (non-WS) emitted token (in DEFAULT_CHANNEL.) */ - protected var previousNonWsToken: Option[Token] = None - - /** The previously emitted token (in DEFAULT_CHANNEL.) */ - protected var previousToken: Option[Token] = None - - // Same original behaviour, just updating `previous{NonWs}Token`. - override def nextToken: Token = { - val token: Token = super.nextToken - if (token.getChannel == Token.DEFAULT_CHANNEL && token.getType != WS) { - previousNonWsToken = Some(token) - } - previousToken = Some(token) - token - } - - def previousNonWsTokenTypeOrEOF(): Int = { - previousNonWsToken.map(_.getType).getOrElse(EOF) - } - - def previousTokenTypeOrEOF(): Int = { - previousToken.map(_.getType).getOrElse(EOF) - } - - def isNumericTokenType(tokenType: Int): Boolean = { - val numericTokenTypes = Set( - DECIMAL_INTEGER_LITERAL, - OCTAL_INTEGER_LITERAL, - HEXADECIMAL_INTEGER_LITERAL, - FLOAT_LITERAL_WITHOUT_EXPONENT, - FLOAT_LITERAL_WITH_EXPONENT - ) - numericTokenTypes.contains(tokenType) - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerPostProcessor.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerPostProcessor.scala deleted file mode 100644 index 0eba4c655137..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/DeprecatedRubyLexerPostProcessor.scala +++ /dev/null @@ -1,74 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyLexer.* -import org.antlr.v4.runtime.Recognizer.EOF -import org.antlr.v4.runtime.misc.Pair -import org.antlr.v4.runtime.{CommonToken, ListTokenSource, Token, TokenSource} - -import scala.:: -import scala.jdk.CollectionConverters.* - -/** Simplifies the token stream obtained from `DeprecatedRubyLexer`. - */ -object DeprecatedRubyLexerPostProcessor { - - def apply(tokenSource: TokenSource): ListTokenSource = { - var tokens = tokenSource.toSeq - - tokens = tokens.mergeConsecutive(NON_EXPANDED_LITERAL_CHARACTER, NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE) - tokens = tokens.mergeConsecutive(EXPANDED_LITERAL_CHARACTER, EXPANDED_LITERAL_CHARACTER_SEQUENCE) - tokens = tokens.filterNot(_.is(WS)) - - new ListTokenSource(tokens.asJava) - } -} - -private implicit class TokenSourceExt(val tokenSource: TokenSource) { - - def toSeq: Seq[Token] = Seq.unfold(tokenSource) { tkSrc => - tkSrc.nextToken() match - case tk if tk.is(EOF) => None - case tk => Some((tk, tkSrc)) - } -} - -private implicit class SeqExt[A](val elems: Seq[A]) { - - /** An order-preserving `groupBy` implemented on top of `Seq`. Each sub-sequence ("chain") contains 1+ elements. If a - * chain contains 2+ elements, then all its elements satisfy `p`. Flattening returns the original sequence. - */ - def chains(p: A => Boolean): Seq[Seq[A]] = elems.foldRight(Nil: Seq[Seq[A]]) { (h, t) => - t match - case chain :: chains if chain.exists(p) && p(h) => (h +: chain) +: chains - case _ => Seq(h) +: t - } - - /** Collapses, according to a merging operation `m`, all chains that verify `p`. - */ - def mergeChains(p: A => Boolean, m: Seq[A] => A): Seq[A] = { - elems.chains(p).flatMap(chain => if (chain.exists(p)) Seq(m(chain)) else chain) - } - -} - -private implicit class TokenSeqExt(val tokens: Seq[Token]) { - - def mergeAs(tokenType: Int): Token = { - val startIndex = tokens.head.getStartIndex - val stopIndex = tokens.last.getStopIndex - val tokenSource = tokens.head.getTokenSource - val inputStream = tokens.head.getInputStream - val channel = tokens.head.getChannel - new CommonToken(new Pair(tokenSource, inputStream), tokenType, channel, startIndex, stopIndex) - } - - def mergeConsecutive(oldTokenType: Int, newTokenType: Int): Seq[Token] = { - tokens.mergeChains(_.is(oldTokenType), _.mergeAs(newTokenType)) - } -} - -private implicit class TokenExt(val token: Token) { - - def is(tokenType: Int): Boolean = token.getType == tokenType - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/HereDocHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/HereDocHandling.scala deleted file mode 100644 index c4d6f25991ca..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/HereDocHandling.scala +++ /dev/null @@ -1,35 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import better.files.EOF - -trait HereDocHandling { this: DeprecatedRubyLexerBase => - - /** @see - * Stack - * Overflow - */ - def heredocEndAhead(partialHeredoc: String): Boolean = - if (this.getCharPositionInLine != 0) { - // If the lexer is not at the start of a line, no end-delimiter can be possible - false - } else { - // Get the delimiter - HereDocHandling.getHereDocDelimiter(partialHeredoc) match - case Some(delimiter) if !delimiter.zipWithIndex.exists { case (c, idx) => this._input.LA(idx + 1) != c } => - // If we get to this point, we know there is an end delimiter ahead in the char stream, make - // sure it is followed by a white space (or the EOF). If we don't do this, then "FOOS" would also - // be considered the end for the delimiter "FOO" - val charAfterDelimiter = this._input.LA(delimiter.length + 1) - charAfterDelimiter == EOF || Character.isWhitespace(charAfterDelimiter) - case _ => false - } - -} - -object HereDocHandling { - - def getHereDocDelimiter(hereDoc: String): Option[String] = - hereDoc.split("\r?\n|\r").headOption.map(_.replaceAll("^<<[~-]\\s*", "")) - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/InterpolationHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/InterpolationHandling.scala deleted file mode 100644 index 2ea50b1bd5f9..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/InterpolationHandling.scala +++ /dev/null @@ -1,21 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import scala.collection.mutable - -trait InterpolationHandling { this: DeprecatedRubyLexerBase => - - private val interpolationEndTokenType = mutable.Stack[Int]() - - def pushInterpolationEndTokenType(endTokenType: Int): Unit = { - interpolationEndTokenType.push(endTokenType) - } - - def popInterpolationEndTokenType(): Int = { - interpolationEndTokenType.pop() - } - - def isEndOfInterpolation: Boolean = { - interpolationEndTokenType.nonEmpty - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/QuotedLiteralHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/QuotedLiteralHandling.scala deleted file mode 100644 index 50daf48988f5..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/QuotedLiteralHandling.scala +++ /dev/null @@ -1,45 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import scala.collection.mutable - -trait QuotedLiteralHandling { this: DeprecatedRubyLexerBase => - - private val delimiters = mutable.Stack[Int]() - private val endTokenTypes = mutable.Stack[Int]() - - private def closingDelimiterFor(char: Int): Int = char match - case '(' => ')' - case '[' => ']' - case '{' => '}' - case '<' => '>' - case c => c - - private def currentOpeningDelimiter: Int = delimiters.top - - private def currentClosingDelimiter: Int = closingDelimiterFor(currentOpeningDelimiter) - - private def isOpeningDelimiter(char: Int): Boolean = char == currentOpeningDelimiter - - private def isClosingDelimiter(char: Int): Boolean = char == currentClosingDelimiter - - def pushQuotedDelimiter(char: Int): Unit = delimiters.push(char) - - def popQuotedDelimiter(): Unit = delimiters.pop() - - def pushQuotedEndTokenType(endTokenType: Int): Unit = endTokenTypes.push(endTokenType) - - def popQuotedEndTokenType(): Int = endTokenTypes.pop() - - def consumeQuotedCharAndMaybePopMode(char: Int): Unit = { - if (isClosingDelimiter(char)) { - popQuotedDelimiter() - - if (delimiters.isEmpty) { - setType(endTokenTypes.pop()) - popMode() - } - } else if (isOpeningDelimiter(char)) { - pushQuotedDelimiter(char) - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexLiteralHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexLiteralHandling.scala deleted file mode 100644 index 2e84700c59ba..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexLiteralHandling.scala +++ /dev/null @@ -1,78 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyLexer.* -import org.antlr.v4.runtime.Recognizer.EOF - -trait RegexLiteralHandling { this: DeprecatedRubyLexerBase => - - /* When encountering '/', we need to decide whether this is a binary operator (e.g. `x / y`) or - * a regular expression delimiter (e.g. `/(eu|us)/`) occurrence. Our approach is to look at the - * previously emitted token and decide accordingly. - */ - private val regexTogglingTokens: Set[Int] = Set( - // When '/' occurs after an opening parenthesis, brace or bracket. - LPAREN, - LCURLY, - LBRACK, - // When '/' occurs after a NL. - NL, - // When '/' occurs after a ','. - COMMA, - // When '/' occurs after a ':'. - COLON, - // When '/' occurs after 'when'. - WHEN, - // When '/' occurs after 'unless'. - UNLESS, - // When '/' occurs after an operator. - EMARK, - EMARKEQ, - EMARKTILDE, - AMP, - AMP2, - AMPDOT, - BAR, - BAR2, - EQ, - EQ2, - EQ3, - CARET, - LTEQGT, - EQTILDE, - GT, - GTEQ, - LT, - LTEQ, - LT2, - GT2, - PLUS, - MINUS, - STAR, - STAR2, - SLASH, - PERCENT, - TILDE, - PLUSAT, - MINUSAT, - ASSIGNMENT_OPERATOR - ) - - /** To be invoked when encountering `/`, deciding if it should emit a `REGULAR_EXPRESSION_START` token. */ - protected def isStartOfRegexLiteral: Boolean = { - val isFirstTokenInTheStream = previousNonWsToken.isEmpty - val isRegexTogglingToken = regexTogglingTokens.contains(previousNonWsTokenTypeOrEOF()) - - isFirstTokenInTheStream || isRegexTogglingToken || isInCommandArgumentPosition - } - - /** Decides if the current `/` is being used as an argument to a command, based on the observation that such literals - * may not start with a WS. E.g. `puts /x/` is valid, but `puts / x/` is not. - */ - private def isInCommandArgumentPosition: Boolean = { - val previousNonWsIsIdentifier = previousNonWsTokenTypeOrEOF() == LOCAL_VARIABLE_IDENTIFIER - val previousIsWs = previousTokenTypeOrEOF() == WS - val nextCharIsWs = _input.LA(1) == ' ' - previousNonWsIsIdentifier && previousIsWs && !nextCharIsWs - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstCreationPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstCreationPass.scala deleted file mode 100644 index 1a93f4b5e9e3..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstCreationPass.scala +++ /dev/null @@ -1,44 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.rubysrc2cpg.Config -import io.joern.rubysrc2cpg.deprecated.astcreation.{AstCreator, ResourceManagedParser} -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser -import io.joern.rubysrc2cpg.deprecated.utils.{PackageContext, PackageTable} -import io.joern.x2cpg.SourceFiles -import io.joern.x2cpg.datastructures.Global -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language.* -import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate - -import scala.jdk.CollectionConverters.EnumerationHasAsScala - -class AstCreationPass( - cpg: Cpg, - parsedFiles: List[(String, DeprecatedRubyParser.ProgramContext)], - packageTable: PackageTable, - config: Config -) extends ForkJoinParallelCpgPass[(String, DeprecatedRubyParser.ProgramContext)](cpg) { - - private val logger = LoggerFactory.getLogger(this.getClass) - - override def generateParts(): Array[(String, DeprecatedRubyParser.ProgramContext)] = parsedFiles.toArray - - override def runOnPart( - diffGraph: DiffGraphBuilder, - fileNameAndContext: (String, DeprecatedRubyParser.ProgramContext) - ): Unit = { - val (fileName, context) = fileNameAndContext - try { - diffGraph.absorb( - new AstCreator(fileName, context, PackageContext(fileName, packageTable), cpg.metaData.root.headOption)( - config.schemaValidation - ).createAst() - ) - } catch { - case ex: Exception => - logger.error(s"Error while processing AST for file - $fileName - ", ex) - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstPackagePass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstPackagePass.scala deleted file mode 100644 index 147a7b154daa..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/AstPackagePass.scala +++ /dev/null @@ -1,67 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import better.files.File -import io.joern.rubysrc2cpg.deprecated.astcreation.{AstCreator, ResourceManagedParser} -import io.joern.rubysrc2cpg.deprecated.utils.{PackageContext, PackageTable} -import io.joern.x2cpg.ValidationMode -import io.joern.x2cpg.datastructures.Global -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.passes.ForkJoinParallelCpgPass -import org.slf4j.LoggerFactory - -import scala.util.{Failure, Success, Try} - -class AstPackagePass( - cpg: Cpg, - tempExtDir: String, - parser: ResourceManagedParser, - packageTable: PackageTable, - inputPath: String -)(implicit withSchemaValidation: ValidationMode) - extends ForkJoinParallelCpgPass[String](cpg) { - - private val logger = LoggerFactory.getLogger(getClass) - - override def generateParts(): Array[String] = - getRubyDependenciesFile(inputPath) ++ getRubyDependenciesFile(tempExtDir) - - override def runOnPart(diffGraph: DiffGraphBuilder, filePath: String): Unit = { - parser.parse(filePath) match - case Failure(exception) => logger.warn(s"Could not parse file: $filePath, skipping", exception); - case Success(programCtx) => - Try( - new AstCreator( - filePath, - programCtx, - PackageContext(resolveModuleNameFromPath(filePath), packageTable), - Option(inputPath) - ).createAst() - ) - - } - - private def getRubyDependenciesFile(inputPath: String): Array[String] = { - val currentDir = File(inputPath) - if (currentDir.exists) { - currentDir.listRecursively.filter(_.extension.exists(_ == ".rb")).map(_.path.toString).toArray - } else { - Array.empty - } - } - - private def resolveModuleNameFromPath(path: String): String = { - if (path.contains(tempExtDir)) { - val moduleNameRegex = Seq("gems", "([^", "]+)", "lib", ".*").mkString(java.io.File.separator).r - moduleNameRegex - .findFirstMatchIn(path) - .map(_.group(1)) - .getOrElse("") - .split(java.io.File.separator) - .last - .split("-") - .head - } else { - path - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/Defines.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/Defines.scala deleted file mode 100644 index 9beeaea5ea8a..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/Defines.scala +++ /dev/null @@ -1,39 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.rubysrc2cpg.deprecated.astcreation.GlobalTypes - -object Defines { - val Any: String = "ANY" - val Object: String = "Object" - - val NilClass: String = "NilClass" - val TrueClass: String = "TrueClass" - val FalseClass: String = "FalseClass" - - val Numeric: String = "Numeric" - val Integer: String = "Integer" - val Float: String = "Float" - - val String: String = "String" - val Symbol: String = "Symbol" - - val Array: String = "Array" - val Hash: String = "Hash" - - val Encoding: String = "Encoding" - val Regexp: String = "Regexp" - - // TODO: The following shall be moved out eventually. - val ModifierRedo: String = "redo" - val ModifierRetry: String = "retry" - var ModifierNext: String = "next" - - // For un-named identifiers and parameters - val TempIdentifier = "tmp" - val TempParameter = "param" - - // Constructor method - val Initialize = "initialize" - - def getBuiltInType(typeInString: String) = s"${GlobalTypes.builtinPrefix}.$typeInString" -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyImportResolverPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyImportResolverPass.scala deleted file mode 100644 index 3ca71e605a9f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyImportResolverPass.scala +++ /dev/null @@ -1,117 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import better.files.File -import io.joern.rubysrc2cpg.deprecated.utils.PackageTable -import io.joern.x2cpg.Defines as XDefines -import io.shiftleft.semanticcpg.language.importresolver.* -import io.joern.x2cpg.passes.frontend.XImportResolverPass -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language.* - -import java.io.File as JFile -import java.util.regex.{Matcher, Pattern} -class RubyImportResolverPass(cpg: Cpg, packageTableInfo: PackageTable) extends XImportResolverPass(cpg) { - - private val pathPattern = Pattern.compile("[\"']([\\w/.]+)[\"']") - - override protected def optionalResolveImport( - fileName: String, - importCall: Call, - importedEntity: String, - importedAs: String, - diffGraph: DiffGraphBuilder - ): Unit = { - - resolveEntities(importedEntity, importCall, fileName).foreach(x => evaluatedImportToTag(x, importCall, diffGraph)) - } - - private def resolveEntities(expEntity: String, importCall: Call, fileName: String): Set[EvaluatedImport] = { - - // TODO - /* Currently we are considering only case where exposed module are Classes, - and the only way to consume them is by creating a new object as we encounter more cases, - This needs to be handled accordingly - */ - - val expResolvedPath = - if (packageTableInfo.getModule(expEntity).nonEmpty || packageTableInfo.getTypeDecl(expEntity).nonEmpty) - expEntity - else if (expEntity.contains(".")) - getResolvedPath(expEntity, fileName) - else if (cpg.file.name(s".*$expEntity.rb").nonEmpty) - getResolvedPath(s"$expEntity.rb", fileName) - else - expEntity - - // TODO Limited ResolvedMethod exposure for now, will open up after looking at more concrete examples - val finalResolved = { - if ( - packageTableInfo.getModule(expResolvedPath).nonEmpty || packageTableInfo.getTypeDecl(expResolvedPath).nonEmpty - ) { - val importNodesFromTypeDecl = packageTableInfo - .getTypeDecl(expEntity) - .flatMap { typeDeclModel => - Seq( - ResolvedMethod(s"${typeDeclModel.fullName}.${XDefines.ConstructorMethodName}", "new"), - ResolvedTypeDecl(typeDeclModel.fullName) - ) - } - .distinct - - val importNodesFromModule = packageTableInfo.getModule(expEntity).flatMap { moduleModel => - Seq(ResolvedTypeDecl(moduleModel.fullName)) - } - (importNodesFromTypeDecl ++ importNodesFromModule).toSet - } else { - val filePattern = s"${Pattern.quote(expResolvedPath)}\\.?.*" - val resolvedTypeDecls = cpg.typeDecl - .where(_.file.name(filePattern)) - .fullName - .flatMap(fullName => - Seq(ResolvedTypeDecl(fullName), ResolvedMethod(s"$fullName.${XDefines.ConstructorMethodName}", "new")) - ) - .toSet - - val resolvedModules = cpg.namespaceBlock - .whereNot(_.nameExact("")) - .where(_.file.name(filePattern)) - .flatMap(module => Seq(ResolvedTypeDecl(module.fullName))) - .toSet - - // Expose methods which are directly present in a file, without any module, TypeDecl - val resolvedMethods = cpg.method - .where(_.file.name(filePattern)) - .where(_.nameExact(":program")) - .astChildren - .astChildren - .isMethod - .flatMap(method => Seq(ResolvedMethod(method.fullName, method.name))) - .toSet - resolvedTypeDecls ++ resolvedModules ++ resolvedMethods - } - }.collectAll[EvaluatedImport].toSet - - finalResolved - } - - def getResolvedPath(expEntity: String, fileName: String) = { - val rawEntity = expEntity.stripPrefix("./") - val matcher = pathPattern.matcher(rawEntity) - val sep = Matcher.quoteReplacement(JFile.separator) - val root = s"$codeRootDir${JFile.separator}" - val currentFile = s"$root$fileName" - val entity = if (matcher.find()) matcher.group(1) else rawEntity - val resolvedPath = better.files - .File( - currentFile.stripSuffix(currentFile.split(sep).lastOption.getOrElse("")), - entity.split("\\.").headOption.getOrElse(entity) - ) - .pathAsString match { - case resPath if entity.endsWith(".rb") => s"$resPath.rb" - case resPath => resPath - } - resolvedPath.stripPrefix(root) - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeHintCallLinker.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeHintCallLinker.scala deleted file mode 100644 index 7c7229fcb893..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeHintCallLinker.scala +++ /dev/null @@ -1,12 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language.* - -class RubyTypeHintCallLinker(cpg: Cpg) extends XTypeHintCallLinker(cpg) { - - override def calls: Iterator[Call] = super.calls.nameNot("^(require).*") - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryPassGenerator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryPassGenerator.scala deleted file mode 100644 index b9dc8c0c80cb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryPassGenerator.scala +++ /dev/null @@ -1,129 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.x2cpg.passes.frontend.* -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate.DiffGraphBuilder -import io.joern.x2cpg.Defines as XDefines - -class RubyTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) - extends XTypeRecoveryPassGenerator[File](cpg, config) { - override protected def generateRecoveryPass(state: XTypeRecoveryState, iteration: Int): XTypeRecovery[File] = - new RubyTypeRecovery(cpg, state, iteration) -} - -private class RubyTypeRecovery(cpg: Cpg, state: XTypeRecoveryState, iteration: Int) - extends XTypeRecovery[File](cpg, state, iteration) { - - override def compilationUnits: Iterator[File] = cpg.file.iterator - - override def generateRecoveryForCompilationUnitTask( - unit: File, - builder: DiffGraphBuilder - ): RecoverForXCompilationUnit[File] = { - new RecoverForRubyFile(cpg, unit, builder, state) - } -} - -private class RecoverForRubyFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, state: XTypeRecoveryState) - extends RecoverForXCompilationUnit[File](cpg, cu, builder, state) { - - /** A heuristic method to determine if a call is a constructor or not. - */ - override protected def isConstructor(c: Call): Boolean = { - isConstructor(c.name) && c.code.charAt(0).isUpper - } - - /** A heuristic method to determine if a call name is a constructor or not. - */ - override protected def isConstructor(name: String): Boolean = - !name.isBlank && (name == "new" || name == XDefines.ConstructorMethodName) - - override def visitImport(i: Import): Unit = for { - resolvedImport <- i.call.tag - alias <- i.importedAs - } { - import io.shiftleft.semanticcpg.language.importresolver.* - EvaluatedImport.tagToEvaluatedImport(resolvedImport).foreach { - case ResolvedTypeDecl(fullName, _) => - symbolTable.append(LocalVar(fullName.split("\\.").lastOption.getOrElse(alias)), fullName) - case _ => super.visitImport(i) - } - } - override def visitIdentifierAssignedToConstructor(i: Identifier, c: Call): Set[String] = { - - def isMatching(cName: String, code: String) = { - val cNameList = cName.split(":program").last.split("\\.").filterNot(_.isEmpty).dropRight(1) - val codeList = code.split("\\(").head.split("[:.]").filterNot(_.isEmpty).dropRight(1) - cNameList sameElements codeList - } - - val constructorPaths = - symbolTable.get(c).filter(isMatching(_, c.code)).map(_.stripSuffix(s"$pathSep${XDefines.ConstructorMethodName}")) - associateTypes(i, constructorPaths) - } - - override def methodReturnValues(methodFullNames: Seq[String]): Set[String] = { - // Check if we have a corresponding member to resolve type - val memberTypes = methodFullNames.flatMap { fullName => - val memberName = fullName.split("\\.").lastOption - if (memberName.isDefined) { - val typeDeclFullName = fullName.stripSuffix(s".${memberName.get}") - cpg.typeDecl.fullName(typeDeclFullName).member.nameExact(memberName.get).typeFullName.l - } else - List.empty - }.toSet - if (memberTypes.nonEmpty) memberTypes else super.methodReturnValues(methodFullNames) - } - - override def visitIdentifierAssignedToCall(i: Identifier, c: Call): Set[String] = { - if (c.name.startsWith("")) { - visitIdentifierAssignedToOperator(i, c, c.name) - } else if (symbolTable.contains(c) && isConstructor(c)) { - visitIdentifierAssignedToConstructor(i, c) - } else if (symbolTable.contains(c)) { - visitIdentifierAssignedToCallRetVal(i, c) - } else if (c.argument.headOption.exists(symbolTable.contains)) { - setCallMethodFullNameFromBase(c) - // Repeat this method now that the call has a type - visitIdentifierAssignedToCall(i, c) - } else if ( - c.argument.headOption - .exists(_.isCall) && c.argument.head - .asInstanceOf[Call] - .name - .equals(".scopeResolution") && c.argument.head - .asInstanceOf[Call] - .argument - .lastOption - .exists(symbolTable.contains) - ) { - setCallMethodFullNameFromBaseScopeResolution(c) - // Repeat this method now that the call has a type - visitIdentifierAssignedToCall(i, c) - } else { - // We can try obtain a return type for this call - visitIdentifierAssignedToCallRetVal(i, c) - } - } - - protected def setCallMethodFullNameFromBaseScopeResolution(c: Call): Set[String] = { - val recTypes = c.argument.headOption - .map { - case x: Call if x.name.equals(".scopeResolution") => - x.argument.lastOption.map(i => symbolTable.get(i)).getOrElse(Set.empty[String]) - } - .getOrElse(Set.empty[String]) - val callTypes = recTypes.map(_.concat(s"$pathSep${c.name}")) - symbolTable.append(c, callTypes) - } - - override protected def visitIdentifierAssignedToTypeRef(i: Identifier, t: TypeRef, rec: Option[String]): Set[String] = - t.typ.referencedTypeDecl - .map(_.fullName.stripSuffix("")) - .map(td => symbolTable.append(CallAlias(i.name, rec), Set(td))) - .headOption - .getOrElse(super.visitIdentifierAssignedToTypeRef(i, t, rec)) - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/utils/PackageTable.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/utils/PackageTable.scala deleted file mode 100644 index 000cdee6af56..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/deprecated/utils/PackageTable.scala +++ /dev/null @@ -1,88 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.utils - -import java.io.File as JFile -import java.util.regex.Pattern -import scala.collection.mutable - -case class MethodTableModel(methodName: String, parentClassPath: String, classType: String) -case class ModuleModel(name: String, fullName: String) -case class TypeDeclModel(name: String, fullName: String) -case class PackageContext(moduleName: String, packageTable: PackageTable) - -class PackageTable { - - val methodTableMap = mutable.HashMap[String, mutable.HashSet[MethodTableModel]]() - val moduleMapping = mutable.HashMap[String, mutable.HashSet[ModuleModel]]() - val typeDeclMapping = mutable.HashMap[String, mutable.HashSet[TypeDeclModel]]() - - def addPackageMethod(moduleName: String, methodName: String, parentClassPath: String, classType: String): Unit = { - val packageMethod = MethodTableModel(methodName, parentClassPath, classType) - methodTableMap.getOrElseUpdate(moduleName, mutable.HashSet.empty[MethodTableModel]) += packageMethod - } - - def addModule(gemOrFileName: String, moduleName: String, modulePath: String): Unit = { - val fName = gemOrFileName.split(Pattern.quote(JFile.separator)).lastOption.getOrElse(gemOrFileName) - moduleMapping.getOrElseUpdate(gemOrFileName, mutable.HashSet.empty[ModuleModel]) += ModuleModel( - moduleName, - s"$fName::program.$modulePath" - ) - } - - def addTypeDecl(gemOrFileName: String, typeDeclName: String, typeDeclPath: String): Unit = { - val fName = gemOrFileName.split(Pattern.quote(JFile.separator)).lastOption.getOrElse(gemOrFileName) - typeDeclMapping.getOrElseUpdate(gemOrFileName, mutable.HashSet.empty[TypeDeclModel]) += TypeDeclModel( - typeDeclName, - s"$fName::program.$typeDeclPath" - ) - } - - def getMethodFullNameUsingName( - packageUsed: List[String] = List(PackageTable.InternalModule), - methodName: String - ): List[String] = - packageUsed - .filter(methodTableMap.contains) - .flatMap { - case PackageTable.InternalModule => - methodTableMap(PackageTable.InternalModule) - .filter(_.methodName == methodName) - .map(method => s"${method.parentClassPath}.$methodName") - case module => - methodTableMap(module) - .filter(_.methodName == methodName) - .map(method => s"$module::program:${method.parentClassPath}$methodName") - } - - def getPackageInfo(moduleName: String): List[MethodTableModel] = { - methodTableMap.get(moduleName) match - case Some(value) => value.toList - case None => List.empty[MethodTableModel] - } - - def getModule(gemOrFileName: String): List[ModuleModel] = { - moduleMapping.get(gemOrFileName) match - case Some(value) => value.toList - case None => List.empty[ModuleModel] - } - - def getTypeDecl(gemOrFileName: String): List[TypeDeclModel] = { - typeDeclMapping.get(gemOrFileName) match - case Some(value) => value.toList - case None => List.empty[TypeDeclModel] - } - - def set(table: PackageTable): Unit = { - methodTableMap.addAll(table.methodTableMap) - moduleMapping.addAll(table.moduleMapping) - typeDeclMapping.addAll(table.typeDeclMapping) - } - def clear(): Unit = { - methodTableMap.clear - moduleMapping.clear - typeDeclMapping.clear - } -} - -object PackageTable { - val InternalModule = "" -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala new file mode 100644 index 000000000000..24776624a3f0 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala @@ -0,0 +1,36 @@ +package io.joern.rubysrc2cpg.parser + +import org.antlr.v4.runtime.ParserRuleContext +import org.antlr.v4.runtime.tree.TerminalNode + +/** General purpose ANTLR parse tree printer. + */ +object AnltrAstPrinter { + private val indentationIncrement = 1 + + private def print(level: Int, sb: StringBuilder, context: ParserRuleContext): StringBuilder = { + val indentation = " ".repeat(level) + val contextName = context.getClass.getSimpleName.stripSuffix("Context") + val nextLevel = level + indentationIncrement + sb.append(s"$indentation$contextName\n") + Option(context.children).foreach(_.forEach { + case c: ParserRuleContext => print(nextLevel, sb, c) + case t: TerminalNode => print(nextLevel, sb, t) + }) + sb + } + + private def print(level: Int, sb: StringBuilder, terminal: TerminalNode): StringBuilder = { + val indentation = " ".repeat(level) + sb.append(s"$indentation${terminal.getText}\n") + sb + } + + /** Pretty-prints an entire `ParserRuleContext` together with its descendants. + * @param context + * the context to pretty-print + * @return + * an indented, multiline string representation + */ + def print(context: ParserRuleContext): String = print(0, new StringBuilder, context).toString() +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala index eb2e0d9ecadc..896f5b9f18d9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala @@ -1,6 +1,6 @@ package io.joern.rubysrc2cpg.parser -import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.TextSpan +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{ProcedureDeclaration, TextSpan, TypeDeclaration} import io.joern.rubysrc2cpg.parser.RubyParser.* import org.antlr.v4.runtime.ParserRuleContext import org.antlr.v4.runtime.misc.Interval @@ -18,14 +18,34 @@ object AntlrContextHelpers { // We need to make sure this doesn't happen when building the `text` field. val startIndex = ctx.getStart.getStartIndex val stopIndex = math.max(startIndex, ctx.getStop.getStopIndex) + + val offset = ctx match { + case x: MethodDefinitionContext => Option(ctx.start.getStartIndex, ctx.stop.getStopIndex + 1) + case x: ClassDefinitionContext => Option(ctx.start.getStartIndex, ctx.stop.getStopIndex + 1) + case x: ModuleDefinitionContext => Option(ctx.start.getStartIndex, ctx.stop.getStopIndex + 1) + case _ => None + } + TextSpan( line = Option(ctx.getStart.getLine), column = Option(ctx.getStart.getCharPositionInLine), lineEnd = Option(ctx.getStop.getLine), columnEnd = Option(ctx.getStop.getCharPositionInLine), + offset = offset, text = ctx.getStart.getInputStream.getText(new Interval(startIndex, stopIndex)) ) } + + /** @return + * true if this token's text is the same as a keyword, false if otherwise. + */ + def isKeyword: Boolean = { + // See RubyParser for why the bounds are used + val minBound = 19 + val maxBound = 56 + val typ = ctx.start.getType + typ >= minBound && typ <= maxBound + } } sealed implicit class CompoundStatementContextHelper(ctx: CompoundStatementContext) { @@ -123,10 +143,10 @@ object AntlrContextHelpers { } sealed implicit class BlockParameterContextHelper(ctx: BlockParameterContext) { - def parameters: List[ParserRuleContext] = Option(ctx.parameterList()).map(_.parameters).getOrElse(List()) + def parameters: List[ParserRuleContext] = Option(ctx.blockParameterList()).map(_.parameters).getOrElse(List()) } - sealed implicit class CommandArgumentContextHelper(ctx: CommandArgumentContext) { + sealed implicit class CommandArgumentContextelper(ctx: CommandArgumentContext) { def arguments: List[ParserRuleContext] = ctx match { case ctx: CommandCommandArgumentListContext => ctx.command() :: Nil case ctx: CommandArgumentCommandArgumentListContext => ctx.commandArgumentList().elements @@ -142,11 +162,44 @@ object AntlrContextHelpers { } } + sealed implicit class SimpleCommandArgumentListContextHelper(ctx: SimpleCommandArgumentListContext) { + def arguments: List[ParserRuleContext] = { + val primaryValues = Option(ctx.primaryValueList()).map(_.primaryValue().asScala.toList).getOrElse(List()) + val associations = Option(ctx.associationList()).map(_.association().asScala.toList).getOrElse(List()) + val argumentLists = Option(ctx.argumentList()).map(_.elements).getOrElse(List()) + primaryValues ++ associations ++ argumentLists + } + } + + sealed implicit class PrimaryValueListWithAssociationContextHelper(ctx: PrimaryValueListWithAssociationContext) { + def elements: List[ParserRuleContext] = { + ctx.children.asScala.collect { + case x: PrimaryValueContext => x + case x: AssociationContext => x + }.toList + } + } + sealed implicit class ModifierStatementContextHelpers(ctx: ModifierStatementContext) { def isUnless: Boolean = Option(ctx.statementModifier().UNLESS()).isDefined def isIf: Boolean = Option(ctx.statementModifier().IF()).isDefined } + sealed implicit class QuotedExpandedArrayElementListContextHelper(ctx: QuotedExpandedArrayElementListContext) { + def elements: List[ParserRuleContext] = ctx.quotedExpandedArrayElement.asScala.toList + } + + sealed implicit class QuotedExpandedArrayElementContextHelper(ctx: QuotedExpandedArrayElementContext) { + def interpolations: List[ParserRuleContext] = ctx + .quotedExpandedArrayElementContent() + .asScala + .filter(x => Option(x.compoundStatement()).isDefined) + .map(_.compoundStatement()) + .toList + def hasInterpolation: Boolean = + ctx.interpolations.nonEmpty + } + sealed implicit class QuotedNonExpandedArrayElementListContextHelper(ctx: QuotedNonExpandedArrayElementListContext) { def elements: List[ParserRuleContext] = ctx.quotedNonExpandedArrayElementContent().asScala.toList } @@ -163,6 +216,27 @@ object AntlrContextHelpers { def parameters: List[ParserRuleContext] = ctx.mandatoryOrOptionalParameter().asScala.toList } + sealed implicit class MandatoryOrOptionalOrGroupedParameterListContextHelper( + ctx: MandatoryOrOptionalOrGroupedParameterListContext + ) { + def parameters: List[ParserRuleContext] = + ctx.mandatoryOrOptionalOrGroupedParameter().asScala.toList + } + + sealed implicit class MandatoryOrGroupedParameterListContextHelper(ctx: MandatoryOrGroupedParameterListContext) { + def parameters: List[ParserRuleContext] = + ctx.mandatoryOrGroupedParameter().asScala.toList + } + + sealed implicit class GroupedParameterListContextHelper(ctx: GroupedParameterListContext) { + def parameters: List[ParserRuleContext] = { + val arrayParameter = Option(ctx.arrayParameter()).toList + val mandatoryParameters = ctx.mandatoryParameter.asScala.toList + + mandatoryParameters ++ arrayParameter + } + } + sealed implicit class MethodParameterPartContextHelper(ctx: MethodParameterPartContext) { def parameters: List[ParserRuleContext] = Option(ctx.parameterList()).map(_.parameters).getOrElse(List()) } @@ -173,18 +247,57 @@ object AntlrContextHelpers { val arrayParameter = Option(ctx.arrayParameter()).toList val hashParameter = Option(ctx.hashParameter()).toList val procParameter = Option(ctx.procParameter()).toList - mandatoryOrOptionals ++ arrayParameter ++ hashParameter ++ procParameter + val mandatoryParams = Option(ctx.mandatoryParameterList()).toList + mandatoryOrOptionals ++ arrayParameter ++ hashParameter ++ procParameter ++ mandatoryParams + } + } + + sealed implicit class BlockParameterListContextHelper(ctx: BlockParameterListContext) { + def parameters: List[ParserRuleContext] = { + val mandatoryOrOptionalOrGrouped = + Option(ctx.mandatoryOrOptionalOrGroupedParameterList()).map(_.parameters).getOrElse(List()) + val arrayParameter = Option(ctx.arrayParameter()).toList + val hashParameter = Option(ctx.hashParameter()).toList + val procParameter = Option(ctx.procParameter()).toList + val mandatoryOrGrouped = + Option(ctx.mandatoryOrGroupedParameterList().asScala).map(_.flatten(_.parameters)).getOrElse(List()) + + mandatoryOrOptionalOrGrouped ++ arrayParameter ++ hashParameter ++ procParameter ++ mandatoryOrGrouped + } + } + + sealed implicit class BracketedArrayElementListContextHelper(ctx: BracketedArrayElementListContext) { + def elements: List[ParserRuleContext] = { + ctx.bracketedArrayElement.asScala.flatMap(_.element).toList + } + } + + sealed implicit class BracketedArrayElementContextHelper(ctx: BracketedArrayElementContext) { + def element: List[ParserRuleContext] = { + ctx.children.asScala + .collect { + case x: OperatorExpressionListContext => x.operatorExpression().asScala + case x: CommandContext => x :: Nil + case x: AssociationListContext => x.associations + case x: SplattingArgumentContext => x :: Nil + case x: IndexingArgumentContext => x :: Nil + case x: IndexingArgumentListContext => x.arguments + case x: HashLiteralContext => x :: Nil + } + .toList + .flatten } } sealed implicit class IndexingArgumentListContextHelper(ctx: IndexingArgumentListContext) { def arguments: List[ParserRuleContext] = ctx match - case ctx: CommandIndexingArgumentListContext => List(ctx.command()) case ctx: OperatorExpressionListIndexingArgumentListContext => ctx.operatorExpressionList().operatorExpression().asScala.toList case ctx: AssociationListIndexingArgumentListContext => ctx.associationList().associations - case ctx: SplattingArgumentIndexingArgumentListContext => ctx.splattingArgument() :: Nil + case ctx: SplattingArgumentIndexingArgumentListContext => ctx.splattingArgument().asScala.toList case ctx: OperatorExpressionListWithSplattingArgumentIndexingArgumentListContext => ctx.splattingArgument() :: Nil + case ctx: IndexingArgumentIndexingArgumentListContext => + ctx.indexingArgument().asScala.toList case ctx => logger.warn(s"IndexingArgumentListContextHelper - Unsupported argument type ${ctx.getClass}") List() @@ -197,25 +310,41 @@ object AntlrContextHelpers { case ctx => logger.warn(s"ArgumentWithParenthesesContextHelper - Unsupported argument type ${ctx.getClass}") List() + + def isArrayArgumentList: Boolean = ctx match { + case ctx: ArgumentListArgumentWithParenthesesContext => ctx.argumentList().isArrayArgumentListContext + case _ => false + } } sealed implicit class ArgumentListContextHelper(ctx: ArgumentListContext) { def elements: List[ParserRuleContext] = ctx match - case ctx: OperatorsArgumentListContext => - val operatorExpressions = ctx.operatorExpressionList().operatorExpression().asScala.toList - val associations = Option(ctx.associationList()).fold(List())(_.association().asScala) - val splatting = Option(ctx.splattingArgument()).toList - val block = Option(ctx.blockArgument()).toList - operatorExpressions ++ associations ++ splatting ++ block - case ctx: AssociationsArgumentListContext => - Option(ctx.associationList()).map(_.associations).getOrElse(List.empty) - case ctx: SplattingArgumentArgumentListContext => - Option(ctx.splattingArgument()).toList + case ctx: ArgumentListItemArgumentListContext => + ctx + .argumentListItem() + .asScala + .flatMap { x => + Option(x.splattingArgument()).toList ++ + Option(x.associationList()).map(_.associations).getOrElse(Nil) ++ + Option(x.blockArgument()).toList ++ + Option(x.operatorExpressionList()).map(_.operatorExpression().asScala.toList).getOrElse(Nil) + } + .sortBy { x => + val span = x.toTextSpan + span.line -> span.column + } + .toList case ctx: BlockArgumentArgumentListContext => Option(ctx.blockArgument()).toList + case ctx: ArrayArgumentListContext => + Option(ctx.indexingArgumentList()).toList + case ctx: SingleCommandArgumentListContext => + Option(ctx.command()).toList case ctx => logger.warn(s"ArgumentListContextHelper - Unsupported element type ${ctx.getClass.getSimpleName}") List() + + def isArrayArgumentListContext: Boolean = ctx.isInstanceOf[ArrayArgumentListContext] } sealed implicit class CommandWithDoBlockContextHelper(ctx: CommandWithDoBlockContext) { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrParser.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrParser.scala index 9fa7f2e47511..4365694780c0 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrParser.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrParser.scala @@ -1,30 +1,65 @@ package io.joern.rubysrc2cpg.parser import better.files.File +import better.files.File.OpenOptions import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.atn.{ATN, ATNConfigSet} +import org.antlr.v4.runtime.atn.{ATN, ATNConfigSet, ProfilingATNSimulator} import org.antlr.v4.runtime.dfa.DFA import org.slf4j.LoggerFactory + +import java.io.FileWriter import java.io.File.separator +import java.nio.file.{Files, Path, StandardOpenOption} import java.util +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} +import scala.collection.concurrent.TrieMap +import scala.collection.mutable import scala.collection.mutable.ListBuffer -import scala.util.Try +import scala.util.{Try, Using} +import flatgraph.help.Table +import flatgraph.help.Table.AvailableWidthProvider +import io.shiftleft.semanticcpg + +/** Summarizes the result of parsing a file. + */ +case class ParserResult(program: Try[RubyParser.ProgramContext], errors: List[String], warnings: List[String]) /** A consumable wrapper for the RubyParser class used to parse the given file and be disposed thereafter. * @param inputDir * the directory of the target to parse. * @param filename * the file path to the file to be parsed. + * @param withDebugging + * if set, will enable the ANTLR debugger, i.e, printing of the parse tree. + * @param maybeParserProfiler + * the parser profiler used to capture profiling information. */ -class AntlrParser(inputDir: File, filename: String) { +class AntlrParser( + inputDir: File, + filename: String, + withDebugging: Boolean = false, + maybeParserProfiler: Option[ParserProfiler] = None +) { - private val charStream = CharStreams.fromFileName(filename) - private val lexer = new RubyLexer(charStream) - private val tokenStream = new CommonTokenStream(RubyLexerPostProcessor(lexer)) - val parser: RubyParser = new RubyParser(tokenStream) + val parser: RubyParser = { + val charStream = CharStreams.fromFileName(filename) + val lexer = new RubyLexer(charStream) + val tokenStream = new CommonTokenStream(RubyLexerPostProcessor(lexer)) + new RubyParser(tokenStream) + } + private var profilerOpt: Option[ProfilingATNSimulator] = None + + parser.setTrace(withDebugging) + if (maybeParserProfiler.isDefined) { + val profiler = new ProfilingATNSimulator(parser) + parser.setInterpreter(profiler) + parser.setProfile(true) + profilerOpt = Option(profiler) + } - def parse(): (Try[RubyParser.ProgramContext], List[String]) = { - val errors = ListBuffer[String]() + def parse(): ParserResult = { + val errors = ListBuffer[String]() + val warnings = ListBuffer[String]() parser.removeErrorListeners() parser.addErrorListener(new ANTLRErrorListener { override def syntaxError( @@ -48,7 +83,19 @@ class AntlrParser(inputDir: File, filename: String) { exact: Boolean, ambigAlts: util.BitSet, configs: ATNConfigSet - ): Unit = {} + ): Unit = { + val atn = recognizer.getATN + val decisionNumber = dfa.decision + val ruleIndex = atn.decisionToState.get(decisionNumber).ruleIndex + val ruleName = recognizer.getRuleNames()(ruleIndex) + + val startToken = recognizer.getTokenStream.get(startIndex) + val stopToken = recognizer.getTokenStream.get(stopIndex) + + warnings.append( + s"Parser ambiguity detected for rule '$ruleName' (decision ${dfa.decision}) from token '${startToken.getText}' [startIndex=$startIndex] to '${stopToken.getText}' [stopIndex=$stopIndex], alternatives: ${ambigAlts.toString}" + ) + } override def reportAttemptingFullContext( recognizer: Parser, @@ -68,8 +115,22 @@ class AntlrParser(inputDir: File, filename: String) { configs: ATNConfigSet ): Unit = {} }) - (Try(parser.program()), errors.toList) + + val program = Try { + val parseStart = System.nanoTime() + val program = parser.program() + val parseTime = System.nanoTime() - parseStart + maybeParserProfiler.foreach(_.captureParseTime(filename, parseTime)) + // If profiling is enabled, read metrics and write accompanying file + profilerOpt.foreach(profiler => + maybeParserProfiler.foreach(_.captureProfilerLogs(parser, inputDir.pathAsString, filename, profiler)) + ) + + program + } + ParserResult(program, errors.toList, warnings.toList) } + } /** A re-usable parser object that clears the ANTLR DFA-cache if it determines that the memory usage is becoming large. @@ -82,27 +143,31 @@ class AntlrParser(inputDir: File, filename: String) { * @param clearLimit * the percentage of used heap to clear the DFA-cache on. */ -class ResourceManagedParser(clearLimit: Double) extends AutoCloseable { +class ResourceManagedParser(clearLimit: Double, debug: Boolean = false, profiling: Boolean = false) + extends AutoCloseable { private val logger = LoggerFactory.getLogger(getClass) private val runtime = Runtime.getRuntime private var maybeDecisionToDFA: Option[Array[DFA]] = None private var maybeAtn: Option[ATN] = None + private val profiler: Option[ParserProfiler] = if profiling then Option(ParserProfiler()) else None def parse(inputFile: File, filename: String): Try[RubyParser.ProgramContext] = { val inputDir = if inputFile.isDirectory then inputFile else inputFile.parent - val antlrParser = AntlrParser(inputDir, filename) + val antlrParser = AntlrParser(inputDir, filename, debug, profiler) val interp = antlrParser.parser.getInterpreter // We need to grab a live instance in order to get the static variables as they are protected from static access maybeDecisionToDFA = Option(interp.decisionToDFA) maybeAtn = Option(interp.atn) val usedMemory = runtime.freeMemory.toDouble / runtime.totalMemory.toDouble - if (usedMemory >= clearLimit) { + if (usedMemory >= clearLimit && !profiling) { // Profiler may need the DFA for approximating time used at decisions logger.debug(s"Runtime memory consumption at $usedMemory, clearing ANTLR DFA cache") clearDFA() } - val (programCtx, errors) = antlrParser.parse() + + val ParserResult(programCtx, errors, warnings) = antlrParser.parse() errors.foreach(logger.warn) + if (profiling) warnings.foreach(logger.warn) programCtx } @@ -118,3 +183,130 @@ class ResourceManagedParser(clearLimit: Double) extends AutoCloseable { override def close(): Unit = clearDFA() } + +class ParserProfiler { + + private val logger = LoggerFactory.getLogger(getClass) + private val ruleCost = TrieMap.empty[String, RuleTimeCost] + private val fileCost = TrieMap.empty[String, Long] + private var projectRoot: Option[Path] = None + + // Note: This is in a shutdown hook to guarantee output is dumped, however it is optional and can readily be + // replaced or dumped at the end of successful parse runs only. + sys.addShutdownHook { + dumpSummary() + } + + /** An object to aggregate the performance cost of a rule. + * @param predictionTime + * the total time in prediction (ns). + * @param lookaheads + * total lookahead operations. + */ + private case class RuleTimeCost(predictionTime: Long, lookaheads: Long) { + def +(o: RuleTimeCost): RuleTimeCost = + this.copy(this.predictionTime + o.predictionTime, this.lookaheads + o.lookaheads) + } + + def captureParseTime(filename: String, nanoTime: Long): Unit = + fileCost.put(filename, nanoTime) + + def captureProfilerLogs( + parser: RubyParser, + inputDir: String, + filename: String, + profiler: ProfilingATNSimulator + ): Unit = { + // Set project root + if projectRoot.isEmpty then projectRoot = Option(Path.of(inputDir)) + + val logFilename = filename.replaceAll("\\.[^.]+$", "") + ".log" + val atn = parser.getATN + Using.resource(FileWriter(logFilename)) { logFile => + logFile.write("Profiling information for file: " + filename + "\n\n") + + var totalTimeInPrediction = 0L + var totalLookaheadOps = 0L + + profiler.getDecisionInfo.foreach { decision => + val decisionNumber = decision.decision + val ruleIndex = atn.decisionToState.get(decisionNumber).ruleIndex + val ruleName = parser.getRuleNames()(ruleIndex) + + logFile.write(s"Decision $decisionNumber ($ruleName):\n") + logFile.write(s" Invocations: ${decision.invocations}\n") + logFile.write(s" Time (ns): ${decision.timeInPrediction}\n") + logFile.write(s" SLL lookahead operations: ${decision.SLL_TotalLook}\n") + logFile.write(s" LL lookahead operations: ${decision.LL_TotalLook}\n") + + val ruleTimeCost = RuleTimeCost(decision.timeInPrediction, decision.SLL_TotalLook + decision.LL_TotalLook) + totalTimeInPrediction += ruleTimeCost.predictionTime + totalLookaheadOps += ruleTimeCost.lookaheads + + ruleCost.updateWith(ruleName) { + case Some(x) => Option(x + ruleTimeCost) + case None => Option(ruleTimeCost) + } + } + logFile.write(s"Total time in prediction: $totalTimeInPrediction ns\n") + logFile.write(s"Total lookahead operations: $totalLookaheadOps\n") + } + } + + private def dumpSummary(): Unit = { + projectRoot match { + case Some(root) => + val conversionFactor = 1 / 1e6 + val timeUnit = "ms" + val summaryPath = root.resolve("antlr_summary.log") + implicit val widthProvider: AvailableWidthProvider = semanticcpg.defaultAvailableWidthProvider + val totalParseTime = fileCost.values.sum + val avgParseTime = totalParseTime / fileCost.size.toDouble + val mostExpensiveFileStr = fileCost.toList.sortBy(_._2).reverse.headOption.map { case (name, time) => + f"Most Expensive File: ${root.relativize(Path.of(name))} (${time * conversionFactor}%.2f $timeUnit)" + } + + val columnNames = Seq("Rule Name", s"Prediction Time ($timeUnit)", "Total Lookaheads") + val rulesByTimeTable = Table( + columnNames = columnNames, + rows = ruleCost.toList.sortBy { case (_, timeCost) => timeCost.predictionTime }.reverse.take(10).map { + case (ruleName, timeCost) => + Seq(ruleName, f"${timeCost.predictionTime * conversionFactor}%.2f", timeCost.lookaheads.toString) + } + ) + val rulesByLookaheadTable = Table( + columnNames = columnNames, + rows = ruleCost.toList.sortBy { case (_, timeCost) => timeCost.lookaheads }.reverse.take(10).map { + case (ruleName, timeCost) => + Seq(ruleName, f"${timeCost.predictionTime * conversionFactor}%.2f", timeCost.lookaheads.toString) + } + ) + + if (Files.exists(summaryPath.getParent)) { + Files.writeString( + summaryPath, + f"""Summary for project at '${root.getFileName}' + |Total Parsed Files: ${fileCost.size} + |Total Parse Time (CPU): ${totalParseTime * conversionFactor}%.2f ($timeUnit) + |Avg. Parse Time Per File: ${avgParseTime * conversionFactor}%.2f ($timeUnit) + |${mostExpensiveFileStr.getOrElse("")} + | + |Most Expensive Rules By Time in Prediction + |========================================== + |${rulesByTimeTable.render} + | + |Most Expensive Rules By Total SLL & LL Lookaheads + |================================================= + |${rulesByLookaheadTable.render} + |""".stripMargin, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ) + } else { + logger.warn(s"${summaryPath.getParent} does not exist. Skipping profile summary dump.") + } + case None => logger.warn("At least one file must be parsed for profiling information to be dumped") + } + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala index 1488305c4269..ac16b39440eb 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala @@ -1,36 +1,1235 @@ package io.joern.rubysrc2cpg.parser +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* +import io.joern.rubysrc2cpg.parser.AntlrContextHelpers.* +import io.joern.rubysrc2cpg.parser.RubyParser.{ + ArrayParameterContext, + CommandWithDoBlockContext, + ConstantVariableReferenceContext, + MandatoryParameterContext, + MethodCallExpressionContext +} +import io.joern.rubysrc2cpg.passes.Defines +import io.joern.rubysrc2cpg.utils.FreshNameGenerator import org.antlr.v4.runtime.ParserRuleContext -import org.antlr.v4.runtime.tree.TerminalNode - -/** General purpose ANTLR parse tree printer. - */ -object AstPrinter { - private val indentationIncrement = 1 - - private def print(level: Int, sb: StringBuilder, context: ParserRuleContext): StringBuilder = { - val indentation = " ".repeat(level) - val contextName = context.getClass.getSimpleName.stripSuffix("Context") - val nextLevel = level + indentationIncrement - sb.append(s"$indentation$contextName\n") - Option(context.children).foreach(_.forEach { - case c: ParserRuleContext => print(nextLevel, sb, c) - case t: TerminalNode => print(nextLevel, sb, t) - }) - sb - } - - private def print(level: Int, sb: StringBuilder, terminal: TerminalNode): StringBuilder = { - val indentation = " ".repeat(level) - sb.append(s"$indentation${terminal.getText}\n") - sb - } - - /** Pretty-prints an entire `ParserRuleContext` together with its descendants. - * @param context - * the context to pretty-print - * @return - * an indented, multiline string representation - */ - def print(context: ParserRuleContext): String = print(0, new StringBuilder, context).toString() +import org.antlr.v4.runtime.tree.{ParseTree, TerminalNode} + +import scala.jdk.CollectionConverters.* + +class AstPrinter extends RubyParserBaseVisitor[String] { + private val ls = "\n" + private val rubyNodeCreator = new RubyNodeCreator + + private val classNameGen = FreshNameGenerator(id => s"") + private val variableNameGen = FreshNameGenerator(id => s"") + + protected def freshClassName(): String = { + classNameGen.fresh + } + + override def defaultResult(): String = "" + + override def visit(tree: ParseTree): String = { + Option(tree).map(super.visit).getOrElse(defaultResult) + } + + override def visitProgram(ctx: RubyParser.ProgramContext): String = { + visit(ctx.compoundStatement()) + } + + override def visitCompoundStatement(ctx: RubyParser.CompoundStatementContext): String = { + ctx.getStatements.map(visit).mkString(ls) + } + + override def visitNextWithoutArguments(ctx: RubyParser.NextWithoutArgumentsContext): String = { + ctx.getText + } + + override def visitGroupingStatement(ctx: RubyParser.GroupingStatementContext): String = { + val stmts = ctx.compoundStatement().getStatements.map(visit) + if (stmts.size == 1) then stmts.head + else stmts.mkString(ls) + } + + override def visitStatements(ctx: RubyParser.StatementsContext): String = { + ctx.statement().asScala.map(visit).toList.mkString(ls) + } + + override def visitWhileExpression(ctx: RubyParser.WhileExpressionContext): String = { + val outputSb = new StringBuilder(ctx.WHILE.getText) + + val condition = visit(ctx.expressionOrCommand) + outputSb.append(s" $condition") + + val body = visit(ctx.doClause()) + + if body != "" then outputSb.append(s"$ls$body") + + outputSb.append(s"$ls${ctx.END.getText}").toString + } + + override def visitUntilExpression(ctx: RubyParser.UntilExpressionContext): String = { + val condition = visit(ctx.expressionOrCommand()) + val body = visit(ctx.doClause()) + + s"${ctx.UNTIL.getText} $condition$ls$body$ls${ctx.END.getText}" + } + + override def visitBeginEndExpression(ctx: RubyParser.BeginEndExpressionContext): String = { + s"${ctx.BEGIN.getText}$ls${visit(ctx.bodyStatement())}$ls${ctx.END.getText}" + } + + override def visitIfExpression(ctx: RubyParser.IfExpressionContext): String = { + val outputSb = new StringBuilder(ctx.IF.getText) + + val condition = visit(ctx.expressionOrCommand()) + outputSb.append(s" $condition") + + val thenBody = visit(ctx.thenClause()) + if thenBody != "" then outputSb.append(s"$ls$thenBody") + + val elsifs = ctx.elsifClause().asScala.map(visit).toList + if elsifs.nonEmpty then outputSb.append(s"$ls${elsifs.mkString(ls)}") + + val elseBody = Option(ctx.elseClause()).map(visit) + if elseBody.isDefined then outputSb.append(s"$ls${elseBody.get}") + + outputSb.append(s"$ls${ctx.END.getText}") + outputSb.toString + } + + override def visitElsifClause(ctx: RubyParser.ElsifClauseContext): String = { + val outputSb = new StringBuilder(ctx.ELSIF.getText) + + val condition = visit(ctx.expressionOrCommand()) + outputSb.append(s" $condition") + + val thenBody = visit(ctx.thenClause()) + if thenBody != "" then outputSb.append(s"$ls$thenBody") + + outputSb.toString + } + + override def visitElseClause(ctx: RubyParser.ElseClauseContext): String = { + val outputSb = new StringBuilder(ctx.ELSE.getText) + + val elseBody = visit(ctx.compoundStatement()) + if elseBody != "" then outputSb.append(s"$ls$elseBody") + + outputSb.toString + } + + override def visitUnlessExpression(ctx: RubyParser.UnlessExpressionContext): String = { + val outputSb = new StringBuilder(ctx.UNLESS.getText) + + val condition = visit(ctx.expressionOrCommand()) + outputSb.append(s" $condition") + + val thenBody = visit(ctx.thenClause()) + if thenBody != "" then outputSb.append(s"$ls$thenBody") + + val elseBody = Option(ctx.elseClause()).map(visit) + if elseBody.isDefined then outputSb.append(s"$ls${elseBody.get}") + + outputSb.append(s"$ls${ctx.END.getText}").toString + } + + override def visitForExpression(ctx: RubyParser.ForExpressionContext): String = { + val forVariable = visit(ctx.forVariable()) + val iterableVariable = visit(ctx.commandOrPrimaryValue()) + val doBlock = visit(ctx.doClause()) + + s"${ctx.FOR.getText} $forVariable ${ctx.IN.getText} $iterableVariable$doBlock$ls${ctx.END.getText}" + } + + override def visitForVariable(ctx: RubyParser.ForVariableContext): String = { + if (ctx.leftHandSide() != null) visit(ctx.leftHandSide()) + else visit(ctx.multipleLeftHandSide()) + } + + override def visitModifierStatement(ctx: RubyParser.ModifierStatementContext): String = { + ctx.statementModifier().getText match + case "if" => + val condition = visit(ctx.expressionOrCommand()) + val thenBody = visit(ctx.statement()) + s"$thenBody if $condition" + case "unless" => + val condition = visit(ctx.expressionOrCommand()) + val thenBody = visit(ctx.statement()) + s"$thenBody unless $condition" + case "while" => + val condition = visit(ctx.expressionOrCommand()) + val body = visit(ctx.statement()) + s"$body while $condition" + case "until" => + val condition = visit(ctx.expressionOrCommand()) + val body = visit(ctx.statement()) + s"$body until $condition" + case "rescue" => + val body = visit(ctx.statement()) + val thenClause = visit(ctx.expressionOrCommand()) + s"$body rescue $thenClause" + case _ => defaultResult() + } + + override def visitTernaryOperatorExpression(ctx: RubyParser.TernaryOperatorExpressionContext): String = { + val condition = visit(ctx.operatorExpression(0)) + val thenBody = visit(ctx.operatorExpression(1)) + val elseBody = visit(ctx.operatorExpression(2)) + + s"$condition ${ctx.QMARK.getText} $thenBody ${ctx.COLON.getText} $elseBody" + } + + override def visitReturnMethodInvocationWithoutParentheses( + ctx: RubyParser.ReturnMethodInvocationWithoutParenthesesContext + ): String = { + val expressions = Option(ctx.primaryValueListWithAssociation().methodInvocationWithoutParentheses()) match { + case Some(methodInvocation) => visit(methodInvocation) :: Nil + case None => ctx.primaryValueListWithAssociation().elements.map(visit).toList + } + + s"return ${expressions.toList.mkString(",")}" + } + + override def visitReturnWithoutArguments(ctx: RubyParser.ReturnWithoutArgumentsContext): String = { + ctx.getText + } + + override def visitNumericLiteral(ctx: RubyParser.NumericLiteralContext): String = { + if ctx.hasSign then s"${ctx.sign.getText}${visit(ctx.unsignedNumericLiteral())}" + else visit(ctx.unsignedNumericLiteral()) + } + + override def visitUnaryExpression(ctx: RubyParser.UnaryExpressionContext): String = { + s"${ctx.unaryOperator().getText}${visit(ctx.primaryValue())}" + } + + override def visitUnaryMinusExpression(ctx: RubyParser.UnaryMinusExpressionContext): String = { + s"${ctx.MINUS().getText}${visit(ctx.primaryValue())}" + } + + override def visitNotExpressionOrCommand(ctx: RubyParser.NotExpressionOrCommandContext): String = { + s"${ctx.NOT().getText} ${visit(ctx.expressionOrCommand())}" + } + + override def visitCommandExpressionOrCommand(ctx: RubyParser.CommandExpressionOrCommandContext): String = { + val methodInvocation = visit(ctx.methodInvocationWithoutParentheses()) + + if (Option(ctx.EMARK()).isDefined) { + s"${ctx.EMARK().getText}$methodInvocation" + } else { + methodInvocation + } + } + + override def visitCommandWithDoBlock(ctx: RubyParser.CommandWithDoBlockContext): String = { + val name = Option(ctx.methodIdentifier()).orElse(Option(ctx.methodName())).map(visit).getOrElse(defaultResult()) + val arguments = ctx.arguments.map(visit).mkString(" ") + val block = visit(ctx.doBlock()) + + s"$name $arguments $block" + } + + override def visitPrimaryOperatorExpression(ctx: RubyParser.PrimaryOperatorExpressionContext): String = { + rubyNodeCreator.visitPrimaryOperatorExpression(ctx) match { + case x: BinaryExpression if x.lhs.text.endsWith("=") && x.op == "*" => + val newLhs = x.lhs match { + case call: SimpleCall => SimpleIdentifier(None)(call.span.spanStart(call.span.text.stripSuffix("="))) + case y => + y + } + val newRhs = { + val oldRhsSpan = x.rhs.span + SplattingRubyNode(x.rhs)(oldRhsSpan.spanStart(s"*${oldRhsSpan.text}")) + } + s"${newLhs.span.text} = ${newRhs.span.text}" + case x => super.visitPrimaryOperatorExpression(ctx) + } + } + + override def visitPowerExpression(ctx: RubyParser.PowerExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.powerOperator.getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitAdditiveExpression(ctx: RubyParser.AdditiveExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.additiveOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitMultiplicativeExpression(ctx: RubyParser.MultiplicativeExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.multiplicativeOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitLogicalAndExpression(ctx: RubyParser.LogicalAndExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.andOperator.getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitLogicalOrExpression(ctx: RubyParser.LogicalOrExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.orOperator.getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitKeywordAndOrExpressionOrCommand(ctx: RubyParser.KeywordAndOrExpressionOrCommandContext): String = { + val lhs = visit(ctx.lhs) + val op = ctx.binOp.getText + val rhs = visit(ctx.rhs) + + s"$lhs $op $rhs" + } + + override def visitShiftExpression(ctx: RubyParser.ShiftExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.bitwiseShiftOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitBitwiseAndExpression(ctx: RubyParser.BitwiseAndExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.bitwiseAndOperator.getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitBitwiseOrExpression(ctx: RubyParser.BitwiseOrExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.bitwiseOrOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitRelationalExpression(ctx: RubyParser.RelationalExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.relationalOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitEqualityExpression(ctx: RubyParser.EqualityExpressionContext): String = { + val lhs = visit(ctx.primaryValue(0)) + val op = ctx.equalityOperator().getText + val rhs = visit(ctx.primaryValue(1)) + + s"$lhs $op $rhs" + } + + override def visitDecimalUnsignedLiteral(ctx: RubyParser.DecimalUnsignedLiteralContext): String = { + ctx.getText + } + + override def visitBinaryUnsignedLiteral(ctx: RubyParser.BinaryUnsignedLiteralContext): String = { + ctx.getText + } + + override def visitOctalUnsignedLiteral(ctx: RubyParser.OctalUnsignedLiteralContext): String = { + ctx.getText + } + + override def visitHexadecimalUnsignedLiteral(ctx: RubyParser.HexadecimalUnsignedLiteralContext): String = { + ctx.getText + } + + override def visitFloatWithExponentUnsignedLiteral( + ctx: RubyParser.FloatWithExponentUnsignedLiteralContext + ): String = { + ctx.getText + } + + override def visitFloatWithoutExponentUnsignedLiteral( + ctx: RubyParser.FloatWithoutExponentUnsignedLiteralContext + ): String = { + ctx.getText + } + + override def visitPureSymbolLiteral(ctx: RubyParser.PureSymbolLiteralContext): String = { + ctx.getText + } + + override def visitSingleQuotedSymbolLiteral(ctx: RubyParser.SingleQuotedSymbolLiteralContext): String = { + ctx.getText + } + + override def visitNilPseudoVariable(ctx: RubyParser.NilPseudoVariableContext): String = { + ctx.getText + } + + override def visitTruePseudoVariable(ctx: RubyParser.TruePseudoVariableContext): String = { + ctx.getText + } + + override def visitFalsePseudoVariable(ctx: RubyParser.FalsePseudoVariableContext): String = { + ctx.getText + } + + override def visitSingleQuotedStringExpression(ctx: RubyParser.SingleQuotedStringExpressionContext): String = { + if (!ctx.isInterpolated) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString("") + } + } + + override def visitQuotedNonExpandedStringLiteral(ctx: RubyParser.QuotedNonExpandedStringLiteralContext): String = { + ctx.getText + } + + override def visitDoubleQuotedStringExpression(ctx: RubyParser.DoubleQuotedStringExpressionContext): String = { + if (!ctx.isInterpolated) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString + } + } + + override def visitDoubleQuotedSymbolLiteral(ctx: RubyParser.DoubleQuotedSymbolLiteralContext): String = { + if (!ctx.isInterpolated) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString + } + } + + override def visitQuotedExpandedStringLiteral(ctx: RubyParser.QuotedExpandedStringLiteralContext): String = { + if (!ctx.isInterpolated) { + ctx.getText + } else { + val b = ctx.interpolations + ctx.children.asScala.map(visit).mkString + } + } + + override def visitQuotedExpandedStringArrayLiteral( + ctx: RubyParser.QuotedExpandedStringArrayLiteralContext + ): String = { + val elements = + if Option(ctx.quotedExpandedArrayElementList()).isDefined then + ctx.quotedExpandedArrayElementList().elements.map(visit).mkString(" ") + else "" + + s"${ctx.QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START.getText}$elements${ctx.QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END.getText}" + } + + override def visitQuotedExpandedSymbolArrayLiteral( + ctx: RubyParser.QuotedExpandedSymbolArrayLiteralContext + ): String = { + val elements = + if Option(ctx.quotedExpandedArrayElementList()).isDefined then + ctx.quotedExpandedArrayElementList().elements.map(visit).mkString(" ") + else "" + + s"${ctx.QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START.getText}$elements${ctx.QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END.getText}" + } + + override def visitQuotedExpandedArrayElement(ctx: RubyParser.QuotedExpandedArrayElementContext): String = { + ctx.quotedExpandedArrayElementContent().asScala.flatMap(_.children.asScala.map(visit)).mkString + } + + override def visitQuotedExpandedLiteralStringContent( + ctx: RubyParser.QuotedExpandedLiteralStringContentContext + ): String = { + Option(ctx.compoundStatement()) match { + case Some(compoundStatement) => + ctx.children.asScala.map(visit).mkString + case None => ctx.getText + } + } + + override def visitRegularExpressionLiteral(ctx: RubyParser.RegularExpressionLiteralContext): String = { + if (ctx.isStatic) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString("") + } + } + + override def visitRegexpLiteralContent(ctx: RubyParser.RegexpLiteralContentContext): String = { + ctx.children.asScala.map(visit).mkString + } + + override def visitQuotedExpandedRegularExpressionLiteral( + ctx: RubyParser.QuotedExpandedRegularExpressionLiteralContext + ): String = { + if (ctx.isStatic) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString + } + } + + override def visitQuotedExpandedExternalCommandLiteral( + ctx: RubyParser.QuotedExpandedExternalCommandLiteralContext + ): String = { + val command = + if ctx.quotedExpandedLiteralStringContent.asScala.nonEmpty then + ctx.quotedExpandedLiteralStringContent.asScala.flatMap(_.children.asScala.map(visit)).mkString("") + else "" + + s"${ctx.QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START.getText}$command${ctx.QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END.getText}" + } + + override def visitDoubleQuotedString(ctx: RubyParser.DoubleQuotedStringContext): String = { + if (!ctx.isInterpolated) { + ctx.getText + } else { + ctx.children.asScala.map(visit).mkString + } + } + + override def visitDoubleQuotedStringContent(ctx: RubyParser.DoubleQuotedStringContentContext): String = { + ctx.children.asScala.map(visit).mkString + } + + override def visitTerminal(node: TerminalNode): String = { + node.getText + } + + override def visitCurlyBracesBlock(ctx: RubyParser.CurlyBracesBlockContext): String = { + val outputSb = new StringBuilder(ctx.LCURLY.getText) + + val params = Option(ctx.blockParameter()).fold(List())(_.parameters).map(visit) + if params.nonEmpty then outputSb.append(s"|${params.mkString(",")}|") + + val body = visit(ctx.compoundStatement()) + if body != "" then outputSb.append(s"$ls$body$ls") + + outputSb.append(ctx.RCURLY.getText).toString + } + + override def visitGroupedParameterList(ctx: RubyParser.GroupedParameterListContext): String = { + s"(${ctx.parameters.map(_.getText).mkString(", ")})" + } + + override def visitDoBlock(ctx: RubyParser.DoBlockContext): String = { + val outputSb = new StringBuilder(ctx.DO.getText) + + val params = Option(ctx.blockParameter()).fold(List())(_.parameters).map(visit).mkString(",") + if params != "" then outputSb.append(s" |$params|") + + outputSb.append(ls) + + val body = visit(ctx.bodyStatement()) + if body != "" then outputSb.append(s"$body$ls") + + outputSb.append(ctx.END.getText).toString + } + + override def visitLocalVariableAssignmentExpression( + ctx: RubyParser.LocalVariableAssignmentExpressionContext + ): String = { + val lhs = visit(ctx.lhs) + val rhs = visit(ctx.rhs) + val op = ctx.assignmentOperator().getText + + s"$lhs $op $rhs" + } + + override def visitSingleAssignmentStatement(ctx: RubyParser.SingleAssignmentStatementContext): String = { + rubyNodeCreator.visitSingleAssignmentStatement(ctx).span.text + } + + override def visitMultipleAssignmentStatement(ctx: RubyParser.MultipleAssignmentStatementContext): String = { + // TODO: fixme - ctx.toTextSpan is being used for individual elements in the building of a MultipleAssignment - needs + // to be fixed so that individual elements span texts can be used to build MultipleAssignment on AstPrinter side. + rubyNodeCreator.visitMultipleAssignmentStatement(ctx).span.text + } + + override def visitMultipleLeftHandSide(ctx: RubyParser.MultipleLeftHandSideContext): String = { + val multiLhs = ctx.multipleLeftHandSideItem.asScala.map(visit).mkString(",") + val packingLhs = Option(ctx.packingLeftHandSide).map(visit).mkString(",") + val procParam = Option(ctx.procParameter).map(visit).mkString(",") + val groupedLhs = Option(ctx.groupedLeftHandSide).map(visit) + + s"$multiLhs $packingLhs $procParam $groupedLhs" + } + + override def visitPackingLeftHandSide(ctx: RubyParser.PackingLeftHandSideContext): String = { + val lhs = visit(ctx.leftHandSide) + val rest = Option(ctx.multipleLeftHandSideItem()).map(_.asScala.map(visit).mkString(",")).getOrElse("") + + s"$lhs $rest" + } + + override def visitMultipleRightHandSide(ctx: RubyParser.MultipleRightHandSideContext): String = { + val rhsStmts = ctx.children.asScala.collect { + case x: RubyParser.SplattingRightHandSideContext => visit(x) :: Nil + case x: RubyParser.OperatorExpressionListContext => (x.operatorExpression.asScala.map(visit).toList) + }.flatten + + if rhsStmts.nonEmpty then rhsStmts.mkString(", ") + else defaultResult() + } + + override def visitSplattingArgument(ctx: RubyParser.SplattingArgumentContext): String = { + val operator = Option(ctx.STAR) match { + case Some(star) => ctx.STAR.getText + case None => ctx.STAR2.getText + } + + s"$operator${visit(ctx.operatorExpression())}" + } + + override def visitAttributeAssignmentExpression(ctx: RubyParser.AttributeAssignmentExpressionContext): String = { + val lhs = visit(ctx.primaryValue()) + val op = ctx.op.getText + val assignmentOp = ctx.assignmentOperator.getText + val memberName = ctx.methodName().getText + val rhs = visit(ctx.operatorExpression()) + + s"$lhs$op$memberName $assignmentOp $rhs" + } + + override def visitSimpleCommand(ctx: RubyParser.SimpleCommandContext): String = { + if (Option(ctx.simpleCommandArgumentList()).map(_.getText).exists(_.startsWith("::"))) { + val memberName = ctx.simpleCommandArgumentList().getText.stripPrefix("::") + val methodIdentifier = visit(ctx.methodIdentifier()) + s"$methodIdentifier::$memberName" + } else if (!ctx.methodIdentifier().isAttrDeclaration) { + val identifierCtx = ctx.methodIdentifier() + val arguments = ctx.simpleCommandArgumentList().arguments.map(visit) + (identifierCtx.getText, arguments) match { + case ("require", List(argument)) => + s"require ${arguments.mkString(",")}" + case ("require_relative", List(argument)) => + s"require_relative ${arguments.mkString(",")}" + case ("require_all", List(argument)) => + s"require_all ${arguments.mkString(",")}" + case ("include", List(argument)) => + s"include ${arguments.mkString(",")}" + case (idAssign, arguments) if idAssign.endsWith("=") => + val argNode = arguments match { + case arg :: Nil => arg + case xs => xs.mkString(", ") + } + s"${idAssign.stripSuffix("=")} = $argNode" + case _ => + s"${visit(identifierCtx)} ${arguments.mkString(",")}" + } + } else { + s"${ctx.simpleCommandArgumentList.arguments.map(visit).mkString(",")}" + } + } + + override def visitSuperWithParentheses(ctx: RubyParser.SuperWithParenthesesContext): String = { + val block = Option(ctx.block()).map(visit) + val arguments = Option(ctx.argumentWithParentheses()).map(_.arguments.map(visit).mkString(",")).getOrElse("") + visitSuperCall(ctx, s"($arguments)", block) + } + + override def visitSuperWithoutParentheses(ctx: RubyParser.SuperWithoutParenthesesContext): String = { + val block = Option(ctx.block()).map(visit) + val arguments = Option(ctx.argumentList()).map(_.elements.map(visit).mkString(",")).getOrElse("") + visitSuperCall(ctx, arguments, block) + } + + private def visitSuperCall(ctx: ParserRuleContext, arguments: String, block: Option[String]): String = { + block match { + case Some(body) => s"super $arguments $body" + case None => s"super $arguments" + } + } + + override def visitIsDefinedExpression(ctx: RubyParser.IsDefinedExpressionContext): String = { + val definedKeyword = visit(ctx.isDefinedKeyword) + val value = visit(ctx.expressionOrCommand()) + s"$definedKeyword($value)" + } + + override def visitIsDefinedCommand(ctx: RubyParser.IsDefinedCommandContext): String = { + val definedKeyword = visit(ctx.isDefinedKeyword) + val value = visit(ctx.primaryValue()) + + s"$definedKeyword $value" + } + + override def visitMethodCallExpression(ctx: RubyParser.MethodCallExpressionContext): String = { + val identifier = visit(ctx.methodOnlyIdentifier()) + s"$identifier" + } + + override def visitMethodCallWithBlockExpression(ctx: RubyParser.MethodCallWithBlockExpressionContext): String = { + ctx.methodIdentifier().getText match { + case Defines.Proc | Defines.Lambda => s"${ctx.methodIdentifier().getText} ${visit(ctx.block())}" + case Defines.Loop => + ctx.block() match { + case b: RubyParser.DoBlockBlockContext => + val body = visit(b.doBlock().bodyStatement) + s"${Defines.Loop} do$ls$body${ls}break if false${ls}end" + case y => + val body = visit(ctx.block()) + s"${Defines.Loop}$ls$body${ls}end" + } + case _ => + val methodIdent = visit(ctx.methodIdentifier) + val body = visit(ctx.block) + + ctx.block() match { + case x: RubyParser.DoBlockBlockContext => s"$methodIdent $body" + case y => s"$methodIdent {$ls$body$ls}" + } + } + } + + override def visitLambdaExpression(ctx: RubyParser.LambdaExpressionContext): String = { + val outputSb = new StringBuilder(ctx.MINUSGT.getText) + + val params = Option(ctx.lambdaExpressionParameterList()) match { + case Some(parameterList) => + Option(parameterList.blockParameterList()).fold(List())(_.parameters).map(visit).mkString(",") + case None => "" + } + + val body = visit(ctx.block()) + + if params != "" then outputSb.append(s"($params)") + if body != "" then outputSb.append(s" $body") + + outputSb.toString + } + + override def visitMethodCallWithParenthesesExpression( + ctx: RubyParser.MethodCallWithParenthesesExpressionContext + ): String = { + val outputSb = new StringBuilder() + + val identifier = visit(ctx.methodIdentifier()) + outputSb.append(identifier) + + val args = ctx.argumentWithParentheses().arguments.map(visit).mkString(",") + + if ctx.argumentWithParentheses().isArrayArgumentList then outputSb.append(s"([$args])") + else outputSb.append(s"($args)") + + if Option(ctx.block).isDefined then outputSb.append(s" ${visit(ctx.block)}") + outputSb.toString + } + + override def visitYieldExpression(ctx: RubyParser.YieldExpressionContext): String = { + val outputSb = new StringBuilder(ctx.YIELD.getText) + val args = Option(ctx.argumentWithParentheses()).iterator.flatMap(_.arguments).map(visit) + if args.nonEmpty then outputSb.append(s"(${args.mkString(",")})") + + outputSb.toString + } + + override def visitYieldMethodInvocationWithoutParentheses( + ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext + ): String = { + val args = ctx.primaryValueListWithAssociation().elements.map(visit).mkString(",") + s"${ctx.YIELD.getText} $args" + } + + override def visitMemberAccessCommand(ctx: RubyParser.MemberAccessCommandContext): String = { + val op = + if Option(ctx.AMPDOT()).isDefined then ctx.AMPDOT().getText + else if Option(ctx.COLON2()).isDefined then ctx.COLON2().getText + else ctx.DOT().getText + + val args = ctx.commandArgument.arguments.map(visit).mkString(", ") + val methodName = visit(ctx.methodName) + val base = visit(ctx.primary()) + + s"$base$op$methodName $args" + } + + override def visitConstantIdentifierVariable(ctx: RubyParser.ConstantIdentifierVariableContext): String = { + ctx.getText + } + + override def visitGlobalIdentifierVariable(ctx: RubyParser.GlobalIdentifierVariableContext): String = { + ctx.getText + } + + override def visitClassIdentifierVariable(ctx: RubyParser.ClassIdentifierVariableContext): String = { + ctx.getText + } + + override def visitInstanceIdentifierVariable(ctx: RubyParser.InstanceIdentifierVariableContext): String = { + ctx.getText + } + + override def visitLocalIdentifierVariable(ctx: RubyParser.LocalIdentifierVariableContext): String = { + ctx.getText + } + + override def visitClassName(ctx: RubyParser.ClassNameContext): String = { + ctx.getText + } + + override def visitMethodIdentifier(ctx: RubyParser.MethodIdentifierContext): String = { + ctx.getText + } + + override def visitMethodOnlyIdentifier(ctx: RubyParser.MethodOnlyIdentifierContext): String = { + ctx.getText + } + + override def visitIsDefinedKeyword(ctx: RubyParser.IsDefinedKeywordContext): String = { + ctx.getText + } + + override def visitLinePseudoVariable(ctx: RubyParser.LinePseudoVariableContext): String = { + ctx.getText + } + + override def visitFilePseudoVariable(ctx: RubyParser.FilePseudoVariableContext): String = { + ctx.getText + } + + override def visitEncodingPseudoVariable(ctx: RubyParser.EncodingPseudoVariableContext): String = { + ctx.getText + } + + override def visitSelfPseudoVariable(ctx: RubyParser.SelfPseudoVariableContext): String = { + ctx.getText + } + + override def visitMemberAccessExpression(ctx: RubyParser.MemberAccessExpressionContext): String = { + val hasArgs = Option(ctx.argumentWithParentheses()).isDefined + val hasBlock = Option(ctx.block()).isDefined + val methodName = ctx.methodName.getText + val isClassDecl = + Option(ctx.primaryValue()).map(_.getText).contains("Class") && Option(ctx.methodName()) + .map(_.getText) + .contains("new") + + if (!hasBlock) { + val target = visit(ctx.primaryValue()) + if (methodName == "new") { + if (!hasArgs) { + return s"$target.$methodName" + } else { + val args = ctx.argumentWithParentheses().arguments.map(visit).mkString(",") + return s"$target.$methodName($args)" + } + } else { + if (!hasArgs) { + return s"$target${ctx.op.getText}$methodName" + } else { + val args = ctx.argumentWithParentheses().arguments.map(visit).mkString(",") + return s"$target${ctx.op.getText}$methodName($args)" + } + } + } + + if (hasBlock && isClassDecl) { + val block = visit(ctx.block()) + } else if (hasBlock) { + val block = visit(ctx.block()) + val target = visit(ctx.primaryValue()) + + if (methodName == "new") { + val callStr = s"$target.$methodName" + + if (!hasArgs) { + return s"$target.$methodName $block" + } else { + val args = ctx.argumentWithParentheses().arguments.map(visit).mkString(",") + return s"$target.$methodName($args) $block" + } + } else { + return s"$target${ctx.op.getText}$methodName $block" + } + } + + defaultResult() + } + + override def visitConstantVariableReference(ctx: RubyParser.ConstantVariableReferenceContext): String = { + s"self::${ctx.CONSTANT_IDENTIFIER().getText}" + } + + override def visitIndexingAccessExpression(ctx: RubyParser.IndexingAccessExpressionContext): String = { + val target = visit(ctx.primaryValue()) + val arg = Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit).mkString(",") + + s"$target${ctx.LBRACK.getText}$arg${ctx.RBRACK.getText}" + } + + override def visitBracketAssignmentExpression(ctx: RubyParser.BracketAssignmentExpressionContext): String = { + val op = ctx.assignmentOperator().getText + + if (op != "=") { + defaultResult() + } + + val lhsBase = visit(ctx.primaryValue()) + val lhsArgs = Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit) + val rhs = visit(ctx.operatorExpression()) + + s"$lhsBase[${lhsArgs.mkString(",")}] $op ${rhs}" + } + + override def visitBracketedArrayLiteral(ctx: RubyParser.BracketedArrayLiteralContext): String = { + val args = Option(ctx.bracketedArrayElementList()).map(_.elements).getOrElse(List()).map(visit).mkString(",") + + s"${ctx.LBRACK.getText}$args${ctx.RBRACK.getText}" + } + + override def visitQuotedNonExpandedStringArrayLiteral( + ctx: RubyParser.QuotedNonExpandedStringArrayLiteralContext + ): String = { + val ctxElements = Option(ctx.quotedNonExpandedArrayElementList()) + + val elements = ctxElements + .map(_.elements) + .getOrElse(List()) + .map(_.getText) + + val sep = + if elements.nonEmpty then + ctxElements + .map(_.NON_EXPANDED_ARRAY_ITEM_SEPARATOR().asScala) + .getOrElse(List()) + .map(_.getText) + .headOption + .getOrElse("") + else "" + + val elementsString = elements.mkString(sep) + + s"${ctx.QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START()}$elementsString${ctx.QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END()}" + } + + override def visitQuotedNonExpandedSymbolArrayLiteral( + ctx: RubyParser.QuotedNonExpandedSymbolArrayLiteralContext + ): String = { + val ctxElements = Option(ctx.quotedNonExpandedArrayElementList()) + + val elements = ctxElements + .map(_.elements) + .getOrElse(List()) + .map(_.getText) + + val sep = + if elements.nonEmpty then + ctxElements + .map(_.NON_EXPANDED_ARRAY_ITEM_SEPARATOR().asScala) + .getOrElse(List()) + .map(_.getText) + .headOption + .getOrElse("") + else "" + + val elementsString = elements.mkString(sep) + + s"${ctx.QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START.getText}$elementsString${ctx.QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END.getText}" + } + + override def visitBoundedRangeExpression(ctx: RubyParser.BoundedRangeExpressionContext): String = { + val lowerBound = visit(ctx.primaryValue(0)) + val upperBound = visit(ctx.primaryValue(1)) + val op = visit(ctx.rangeOperator()) + + s"$lowerBound$op$upperBound" + } + + override def visitEndlessRangeExpression(ctx: RubyParser.EndlessRangeExpressionContext): String = { + val lowerBound = visit(ctx.primaryValue) + val op = ctx.rangeOperator().getText + + s"$lowerBound${op}Float::INFINITY" + } + + override def visitBeginlessRangeExpression(ctx: RubyParser.BeginlessRangeExpressionContext): String = { + val upperBound = visit(ctx.primaryValue) + val op = ctx.rangeOperator().getText + + s"-Float::INFINITY$op$upperBound" + } + + override def visitRangeOperator(ctx: RubyParser.RangeOperatorContext): String = { + ctx.getText + } + + override def visitHashLiteral(ctx: RubyParser.HashLiteralContext): String = { + val outputSb = new StringBuilder(ctx.LCURLY.getText) + val assocList = Option(ctx.associationList()).map(_.associations).getOrElse(List()).map(visit).mkString(",") + if assocList != "" then outputSb.append(s"$assocList") + outputSb.append(ctx.RCURLY.getText) + outputSb.toString + } + + override def visitAssociationElement(ctx: RubyParser.AssociationElementContext): String = { + val assocOp = Option(ctx.COLON()) match { + case Some(colon) => ":" + case None => "=>" + } + + val opExpression = visit(ctx.operatorExpression()) + + ctx.associationKey().getText match { + case "if" => + s"${ctx.associationKey().getText}$assocOp $opExpression" + case _ => + val assocKey = visit(ctx.associationKey()) + s"$assocKey$assocOp $opExpression" + } + } + + override def visitAssociationHashArgument(ctx: RubyParser.AssociationHashArgumentContext): String = { + val identifierName = Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText) + + val body = identifierName match { + case Some(identifierName) => identifierName + case None => + if ctx.LPAREN() == null then visit(ctx.methodCallsWithParentheses()) + else visit(ctx.methodInvocationWithoutParentheses()) + } + + s"${ctx.STAR2().getText}$body" + } + + override def visitModuleDefinition(ctx: RubyParser.ModuleDefinitionContext): String = { + val outputSb = new StringBuilder(ctx.MODULE.getText) + + val (nonFieldStmts, fields) = rubyNodeCreator.genInitFieldStmts(ctx.bodyStatement()) + + val moduleName = visit(ctx.classPath()) + + outputSb.append(s" $moduleName$ls") + if fields.nonEmpty then outputSb.append(fields.mkString(ls)) + + outputSb.append(nonFieldStmts.span.text).append(s"$ls${ctx.END.getText}").toString + } + + override def visitSingletonClassDefinition(ctx: RubyParser.SingletonClassDefinitionContext): String = { + val outputSb = new StringBuilder() + + val baseClass = Option(ctx.commandOrPrimaryValueClass()).map(visit) + val body = visit(ctx.bodyStatement()) + + baseClass match { + case Some(baseClass) if baseClass == "self" => + outputSb.append(ctx.CLASS.getText).append(s" << $baseClass.${freshClassName()}") + if body != "" then outputSb.append(s"$ls$body") + outputSb.append(s"$ls${ctx.END.getText}") + outputSb.toString + case Some(baseClass) => + outputSb.append(ctx.CLASS.getText).append(s" << $baseClass") + if body != "" then outputSb.append(s"$ls$body") + outputSb.append(s"$ls${ctx.END.getText}").toString + case None => + s"${ctx.CLASS.getText} ${freshClassName()}$ls$body$ls${ctx.END.getText}" + } + } + + override def visitClassDefinition(ctx: RubyParser.ClassDefinitionContext): String = { + val (nonFieldStmts, fields) = rubyNodeCreator.genInitFieldStmts(ctx.bodyStatement()) + + val stmts = rubyNodeCreator.lowerAliasStatementsToMethods(nonFieldStmts) + + val classBody = rubyNodeCreator.filterNonAllowedTypeDeclChildren(stmts) + val className = visit(ctx.classPath()) + + s"class $className$ls${classBody.span.text}${ls}end" + } + + override def visitMethodDefinition(ctx: RubyParser.MethodDefinitionContext): String = { + val outputSb = new StringBuilder(s"${ctx.DEF.getText} ${ctx.definedMethodName.getText}") + + val params = Option(ctx.methodParameterPart().parameterList()).fold(List())(_.parameters).map(visit) + if params.nonEmpty then outputSb.append(s"(${params.mkString(",")})") + + val methodBody = visit(ctx.bodyStatement()) + if methodBody != "" then outputSb.append(s"$ls$methodBody") + + outputSb.append(s"$ls${ctx.END.getText}").toString + } + + override def visitEndlessMethodDefinition(ctx: RubyParser.EndlessMethodDefinitionContext): String = { + val outputSb = new StringBuilder(s"${ctx.DEF.getText} ${ctx.definedMethodName.getText}") + + val params = Option(ctx.parameterList()).fold(List())(_.parameters).map(visit) + if params.nonEmpty then outputSb.append(s"${ctx.LPAREN.getText}${params.mkString(",")}${ctx.RPAREN.getText}") + + outputSb.append(s" ${ctx.EQ.getText}") + val body = visit(ctx.statement()) + if body != "" then outputSb.append(s" $body") + + outputSb.toString + } + + override def visitSingletonMethodDefinition(ctx: RubyParser.SingletonMethodDefinitionContext): String = { + val target = visit(ctx.singletonObject()) + val op = ctx.op.getText + val methodName = ctx.definedMethodName().getText + val params = Option(ctx.methodParameterPart().parameterList()).fold(List())(_.parameters).map(visit).mkString(",") + val body = visit(ctx.bodyStatement()) + + if Option(ctx.methodParameterPart().LPAREN()).isDefined then + s"${ctx.DEF.getText} $target$op$methodName ($params)$ls$body$ls${ctx.END.getText}" + else s"${ctx.DEF.getText} $target$op$methodName $params$ls$body$ls${ctx.END.getText}" + } + + override def visitProcParameter(ctx: RubyParser.ProcParameterContext): String = { + val identName = + Option(ctx.procParameterName()).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText) + s"${ctx.AMP().getText}$identName" + } + + override def visitHashParameter(ctx: RubyParser.HashParameterContext): String = { + val identName = Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText) + s"${ctx.STAR2().getText}$identName" + } + + override def visitArrayParameter(ctx: RubyParser.ArrayParameterContext): String = { + val identName = Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText) + if identName == ctx.STAR.getText then ctx.STAR.getText + else s"${ctx.STAR.getText}$identName" + } + + override def visitOptionalParameter(ctx: RubyParser.OptionalParameterContext): String = { + val paramName = ctx.optionalParameterName().LOCAL_VARIABLE_IDENTIFIER().getText + val value = visit(ctx.operatorExpression()) + val op = + if Option(ctx.COLON()).isDefined then ctx.COLON().getText + else ctx.EQ().getText + + s"$paramName$op$value" + } + + override def visitMandatoryParameter(ctx: RubyParser.MandatoryParameterContext): String = { + val paramName = ctx.LOCAL_VARIABLE_IDENTIFIER().getText + val op = Option(ctx.COLON) match { + case Some(colon) => ctx.COLON.getText + case None => "" + } + + s"$paramName$op" + } + + override def visitVariableLeftHandSide(ctx: RubyParser.VariableLeftHandSideContext): String = { + s"${ctx.getText}" + } + + override def visitBodyStatement(ctx: RubyParser.BodyStatementContext): String = { + val body = visit(ctx.compoundStatement()) + val rescueClause = Option(ctx.rescueClause.asScala).fold(List())(_.map(visit)) + val elseClause = Option(ctx.elseClause()).map(visit).getOrElse("") + val ensureClause = Option(ctx.ensureClause).map(visit).getOrElse("") + + if (rescueClause.isEmpty && elseClause.isEmpty && ensureClause.isEmpty) { + body + } else { + val outputSb = new StringBuilder(body) + if rescueClause.nonEmpty then outputSb.append(s"$ls${rescueClause.mkString(ls)}") + if elseClause.nonEmpty then outputSb.append(s"$elseClause$ls") + if ensureClause.nonEmpty then outputSb.append(s"$ensureClause") + + outputSb.toString + } + } + + override def visitExceptionClassList(ctx: RubyParser.ExceptionClassListContext): String = { + Option(ctx.multipleRightHandSide()).map(visitMultipleRightHandSide).getOrElse(visit(ctx.operatorExpression())) + } + + override def visitRescueClause(ctx: RubyParser.RescueClauseContext): String = { + val outputSb = new StringBuilder(ctx.RESCUE().getText) + val exceptionClassList = Option(ctx.exceptionClassList).map(visit).getOrElse("") + val variables = Option(ctx.exceptionVariableAssignment).map(visit).getOrElse("") + val thenClause = visit(ctx.thenClause) + + val thenKeyword = + if Option(ctx.thenClause().THEN()).isDefined then s" ${ctx.thenClause().THEN().getText}" + else "" + + if exceptionClassList != "" then outputSb.append(s" $exceptionClassList") + if variables != "" then outputSb.append(s" => $variables") + + outputSb.append(thenKeyword) + + if thenClause != "" then outputSb.append(s"\n${thenClause}") + + outputSb.toString() + } + + override def visitEnsureClause(ctx: RubyParser.EnsureClauseContext): String = { + val stmt = visit(ctx.compoundStatement) + s"${ctx.ENSURE().getText}$ls$stmt" + } + + override def visitCaseWithExpression(ctx: RubyParser.CaseWithExpressionContext): String = { + val outputSb = new StringBuilder(ctx.CASE.getText) + + val expression = Option(ctx.expressionOrCommand).map(visit) + if expression.isDefined then outputSb.append(s" ${expression.get}") + + val whenClauses = Option(ctx.whenClause().asScala).fold(List())(_.map(visit)) + if whenClauses.nonEmpty then outputSb.append(s"$ls${whenClauses.mkString}") + + val elseClause = Option(ctx.elseClause()).map(visit) + if elseClause.isDefined then outputSb.append(s"${elseClause.get}$ls") + + outputSb.append(s"${ctx.END.getText}").toString + } + + override def visitCaseWithoutExpression(ctx: RubyParser.CaseWithoutExpressionContext): String = { + val whenClauses = Option(ctx.whenClause().asScala).fold(List())(_.map(visit)).mkString(ls) + val elseClause = Option(ctx.elseClause()).map(visit) + + val op = + if Option(ctx.SEMI()).isDefined then ";" + else ls + s"${ctx.CASE().getText}$op$whenClauses$elseClause" + } + + override def visitWhenClause(ctx: RubyParser.WhenClauseContext): String = { + val outputSb = new StringBuilder(ctx.WHEN.getText) + + val whenArgs = ctx.whenArgument() + val matchArgs = + Option(whenArgs.operatorExpressionList()).iterator.flatMap(_.operatorExpression().asScala).map(visit) + val matchSplatArg = Option(whenArgs.splattingArgument()).map(visit) + val thenClause = visit(ctx.thenClause()) + + if matchArgs.nonEmpty then + val matchArgsStr = matchArgs.mkString(",") + outputSb.append(s" $matchArgsStr") + + if matchSplatArg.isDefined then outputSb.append(s", ${matchSplatArg.get}") + else if matchSplatArg.isDefined then outputSb.append(s" ${matchSplatArg.get}") + + if Option(ctx.thenClause().THEN).isDefined then outputSb.append(s" ${ctx.thenClause.THEN.getText}") + if thenClause != "" then outputSb.append(s"$ls$thenClause") + + outputSb.append(ls).toString + } + + override def visitAssociationKey(ctx: RubyParser.AssociationKeyContext): String = { + Option(ctx.operatorExpression()) match { + case Some(ctx) => visit(ctx) + case None => ctx.getText + } + } + + override def visitAliasStatement(ctx: RubyParser.AliasStatementContext): String = { + s"${ctx.ALIAS.getText} ${ctx.oldName.getText} ${ctx.newName.getText}" + } + + override def visitBreakWithoutArguments(ctx: RubyParser.BreakWithoutArgumentsContext): String = { + ctx.BREAK.getText + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/KeywordHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/KeywordHandling.scala new file mode 100644 index 000000000000..6de85ae30c7c --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/KeywordHandling.scala @@ -0,0 +1,76 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.parser.RubyLexer.* + +trait KeywordHandling { this: RubyLexerBase => + + val keywordMap: Map[String, Int] = Map( + "LINE__" -> LINE__, + "ENCODING__" -> ENCODING__, + "FILE__" -> FILE__, + "BEGIN_" -> BEGIN_, + "END_" -> END_, + "ALIAS" -> ALIAS, + "AND" -> AND, + "BEGIN" -> BEGIN, + "BREAK" -> BREAK, + "CASE" -> CASE, + "CLASS" -> CLASS, + "DEF" -> DEF, + "IS_DEFINED" -> IS_DEFINED, + "DO" -> DO, + "ELSE" -> ELSE, + "ELSIF" -> ELSIF, + "END" -> END, + "ENSURE" -> ENSURE, + "FOR" -> FOR, + "FALSE" -> FALSE, + "IF" -> IF, + "IN" -> IN, + "MODULE" -> MODULE, + "NEXT" -> NEXT, + "NIL" -> NIL, + "NOT" -> NOT, + "OR" -> OR, + "REDO" -> REDO, + "RESCUE" -> RESCUE, + "RETRY" -> RETRY, + "RETURN" -> RETURN, + "SELF" -> SELF, + "SUPER" -> SUPER, + "THEN" -> THEN, + "TRUE" -> TRUE, + "UNDEF" -> UNDEF, + "UNLESS" -> UNLESS, + "UNTIL" -> UNTIL, + "WHEN" -> WHEN, + "WHILE" -> WHILE, + "YIELD" -> YIELD + ) + + private def isPreviousTokenColonOrDot: Boolean = { + val previousToken = previousTokenTypeOrEOF() + previousToken == RubyLexer.DOT || previousToken == RubyLexer.COLON || previousToken == RubyLexer.COLON2 + } + + private def isNextTokenColonOrDot: Boolean = { + _input.LA(1) == '.' || _input.LA(1) == ':' + } + + def setKeywordTokenType(): Unit = { + val tokenText = getText + if (tokenText == null) { + return + } + + if ( + isPreviousTokenColonOrDot || (isNextTokenColonOrDot && tokenText.toUpperCase != "SELF") || !keywordMap.contains( + tokenText.toUpperCase + ) + ) { + setType(RubyLexer.LOCAL_VARIABLE_IDENTIFIER) + } else { + keywordMap.get(tokenText.toUpperCase).foreach(setType) + } + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala index d3aea681c180..e63249d965ba 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala @@ -10,6 +10,7 @@ abstract class RubyLexerBase(input: CharStream) with RegexLiteralHandling with InterpolationHandling with QuotedLiteralHandling + with KeywordHandling with HereDocHandling { /** The previously (non-WS) emitted token (in DEFAULT_CHANNEL.) */ diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index 4f5ec2560ea3..b4b8b155e9e0 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -1,20 +1,26 @@ package io.joern.rubysrc2cpg.parser -import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* +import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{Block, *} import io.joern.rubysrc2cpg.parser.AntlrContextHelpers.* -import io.joern.rubysrc2cpg.parser.RubyParser.{CommandWithDoBlockContext, ConstantVariableReferenceContext} +import io.joern.rubysrc2cpg.parser.RubyParser.* import io.joern.rubysrc2cpg.passes.Defines -import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType +import io.joern.rubysrc2cpg.passes.Defines.{Self, getBuiltInType} +import io.joern.rubysrc2cpg.passes.GlobalTypes.builtinPrefix import io.joern.rubysrc2cpg.utils.FreshNameGenerator -import io.joern.x2cpg.Defines as XDefines +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.ImportsPass +import org.antlr.v4.runtime.ParserRuleContext import org.antlr.v4.runtime.tree.{ParseTree, RuleNode} import org.slf4j.LoggerFactory +import scala.annotation.tailrec import scala.jdk.CollectionConverters.* /** Converts an ANTLR Ruby Parse Tree into the intermediate Ruby AST. */ -class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { +class RubyNodeCreator( + variableNameGen: FreshNameGenerator[String] = FreshNameGenerator(id => s""), + procParamGen: FreshNameGenerator[Left[String, Nothing]] = FreshNameGenerator(id => Left(s"")) +) extends RubyParserBaseVisitor[RubyExpression] { private val logger = LoggerFactory.getLogger(getClass) private val classNameGen = FreshNameGenerator(id => s"") @@ -23,26 +29,30 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { SimpleIdentifier(None)(span.spanStart(classNameGen.fresh)) } - private def defaultTextSpan(code: String = ""): TextSpan = TextSpan(None, None, None, None, code) + private def defaultTextSpan(code: String = ""): TextSpan = TextSpan(None, None, None, None, None, code) - override def defaultResult(): RubyNode = Unknown()(defaultTextSpan()) + override def defaultResult(): RubyExpression = Unknown()(defaultTextSpan()) - override protected def shouldVisitNextChild(node: RuleNode, currentResult: RubyNode): Boolean = + override protected def shouldVisitNextChild(node: RuleNode, currentResult: RubyExpression): Boolean = currentResult.isInstanceOf[Unknown] - override def visit(tree: ParseTree): RubyNode = { + override def visit(tree: ParseTree): RubyExpression = { Option(tree).map(super.visit).getOrElse(defaultResult()) } - override def visitProgram(ctx: RubyParser.ProgramContext): RubyNode = { + override def visitProgram(ctx: RubyParser.ProgramContext): RubyExpression = { visit(ctx.compoundStatement()) } - override def visitCompoundStatement(ctx: RubyParser.CompoundStatementContext): RubyNode = { + override def visitCompoundStatement(ctx: RubyParser.CompoundStatementContext): RubyExpression = { StatementList(ctx.getStatements.map(visit))(ctx.toTextSpan) } - override def visitGroupingStatement(ctx: RubyParser.GroupingStatementContext): RubyNode = { + override def visitNextWithoutArguments(ctx: RubyParser.NextWithoutArgumentsContext): RubyExpression = { + NextExpression()(ctx.toTextSpan) + } + + override def visitGroupingStatement(ctx: RubyParser.GroupingStatementContext): RubyExpression = { // When there's only 1 statement, we can use it directly, instead of wrapping it in a StatementList. val statements = ctx.compoundStatement().getStatements.map(visit) if (statements.size == 1) { @@ -52,27 +62,27 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitStatements(ctx: RubyParser.StatementsContext): RubyNode = { + override def visitStatements(ctx: RubyParser.StatementsContext): RubyExpression = { StatementList(ctx.statement().asScala.map(visit).toList)(ctx.toTextSpan) } - override def visitWhileExpression(ctx: RubyParser.WhileExpressionContext): RubyNode = { + override def visitWhileExpression(ctx: RubyParser.WhileExpressionContext): RubyExpression = { val condition = visit(ctx.expressionOrCommand()) val body = visit(ctx.doClause()) WhileExpression(condition, body)(ctx.toTextSpan) } - override def visitUntilExpression(ctx: RubyParser.UntilExpressionContext): RubyNode = { + override def visitUntilExpression(ctx: RubyParser.UntilExpressionContext): RubyExpression = { val condition = visit(ctx.expressionOrCommand()) val body = visit(ctx.doClause()) UntilExpression(condition, body)(ctx.toTextSpan) } - override def visitBeginEndExpression(ctx: RubyParser.BeginEndExpressionContext): RubyNode = { + override def visitBeginEndExpression(ctx: RubyParser.BeginEndExpressionContext): RubyExpression = { visit(ctx.bodyStatement()) } - override def visitIfExpression(ctx: RubyParser.IfExpressionContext): RubyNode = { + override def visitIfExpression(ctx: RubyParser.IfExpressionContext): RubyExpression = { val condition = visit(ctx.expressionOrCommand()) val thenBody = visit(ctx.thenClause()) val elsifs = ctx.elsifClause().asScala.map(visit).toList @@ -80,34 +90,34 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { IfExpression(condition, thenBody, elsifs, elseBody)(ctx.toTextSpan) } - override def visitElsifClause(ctx: RubyParser.ElsifClauseContext): RubyNode = { + override def visitElsifClause(ctx: RubyParser.ElsifClauseContext): RubyExpression = { ElsIfClause(visit(ctx.expressionOrCommand()), visit(ctx.thenClause()))(ctx.toTextSpan) } - override def visitElseClause(ctx: RubyParser.ElseClauseContext): RubyNode = { + override def visitElseClause(ctx: RubyParser.ElseClauseContext): RubyExpression = { ElseClause(visit(ctx.compoundStatement()))(ctx.toTextSpan) } - override def visitUnlessExpression(ctx: RubyParser.UnlessExpressionContext): RubyNode = { + override def visitUnlessExpression(ctx: RubyParser.UnlessExpressionContext): RubyExpression = { val condition = visit(ctx.expressionOrCommand()) val thenBody = visit(ctx.thenClause()) val elseBody = Option(ctx.elseClause()).map(visit) UnlessExpression(condition, thenBody, elseBody)(ctx.toTextSpan) } - override def visitForExpression(ctx: RubyParser.ForExpressionContext): RubyNode = { + override def visitForExpression(ctx: RubyParser.ForExpressionContext): RubyExpression = { val forVariable = visit(ctx.forVariable()) val iterableVariable = visit(ctx.commandOrPrimaryValue()) val doBlock = visit(ctx.doClause()) ForExpression(forVariable, iterableVariable, doBlock)(ctx.toTextSpan) } - override def visitForVariable(ctx: RubyParser.ForVariableContext): RubyNode = { + override def visitForVariable(ctx: RubyParser.ForVariableContext): RubyExpression = { if (ctx.leftHandSide() != null) visit(ctx.leftHandSide()) else visit(ctx.multipleLeftHandSide()) } - override def visitModifierStatement(ctx: RubyParser.ModifierStatementContext): RubyNode = { + override def visitModifierStatement(ctx: RubyParser.ModifierStatementContext): RubyExpression = { ctx.statementModifier().getText match case "if" => val condition = visit(ctx.expressionOrCommand()) @@ -141,7 +151,19 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { Unknown()(ctx.toTextSpan) } - override def visitTernaryOperatorExpression(ctx: RubyParser.TernaryOperatorExpressionContext): RubyNode = { + override def visitCommandTernaryOperatorExpression(ctx: CommandTernaryOperatorExpressionContext): RubyExpression = { + val condition = visit(ctx.operatorExpression(0)) + val thenBody = visit(ctx.operatorExpression(1)) + val elseBody = visit(ctx.operatorExpression(2)) + IfExpression( + condition, + thenBody, + List.empty, + Option(ElseClause(StatementList(elseBody :: Nil)(elseBody.span))(elseBody.span)) + )(ctx.toTextSpan) + } + + override def visitTernaryOperatorExpression(ctx: RubyParser.TernaryOperatorExpressionContext): RubyExpression = { val condition = visit(ctx.operatorExpression(0)) val thenBody = visit(ctx.operatorExpression(1)) val elseBody = visit(ctx.operatorExpression(2)) @@ -155,12 +177,20 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { override def visitReturnMethodInvocationWithoutParentheses( ctx: RubyParser.ReturnMethodInvocationWithoutParenthesesContext - ): RubyNode = { - val expressions = ctx.primaryValueList().primaryValue().asScala.map(visit).toList + ): RubyExpression = { + val expressions = Option(ctx.primaryValueListWithAssociation().methodInvocationWithoutParentheses()) match { + case Some(methodInvocation) => visit(methodInvocation) :: Nil + case None => ctx.primaryValueListWithAssociation().elements.map(visit).toList + } + ReturnExpression(expressions)(ctx.toTextSpan) } - override def visitNumericLiteral(ctx: RubyParser.NumericLiteralContext): RubyNode = { + override def visitReturnWithoutArguments(ctx: RubyParser.ReturnWithoutArgumentsContext): RubyExpression = { + ReturnExpression(Nil)(ctx.toTextSpan) + } + + override def visitNumericLiteral(ctx: RubyParser.NumericLiteralContext): RubyExpression = { if (ctx.hasSign) { UnaryExpression(ctx.sign.getText, visit(ctx.unsignedNumericLiteral()))(ctx.toTextSpan) } else { @@ -168,19 +198,19 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitUnaryExpression(ctx: RubyParser.UnaryExpressionContext): RubyNode = { + override def visitUnaryExpression(ctx: RubyParser.UnaryExpressionContext): RubyExpression = { UnaryExpression(ctx.unaryOperator().getText, visit(ctx.primaryValue()))(ctx.toTextSpan) } - override def visitUnaryMinusExpression(ctx: RubyParser.UnaryMinusExpressionContext): RubyNode = { + override def visitUnaryMinusExpression(ctx: RubyParser.UnaryMinusExpressionContext): RubyExpression = { UnaryExpression(ctx.MINUS().getText, visit(ctx.primaryValue()))(ctx.toTextSpan) } - override def visitNotExpressionOrCommand(ctx: RubyParser.NotExpressionOrCommandContext): RubyNode = { + override def visitNotExpressionOrCommand(ctx: RubyParser.NotExpressionOrCommandContext): RubyExpression = { UnaryExpression(ctx.NOT().getText, visit(ctx.expressionOrCommand()))(ctx.toTextSpan) } - override def visitCommandExpressionOrCommand(ctx: RubyParser.CommandExpressionOrCommandContext): RubyNode = { + override def visitCommandExpressionOrCommand(ctx: RubyParser.CommandExpressionOrCommandContext): RubyExpression = { val methodInvocation = visit(ctx.methodInvocationWithoutParentheses()) if (Option(ctx.EMARK()).isDefined) { UnaryExpression(ctx.EMARK().getText, methodInvocation)(ctx.toTextSpan) @@ -189,147 +219,151 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitCommandWithDoBlock(ctx: CommandWithDoBlockContext): RubyNode = { + override def visitCommandWithDoBlock(ctx: CommandWithDoBlockContext): RubyExpression = { val name = Option(ctx.methodIdentifier()).orElse(Option(ctx.methodName())).map(visit).getOrElse(defaultResult()) val arguments = ctx.arguments.map(visit) val block = visit(ctx.doBlock()).asInstanceOf[Block] SimpleCallWithBlock(name, arguments, block)(ctx.toTextSpan) } - override def visitHereDocs(ctx: RubyParser.HereDocsContext): RubyNode = { + override def visitHereDocs(ctx: RubyParser.HereDocsContext): RubyExpression = { HereDocNode(ctx.hereDoc().getText)(ctx.toTextSpan) } - override def visitPrimaryOperatorExpression(ctx: RubyParser.PrimaryOperatorExpressionContext): RubyNode = { + override def visitPrimaryOperatorExpression(ctx: RubyParser.PrimaryOperatorExpressionContext): RubyExpression = { super.visitPrimaryOperatorExpression(ctx) match { - case x: BinaryExpression if x.lhs.text.endsWith("=") && x.op == "*" => + case expr @ BinaryExpression(SimpleCall(lhs: SimpleIdentifier, Nil), "*", rhs) if lhs.text.endsWith("=") => // fixme: This workaround handles a parser ambiguity with method identifiers having `=` and assignments with // splatting on the RHS. The Ruby parser gives precedence to assignments over methods called with this suffix - // however - val newLhs = x.lhs match { - case call: SimpleCall => SimpleIdentifier(None)(call.span.spanStart(call.span.text.stripSuffix("="))) - case y => - logger.warn(s"Unhandled class in repacking of primary operator expression ${y.getClass}") - y - } - val newRhs = { - val oldRhsSpan = x.rhs.span - SplattingRubyNode(x.rhs)(oldRhsSpan.spanStart(s"*${oldRhsSpan.text}")) - } - SingleAssignment(newLhs, "=", newRhs)(x.span) + // however. See https://github.com/joernio/joern/issues/4775 + val newLhs = SimpleIdentifier(None)(lhs.span.spanStart(lhs.span.text.stripSuffix("="))) + val newRhs = SplattingRubyNode(rhs)(rhs.span.spanStart(s"*${rhs.span.text}")) + SingleAssignment(newLhs, "=", newRhs)(expr.span) case x => x } } - override def visitPowerExpression(ctx: RubyParser.PowerExpressionContext): RubyNode = { + override def visitPowerExpression(ctx: RubyParser.PowerExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.powerOperator.getText, visit(ctx.primaryValue(1)))(ctx.toTextSpan) } - override def visitAdditiveExpression(ctx: RubyParser.AdditiveExpressionContext): RubyNode = { + override def visitAdditiveExpression(ctx: RubyParser.AdditiveExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.additiveOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitMultiplicativeExpression(ctx: RubyParser.MultiplicativeExpressionContext): RubyNode = { + override def visitMultiplicativeExpression(ctx: RubyParser.MultiplicativeExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.multiplicativeOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitLogicalAndExpression(ctx: RubyParser.LogicalAndExpressionContext): RubyNode = { - BinaryExpression(visit(ctx.primaryValue(0)), ctx.andOperator.getText, visit(ctx.primaryValue(1)))(ctx.toTextSpan) + override def visitLogicalAndExpression(ctx: RubyParser.LogicalAndExpressionContext): RubyExpression = { + val rhs = Option(ctx.RETURN()) match { + case Some(returnExpr) => ReturnExpression(List.empty)(ctx.toTextSpan.spanStart(returnExpr.toString)) + case None => visit(ctx.primaryValue(1)) + } + + BinaryExpression(visit(ctx.primaryValue(0)), ctx.andOperator.getText, rhs)(ctx.toTextSpan) } - override def visitLogicalOrExpression(ctx: RubyParser.LogicalOrExpressionContext): RubyNode = { - BinaryExpression(visit(ctx.primaryValue(0)), ctx.orOperator.getText, visit(ctx.primaryValue(1)))(ctx.toTextSpan) + override def visitLogicalOrExpression(ctx: RubyParser.LogicalOrExpressionContext): RubyExpression = { + val rhs = Option(ctx.RETURN()) match { + case Some(returnExpr) => ReturnExpression(List.empty)(ctx.toTextSpan.spanStart(returnExpr.toString)) + case None => visit(ctx.primaryValue(1)) + } + + BinaryExpression(visit(ctx.primaryValue(0)), ctx.orOperator.getText, rhs)(ctx.toTextSpan) } override def visitKeywordAndOrExpressionOrCommand( ctx: RubyParser.KeywordAndOrExpressionOrCommandContext - ): RubyNode = { + ): RubyExpression = { BinaryExpression(visit(ctx.lhs), ctx.binOp.getText, visit(ctx.rhs))(ctx.toTextSpan) } - override def visitShiftExpression(ctx: RubyParser.ShiftExpressionContext): RubyNode = { + override def visitShiftExpression(ctx: RubyParser.ShiftExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.bitwiseShiftOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitBitwiseAndExpression(ctx: RubyParser.BitwiseAndExpressionContext): RubyNode = { + override def visitBitwiseAndExpression(ctx: RubyParser.BitwiseAndExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.bitwiseAndOperator.getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitBitwiseOrExpression(ctx: RubyParser.BitwiseOrExpressionContext): RubyNode = { + override def visitBitwiseOrExpression(ctx: RubyParser.BitwiseOrExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.bitwiseOrOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitRelationalExpression(ctx: RubyParser.RelationalExpressionContext): RubyNode = { + override def visitRelationalExpression(ctx: RubyParser.RelationalExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.relationalOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitEqualityExpression(ctx: RubyParser.EqualityExpressionContext): RubyNode = { + override def visitEqualityExpression(ctx: RubyParser.EqualityExpressionContext): RubyExpression = { BinaryExpression(visit(ctx.primaryValue(0)), ctx.equalityOperator().getText, visit(ctx.primaryValue(1)))( ctx.toTextSpan ) } - override def visitDecimalUnsignedLiteral(ctx: RubyParser.DecimalUnsignedLiteralContext): RubyNode = { + override def visitDecimalUnsignedLiteral(ctx: RubyParser.DecimalUnsignedLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Integer))(ctx.toTextSpan) } - override def visitBinaryUnsignedLiteral(ctx: RubyParser.BinaryUnsignedLiteralContext): RubyNode = { + override def visitBinaryUnsignedLiteral(ctx: RubyParser.BinaryUnsignedLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Integer))(ctx.toTextSpan) } - override def visitOctalUnsignedLiteral(ctx: RubyParser.OctalUnsignedLiteralContext): RubyNode = { + override def visitOctalUnsignedLiteral(ctx: RubyParser.OctalUnsignedLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Integer))(ctx.toTextSpan) } - override def visitHexadecimalUnsignedLiteral(ctx: RubyParser.HexadecimalUnsignedLiteralContext): RubyNode = { + override def visitHexadecimalUnsignedLiteral(ctx: RubyParser.HexadecimalUnsignedLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Integer))(ctx.toTextSpan) } override def visitFloatWithExponentUnsignedLiteral( ctx: RubyParser.FloatWithExponentUnsignedLiteralContext - ): RubyNode = { + ): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Float))(ctx.toTextSpan) } override def visitFloatWithoutExponentUnsignedLiteral( ctx: RubyParser.FloatWithoutExponentUnsignedLiteralContext - ): RubyNode = { + ): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Float))(ctx.toTextSpan) } - override def visitPureSymbolLiteral(ctx: RubyParser.PureSymbolLiteralContext): RubyNode = { + override def visitPureSymbolLiteral(ctx: RubyParser.PureSymbolLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Symbol))(ctx.toTextSpan) } - override def visitSingleQuotedSymbolLiteral(ctx: RubyParser.SingleQuotedSymbolLiteralContext): RubyNode = { + override def visitSingleQuotedSymbolLiteral(ctx: RubyParser.SingleQuotedSymbolLiteralContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.Symbol))(ctx.toTextSpan) } - override def visitNilPseudoVariable(ctx: RubyParser.NilPseudoVariableContext): RubyNode = { + override def visitNilPseudoVariable(ctx: RubyParser.NilPseudoVariableContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.NilClass))(ctx.toTextSpan) } - override def visitTruePseudoVariable(ctx: RubyParser.TruePseudoVariableContext): RubyNode = { + override def visitTruePseudoVariable(ctx: RubyParser.TruePseudoVariableContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.TrueClass))(ctx.toTextSpan) } - override def visitFalsePseudoVariable(ctx: RubyParser.FalsePseudoVariableContext): RubyNode = { + override def visitFalsePseudoVariable(ctx: RubyParser.FalsePseudoVariableContext): RubyExpression = { StaticLiteral(getBuiltInType(Defines.FalseClass))(ctx.toTextSpan) } - override def visitSingleQuotedStringExpression(ctx: RubyParser.SingleQuotedStringExpressionContext): RubyNode = { + override def visitSingleQuotedStringExpression( + ctx: RubyParser.SingleQuotedStringExpressionContext + ): RubyExpression = { if (!ctx.isInterpolated) { StaticLiteral(getBuiltInType(Defines.String))(ctx.toTextSpan) } else { @@ -337,11 +371,26 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitQuotedNonExpandedStringLiteral(ctx: RubyParser.QuotedNonExpandedStringLiteralContext): RubyNode = { + override def visitQuotedNonExpandedStringLiteral( + ctx: RubyParser.QuotedNonExpandedStringLiteralContext + ): RubyExpression = { StaticLiteral(getBuiltInType(Defines.String))(ctx.toTextSpan) } - override def visitDoubleQuotedStringExpression(ctx: RubyParser.DoubleQuotedStringExpressionContext): RubyNode = { + override def visitQuotedExpandedStringArrayLiteral( + ctx: RubyParser.QuotedExpandedStringArrayLiteralContext + ): RubyExpression = { + val elements = + if Option(ctx.quotedExpandedArrayElementList()).isDefined then + ctx.quotedExpandedArrayElementList().elements.map(visit) + else List.empty + + ArrayLiteral(elements)(ctx.toTextSpan) + } + + override def visitDoubleQuotedStringExpression( + ctx: RubyParser.DoubleQuotedStringExpressionContext + ): RubyExpression = { if (!ctx.isInterpolated) { StaticLiteral(getBuiltInType(Defines.String))(ctx.toTextSpan) } else { @@ -349,7 +398,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitDoubleQuotedSymbolLiteral(ctx: RubyParser.DoubleQuotedSymbolLiteralContext): RubyNode = { + override def visitDoubleQuotedSymbolLiteral(ctx: RubyParser.DoubleQuotedSymbolLiteralContext): RubyExpression = { if (!ctx.isInterpolated) { StaticLiteral(getBuiltInType(Defines.Symbol))(ctx.toTextSpan) } else { @@ -357,7 +406,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitQuotedExpandedStringLiteral(ctx: RubyParser.QuotedExpandedStringLiteralContext): RubyNode = { + override def visitQuotedExpandedStringLiteral(ctx: RubyParser.QuotedExpandedStringLiteralContext): RubyExpression = { if (!ctx.isInterpolated) { StaticLiteral(getBuiltInType(Defines.String))(ctx.toTextSpan) } else { @@ -365,7 +414,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitRegularExpressionLiteral(ctx: RubyParser.RegularExpressionLiteralContext): RubyNode = { + override def visitRegularExpressionLiteral(ctx: RubyParser.RegularExpressionLiteralContext): RubyExpression = { if (ctx.isStatic) { StaticLiteral(getBuiltInType(Defines.Regexp))(ctx.toTextSpan) } else { @@ -375,7 +424,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { override def visitQuotedExpandedRegularExpressionLiteral( ctx: RubyParser.QuotedExpandedRegularExpressionLiteralContext - ): RubyNode = { + ): RubyExpression = { if (ctx.isStatic) { StaticLiteral(getBuiltInType(Defines.Regexp))(ctx.toTextSpan) } else { @@ -383,27 +432,110 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitCurlyBracesBlock(ctx: RubyParser.CurlyBracesBlockContext): RubyNode = { - val parameters = Option(ctx.blockParameter()).fold(List())(_.parameters).map(visit) - val body = visit(ctx.compoundStatement()) - Block(parameters, body)(ctx.toTextSpan) + override def visitQuotedExpandedExternalCommandLiteral( + ctx: RubyParser.QuotedExpandedExternalCommandLiteralContext + ): RubyExpression = { + val commandLiteral = + if ctx.quotedExpandedLiteralStringContent.asScala.nonEmpty then + StaticLiteral(Defines.String)(ctx.quotedExpandedLiteralStringContent.asScala.toList.map(_.toTextSpan).head) + else StaticLiteral(Defines.String)(ctx.toTextSpan.spanStart()) + + SimpleCall(SimpleIdentifier()(ctx.toTextSpan.spanStart("exec")), List(commandLiteral))(ctx.toTextSpan) } - override def visitDoBlock(ctx: RubyParser.DoBlockContext): RubyNode = { + override def visitCurlyBracesBlock(ctx: RubyParser.CurlyBracesBlockContext): RubyExpression = { + val parameters = + Option(ctx.blockParameter()).fold(List())(_.parameters).map(visit).sortBy(x => (x.span.line, x.span.column)) + + val assignments = parameters.collect { case x: GroupedParameter => + x.multipleAssignment + } + + val body = visit(ctx.compoundStatement()) + val bodyWithAssignments = StatementList(assignments ++ body.asStatementList.statements)(body.span) + + Block(parameters, bodyWithAssignments)(ctx.toTextSpan) + } + + override def visitGroupedParameterList(ctx: RubyParser.GroupedParameterListContext): RubyExpression = { + val freshTmpVar = variableNameGen.fresh + val tmpMandatoryParam = MandatoryParameter(freshTmpVar)(ctx.toTextSpan.spanStart(freshTmpVar)) + + val singleAssignments = ctx.parameters.map { param => + val rhsSplattingNode = SplattingRubyNode(tmpMandatoryParam)(ctx.toTextSpan.spanStart(s"*$freshTmpVar")) + val lhs = param match { + case x: MandatoryParameterContext => SimpleIdentifier()(ctx.toTextSpan.spanStart(x.getText)) + case x: ArrayParameterContext => + SplattingRubyNode(SimpleIdentifier()(ctx.toTextSpan.spanStart(x.getText.stripPrefix("*"))))( + ctx.toTextSpan.spanStart(s"${x.getText}") + ) + case x => + logger.warn(s"Invalid parameter type in grouped parameter list: ${x.getClass}") + defaultResult() + } + SingleAssignment(lhs, "=", rhsSplattingNode)( + ctx.toTextSpan.spanStart(s"${lhs.span.text} = ${rhsSplattingNode.span.text}") + ) + } + + GroupedParameter( + tmpMandatoryParam.span.text, + tmpMandatoryParam, + GroupedParameterDesugaring(singleAssignments)(ctx.toTextSpan) + )(ctx.toTextSpan) + } + + override def visitDoBlock(ctx: RubyParser.DoBlockContext): RubyExpression = { val parameters = Option(ctx.blockParameter()).fold(List())(_.parameters).map(visit) val body = visit(ctx.bodyStatement()) Block(parameters, body)(ctx.toTextSpan) } override def visitLocalVariableAssignmentExpression( ctx: RubyParser.LocalVariableAssignmentExpressionContext - ): RubyNode = { + ): RubyExpression = { val lhs = visit(ctx.lhs) val rhs = visit(ctx.rhs) val op = ctx.assignmentOperator().getText - SingleAssignment(lhs, op, rhs)(ctx.toTextSpan) + + if op == "||=" || op == "&&=" then lowerAssignmentOperator(lhs, rhs, op, ctx.toTextSpan) + else SingleAssignment(lhs, op, rhs)(ctx.toTextSpan) + } + + override def visitSingleAssignmentStatement(ctx: RubyParser.SingleAssignmentStatementContext): RubyExpression = { + val lhs = + if Option(ctx.CONSTANT_IDENTIFIER()).isDefined then + MemberAccess( + SelfIdentifier()(ctx.toTextSpan.spanStart(Defines.Self)), + ctx.COLON2().getText, + ctx.CONSTANT_IDENTIFIER().getText + )(ctx.toTextSpan.spanStart(s"$Self::${ctx.CONSTANT_IDENTIFIER().getText}")) + else if Option(ctx.variable()).isDefined then visit(ctx.variable()) + else if Option(ctx.indexingArgumentList()).isDefined then + val target = visit(ctx.primary()) + val args = + Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit) + IndexAccess(target, args)( + ctx.toTextSpan.spanStart(s"${target.span.text}[${args.map(_.span.text).mkString(",")}]") + ) + else + val memberAccessOp = Option(ctx.DOT) match { + case Some(dot) => ctx.DOT().getText + case None => ctx.COLON2().getText + } + val target = visit(ctx.primary) + val methodName = visit(ctx.methodName) + MemberAccess(target, memberAccessOp, methodName.span.text)( + ctx.toTextSpan.spanStart(s"${target.span.text}$memberAccessOp${methodName.span.text}") + ) + + val rhs = visit(ctx.methodInvocationWithoutParentheses()) + val op = ctx.assignmentOperator().getText + + if op == "||=" || op == "&&=" then lowerAssignmentOperator(lhs, rhs, op, ctx.toTextSpan) + else SingleAssignment(lhs, op, rhs)(ctx.toTextSpan) } - private def flattenStatementLists(x: List[RubyNode]): List[RubyNode] = { + private def flattenStatementLists(x: List[RubyExpression]): List[RubyExpression] = { x match { case (head: StatementList) :: xs => head.statements ++ flattenStatementLists(xs) case head :: tail => head +: flattenStatementLists(tail) @@ -411,7 +543,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitMultipleAssignmentStatement(ctx: RubyParser.MultipleAssignmentStatementContext): RubyNode = { + override def visitMultipleAssignmentStatement(ctx: RubyParser.MultipleAssignmentStatementContext): RubyExpression = { /** Recursively expand and duplicate splatting nodes so that they line up with what they consume. * @@ -420,7 +552,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { * @param expandSize * how many more duplicates to create. */ - def slurp(nodes: List[RubyNode], expandSize: Int): List[RubyNode] = nodes match { + def slurp(nodes: List[RubyExpression], expandSize: Int): List[RubyExpression] = nodes match { case (head: SplattingRubyNode) :: tail if expandSize > 0 => head :: slurp(head :: tail, expandSize - 1) case head :: tail => head :: slurp(tail, expandSize) case Nil => List.empty @@ -492,10 +624,10 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } else { defaultAssignments } - MultipleAssignment(assignments)(ctx.toTextSpan) + DefaultMultipleAssignment(assignments)(ctx.toTextSpan) } - override def visitMultipleLeftHandSide(ctx: RubyParser.MultipleLeftHandSideContext): RubyNode = { + override def visitMultipleLeftHandSide(ctx: RubyParser.MultipleLeftHandSideContext): RubyExpression = { val multiLhsItems = ctx.multipleLeftHandSideItem.asScala.map(visit).toList val packingLHSNodes = Option(ctx.packingLeftHandSide) .map(visit) @@ -510,36 +642,47 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { StatementList(statements)(ctx.toTextSpan) } - override def visitPackingLeftHandSide(ctx: RubyParser.PackingLeftHandSideContext): RubyNode = { - val splatNode = SplattingRubyNode(visit(ctx.leftHandSide))(ctx.toTextSpan) + override def visitPackingLeftHandSide(ctx: RubyParser.PackingLeftHandSideContext): RubyExpression = { + val splatNode = Option(ctx.leftHandSide()) match { + case Some(lhs) => SplattingRubyNode(visit(lhs))(ctx.toTextSpan) + case None => + SplattingRubyNode(MandatoryParameter("_")(ctx.toTextSpan.spanStart("_")))(ctx.toTextSpan.spanStart("*_")) + } + Option(ctx.multipleLeftHandSideItem()).map(_.asScala.map(visit).toList).getOrElse(List.empty) match { case Nil => splatNode case xs => StatementList(splatNode +: xs)(ctx.toTextSpan) } } - override def visitMultipleRightHandSide(ctx: RubyParser.MultipleRightHandSideContext): RubyNode = { - val rhsSplatting = Option(ctx.splattingRightHandSide()).map(_.splattingArgument()).map(visit).toList - Option(ctx.operatorExpressionList()) - .map(x => StatementList(x.operatorExpression().asScala.map(visit).toList ++ rhsSplatting)(ctx.toTextSpan)) - .getOrElse(defaultResult()) + override def visitMultipleRightHandSide(ctx: RubyParser.MultipleRightHandSideContext): RubyExpression = { + val rhsStmts = ctx.children.asScala.collect { + case x: SplattingRightHandSideContext => visit(x) :: Nil + case x: OperatorExpressionListContext => x.operatorExpression.asScala.map(visit).toList + }.flatten + + if rhsStmts.nonEmpty then StatementList(rhsStmts.toList)(ctx.toTextSpan) + else defaultResult() } - override def visitSplattingArgument(ctx: RubyParser.SplattingArgumentContext): RubyNode = { + override def visitSplattingArgument(ctx: RubyParser.SplattingArgumentContext): RubyExpression = { SplattingRubyNode(visit(ctx.operatorExpression()))(ctx.toTextSpan) } - override def visitAttributeAssignmentExpression(ctx: RubyParser.AttributeAssignmentExpressionContext): RubyNode = { - val lhs = visit(ctx.primaryValue()) - val op = ctx.op.getText - val memberName = ctx.methodName.getText - val rhs = visit(ctx.operatorExpression()) - AttributeAssignment(lhs, op, memberName, rhs)(ctx.toTextSpan) + override def visitAttributeAssignmentExpression( + ctx: RubyParser.AttributeAssignmentExpressionContext + ): RubyExpression = { + val lhs = visit(ctx.primaryValue()) + val op = ctx.op.getText + val assignmentOperator = ctx.assignmentOperator().getText + val memberName = ctx.methodName.getText + val rhs = visit(ctx.operatorExpression()) + AttributeAssignment(lhs, op, memberName, assignmentOperator, rhs)(ctx.toTextSpan) } - override def visitSimpleCommand(ctx: RubyParser.SimpleCommandContext): RubyNode = { - if (Option(ctx.commandArgument()).map(_.getText).exists(_.startsWith("::"))) { - val memberName = ctx.commandArgument().getText.stripPrefix("::") + override def visitSimpleCommand(ctx: RubyParser.SimpleCommandContext): RubyExpression = { + if (Option(ctx.simpleCommandArgumentList()).map(_.getText).exists(_.startsWith("::"))) { + val memberName = ctx.simpleCommandArgumentList().getText.stripPrefix("::") if (memberName.headOption.exists(_.isUpper)) { // Constant accesses are upper-case 1st letter MemberAccess(visit(ctx.methodIdentifier()), "::", memberName)(ctx.toTextSpan) } else { @@ -547,53 +690,85 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } else if (!ctx.methodIdentifier().isAttrDeclaration) { val identifierCtx = ctx.methodIdentifier() - val arguments = ctx.commandArgument().arguments.map(visit) + val arguments = ctx.simpleCommandArgumentList().arguments.map(visit) (identifierCtx.getText, arguments) match { - case ("require", List(argument)) => - RequireCall(visit(identifierCtx), argument)(ctx.toTextSpan) - case ("require_relative", List(argument)) => - RequireCall(visit(identifierCtx), argument, true)(ctx.toTextSpan) - case ("require_all", List(argument)) => - RequireCall(visit(identifierCtx), argument, true, true)(ctx.toTextSpan) + case (requireLike, List(argument)) if ImportsPass.ImportCallNames.contains(requireLike) => + val isRelative = requireLike == "require_relative" || requireLike == "require_all" + val isWildcard = requireLike == "require_all" + RequireCall(visit(identifierCtx), argument, isRelative, isWildcard)(ctx.toTextSpan) case ("include", List(argument)) => IncludeCall(visit(identifierCtx), argument)(ctx.toTextSpan) + case ("raise", List(argument: LiteralExpr)) => + val simpleErrorId = + SimpleIdentifier(Option(s"$builtinPrefix.StandardError"))(argument.span.spanStart("StandardError")) + val implicitSimpleErrInst = SimpleObjectInstantiation(simpleErrorId, argument :: Nil)( + argument.span.spanStart(s"StandardError.new(${argument.text})") + ) + RaiseCall(visit(identifierCtx), implicitSimpleErrInst :: Nil)(ctx.toTextSpan) + case ("raise", _) => + RaiseCall(visit(identifierCtx), arguments)(ctx.toTextSpan) case (idAssign, arguments) if idAssign.endsWith("=") => // fixme: This workaround handles a parser ambiguity with method identifiers having `=` and assignments. // The Ruby parser gives precedence to assignments over methods called with this suffix however val lhsIdentifier = SimpleIdentifier(None)(identifierCtx.toTextSpan.spanStart(idAssign.stripSuffix("="))) val argNode = arguments match { case arg :: Nil => arg - case xs => ArrayLiteral(xs)(ctx.commandArgument().toTextSpan) + case xs => ArrayLiteral(xs)(ctx.simpleCommandArgumentList().toTextSpan) } SingleAssignment(lhsIdentifier, "=", argNode)(ctx.toTextSpan) case _ => SimpleCall(visit(identifierCtx), arguments)(ctx.toTextSpan) } } else { - FieldsDeclaration(ctx.commandArgument().arguments.map(visit))(ctx.toTextSpan) + FieldsDeclaration(ctx.simpleCommandArgumentList().arguments.map(visit))(ctx.toTextSpan) + } + } + + override def visitSuperWithParentheses(ctx: RubyParser.SuperWithParenthesesContext): RubyExpression = { + val block = Option(ctx.block()).map(visit) + val arguments = + Option(ctx.argumentWithParentheses()).map(_.arguments.map(visit)).getOrElse(Nil) + visitSuperCall(ctx, arguments, block) + } + + override def visitSuperWithoutParentheses(ctx: RubyParser.SuperWithoutParenthesesContext): RubyExpression = { + val block = Option(ctx.block()).map(visit) + val arguments = Option(ctx.argumentList()).map(_.elements.map(visit)).getOrElse(Nil) + visitSuperCall(ctx, arguments, block) + } + + private def visitSuperCall( + ctx: ParserRuleContext, + arguments: List[RubyExpression], + block: Option[RubyExpression] + ): RubyExpression = { + val callName = SimpleIdentifier()(ctx.toTextSpan.spanStart("super")) + block match { + case Some(body) => SimpleCallWithBlock(callName, arguments, body.asInstanceOf[Block])(ctx.toTextSpan) + case None => SimpleCall(callName, arguments)(ctx.toTextSpan) } } - override def visitIsDefinedExpression(ctx: RubyParser.IsDefinedExpressionContext): RubyNode = { + override def visitIsDefinedExpression(ctx: RubyParser.IsDefinedExpressionContext): RubyExpression = { SimpleCall(visit(ctx.isDefinedKeyword), visit(ctx.expressionOrCommand()) :: Nil)(ctx.toTextSpan) } - override def visitIsDefinedCommand(ctx: RubyParser.IsDefinedCommandContext): RubyNode = { + override def visitIsDefinedCommand(ctx: RubyParser.IsDefinedCommandContext): RubyExpression = { SimpleCall(visit(ctx.isDefinedKeyword), visit(ctx.primaryValue()) :: Nil)(ctx.toTextSpan) } - override def visitMethodCallExpression(ctx: RubyParser.MethodCallExpressionContext): RubyNode = { + override def visitMethodCallExpression(ctx: RubyParser.MethodCallExpressionContext): RubyExpression = { SimpleCall(visit(ctx.methodOnlyIdentifier()), List())(ctx.toTextSpan) } - override def visitMethodCallWithBlockExpression(ctx: RubyParser.MethodCallWithBlockExpressionContext): RubyNode = { + override def visitMethodCallWithBlockExpression( + ctx: RubyParser.MethodCallWithBlockExpressionContext + ): RubyExpression = { ctx.methodIdentifier().getText match { case Defines.Proc | Defines.Lambda => ProcOrLambdaExpr(visit(ctx.block()).asInstanceOf[Block])(ctx.toTextSpan) case Defines.Loop => DoWhileExpression( - SimpleIdentifier(Option(Defines.getBuiltInType(Defines.TrueClass)))( - ctx.methodIdentifier().toTextSpan.spanStart("true") - ), + StaticLiteral(Defines.getBuiltInType(Defines.TrueClass))(ctx.methodIdentifier().toTextSpan.spanStart("true")), ctx.block() match { case b: RubyParser.DoBlockBlockContext => visit(b.doBlock().bodyStatement()) @@ -609,103 +784,145 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitLambdaExpression(ctx: RubyParser.LambdaExpressionContext): RubyNode = { - val parameters = Option(ctx.parameterList()).fold(List())(_.parameters).map(visit) - val body = visit(ctx.block()) + override def visitLambdaExpression(ctx: RubyParser.LambdaExpressionContext): RubyExpression = { + val parameters = Option(ctx.lambdaExpressionParameterList()) match { + case Some(parameterList) => Option(parameterList.blockParameterList()).fold(List())(_.parameters).map(visit) + case None => List() + } + + val body = visit(ctx.block()).asInstanceOf[Block] ProcOrLambdaExpr(Block(parameters, body)(ctx.toTextSpan))(ctx.toTextSpan) } override def visitMethodCallWithParenthesesExpression( ctx: RubyParser.MethodCallWithParenthesesExpressionContext - ): RubyNode = { + ): RubyExpression = { + val callArgs = ctx + .argumentWithParentheses() + .arguments + .map { + case x: BlockArgumentContext => + if Option(x.operatorExpression()).isDefined then visit(x) + else SimpleIdentifier()(ctx.toTextSpan.spanStart(procParamGen.current.value)) + case x => visit(x) + } + .sortBy(x => (x.line, x.column)) + + val args = + if (ctx.argumentWithParentheses().isArrayArgumentList) then + ArrayLiteral(callArgs)(ctx.toTextSpan.spanStart(callArgs.map(_.span.text).mkString(", "))) :: Nil + else callArgs + if (Option(ctx.block()).isDefined) { - SimpleCallWithBlock( - visit(ctx.methodIdentifier()), - ctx.argumentWithParentheses().arguments.map(visit), - visit(ctx.block()).asInstanceOf[Block] - )(ctx.toTextSpan) + SimpleCallWithBlock(visit(ctx.methodIdentifier()), args, visit(ctx.block()).asInstanceOf[Block])(ctx.toTextSpan) } else { - SimpleCall(visit(ctx.methodIdentifier()), ctx.argumentWithParentheses().arguments.map(visit))(ctx.toTextSpan) + SimpleCall(visit(ctx.methodIdentifier()), args)(ctx.toTextSpan) } } - override def visitYieldExpression(ctx: RubyParser.YieldExpressionContext): RubyNode = { - val arguments = Option(ctx.argumentWithParentheses()).iterator.flatMap(_.arguments).map(visit).toList + override def visitYieldExpression(ctx: RubyParser.YieldExpressionContext): RubyExpression = { + val arguments = Option(ctx.argumentWithParentheses()).iterator + .flatMap(_.arguments) + .map(visit) + .toList YieldExpr(arguments)(ctx.toTextSpan) } override def visitYieldMethodInvocationWithoutParentheses( ctx: RubyParser.YieldMethodInvocationWithoutParenthesesContext - ): RubyNode = { - val arguments = ctx.primaryValueList().primaryValue().asScala.map(visit).toList + ): RubyExpression = { + val arguments = ctx.primaryValueListWithAssociation().elements.map(visit).toList YieldExpr(arguments)(ctx.toTextSpan) } - override def visitMemberAccessCommand(ctx: RubyParser.MemberAccessCommandContext): RubyNode = { - val arg = visit(ctx.commandArgument()) - val methodName = visit(ctx.methodName()) - val base = visit(ctx.primary()) - MemberCall(base, ".", methodName.text, List(arg))(ctx.toTextSpan) + override def visitMemberAccessCommand(ctx: RubyParser.MemberAccessCommandContext): RubyExpression = { + val args = ctx.commandArgument.arguments.map(visit) + val base = visit(ctx.primary()) + + if (ctx.methodName().getText == "new") { + base match { + case SingleAssignment(lhs, op, rhs) => + // fixme: Parser packaging arguments from a parenthesis-less object instantiation is odd + val assignSpan = base.span.spanStart(s"${base.span.text}.new") + val rhsSpan = rhs.span.spanStart(s"${rhs.span.text}.new") + SingleAssignment(lhs, op, SimpleObjectInstantiation(rhs, args)(rhsSpan))(assignSpan) + case _ => SimpleObjectInstantiation(base, args)(ctx.toTextSpan) + } + } else { + val methodName = visit(ctx.methodName()) + MemberCall(base, ".", methodName.text, args)(ctx.toTextSpan) + } } - override def visitConstantIdentifierVariable(ctx: RubyParser.ConstantIdentifierVariableContext): RubyNode = { + override def visitConstantIdentifierVariable(ctx: RubyParser.ConstantIdentifierVariableContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitGlobalIdentifierVariable(ctx: RubyParser.GlobalIdentifierVariableContext): RubyNode = { + override def visitGlobalIdentifierVariable(ctx: RubyParser.GlobalIdentifierVariableContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitClassIdentifierVariable(ctx: RubyParser.ClassIdentifierVariableContext): RubyNode = { + override def visitClassIdentifierVariable(ctx: RubyParser.ClassIdentifierVariableContext): RubyExpression = { ClassFieldIdentifier()(ctx.toTextSpan) } - override def visitInstanceIdentifierVariable(ctx: RubyParser.InstanceIdentifierVariableContext): RubyNode = { + override def visitInstanceIdentifierVariable(ctx: RubyParser.InstanceIdentifierVariableContext): RubyExpression = { InstanceFieldIdentifier()(ctx.toTextSpan) } - override def visitLocalIdentifierVariable(ctx: RubyParser.LocalIdentifierVariableContext): RubyNode = { - SimpleIdentifier()(ctx.toTextSpan) + override def visitLocalIdentifierVariable(ctx: RubyParser.LocalIdentifierVariableContext): RubyExpression = { + // Sometimes pseudo variables aren't given precedence in the parser, so we double-check here + ctx.getText match { + case "nil" => StaticLiteral(getBuiltInType(Defines.NilClass))(ctx.toTextSpan) + case "true" => StaticLiteral(getBuiltInType(Defines.TrueClass))(ctx.toTextSpan) + case "false" => StaticLiteral(getBuiltInType(Defines.FalseClass))(ctx.toTextSpan) + case "public" => PublicModifier()(ctx.toTextSpan) + case "private" => PrivateModifier()(ctx.toTextSpan) + case "protected" => ProtectedModifier()(ctx.toTextSpan) + case _ => SimpleIdentifier()(ctx.toTextSpan) + } } - override def visitClassName(ctx: RubyParser.ClassNameContext): RubyNode = { + override def visitClassName(ctx: RubyParser.ClassNameContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitMethodIdentifier(ctx: RubyParser.MethodIdentifierContext): RubyNode = { + override def visitMethodIdentifier(ctx: RubyParser.MethodIdentifierContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitMethodOnlyIdentifier(ctx: RubyParser.MethodOnlyIdentifierContext): RubyNode = { + override def visitMethodOnlyIdentifier(ctx: RubyParser.MethodOnlyIdentifierContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitIsDefinedKeyword(ctx: RubyParser.IsDefinedKeywordContext): RubyNode = { + override def visitIsDefinedKeyword(ctx: RubyParser.IsDefinedKeywordContext): RubyExpression = { SimpleIdentifier()(ctx.toTextSpan) } - override def visitLinePseudoVariable(ctx: RubyParser.LinePseudoVariableContext): RubyNode = { + override def visitLinePseudoVariable(ctx: RubyParser.LinePseudoVariableContext): RubyExpression = { SimpleIdentifier(Some(getBuiltInType(Defines.Integer)))(ctx.toTextSpan) } - override def visitFilePseudoVariable(ctx: RubyParser.FilePseudoVariableContext): RubyNode = { + override def visitFilePseudoVariable(ctx: RubyParser.FilePseudoVariableContext): RubyExpression = { SimpleIdentifier(Some(getBuiltInType(Defines.String)))(ctx.toTextSpan) } - override def visitEncodingPseudoVariable(ctx: RubyParser.EncodingPseudoVariableContext): RubyNode = { + override def visitEncodingPseudoVariable(ctx: RubyParser.EncodingPseudoVariableContext): RubyExpression = { SimpleIdentifier(Some(getBuiltInType(Defines.Encoding)))(ctx.toTextSpan) } - override def visitSelfPseudoVariable(ctx: RubyParser.SelfPseudoVariableContext): RubyNode = { + override def visitSelfPseudoVariable(ctx: RubyParser.SelfPseudoVariableContext): RubyExpression = { SelfIdentifier()(ctx.toTextSpan) } - override def visitMemberAccessExpression(ctx: RubyParser.MemberAccessExpressionContext): RubyNode = { + override def visitMemberAccessExpression(ctx: RubyParser.MemberAccessExpressionContext): RubyExpression = { val hasArguments = Option(ctx.argumentWithParentheses()).isDefined val hasBlock = Option(ctx.block()).isDefined - val isClassDecl = Option(ctx.primaryValue()).map(_.getText).contains("Class") && Option(ctx.methodName()) - .map(_.getText) - .contains("new") + val isClassDecl = + Option(ctx.primaryValue()).map(_.getText).contains("Class") && Option(ctx.methodName()) + .map(_.getText) + .contains("new") + val methodName = ctx.methodName().getText if (!hasBlock) { @@ -719,14 +936,16 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } else { if (!hasArguments) { if (methodName.headOption.exists(_.isUpper)) { + // This would be a symbol-like member return MemberAccess(target, ctx.op.getText, methodName)(ctx.toTextSpan) } else { - return MemberCall(target, ctx.op.getText, methodName, Nil)(ctx.toTextSpan) + // Approximate this as a field-load + return MemberAccess(target, ctx.op.getText, methodName)(ctx.toTextSpan) } } else { - return MemberCall(target, ctx.op.getText, methodName, ctx.argumentWithParentheses().arguments.map(visit))( - ctx.toTextSpan - ) + val args = ctx.argumentWithParentheses().arguments.map(visit) + if methodName == "[]" then return IndexAccess(target, args)(ctx.toTextSpan) + else return MemberCall(target, ctx.op.getText, methodName, args)(ctx.toTextSpan) } } } @@ -741,16 +960,18 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { if (!hasArguments) { return ObjectInstantiationWithBlock(target, List.empty, block)(ctx.toTextSpan) } else { - return ObjectInstantiationWithBlock(target, ctx.argumentWithParentheses().arguments.map(visit), block)( - ctx.toTextSpan - ) + val args = ctx.argumentWithParentheses().arguments.map(visit) + return ObjectInstantiationWithBlock(target, args, block)(ctx.toTextSpan) } } else { return MemberCallWithBlock( target, ctx.op.getText, methodName, - Option(ctx.argumentWithParentheses()).map(_.arguments).getOrElse(List()).map(visit), + Option(ctx.argumentWithParentheses()) + .map(_.arguments) + .getOrElse(List()) + .map(visit), visit(ctx.block()).asInstanceOf[Block] )(ctx.toTextSpan) } @@ -760,26 +981,82 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { Unknown()(ctx.toTextSpan) } - override def visitConstantVariableReference(ctx: ConstantVariableReferenceContext): RubyNode = { + override def visitConstantVariableReference(ctx: ConstantVariableReferenceContext): RubyExpression = { MemberAccess(SelfIdentifier()(ctx.toTextSpan.spanStart(Defines.Self)), "::", ctx.CONSTANT_IDENTIFIER().getText)( ctx.toTextSpan ) } - override def visitIndexingAccessExpression(ctx: RubyParser.IndexingAccessExpressionContext): RubyNode = { + override def visitIndexingAccessExpression(ctx: RubyParser.IndexingAccessExpressionContext): RubyExpression = { IndexAccess( visit(ctx.primaryValue()), Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit) )(ctx.toTextSpan) } - override def visitBracketedArrayLiteral(ctx: RubyParser.BracketedArrayLiteralContext): RubyNode = { - ArrayLiteral(Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit))(ctx.toTextSpan) + override def visitBracketAssignmentExpression(ctx: RubyParser.BracketAssignmentExpressionContext): RubyExpression = { + val lhsBase = visit(ctx.primaryValue()) + val lhsArgs = Option(ctx.indexingArgumentList()).map(_.arguments).getOrElse(List()).map(visit) + + val lhs = IndexAccess(lhsBase, lhsArgs)( + ctx.toTextSpan.spanStart(s"${lhsBase.span.text}[${lhsArgs.map(_.span.text).mkString(", ")}]") + ) + val op = ctx.assignmentOperator().getText + val rhs = visit(ctx.operatorExpression()) + + if op == "||=" || op == "&&=" then lowerAssignmentOperator(lhs, rhs, op, ctx.toTextSpan) + else SingleAssignment(lhs, op, rhs)(ctx.toTextSpan) + } + + /** Lowers the `||=` and `&&=` assignment operators to the respective `.nil?` checks + */ + private def lowerAssignmentOperator( + lhs: RubyExpression, + rhs: RubyExpression, + op: String, + span: TextSpan + ): RubyExpression = { + val condition = nilCheckCondition(lhs, op, "nil?", span) + val thenClause = nilCheckThenClause(lhs, rhs, span) + nilCheckIfStatement(condition, thenClause, span) + } + + /** Generates the requried `.nil?` check condition used in the lowering of `||=` and `&&=` + */ + private def nilCheckCondition(lhs: RubyExpression, op: String, memberName: String, span: TextSpan): RubyExpression = { + val memberAccess = + MemberAccess(lhs, op = ".", memberName = "nil?")(span.spanStart(s"${lhs.span.text}.nil?")) + if op == "||=" then memberAccess + else UnaryExpression(op = "!", expression = memberAccess)(span.spanStart(s"!${memberAccess.span.text}")) + } + + /** Generates the assignment and the `thenClause` used in the lowering of `||=` and `&&=` + */ + private def nilCheckThenClause(lhs: RubyExpression, rhs: RubyExpression, span: TextSpan): RubyExpression = { + StatementList(List(SingleAssignment(lhs, "=", rhs)(span.spanStart(s"${lhs.span.text} = ${rhs.span.text}"))))( + span.spanStart(s"${lhs.span.text} = ${rhs.span.text}") + ) + } + + /** Generates the if statement for the lowering of `||=` and `&&=` + */ + private def nilCheckIfStatement( + condition: RubyExpression, + thenClause: RubyExpression, + span: TextSpan + ): RubyExpression = { + IfExpression(condition = condition, thenClause = thenClause, elsifClauses = List.empty, elseClause = None)( + span.spanStart(s"if ${condition.span.text} then ${thenClause.span.text} end") + ) + } + + override def visitBracketedArrayLiteral(ctx: RubyParser.BracketedArrayLiteralContext): RubyExpression = { + ArrayLiteral(Option(ctx.bracketedArrayElementList()).map(_.elements).getOrElse(List()).map(visit))(ctx.toTextSpan) } override def visitQuotedNonExpandedStringArrayLiteral( ctx: RubyParser.QuotedNonExpandedStringArrayLiteralContext - ): RubyNode = { + ): RubyExpression = { val elements = Option(ctx.quotedNonExpandedArrayElementList()) .map(_.elements) .getOrElse(List()) @@ -789,7 +1066,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { override def visitQuotedNonExpandedSymbolArrayLiteral( ctx: RubyParser.QuotedNonExpandedSymbolArrayLiteralContext - ): RubyNode = { + ): RubyExpression = { val elements = Option(ctx.quotedNonExpandedArrayElementList()) .map(_.elements) .getOrElse(List()) @@ -797,7 +1074,46 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { ArrayLiteral(elements)(ctx.toTextSpan) } - override def visitRangeExpression(ctx: RubyParser.RangeExpressionContext): RubyNode = { + override def visitQuotedExpandedSymbolArrayLiteral( + ctx: RubyParser.QuotedExpandedSymbolArrayLiteralContext + ): RubyExpression = { + if (Option(ctx.quotedExpandedArrayElementList).isDefined) { + ArrayLiteral(ctx.quotedExpandedArrayElementList().elements.map(visit))(ctx.toTextSpan) + } else { + ArrayLiteral(List())(ctx.toTextSpan) + } + } + + override def visitQuotedExpandedArrayElement(ctx: RubyParser.QuotedExpandedArrayElementContext): RubyExpression = { + val literalType = findParent(ctx) match { + case Some(parentCtx) => + parentCtx match + case x: QuotedExpandedStringArrayLiteralContext => Defines.String + case x: QuotedExpandedSymbolArrayLiteralContext => Defines.Symbol + case _ => logger.warn("Cannot determine type, defaulting to String"); Defines.String + case _ => logger.warn("Cannot determine type, defaulting to String"); Defines.String + } + + if (ctx.hasInterpolation) { + DynamicLiteral(literalType, ctx.interpolations.map(visit))(ctx.toTextSpan) + } else { + StaticLiteral(literalType)(ctx.toTextSpan) + } + } + + @tailrec + private def findParent(ctx: ParserRuleContext): Option[ParserRuleContext] = { + ctx match { + case x: QuotedExpandedSymbolArrayLiteralContext => Option(ctx) + case x: QuotedExpandedStringArrayLiteralContext => Option(ctx) + case null => Option(ctx) + case _ => + if ctx.parent != null then findParent(ctx.parent.asInstanceOf[ParserRuleContext]) + else None + } + } + + override def visitBoundedRangeExpression(ctx: RubyParser.BoundedRangeExpressionContext): RubyExpression = { RangeExpression( visit(ctx.primaryValue(0)), visit(ctx.primaryValue(1)), @@ -805,15 +1121,48 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { )(ctx.toTextSpan) } - override def visitRangeOperator(ctx: RubyParser.RangeOperatorContext): RubyNode = { + override def visitEndlessRangeExpression(ctx: RubyParser.EndlessRangeExpressionContext): RubyExpression = { + val infinityUpperBound = + MemberAccess( + SimpleIdentifier(Option(getBuiltInType(Defines.Float)))(ctx.toTextSpan.spanStart("Float")), + "::", + "INFINITY" + )(ctx.toTextSpan.spanStart("Float::INFINITY")) + + RangeExpression( + visit(ctx.primaryValue), + infinityUpperBound, + visit(ctx.rangeOperator()).asInstanceOf[RangeOperator] + )(ctx.toTextSpan) + } + + override def visitBeginlessRangeExpression(ctx: RubyParser.BeginlessRangeExpressionContext): RubyExpression = { + val lowerBoundInfinity = + UnaryExpression( + "-", + MemberAccess( + SimpleIdentifier(Option(getBuiltInType(Defines.Float)))(ctx.toTextSpan.spanStart("Float")), + "::", + "INFINITY" + )(ctx.toTextSpan.spanStart("Float::INFINITY")) + )(ctx.toTextSpan.spanStart("-Float::INFINITY")) + + RangeExpression( + lowerBoundInfinity, + visit(ctx.primaryValue), + visit(ctx.rangeOperator()).asInstanceOf[RangeOperator] + )(ctx.toTextSpan) + } + + override def visitRangeOperator(ctx: RubyParser.RangeOperatorContext): RubyExpression = { RangeOperator(Option(ctx.DOT2()).isEmpty)(ctx.toTextSpan) } - override def visitHashLiteral(ctx: RubyParser.HashLiteralContext): RubyNode = { + override def visitHashLiteral(ctx: RubyParser.HashLiteralContext): RubyExpression = { HashLiteral(Option(ctx.associationList()).map(_.associations).getOrElse(List()).map(visit))(ctx.toTextSpan) } - override def visitAssociation(ctx: RubyParser.AssociationContext): RubyNode = { + override def visitAssociationElement(ctx: RubyParser.AssociationElementContext): RubyExpression = { ctx.associationKey().getText match { case "if" => Association(SimpleIdentifier()(ctx.toTextSpan.spanStart("if")), visit(ctx.operatorExpression()))(ctx.toTextSpan) @@ -822,24 +1171,66 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitModuleDefinition(ctx: RubyParser.ModuleDefinitionContext): RubyNode = { + override def visitAssociationHashArgument(ctx: RubyParser.AssociationHashArgumentContext): RubyExpression = { + val identifierName = Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText) + + identifierName match { + case Some(identName) => + SplattingRubyNode(SimpleIdentifier()(ctx.toTextSpan.spanStart(identName)))(ctx.toTextSpan) + case None => + if ctx.LPAREN() == null then SplattingRubyNode(visit(ctx.methodCallsWithParentheses()))(ctx.toTextSpan) + else SplattingRubyNode(visit(ctx.methodInvocationWithoutParentheses()))(ctx.toTextSpan) + } + } + + override def visitModuleDefinition(ctx: RubyParser.ModuleDefinitionContext): RubyExpression = { val (nonFieldStmts, fields) = genInitFieldStmts(ctx.bodyStatement()) - val moduleName = visit(ctx.classPath()) + val (moduleName, namespaceDecl) = ctx.classPath match { + case x: NestedClassPathContext => + (SimpleIdentifier()(ctx.toTextSpan.spanStart(x.CONSTANT_IDENTIFIER().getText)), namespaceDeclaration(x)) + case _ => (visit(ctx.classPath()), None) + } + val memberCall = createBodyMemberCall(moduleName.span.text, ctx.toTextSpan) - ModuleDeclaration(visit(ctx.classPath()), nonFieldStmts, fields, Option(memberCall))(ctx.toTextSpan) - } + ModuleDeclaration(moduleName, nonFieldStmts, fields, Option(memberCall), namespaceDecl)(ctx.toTextSpan) + } + + override def visitSingletonClassDefinition(ctx: RubyParser.SingletonClassDefinitionContext): RubyExpression = { + val baseClass = Option(ctx.commandOrPrimaryValueClass()).map(visit) + val body = visit(ctx.bodyStatement()).asInstanceOf[StatementList] + + baseClass match { + case Some(baseClass) => + baseClass match { + case x: SelfIdentifier => + SingletonClassDeclaration(freshClassName(ctx.toTextSpan), Option(baseClass), body)(ctx.toTextSpan) + case x => + val stmts = body.statements.map { + case x: MethodDeclaration => + val memberAccess = + MemberAccess(baseClass, ".", x.methodName)( + x.span.spanStart(s"${baseClass.span.text}.${x.methodName}") + ) + val singletonBlockMethod = + SingletonObjectMethodDeclaration(x.methodName, x.parameters, x.body, baseClass)(x.span) + SingleAssignment(memberAccess, "=", singletonBlockMethod)( + ctx.toTextSpan.spanStart(s"${memberAccess.span.text} = ${x.span.text}") + ) + case x => x + } - override def visitSingletonClassDefinition(ctx: RubyParser.SingletonClassDefinitionContext): RubyNode = { - SingletonClassDeclaration( - freshClassName(ctx.toTextSpan), - Option(ctx.commandOrPrimaryValueClass()).map(visit), - visit(ctx.bodyStatement()) - )(ctx.toTextSpan) + SingletonStatementList(stmts)(ctx.toTextSpan) + } + case None => + SingletonClassDeclaration(freshClassName(ctx.toTextSpan), baseClass, body)(ctx.toTextSpan) + } } - private def findFieldsInMethodDecls(methodDecls: List[MethodDeclaration]): List[RubyNode & RubyFieldIdentifier] = { + private def findFieldsInMethodDecls( + methodDecls: List[MethodDeclaration] + ): List[RubyExpression & RubyFieldIdentifier] = { // TODO: Handle case where body of method is not a StatementList methodDecls .flatMap { x => @@ -851,21 +1242,21 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { case _ => List.empty } } - .collect { case x: (RubyNode & RubyFieldIdentifier) => + .collect { case x: (RubyExpression & RubyFieldIdentifier) => x } } - private def genInitFieldStmts( + def genInitFieldStmts( ctxBodyStatement: RubyParser.BodyStatementContext - ): (RubyNode, List[RubyNode & RubyFieldIdentifier]) = { + ): (RubyExpression, List[RubyExpression & RubyFieldIdentifier]) = { val loweredClassDecls = lowerSingletonClassDeclarations(ctxBodyStatement) /** Generates SingleAssignment RubyNodes for list of fields and fields found in method decls */ def genSingleAssignmentStmtList( - fields: List[RubyNode], - fieldsInMethodDecls: List[RubyNode] + fields: List[RubyExpression], + fieldsInMethodDecls: List[RubyExpression] ): List[SingleAssignment] = { (fields ++ fieldsInMethodDecls).map { x => SingleAssignment(x, "=", StaticLiteral(getBuiltInType(Defines.NilClass))(x.span.spanStart("nil")))( @@ -876,7 +1267,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { /** Partition RubyFields into InstanceFieldIdentifiers and ClassFieldIdentifiers */ - def partitionRubyFields(fields: List[RubyNode]): (List[RubyNode], List[RubyNode]) = { + def partitionRubyFields(fields: List[RubyExpression]): (List[RubyExpression], List[RubyExpression]) = { fields.partition { case _: InstanceFieldIdentifier => true case _ => false @@ -886,8 +1277,8 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { loweredClassDecls match { case stmtList: StatementList => val (rubyFieldIdentifiers, otherStructures) = stmtList.statements.partition { - case x: (RubyNode & RubyFieldIdentifier) => true - case _ => false + case x: (RubyExpression & RubyFieldIdentifier) => true + case _ => false } val (fieldAssignments, rest) = otherStructures .map { @@ -933,7 +1324,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { ( StatementList(bodyMethod +: rest)(bodyMethod.span), - combinedFields.asInstanceOf[List[RubyNode & RubyFieldIdentifier]] + combinedFields.asInstanceOf[List[RubyExpression & RubyFieldIdentifier]] ) case decls => (decls, List.empty) } @@ -945,7 +1336,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { * @return * the class body as a statement list. */ - private def lowerAliasStatementsToMethods(classBody: RubyNode): StatementList = { + def lowerAliasStatementsToMethods(classBody: RubyExpression): StatementList = { val classBodyStmts = classBody match { case StatementList(stmts) => stmts @@ -995,14 +1386,14 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { * - `initialize` MethodDeclaration with all non-allowed children nodes added * - list of all nodes allowed directly under type decl */ - private def filterNonAllowedTypeDeclChildren(stmts: StatementList): RubyNode = { + def filterNonAllowedTypeDeclChildren(stmts: StatementList): RubyExpression = { val (initMethod, nonInitStmts) = stmts.statements.partition { case x: MethodDeclaration if x.methodName == Defines.Initialize => true case _ => false } val (allowedTypeDeclChildren, nonAllowedTypeDeclChildren) = nonInitStmts.partition { - case x: AllowedTypeDeclarationChild => true + case _: AllowedTypeDeclarationChild => true case _ => false } @@ -1024,37 +1415,79 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { ) } - StatementList(otherTypeDeclChildren ++ updatedBodyMethod)(stmts.span) + val otherTypeDeclChildrenSpan = otherTypeDeclChildren match { + case head :: tail => s"\n${head.span.text.concat(tail.map(_.span.text).mkString("\n"))}" + case _ => "" + } + + val initMethodSpanText = initMethod match { + case head :: _ => s"\n${head.span.text}" + case _ => "" + } + + StatementList(initMethod ++ otherTypeDeclChildren ++ updatedBodyMethod)( + stmts.span.spanStart( + updatedBodyMethod.headOption + .map(x => x.span.text) + .getOrElse("") + initMethodSpanText + otherTypeDeclChildrenSpan + ) + ) } - override def visitClassDefinition(ctx: RubyParser.ClassDefinitionContext): RubyNode = { + override def visitClassDefinition(ctx: RubyParser.ClassDefinitionContext): RubyExpression = { val (nonFieldStmts, fields) = genInitFieldStmts(ctx.bodyStatement()) val stmts = lowerAliasStatementsToMethods(nonFieldStmts) val classBody = filterNonAllowedTypeDeclChildren(stmts) - val className = visit(ctx.classPath()) + + val (className, namespaceDecl) = ctx.classPath match { + case x: NestedClassPathContext => + (SimpleIdentifier()(ctx.toTextSpan.spanStart(x.CONSTANT_IDENTIFIER().getText)), namespaceDeclaration(x)) + case _ => (visit(ctx.classPath()), None) + } val memberCall = createBodyMemberCall(className.span.text, ctx.toTextSpan) ClassDeclaration( - visit(ctx.classPath()), + className, Option(ctx.commandOrPrimaryValueClass()).map(visit), classBody, fields, - Option(memberCall) + Option(memberCall), + namespaceDecl )(ctx.toTextSpan) } - private def createBodyMemberCall(name: String, textSpan: TextSpan): MemberCall = { - MemberCall( + private def namespaceDeclaration(ctx: RubyParser.NestedClassPathContext): Option[List[String]] = { + val namespaces = ctx.classPath match { + case x: NestedClassPathContext => buildNestedClassPath(ctx.classPath.asInstanceOf[NestedClassPathContext]) + case x: ClassNameContext => ctx.classPath().getText + } + + Option(namespaces.split("\\s+").toList) + } + + private def buildNestedClassPath(ctx: RubyParser.NestedClassPathContext): String = { + Option(ctx.classPath()) match { + case Some(cp) => + cp match { + case x: NestedClassPathContext => s"${buildNestedClassPath(x)} ${ctx.CONSTANT_IDENTIFIER()}" + case x: ClassNameContext => s"${visit(ctx.classPath).span.text} ${ctx.CONSTANT_IDENTIFIER().getText}" + } + case None => + ctx.CONSTANT_IDENTIFIER().getText + } + + } + + private def createBodyMemberCall(name: String, textSpan: TextSpan): TypeDeclBodyCall = { + TypeDeclBodyCall( MemberAccess(SelfIdentifier()(textSpan.spanStart(Defines.Self)), "::", name)( textSpan.spanStart(s"${Defines.Self}::$name") ), - "::", - Defines.TypeDeclBody, - List.empty - )(textSpan.spanStart(s"${Defines.Self}::$name::")) + name + )(textSpan.spanStart(s"${Defines.Self}::$name::${Defines.TypeDeclBody}")) } /** Lowers all MethodDeclaration found in SingletonClassDeclaration to SingletonMethodDeclaration. @@ -1063,7 +1496,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { * @return * RubyNode with lowered MethodDeclarations where required */ - private def lowerSingletonClassDeclarations(ctx: RubyParser.BodyStatementContext): RubyNode = { + private def lowerSingletonClassDeclarations(ctx: RubyParser.BodyStatementContext): RubyExpression = { visit(ctx) match { case stmtList: StatementList => StatementList(stmtList.statements.flatMap { @@ -1089,15 +1522,17 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitMethodDefinition(ctx: RubyParser.MethodDefinitionContext): RubyNode = { - MethodDeclaration( - ctx.definedMethodName().getText, - Option(ctx.methodParameterPart().parameterList()).fold(List())(_.parameters).map(visit), - visit(ctx.bodyStatement()) - )(ctx.toTextSpan) + override def visitMethodDefinition(ctx: RubyParser.MethodDefinitionContext): RubyExpression = { + val params = + Option(ctx.methodParameterPart().parameterList()) + .fold(List())(_.parameters) + .map(visit) + .sortBy(x => (x.span.line, x.span.column)) + + MethodDeclaration(ctx.definedMethodName().getText, params, visit(ctx.bodyStatement()))(ctx.toTextSpan) } - override def visitEndlessMethodDefinition(ctx: RubyParser.EndlessMethodDefinitionContext): RubyNode = { + override def visitEndlessMethodDefinition(ctx: RubyParser.EndlessMethodDefinitionContext): RubyExpression = { val body = visit(ctx.statement()) match { case x: StatementList => x case x => StatementList(x :: Nil)(x.span) @@ -1109,7 +1544,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { )(ctx.toTextSpan) } - override def visitSingletonMethodDefinition(ctx: RubyParser.SingletonMethodDefinitionContext): RubyNode = { + override def visitSingletonMethodDefinition(ctx: RubyParser.SingletonMethodDefinitionContext): RubyExpression = { SingletonMethodDeclaration( visit(ctx.singletonObject()), ctx.definedMethodName().getText, @@ -1118,32 +1553,38 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { )(ctx.toTextSpan) } - override def visitProcParameter(ctx: RubyParser.ProcParameterContext): RubyNode = { - ProcParameter( - Option(ctx.procParameterName).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText()) - )(ctx.toTextSpan) + override def visitProcParameter(ctx: RubyParser.ProcParameterContext): RubyExpression = { + val procParamName = + Option(ctx.procParameterName()).map(_.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText()).getOrElse(ctx.getText) match { + case "&" => + procParamGen.fresh.value + case x => x + } + + ProcParameter(procParamName)(ctx.toTextSpan) } - override def visitHashParameter(ctx: RubyParser.HashParameterContext): RubyNode = { - HashParameter(Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText))(ctx.toTextSpan) + override def visitHashParameter(ctx: RubyParser.HashParameterContext): RubyExpression = { + val identifierName = Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText) + HashParameter(identifierName)(ctx.toTextSpan) } - override def visitArrayParameter(ctx: RubyParser.ArrayParameterContext): RubyNode = { + override def visitArrayParameter(ctx: RubyParser.ArrayParameterContext): RubyExpression = { ArrayParameter(Option(ctx.LOCAL_VARIABLE_IDENTIFIER()).map(_.getText).getOrElse(ctx.getText))(ctx.toTextSpan) } - override def visitOptionalParameter(ctx: RubyParser.OptionalParameterContext): RubyNode = { + override def visitOptionalParameter(ctx: RubyParser.OptionalParameterContext): RubyExpression = { OptionalParameter( ctx.optionalParameterName().LOCAL_VARIABLE_IDENTIFIER().toString, visit(ctx.operatorExpression()) )(ctx.toTextSpan) } - override def visitMandatoryParameter(ctx: RubyParser.MandatoryParameterContext): RubyNode = { + override def visitMandatoryParameter(ctx: RubyParser.MandatoryParameterContext): RubyExpression = { MandatoryParameter(ctx.LOCAL_VARIABLE_IDENTIFIER().toString)(ctx.toTextSpan) } - override def visitVariableLeftHandSide(ctx: RubyParser.VariableLeftHandSideContext): RubyNode = { + override def visitVariableLeftHandSide(ctx: RubyParser.VariableLeftHandSideContext): RubyExpression = { if (Option(ctx.primary()).isEmpty) { MandatoryParameter(ctx.toTextSpan.text)(ctx.toTextSpan) } else { @@ -1152,7 +1593,7 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitBodyStatement(ctx: RubyParser.BodyStatementContext): RubyNode = { + override def visitBodyStatement(ctx: RubyParser.BodyStatementContext): RubyExpression = { val body = visit(ctx.compoundStatement()) val rescueClauses = Option(ctx.rescueClause.asScala).fold(List())(_.map(visit).toList).collect { case x: RescueClause => x } @@ -1166,36 +1607,36 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { } } - override def visitExceptionClassList(ctx: RubyParser.ExceptionClassListContext): RubyNode = { + override def visitExceptionClassList(ctx: RubyParser.ExceptionClassListContext): RubyExpression = { Option(ctx.multipleRightHandSide()).map(visitMultipleRightHandSide).getOrElse(visit(ctx.operatorExpression())) } - override def visitRescueClause(ctx: RubyParser.RescueClauseContext): RubyNode = { + override def visitRescueClause(ctx: RubyParser.RescueClauseContext): RubyExpression = { val exceptionClassList = Option(ctx.exceptionClassList).map(visit) val variables = Option(ctx.exceptionVariableAssignment).map(visit) val thenClause = visit(ctx.thenClause) RescueClause(exceptionClassList, variables, thenClause)(ctx.toTextSpan) } - override def visitEnsureClause(ctx: RubyParser.EnsureClauseContext): RubyNode = { + override def visitEnsureClause(ctx: RubyParser.EnsureClauseContext): RubyExpression = { EnsureClause(visit(ctx.compoundStatement()))(ctx.toTextSpan) } - override def visitCaseWithExpression(ctx: RubyParser.CaseWithExpressionContext): RubyNode = { + override def visitCaseWithExpression(ctx: RubyParser.CaseWithExpressionContext): RubyExpression = { val expression = Option(ctx.expressionOrCommand()).map(visit) val whenClauses = Option(ctx.whenClause().asScala).fold(List())(_.map(visit).toList) val elseClause = Option(ctx.elseClause()).map(visit) CaseExpression(expression, whenClauses, elseClause)(ctx.toTextSpan) } - override def visitCaseWithoutExpression(ctx: RubyParser.CaseWithoutExpressionContext): RubyNode = { + override def visitCaseWithoutExpression(ctx: RubyParser.CaseWithoutExpressionContext): RubyExpression = { val expression = None val whenClauses = Option(ctx.whenClause().asScala).fold(List())(_.map(visit).toList) val elseClause = Option(ctx.elseClause()).map(visit) CaseExpression(expression, whenClauses, elseClause)(ctx.toTextSpan) } - override def visitWhenClause(ctx: RubyParser.WhenClauseContext): RubyNode = { + override def visitWhenClause(ctx: RubyParser.WhenClauseContext): RubyExpression = { val whenArgs = ctx.whenArgument() val matchArgs = Option(whenArgs.operatorExpressionList()).iterator.flatMap(_.operatorExpression().asScala).map(visit).toList @@ -1204,20 +1645,20 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] { WhenClause(matchArgs, matchSplatArg, thenClause)(ctx.toTextSpan) } - override def visitAssociationKey(ctx: RubyParser.AssociationKeyContext): RubyNode = { - if (Option(ctx.operatorExpression()).isDefined) { - visit(ctx.operatorExpression()) - } else { - SimpleIdentifier()(ctx.toTextSpan) + override def visitAssociationKey(ctx: RubyParser.AssociationKeyContext): RubyExpression = { + Option(ctx.operatorExpression()) match { + case Some(ctx) if ctx.isKeyword => SimpleIdentifier()(ctx.toTextSpan) + case Some(ctx) => visit(ctx) + case None => SimpleIdentifier()(ctx.toTextSpan) } } - override def visitAliasStatement(ctx: RubyParser.AliasStatementContext): RubyNode = { + override def visitAliasStatement(ctx: RubyParser.AliasStatementContext): RubyExpression = { AliasStatement(ctx.oldName.getText, ctx.newName.getText)(ctx.toTextSpan) } - override def visitBreakWithoutArguments(ctx: RubyParser.BreakWithoutArgumentsContext): RubyNode = { - BreakStatement()(ctx.toTextSpan) + override def visitBreakWithoutArguments(ctx: RubyParser.BreakWithoutArgumentsContext): RubyExpression = { + BreakExpression()(ctx.toTextSpan) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala index 234a844522df..8e43529dc25e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/AstCreationPass.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.passes +import flatgraph.DiffGraphApplier import io.joern.rubysrc2cpg.astcreation.AstCreator import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes @@ -7,7 +8,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate class AstCreationPass(cpg: Cpg, astCreators: List[AstCreator]) extends ForkJoinParallelCpgPass[AstCreator](cpg) { @@ -32,7 +32,7 @@ class AstCreationPass(cpg: Cpg, astCreators: List[AstCreator]) extends ForkJoinP .astParentFullName(NamespaceTraversal.globalNamespaceName) .isExternal(true) diffGraph.addNode(emptyType).addNode(anyType) - BatchedUpdate.applyDiff(cpg.graph, diffGraph) + DiffGraphApplier.applyDiff(cpg.graph, diffGraph) } override def runOnPart(diffGraph: DiffGraphBuilder, astCreator: AstCreator): Unit = { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala index 9a1f5b4987ff..07fdee3faa2b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPass.scala @@ -16,5 +16,16 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) { case None => Seq() } - override protected val configFileFilters: List[File => Boolean] = List(validGemfilePaths.contains) + override protected val configFileFilters: List[File => Boolean] = List( + // Gemfiles + validGemfilePaths.contains, + extensionFilter(".ini"), + // YAML files + extensionFilter(".yaml"), + extensionFilter(".yml"), + // XML files + extensionFilter(".xml"), + // ERB files + extensionFilter(".erb") + ) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala index c571804387f3..11c70b4334b3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala @@ -24,12 +24,10 @@ object Defines { val Initialize: String = "initialize" val TypeDeclBody: String = "" - val Program: String = ":program" + val Main: String = "
" val Resolver: String = "" - val AnonymousProcParameter = "" - def getBuiltInType(typeInString: String) = s"${GlobalTypes.kernelPrefix}.$typeInString" object RubyOperators { @@ -192,7 +190,8 @@ object GlobalTypes { /* Source: https://ruby-doc.org/3.2.2/Kernel.html * - * We comment-out methods that require an explicit "receiver" (target of member access.) + * We comment-out methods that require an explicit "receiver" (target of member access) and those that may be commonly + * shadowed. */ val kernelFunctions: Set[String] = Set( "Array", @@ -252,7 +251,7 @@ object GlobalTypes { "require", "require_all", "require_relative", - "select", +// "select", "set_trace_func", "sleep", "spawn", diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencyPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencyPass.scala index 9413191ec7c6..d7fd9648a42f 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencyPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencyPass.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.passes +import flatgraph.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{ConfigFile, NewDependency} import io.shiftleft.passes.ForkJoinParallelCpgPass @@ -12,6 +13,18 @@ import io.shiftleft.semanticcpg.language.* */ class DependencyPass(cpg: Cpg) extends ForkJoinParallelCpgPass[ConfigFile](cpg) { + /** Adds all necessary initial core gems. + */ + override def init(): Unit = { + val diffGraph = Cpg.newDiffGraphBuilder + DependencyPass.CORE_GEMS + .map { coreGemName => + NewDependency().name(coreGemName).version(DependencyPass.CORE_GEM_VERSION) + } + .foreach(diffGraph.addNode) + flatgraph.DiffGraphApplier.applyDiff(cpg.graph, diffGraph) + } + /** @return * the Gemfiles, while preferring `Gemfile.lock` files if present. */ @@ -88,3 +101,107 @@ class DependencyPass(cpg: Cpg) extends ForkJoinParallelCpgPass[ConfigFile](cpg) } } + +object DependencyPass { + val CORE_GEM_VERSION: String = "3.0.0" + // Scraped from: https://ruby-doc.org/stdlib-$CORE_GEM_VERSION/ + // These gems require explicit import but no entry required in `Gemsfile` + val CORE_GEMS: Set[String] = Set( + "abbrev", + "base64", + "benchmark", + "bigdecimal", + "bundler", + "cgi", + "coverage", + "csv", + "date", + "dbm", + "debug", + "delegate", + "did_you_mean", + "digest", + "drb", + "English", + "erb", + "etc", + "extmk", + "fcntl", + "fiddle", + "fileutils", + "find", + "forwardable", + "gdbm", + "getoptlong", + "io/console", + "io/nonblock", + "io/wait", + "ipaddr", + "irb", + "json", + "logger", + "matrix", + "minitest", + "mkmf", + "monitor", + "mutex_m", + "net/ftp", + "net/http", + "net/imap", + "net/pop", + "net/protocol", + "net/smtp", + "nkf", + "objspace", + "observer", + "open-uri", + "open3", + "openssl", + "optparse", + "ostruct", + "pathname", + "power_assert", + "pp", + "prettyprint", + "prime", + "pstore", + "psych", + "pty", + "racc", + "racc/parser", + "rake", + "rbs", + "readline", + "readline", + "reline", + "resolv", + "resolv-replace", + "rexml", + "rinda", + "ripper", + "rss", + "rubygems", + "securerandom", + "set", + "shellwords", + "singleton", + "socket", + "stringio", + "strscan", + "syslog", + "tempfile", + "test-unit", + "time", + "timeout", + "tmpdir", + "tracer", + "tsort", + "typeprof", + "un", + "uri", + "weakref", + "win32ole", + "yaml", + "zlib" + ) +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencySummarySolverPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencySummarySolverPass.scala index 30f4908bbad0..f5bb6e11e890 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencySummarySolverPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/DependencySummarySolverPass.scala @@ -16,7 +16,7 @@ class DependencySummarySolverPass(cpg: Cpg, dependencySummary: RubyProgramSummar override def runOnPart(diffGraph: DiffGraphBuilder, dependency: Dependency): Unit = { dependencySummary.namespaceToType.filter(_._1.startsWith(dependency.name)).flatMap(_._2).foreach { x => val typeDeclName = - if x.name.endsWith(RDefines.Program) then RDefines.Program + if x.name.endsWith(RDefines.Main) then RDefines.Main else x.name.split("[.]").lastOption.getOrElse(Defines.Unknown) val dependencyTypeDecl = TypeDeclStubCreator diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImplicitRequirePass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImplicitRequirePass.scala deleted file mode 100644 index f556b25457fb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImplicitRequirePass.scala +++ /dev/null @@ -1,103 +0,0 @@ -package io.joern.rubysrc2cpg.passes - -import io.joern.rubysrc2cpg.datastructures.{RubyProgramSummary, RubyType} -import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes, Operators} -import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language.* -import org.apache.commons.text.CaseUtils - -import scala.collection.mutable - -/** In some Ruby frameworks, it is common to have an autoloader library that implicitly loads requirements onto the - * stack. This pass makes these imports explicit. The most popular one is Zeitwerk which we check in `Gemsfile.lock` to enable this pass. - */ -class ImplicitRequirePass(cpg: Cpg, programSummary: RubyProgramSummary) extends ForkJoinParallelCpgPass[Method](cpg) { - - private val importCallName: String = "require" - private val typeToPath = mutable.Map.empty[String, String] - - override def init(): Unit = { - programSummary.pathToType - .map { case (path, types) => - // zeitwerk will match types that share the name of the path. - // This match is insensitive to camel case, i.e, foo_bar will match type FooBar. - val fileName = path.split('/').last - path -> types.filter { t => - val typeName = t.name.split("[.]").last - typeName == fileName || typeName == CaseUtils.toCamelCase(fileName, true, '_', '-') - } - } - .foreach { case (path, types) => - types.foreach { typ => typeToPath.put(typ.name, path) } - } - } - - override def generateParts(): Array[Method] = - cpg.method.isModule.whereNot(_.astChildren.isCall.nameExact(importCallName)).toArray - - /** Collects methods within a module. - */ - private def findMethodsViaAstChildren(module: Method): Iterator[Method] = { - Iterator(module) ++ module.astChildren.flatMap { - case x: TypeDecl => x.method.flatMap(findMethodsViaAstChildren) - case x: Method => Iterator(x) ++ x.astChildren.collectAll[Method].flatMap(findMethodsViaAstChildren) - case _ => Iterator.empty - } - } - - override def runOnPart(builder: DiffGraphBuilder, part: Method): Unit = { - findMethodsViaAstChildren(part).ast.isCall - .flatMap { - case x if x.name == Operators.alloc => - x.argument.isIdentifier - case x => - x.receiver.fieldAccess.fieldIdentifier - } - .map { - case fi: FieldIdentifier => fi -> programSummary.matchingTypes(fi.canonicalName) - case i: Identifier => i -> programSummary.matchingTypes(i.name) - } - .distinct - .foreach { case (identifier, rubyTypes) => - val requireCalls = rubyTypes.flatMap { rubyType => - typeToPath.get(rubyType.name) match { - case Some(path) - if identifier.file.name - .map(_.replace("\\", "/")) - .headOption - .exists(x => rubyType.name.startsWith(x)) => - None // do not add an import to a file that defines the type - case Some(path) => Option(createRequireCall(builder, rubyType, path)) - case None => None - } - } - val startIndex = part.block.astChildren.size - requireCalls.zipWithIndex.foreach { case (call, idx) => - call.order(startIndex + idx) - builder.addEdge(part.block, call, EdgeTypes.AST) - } - } - } - - private def createRequireCall(builder: DiffGraphBuilder, rubyType: RubyType, path: String): NewCall = { - val requireCallNode = NewCall() - .name(importCallName) - .code(s"$importCallName '$path'") - .methodFullName(s"__builtin:$importCallName") - .dispatchType(DispatchTypes.DYNAMIC_DISPATCH) - .typeFullName(Defines.Any) - val receiverIdentifier = - NewIdentifier().name(importCallName).code(importCallName).typeFullName(Defines.Any).argumentIndex(0).order(1) - val pathLiteralNode = NewLiteral().code(s"'$path'").typeFullName("__builtin.String").argumentIndex(1).order(2) - builder.addNode(requireCallNode) - builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.AST) - builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.ARGUMENT) - builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.RECEIVER) - builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.AST) - builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.ARGUMENT) - requireCallNode - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/DependencyDownloader.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/DependencyDownloader.scala index 0b6497bad368..586bdb873c05 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/DependencyDownloader.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/DependencyDownloader.scala @@ -2,7 +2,7 @@ package io.joern.rubysrc2cpg.utils import better.files.File import io.joern.rubysrc2cpg.datastructures.RubyProgramSummary -import io.joern.rubysrc2cpg.passes.Defines +import io.joern.rubysrc2cpg.passes.{Defines, DependencyPass} import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg, parser} import io.joern.x2cpg.utils.ConcurrentTaskUtil import io.shiftleft.codepropertygraph.generated.Cpg @@ -34,10 +34,15 @@ class DependencyDownloader(cpg: Cpg) { */ def download(): RubyProgramSummary = { File.temporaryDirectory("joern-rubysrc2cpg").apply { dir => - cpg.dependency.filterNot(_.name == Defines.Resolver).foreach { dependency => - Try(Thread.sleep(100)) // Rate limit - downloadDependency(dir, dependency) - } + cpg.dependency + .filterNot(dep => + dep.name == Defines.Resolver || + (DependencyPass.CORE_GEMS.contains(dep.name) && DependencyPass.CORE_GEM_VERSION == dep.version) + ) + .foreach { dependency => + Try(Thread.sleep(100)) // Rate limit + downloadDependency(dir, dependency) + } untarDependencies(dir) summarizeDependencies(dir / "lib") } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala index a7e77e248d8a..fc87a4a4c89e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/FreshNameGenerator.scala @@ -7,4 +7,8 @@ class FreshNameGenerator[T](template: Int => T) { counter += 1 name } + + def current: T = { + template(counter - 1) + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/CallTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/CallTests.scala index d54cff93bd70..df3264db6379 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/CallTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/CallTests.scala @@ -287,8 +287,7 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true, withDataF | |x = 1 |foo = Foo.new - |y = foo - | .bar(1) + |y = foo.bar(1) |puts y |""".stripMargin) @@ -296,25 +295,13 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true, withDataF val sink = cpg.call.name("puts").argument(1).l val List(flow) = sink.reachableByFlows(src).map(flowToResultPairs).distinct.sortBy(_.length).l flow shouldBe List( - ( - """|foo - | .bar(1)""".stripMargin, - 11 - ), + ("foo.bar(1)", 10), ("bar(self, x)", 3), ("return x", 4), ("RET", 3), - ( - """|foo - | .bar(1)""".stripMargin, - 10 - ), - ( - """|y = foo - | .bar(1)""".stripMargin, - 10 - ), - ("puts y", 12) + ("foo.bar(1)", 10), + ("y = foo.bar(1)", 10), + ("puts y", 11) ) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/ProcParameterAndYieldTests.scala index 04ca9426a463..3aeece7aed74 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/ProcParameterAndYieldTests.scala @@ -95,7 +95,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture(withPostProcessing sink2.reachableByFlows(src2).size shouldBe 2 } - "Data flow through invocationWithBlockOnlyPrimary usage" in { + "Data flow through invocationWithBlockOnlyPrimary usage" ignore { val cpg = code(""" |def hello(&block) | block.call @@ -110,7 +110,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture(withPostProcessing sink.reachableByFlows(source).size shouldBe 1 } - "Data flow through invocationWithBlockOnlyPrimary and method name starting with capital usage" in { + "Data flow through invocationWithBlockOnlyPrimary and method name starting with capital usage" ignore { val cpg = code(""" |def Hello(&block) | block.call @@ -126,7 +126,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture(withPostProcessing } // Works in deprecated - "Data flow for yield block specified along with the call" in { + "Data flow for yield block specified along with the call" ignore { val cpg = code(""" |x=10 |def foo(x) @@ -168,7 +168,7 @@ class ProcParameterAndYieldTests extends RubyCode2CpgFixture(withPostProcessing sink.reachableByFlows(source).size shouldBe 2 } - "flow through a proc definition with non-empty block and zero parameters" in { + "flow through a proc definition with non-empty block and zero parameters" ignore { val cpg = code(""" |x=10 |y = x diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala index 7f20321567c8..8ee26a108469 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/SingleAssignmentTests.scala @@ -15,13 +15,13 @@ class SingleAssignmentTests extends RubyCode2CpgFixture(withPostProcessing = tru |""".stripMargin) val source = cpg.literal.l val sink = cpg.method.name("puts").callIn.argument.l - val flows = sink.reachableByFlows(source).map(flowToResultPairs).distinct.sortBy(_.length).l - val List(flow1, flow2, flow3, flow4, flow5) = flows - flow1 shouldBe List(("y = 1", 2), ("puts y", 3)) - flow2 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4)) - flow3 shouldBe List(("y = 1", 2), ("puts y", 3), ("puts x", 4)) - flow4 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("z = x = y = 1", 2), ("puts z", 5)) - flow5 shouldBe List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4), ("puts z", 5)) + val flows = sink.reachableByFlows(source).map(flowToResultPairs).distinct.l + flows.size shouldBe 5 + flows should contain(List(("y = 1", 2), ("puts y", 3))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4))) + flows should contain(List(("y = 1", 2), ("puts y", 3), ("puts x", 4))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("z = x = y = 1", 2), ("puts z", 5))) + flows should contain(List(("y = 1", 2), ("x = y = 1", 2), ("puts x", 4), ("puts z", 5))) } "flow through expressions" in { @@ -47,6 +47,21 @@ class SingleAssignmentTests extends RubyCode2CpgFixture(withPostProcessing = tru sink.reachableByFlows(src).l.size shouldBe 2 } + "flow through **=" in { + val cpg = code(""" + |x = 5 + |call1(x**=2) + |call2(x) + |""".stripMargin) + + val source = cpg.literal("2").l + val call1 = cpg.call("call1") + val call2 = cpg.call("call2") + + call1.reachableBy(source).l shouldBe source + call2.reachableBy(source).l shouldBe source + } + "Data flow through grouping expression" in { val cpg = code(""" |x = 0 diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/dataflow/DataFlowTests.scala deleted file mode 100644 index d95eaf938596..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/dataflow/DataFlowTests.scala +++ /dev/null @@ -1,2633 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.dataflow - -import io.joern.dataflowengineoss.language.* -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class DataFlowTests - extends RubyCode2CpgFixture(withPostProcessing = true, withDataFlow = true, useDeprecatedFrontend = true) { - - "Data flow through if-elseif-else" should { - val cpg = code(""" - |x = 2 - |a = x - |b = 0 - | - |if a > 2 - | b = a + 3 - |elsif a > 4 - | b = a + 5 - |elsif a > 8 - | b = a + 5 - |else - | b = a + 9 - |end - | - |puts(b) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Flow via call" should { - val cpg = code(""" - |def print(content) - |puts content - |end - | - |def main - |n = 1 - |print( n ) - |end - |""".stripMargin) - - "be found" in { - implicit val resolver: ICallResolver = NoResolve - val src = cpg.identifier.name("n").where(_.inCall.name("print")).l - val sink = cpg.method.name("puts").callIn.argument(1).l - sink.reachableByFlows(src).size shouldBe 1 - } - } - - "Explicit return via call with initialization" should { - val cpg = code(""" - |def add(p) - |q = 5 - |q = p - |return q - |end - | - |n = 1 - |ret = add(n) - |puts ret - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("n").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Implicit return via call with initialization" should { - val cpg = code(""" - |def add(p) - |q = 5 - |q = p - |q - |end - | - |n = 1 - |ret = add(n) - |puts ret - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("n").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Implicit return in if-else block" should { - val cpg = code(""" - |def foo(arg) - |if arg > 1 - | arg + 1 - |else - | arg + 10 - |end - |end - | - |x = 1 - |y = foo x - |puts y - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Implicit return in if-else block and underlying function call" should { - val cpg = code(""" - |def add(arg) - |arg + 100 - |end - | - |def foo(arg) - |if arg > 1 - | add(arg) - |else - | add(arg) - |end - |end - | - |x = 1 - |y = foo x - |puts y - | - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Return via call w/o initialization" should { - val cpg = code(""" - |def add(p) - |q = p - |return q - |end - | - |n = 1 - |ret = add(n) - |puts ret - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("n").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow in a while loop" should { - val cpg = code(""" - |i = 0 - |num = 5 - | - |while i < num do - | num = i + 3 - |end - |puts num - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("i").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 3 - } - } - - "Data flow in a while modifier" should { - val cpg = code(""" - |i = 0 - |num = 5 - |begin - | num = i + 3 - |end while i < num - |puts num - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("i").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 3 - } - } - - "Data flow through expressions" should { - val cpg = code(""" - |a = 1 - |b = a+3 - |c = 2 + b%6 - |d = c + b & !c + -b - |e = c/d + b || d - |f = c - d & ~e - |g = f-c%d - +d - |h = g**5 << b*g - |i = b && c || e > g - |j = b>c ? (e+-6) : (f +5) - |k = i..h - |l = j...g - | - |puts l - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("a").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through multiple assignments" should { - val cpg = code(""" - |x = 1 - |y = 2 - |c, d = x, y - |puts c - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through multiple assignments with grouping" should { - val cpg = code(""" - |x = 1 - |y = 2 - |(c, d) = x, y - |puts c - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through multiple assignments with multi level grouping" ignore { - val cpg = code(""" - |x = 1 - |y = 2 - |z = 3 - |a,(b,c) = z,y,x - |puts a - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - "Data flow through multiple assignments with grouping and method in RHS" should { - val cpg = code(""" - |def foo() - |x = 1 - |return x - |end - | - |b = 2 - |(c, d) = foo, b - |puts c - | - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through single LHS and splatting RHS" should { - val cpg = code(""" - |x=1 - |y=*x - |puts y - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through class method" should { - val cpg = code(""" - |class MyClass - | def print(text) - | puts text - | end - |end - | - | - |x = "some text" - |inst = MyClass.new - |inst.print(x) - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through class member" should { - val cpg = code(""" - |class MyClass - | @instanceVariable - | - | def initialize(value) - | @instanceVariable = value - | end - | - | def getValue() - | @instanceVariable - | end - |end - | - |x = 12345 - |inst = MyClass.new(x) - |y = inst.getValue - |puts y - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through module method" should { - val cpg = code(""" - |module MyModule - | def MyModule.print(text) - | puts text - | end - |end - | - |x = "some text" - | - |MyModule::print(x) - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 2 - } - } - - "Data flow through yield with argument having parenthesis" should { - val cpg = code(""" - |def yield_with_arguments - | a = "something" - | yield(a) - |end - | - |yield_with_arguments { |arg| puts "Argument is #{arg}" } - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("a").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 2 - } - } - - "Data flow through yield with argument without parenthesis and multiple yield blocks" should { - val cpg = code(""" - |def yield_with_arguments - | x = "something" - | y = "something_else" - | yield(x,y) - |end - | - |yield_with_arguments { |arg1, arg2| puts "Yield block 1 #{arg1} and #{arg2}" } - |yield_with_arguments { |arg1, arg2| puts "Yield block 2 #{arg2} and #{arg1}" } - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 4 - } - } - - "Data flow through yield without argument" should { - val cpg = code(""" - |x = 1 - |def yield_method - | yield - |end - |yield_method { puts x } - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 1 - } - } - - "Data flow coming out of yield without argument" should { - val cpg = code(""" - |def foo - | x=10 - | z = yield - | puts z - |end - | - |x = 100 - |foo{ x + 10 } - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 1 - } - } - // TODO: - "Data flow through yield with argument and multiple yield blocks" ignore { - val cpg = code(""" - |def yield_with_arguments - | x = "something" - | y = "something_else" - | yield(x) - | yield(y) - |end - | - |yield_with_arguments { |arg| puts "Yield block 1 #{arg}" } - |yield_with_arguments { |arg| puts "Yield block 2 #{arg}" } - |""".stripMargin) - - "be found" in { - val src1 = cpg.identifier.name("x").l - val sink1 = cpg.call.name("puts").l - sink1.reachableByFlows(src1).size shouldBe 2 - - val src2 = cpg.identifier.name("y").l - val sink2 = cpg.call.name("puts").l - sink2.reachableByFlows(src2).size shouldBe 2 - } - } - - "Data flow through a until loop" should { - val cpg = code(""" - |i = 0 - |num = 5 - | - |until i < num - | num = i + 3 - |end - |puts num - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("i").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 3 - } - } - - "Data flow in through until modifier" should { - val cpg = code(""" - |i = 0 - |num = 5 - |begin - | num = i + 3 - |end until i < num - |puts num - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("i").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).l.size shouldBe 3 - } - } - - "Data flow through unless-else" should { - val cpg = code(""" - |x = 2 - |a = x - |b = 0 - | - |unless a > 2 - | b = a + 3 - |else - | b = a + 9 - |end - | - |puts(b) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through case statement" should { - val cpg = code(""" - |x = 2 - |b = x - | - |case b - |when 1 - | puts b - |when 2 - | puts b - |when 3 - | puts b - |else - | puts b - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 8 - } - } - - "Data flow through do-while loop" should { - val cpg = code(""" - |x = 0 - |num = -1 - |loop do - | num = x + 1 - | x = x + 1 - | if x > 10 - | break - | end - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for loop" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | y = x + i - | num = y*i - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for loop simple" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | num = x - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and next AFTER statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | num = x - | next if i % 2 == 0 - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and next BEFORE statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | next if i % 2 == 0 - | num = x - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and redo AFTER statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | num = x - | redo if i % 2 == 0 - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and redo BEFORE statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | redo if i % 2 == 0 - | num = x - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and retry AFTER statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | num = x - | retry if i % 2 == 0 - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through for and retry BEFORE statement" should { - val cpg = code(""" - |x = 0 - |arr = [1,2,3,4,5] - |num = 0 - |for i in arr do - | retry if i % 2 == 0 - | num = x - |end - |puts num - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through grouping expression" should { - val cpg = code(""" - |x = 0 - |y = (x==0) - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through variable assigned a scoped constant" should { - val cpg = code(""" - |MyConst = 10 - |x = ::MyConst - |puts x - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through variable assigned a chained scoped constant" should { - val cpg = code(""" - |MyConst = 10 - |x = ::MyConst - |puts x - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through array constructor expressionsOnlyIndexingArguments" should { - val cpg = code(""" - |x = 1 - |array = [x,2] - |puts x - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 3 - } - } - - "Data flow through array constructor splattingOnlyIndexingArguments" should { - val cpg = code(""" - |def foo(*splat_args) - |array = [*splat_args] - |puts array - |end - | - |x = 1 - |y = 2 - |y = foo(x,y) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through array constructor expressionsAndSplattingIndexingArguments" should { - val cpg = code(""" - |def foo(*splat_args) - |array = [1,2,*splat_args] - |puts array - |end - | - |x = 3 - |foo(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through array constructor associationsOnlyIndexingArguments" should { - val cpg = code(""" - |def foo(arg) - |array = [1 => arg, 2 => arg] - |puts array - |end - | - |x = 3 - |foo(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through array constructor commandOnlyIndexingArguments" should { - val cpg = code(""" - |def increment(arg) - |return arg + 1 - |end - | - |x = 1 - |array = [ increment(x), increment(x+1)] - |puts array - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 3 - } - } - - "Data flow through hash constructor" should { - val cpg = code(""" - |def foo(arg) - |hash = {1 => arg, 2 => arg} - |puts hash - |end - | - |x = 3 - |foo(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through string interpolation" should { - val cpg = code(""" - |x = 1 - |str = "The source is #{x}" - |puts str - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through indexingExpressionPrimary" should { - val cpg = code(""" - |x = [1,2,3] - |y = x[0] - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through methodOnlyIdentifier usage" should { - val cpg = code(""" - |x = 1 - |y = SomeConstant! + x - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through chainedInvocationPrimary usage" should { - val cpg = code(""" - |x = 1 - | - |[x, x+1].each do |number| - | puts "#{number} was passed to the block" - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - // TODO: - "Data flow coming out of chainedInvocationPrimary usage" ignore { - val cpg = code(""" - |x = 1 - |y = 10 - |[x, x+1].each do |number| - | y += x - |end - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through chainedInvocationPrimary without arguments to block usage" should { - val cpg = code(""" - |x = 1 - | - |[1,2,3].each do - | puts "Right here #{x}" - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 1 - } - } - - "Data flow through invocationWithBlockOnlyPrimary usage" should { - val cpg = code(""" - |def hello(&block) - | block.call - |end - | - |x = "hello" - |hello { puts x } - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow through invocationWithBlockOnlyPrimary and method name starting with capital usage" should { - val cpg = code(""" - |def Hello(&block) - | block.call - |end - |x = "hello" - |Hello = "this should not be used" - |Hello { puts x } - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in begin" should { - val cpg = code(""" - |x = 1 - |begin - | puts x - |rescue SomeException - | puts "SomeException occurred" - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts "Catch-all block" - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in else" should { - val cpg = code(""" - |x = 1 - |begin - | puts "In begin" - |rescue SomeException - | puts "SomeException occurred" - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts "Catch-all block" - |else - | puts x - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in rescue" should { - val cpg = code(""" - |x = 1 - |begin - | puts "in begin" - |rescue SomeException - | puts x - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts "Catch-all block" - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in rescue with exception var" should { - val cpg = code(""" - |begin - | puts "in begin" - |rescue SomeException - | puts "SomeException occurred" - |rescue => x - | y = x - | puts "Caught exception in variable #{y}" - |rescue - | puts "Catch-all block" - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - "Data flow for begin/rescue with sink in catch-all rescue" should { - val cpg = code(""" - |x = 1 - |begin - | puts "in begin" - |rescue SomeException - | puts "SomeException occurred" - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts x - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in ensure" should { - val cpg = code(""" - |x = 1 - |begin - | puts "in begin" - |rescue SomeException - | puts "SomeException occurred" - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts "In rescue all" - |ensure - | puts x - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with data flow through the exception" should { - val cpg = code(""" - |x = "Exception message: " - |begin - |1/0 - |rescue ZeroDivisionError => e - | y = x + e.message - | puts y - |end - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with data flow through block with multiple exceptions being caught" should { - val cpg = code(""" - |x = 1 - |y = 10 - |begin - |1/0 - |rescue SystemCallError, ZeroDivisionError - | y = x + 100 - |end - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in function without begin" should { - val cpg = code(""" - |def foo(arg) - | puts "in begin" - |rescue SomeException - | return arg - |rescue => exvar - | puts "Caught exception in variable #{exvar}" - |rescue - | puts "Catch-all block" - |end - | - |x = 1 - |y = foo x - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in function without begin and sink in rescue with exception" should { - val cpg = code(""" - |def foo(arg) - | puts "in begin" - |rescue SomeException - | puts "SomeException occurred #{arg}" - |rescue => exvar - | puts "Caught exception in variable #{exvar}" - |rescue - | puts "Catch-all block" - |end - | - |x = 1 - |foo x - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in function without begin and sink in catch-call rescue" should { - val cpg = code(""" - |def foo(arg) - | puts "in begin" - | raise "This is an exception" - |rescue - | puts "Catch-all block. Arg is #{arg}" - |end - | - |x = 1 - |foo x - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in function without begin with return from begin" should { - val cpg = code(""" - |def foo(arg) - | puts "in begin" - | return arg - |rescue SomeException - | puts "Caught SomeException" - |rescue => exvar - | puts "Caught exception in variable #{exvar}" - |rescue - | puts "Catch-all block" - |end - | - |x = 1 - |y = foo x - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for begin/rescue with sink in function within do block" ignore { - val cpg = code(""" - |def foo(arg) - | puts "in begin" - | arg do |y| - | return y - |rescue SomeException - | puts "Caught SomeException" - |rescue => exvar - | puts "Caught exception in variable #{exvar}" - |rescue - | puts "Catch-all block" - |end - | - |x = 1 - |z = foo x - |puts z - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").lineNumber(8).l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - "Data flow through array assignments" should { - val cpg = code(""" - |x = 10 - |array = [0, 1] - |array[0] = x - |puts array - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through chained scoped constant reference" should { - val cpg = code(""" - |module SomeModule - |SomeConstant = 1 - |end - | - |x = 1 - |y = SomeModule::SomeConstant * x - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through scopedConstantAccessSingleLeftHandSide" should { - val cpg = code(""" - |SomeConstant = 1 - | - |x = 1 - |::SomeConstant = x - |y = ::SomeConstant + 10 - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through xdotySingleLeftHandSide through a constant on left of the ::" should { - val cpg = code(""" - |module SomeModule - |SomeConstant = 100 - |end - | - |x = 2 - |SomeModule::SomeConstant = x - |y = SomeModule::SomeConstant - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - // TODO: - "Data flow through xdotySingleLeftHandSide through a local on left of the ::" ignore { - val cpg = code(""" - |module SomeModule - |SomeConstant = 100 - |end - | - |x = 2 - |local = SomeModule - |local::SomeConstant = x - |y = SomeModule::SomeConstant - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through packing left hand side through the first identifier" should { - val cpg = code(""" - |x = 1 - |p = 2 - |*y = x,p - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through packing left hand side through beyond the first identifier" should { - val cpg = code(""" - |x = 1 - |y = 2 - |z = 3 - |*a = z,y,x - |puts a - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through packing left hand side with unequal RHS" should { - val cpg = code(""" - |x = 1 - |y = 2 - |z = 3 - |p,*a = z,y,x - |puts a - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through single LHS and multiple RHS" should { - val cpg = code(""" - |x = 1 - |y = 2 - |z = 3 - |a = z,y,x - |puts a - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through argsAndDoBlockAndMethodIdCommandWithDoBlock" should { - val cpg = code(""" - |def foo (blockArg,&block) - |block.call(blockArg) - |end - | - |x = 10 - |foo :a_symbol do |arg| - | y = x + arg.length - | puts y - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through primaryMethodArgsDoBlockCommandWithDoBlock" should { - val cpg = code(""" - |module FooModule - |def foo (blockArg,&block) - |block.call(blockArg) - |end - |end - | - |x = 10 - |FooModule.foo :a_symbol do |arg| - | y = x + arg.length - | puts y - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow with super usage" should { - val cpg = code(""" - |class BaseClass - | def doSomething(arg) - | return arg + 10 - | end - |end - | - |class DerivedClass < BaseClass - | def doSomething(arg) - | super(arg) - | end - |end - | - |x = 1 - |object = DerivedClass.new - |y = object.doSomething(x) - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through blockExprAssocTypeArguments" should { - val cpg = code(""" - |def foo(*args) - |puts args - |end - | - |x = "value1" - |foo(key1: x, key2: "value2") - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through blockSplattingTypeArguments" should { - val cpg = code(""" - |def foo(arg) - |puts arg - |end - | - |x = 1 - |foo(*x) - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through blockSplattingExprAssocTypeArguments without block" should { - val cpg = code(""" - |def foo(*arg) - |puts arg - |end - | - |x = 1 - |foo( x+1, key1: x*2, key2: x*3 ) - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through blockSplattingTypeArguments without block" should { - val cpg = code(""" - |def foo (blockArg,&block) - |block.call(blockArg) - |end - | - |x = 10 - |foo(*x do |arg| - | y = x + arg - | puts y - |end - |) - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - // TODO: - "Data flow through blockExprAssocTypeArguments with block argument in the wrapper function" ignore { - val cpg = code(""" - |def foo (blockArg,&block) - |block.call(blockArg) - |end - | - |def foo_wrap (blockArg,&block) - |foo(blockArg,&block) - |end - | - | - |x = 10 - |foo_wrap x do |arg| - | y = 100 + arg - | puts y - |end - | - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through grouping expression with negation" should { - val cpg = code(""" - |def foo(arg) - |return arg - |end - | - |x = false - |y = !(foo x) - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through break with args" should { - val cpg = code(""" - |x = 1 - |arr = [x, 2, 3] - |y = arr.each do |num| - | break num if num < 2 - | puts num - |end - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 4 - } - } - - // TODO: - "Data flow through next with args" ignore { - val cpg = code(""" - |x = 10 - |a = [1, 2, 3] - |y = a.map do |num| - | next x if num.even? - | num - |end - | - |puts y - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - // TODO: - "Data flow through a global variable" ignore { - val cpg = code(""" - |def foo(arg) - | loop do - | arg += 1 - | if arg > 3 - | $y = arg - | return - | end - | end - |end - | - |x = 1 - |foo x - |puts $y - | - | - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow using a keyword" should { - val cpg = code(""" - |class MyClass - |end - | - |x = MyClass.new - |y = x.class - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through variable params" should { - val cpg = code(""" - |def foo(*args) - | return args - |end - | - |x = 1 - |y = foo(x, "another param") - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through optional params" should { - val cpg = code(""" - |def foo(arg=10) - | return arg + 10 - |end - | - |x = 1 - |y = foo(x) - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow across files" should { - val cpg = code( - """ - |def my_func(x) - | puts x - |end - |""".stripMargin, - "foo.rb" - ) - .moreCode( - """ - |require_relative 'foo.rb' - |x = 1 - |my_func(x) - |""".stripMargin, - "bar.rb" - ) - - "be found in" in { - val source = cpg.literal.code("1").l - val sink = cpg.call.name("puts").argument(1).l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - "Across the file data flow test" should { - val cpg = code( - """ - |def foo(arg) - | puts arg - | loop do - | arg += 1 - | if arg > 3 - | puts arg - | return - | end - | end - |end - |""".stripMargin, - "foo.rb" - ) - .moreCode( - """ - |require_relative 'foo.rb' - |x = 1 - |foo x - |""".stripMargin, - "bar.rb" - ) - - "be found in" in { - val source = cpg.literal.code("1").l - val sink = cpg.call.name("puts").argument(1).lineNumber(3).l - sink.reachableByFlows(source).size shouldBe 1 - val src = cpg.identifier("x").lineNumber(3).l - sink.reachableByFlows(src).size shouldBe 1 - } - - // TODO: Need to be fixed. - "be found for sink in nested block" ignore { - val src = cpg.identifier("x").lineNumber(3).l - val sink = cpg.call.name("puts").argument(1).lineNumber(7).l - sink.reachableByFlows(src).size shouldBe 1 - } - } - - "Data flows for pseudo variable identifiers" should { - "Data flow for __LINE__ variable identifier" should { - val cpg = code(""" - |x=1 - |a=x+__LINE__ - |puts a - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - } - - "Data flow for chained command with do-block with parentheses" should { - val cpg = code(""" - |def foo() - | yield if block_given? - |end - | - |y = foo do - | x = 1 - | [x+1,x+2] - |end.sum(10) - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for chained command with do-block without parentheses" should { - val cpg = code(""" - |def foo() - | yield if block_given? - |end - | - |y = foo do - | x = 1 - | [x+1,x+2] - |end.sum 10 - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow for yield block specified alongwith the call" should { - val cpg = code(""" - |x=10 - |def foo(x) - | a = yield - | puts a - |end - | - |foo(x) { - | x + 2 - |} - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 1 - /* - * TODO the flow count shows 1 since the origin is considered as x + 2 - * The actual origin is x=10. However, this is not considered since there is - * no REACHING_DEF edge from the x of 'x=10' to the x of 'x + 2'. - * There are already other disabled data flow test cases for this problem. Once solved, it should - * be possible to set the required count to 2 - */ - - } - } - - "Data flows through range operators" should { - val cpg = code(""" - |x = 10 - |y=0 - |for i in 1...10 do - | x += i - | if (x > 10) - | y = x - | end - |end - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - "Data flow through unless modifier" should { - val cpg = code(""" - |x = 1 - | - |x += 2 unless x.zero? - | puts(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - "Data flow through invocation or command with EMARK" should { - val cpg = code(""" - |x=12 - |def woo(x) - | return x == 10 - |end - | - |if !woo x - | puts x - |else - | puts "No" - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - // TODO: - "Data flow through overloaded operator method" ignore { - val cpg = code(""" - |class Foo - | @@x = 1 - | def +(y) - | @@x + y - | end - |end - | - |y = Foo.new + 1 - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.member.name("@@x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - // TODO: - "Data flow through assignment-like method identifier" ignore { - val cpg = code(""" - |class Foo - | @@x = 1 - | def CONST=(y) - | return @@x == y - | end - |end - |puts Foo::CONST= 2 - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.member.name("@@x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - // TODO: - "Data flow through a when argument context" ignore { - val cpg = code(""" - |x = 10 - | - |case x - | - |when 1..5 - | y = x - |when 5..10 - | z = x - |when 10..15 - | w = x - |else - | _p = x - |end - | - |puts _p - |puts w - |puts y - |puts z - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - "Data flow through ensureClause" should { - val cpg = code(""" - |begin - | x = File.open("myFile.txt", "r") - | x << "#{content} \n" - |rescue - | x = "pqr" - |ensure - | x = "abc" - | y = x - |end - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 // flow through the rescue is not a flow - } - } - - "Data flow through begin-else" should { - val cpg = code(""" - |begin - | x = File.open("myFile.txt", "r") - | x << "#{content} \n" - |rescue - | x = "pqr" - |else - | y = x - |ensure - | x = "abc" - |end - | - |puts y - |""".stripMargin).moreCode( - """ - |My file - |""".stripMargin, - "myFile.txt" - ) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - "Data flow through block argument context" should { - val cpg = code(""" - |x=10 - |y=0 - |def foo(n, &block) - | woo(n, &block) - |end - | - |def woo(n, &block) - | n.times {yield} - |end - | - |foo(5) { - | y = x - |} - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Data flow through block splatting type arguments context" should { - val cpg = code(""" - |x=10 - |y=0 - |def foo(*n, &block) - | woo(*n, &block) - |end - | - |def woo(n, &block) - | n.times {yield} - |end - | - |foo(5) { - | y = x - |} - | - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Flow through tainted object" should { - val cpg = code(""" - |def put_req(api_endpoint, params) - | puts "Hitting " + api_endpoint + " with params: " + params - |end - |class TestClient - | def get_event_data(accountId) - | payload = accountId - | r = put_req( - | "https://localhost:8080/v3/users/me/", - | params=payload - | ) - | end - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("accountId").l - val sink = cpg.call.name("put_req").l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - // TODO: - "Flow for a global variable" ignore { - val cpg = code(""" - |$person_height = 6 - |class Person - | def height_in_cm - | puts $person_height * 30 - | end - |end - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("$person_height").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "Flow for nested puts calls" should { - val cpg = code(""" - |x=10 - |def put_name(x) - | puts x - |end - |def nested_put(x) - | put_name(x) - |end - |def double_nested_put(x) - | nested_put(x) - |end - |double_nested_put(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 5 - } - } - - "Data flow through a keyword? named method usage" should { - val cpg = code(""" - |x = 1 - |y = x.nil? - |puts y - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 2 - } - } - - "Data flow through a keyword inside a association" should { - val cpg = code(""" - |def foo(arg) - |puts arg - |end - | - |x = 1 - |foo if: x.nil? - |""".stripMargin) - - "be found" in { - val src = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(src).size shouldBe 2 - } - } - - "Data flow through a regex interpolation" should { - val cpg = code(s""" - |x="abc" - |y=/x#{x}b/ - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a regex interpolation with multiple expressions" should { - val cpg = code(""" - |x="abc" - |y=/x#{x}b#{x+'z'}b{x+'y'+'z'}w/ - |puts y - |""".stripMargin) - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 3 - } - } - - "flow through a proc definition using Proc.new and flow originating within the proc" should { - val cpg = code(""" - |y = Proc.new { - |x=1 - |x - |} - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a proc definition with non-empty block and zero parameters" ignore { - val cpg = code(""" - |x=10 - |y = x - |-> { - |puts y - |}.call - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a proc definition with non-empty block and non-zero parameters" should { - val cpg = code(""" - |x=10 - |-> (arg){ - |puts arg - |}.call(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a method call with safe navigation operator with parantheses" should { - val cpg = code(""" - |class Foo - | def bar(arg) - | return arg - | end - |end - |x=1 - |foo = Foo.new - |y = foo&.bar(x) - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a method call with safe navigation operator without parantheses" should { - val cpg = code(""" - |class Foo - | def bar(arg) - | return arg - | end - |end - |x=1 - |foo = Foo.new - |y = foo&.bar x - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through a method call present in next line, with the second line starting with `.`" should { - val cpg = code(""" - |class Foo - | def bar(x) - | return x - | end - |end - | - |x = 1 - |foo = Foo.new - |y = foo - | .bar(1) - |puts y - |""".stripMargin) - - "find flow to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - "flow through a method call present in next line, with the first line ending with `.`" should { - val cpg = code(""" - |class Foo - | def bar(x) - | return x - | end - |end - | - |x = 1 - |foo = Foo.new - |y = foo. - | bar(1) - |puts y - |""".stripMargin) - - "find flow to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 1 - } - } - - "flow through statement when regular expression literal passed after `when`" should { - val cpg = code(""" - |x = 2 - |a = 2 - | - |case a - | when /^ch/ - | b = x - | puts b - |end - |""".stripMargin) - - "find flows to the sink" in { - - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through interpolated double-quoted string literal " should { - val cpg = code(""" - |x = "foo" - |y = :"bar #{x}" - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through conditional return statement" should { - val cpg = code(""" - |class Foo - | def bar(value) - | j = 0 - | return(value) unless j == 0 - | end - |end - | - |x = 10 - |foo = Foo.new - |y = foo.bar(x) - |puts y - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through statement with ternary operator with multiple line" in { - val cpg = code(""" - |x = 2 - |y = 3 - |z = 4 - | - |w = x == 2 ? - | y - | : z - |puts y - |""".stripMargin) - - val source = cpg.identifier.name("y").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through endless method" in { - val cpg = code(""" - |def multiply(a,b) = a*b - |x = 10 - |y = multiply(3,x) - |puts y - |""".stripMargin) - - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through symbol literal defined using \\:" should { - val cpg = code(""" - |def foo(arg) - |hash = {:y => arg} - |puts hash - |end - | - |x = 3 - |foo(x) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - } - - "flow through %w array" in { - val cpg = code(""" - |a = %w[b c] - |puts a - |""".stripMargin) - - val source = cpg.literal.code("b").l - val sink = cpg.call.name("puts").l - val List(flow) = sink.reachableByFlows(source).map(flowToResultPairs).distinct.sortBy(_.length).l - flow shouldBe List(("[b, c]", 2), ("[b, c]", -1), ("a = %w[b c]", 2), ("puts a", 3)) - } - - "flow through hash containing splatting literal" in { - val cpg = code(""" - |x={:y=>1} - |z = { - |**x - |} - |puts z - |""".stripMargin) - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "dataflow in method defined under class << self block" ignore { - // Marked it ignored as flow from identifier "firstName" to call "puts" is missing - val cpg = code(""" - class MyClass - | - | class << self - | def printPII - | firstName="somename" - | puts "log PII #{firstName}" - | end - | end - |end - | - |MyClass.printPII""".stripMargin) - - val source = cpg.identifier.name("firstName").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through association identifier" in { - val cpg = code(""" - |def foo(a:) - | puts a - |end - | - |x =1 - |foo(a:x) - |""".stripMargin) - - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through special prefix methods" in { - /* We only check private_class_method here. The mechanism is similar to others: - * attr_reader - * attr_writer - * attr_accessor - * remove_method - * public_class_method - * private - * protected - */ - val cpg = code(""" - |class Foo - | z = 1 - | private_class_method def self.bar(x) - | x - | end - | - | y = self.bar(z) - | puts y - |end - |""".stripMargin) - - val source = cpg.identifier.name("z").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through %i array" in { - val cpg = code(""" - |a = %i[b - | c] - |puts a - |""".stripMargin) - - val source = cpg.literal.code("b").l - val sink = cpg.call.name("puts").l - val List(flow) = sink.reachableByFlows(source).map(flowToResultPairs).distinct.sortBy(_.length).l - flow shouldBe List( - ("[b, c]", 2), - ("[b, c]", -1), - ( - """|a = %i[b - | c]""".stripMargin, - 2 - ), - ("puts a", 4) - ) - } - - "flow through array constructor using []" in { - val cpg = code(""" - |x=1 - |y=x - |z = Array[y,2] - |puts "#{z}" - |""".stripMargin) - - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } - - "flow through array constructor using [] and command in []" in { - val cpg = code(""" - |def foo(arg) - |return arg - |end - | - |x=1 - |y=x - |z = Array[foo y] - |puts "#{z}" - |""".stripMargin) - - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).size shouldBe 2 - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ArrayTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ArrayTests.scala deleted file mode 100644 index 7899dddd5287..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ArrayTests.scala +++ /dev/null @@ -1,319 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class ArrayTests extends RubyParserAbstractTest { - - "An empty array literal" should { - - "be parsed as a primary expression" when { - - "it uses the traditional [, ] delimiters" in { - val code = "[]" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | BracketedArrayConstructor - | [ - | ]""".stripMargin - } - - "it uses the %w[ ] delimiters" in { - val code = "%w[]" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w[ - | ]""".stripMargin - } - - "it uses the %W< > delimiters" in { - val code = "%W<>" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | ExpandedWordArrayConstructor - | %W< - | >""".stripMargin - } - - "it uses the %i[ ] delimiters" in { - val code = "%i[]" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedSymbolArrayConstructor - | %i[ - | ]""".stripMargin - } - - "it uses the %I{ } delimiters" in { - val code = "%I{}" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | ExpandedSymbolArrayConstructor - | %I{ - | }""".stripMargin - } - } - } - - "A non-empty word array literal" should { - - "be parsed as a primary expression" when { - - "it uses the %w[ ] delimiters" in { - val code = "%w[x y z]" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w[ - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | NonExpandedArrayElement - | z - | ]""".stripMargin - } - - "it uses the %w( ) delimiters" in { - val code = "%w(x y z)" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w( - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | NonExpandedArrayElement - | z - | )""".stripMargin - } - - "it uses the %w{ } delimiters" in { - val code = "%w{x y z}" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w{ - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | NonExpandedArrayElement - | z - | }""".stripMargin - } - - "it uses the %w< > delimiters" in { - val code = "%w" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w< - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | \ - | y - | >""".stripMargin - } - - "it uses the %w- - delimiters" in { - val code = "%w-x y z-" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w- - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | NonExpandedArrayElement - | z - | -""".stripMargin - } - - "it spans multiple lines" in { - val code = - """%w( - | bob - | cod - | dod - |)""".stripMargin - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedWordArrayConstructor - | %w( - | NonExpandedArrayElements - | NonExpandedArrayElement - | b - | o - | b - | NonExpandedArrayElement - | c - | o - | d - | NonExpandedArrayElement - | d - | o - | d - | )""".stripMargin - - } - - "it uses the %W( ) delimiters and contains a numeric interpolation" in { - val code = "%W(x#{1})" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | ExpandedWordArrayConstructor - | %W( - | ExpandedArrayElements - | ExpandedArrayElement - | x - | DelimitedArrayItemInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | )""".stripMargin - } - - "it spans multiple lines and contains a numeric interpolation" in { - val code = - """%W[ - | x#{0} - |]""".stripMargin - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | ExpandedWordArrayConstructor - | %W[ - | ExpandedArrayElements - | ExpandedArrayElement - | x - | DelimitedArrayItemInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | } - | ]""".stripMargin - } - } - } - - "A non-empty symbol array literal" should { - - "be parsed as a primary expression" when { - - "it uses the %i< > delimiters" in { - val code = "%i" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedSymbolArrayConstructor - | %i< - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | >""".stripMargin - } - - "it uses the %i{ } delimiters" in { - val code = "%i{x\\ y}" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedSymbolArrayConstructor - | %i{ - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | \ - | y - | }""".stripMargin - } - - "it uses the %i[ ] delimiters nestedly" in { - val code = "%i[x [y]]" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedSymbolArrayConstructor - | %i[ - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | [ - | y - | ] - | ]""".stripMargin - - } - - "it uses the %i( ) delimiters in a multi-line fashion" in { - val code = - """%i( - |x y - |z - |)""".stripMargin - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | NonExpandedSymbolArrayConstructor - | %i( - | NonExpandedArrayElements - | NonExpandedArrayElement - | x - | NonExpandedArrayElement - | y - | NonExpandedArrayElement - | z - | )""".stripMargin - } - - "it uses the %I( ) delimiters and contains a numeric interpolation" in { - val code = "%I(x#{0} x1)" - printAst(_.primary(), code) shouldEqual - """ArrayConstructorPrimary - | ExpandedSymbolArrayConstructor - | %I( - | ExpandedArrayElements - | ExpandedArrayElement - | x - | DelimitedArrayItemInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | } - | ExpandedArrayElement - | x - | 1 - | )""".stripMargin - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/AssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/AssignmentTests.scala deleted file mode 100644 index 4ba931015782..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/AssignmentTests.scala +++ /dev/null @@ -1,81 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class AssignmentTests extends RubyParserAbstractTest { - - "single assignment" should { - - "be parsed as a statement" when { - - "it contains no whitespace before `=`" in { - val code = "x=1" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | SingleAssignmentExpression - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | x - | = - | MultipleRightHandSide - | ExpressionOrCommands - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1""".stripMargin - } - } - } - - "multiple assignment" should { - - "be parsed as a statement" when { - "two identifiers are assigned an array containing two calls" in { - val code = "p, q = [foo(), bar()]" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultipleAssignmentExpression - | MultipleLeftHandSideAndpackingLeftHandSideMultipleLeftHandSide - | MultipleLeftHandSideItem - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | p - | , - | MultipleLeftHandSideItem - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | q - | = - | MultipleRightHandSide - | ExpressionOrCommands - | ExpressionExpressionOrCommand - | PrimaryExpression - | ArrayConstructorPrimary - | BracketedArrayConstructor - | [ - | ExpressionsOnlyIndexingArguments - | Expressions - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | BlankArgsArgumentsWithParentheses - | ( - | ) - | , - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | bar - | BlankArgsArgumentsWithParentheses - | ( - | ) - | ]""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginExpressionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginExpressionTests.scala deleted file mode 100644 index 17ec8f53514b..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginExpressionTests.scala +++ /dev/null @@ -1,58 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class BeginExpressionTests extends RubyParserAbstractTest { - - "A begin expression" should { - - "be parsed as a primary expression" when { - - "it contains a `rescue` clause with both exception class and exception variable" in { - val code = """begin - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin - - printAst(_.primary(), code) shouldBe - """BeginExpressionPrimary - | BeginExpression - | begin - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultiplicativeExpression - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | / - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | RescueClause - | rescue - | ExceptionClass - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | ZeroDivisionError - | ExceptionVariableAssignment - | => - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | e - | ThenClause - | CompoundStatement - | end""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginStatementTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginStatementTests.scala deleted file mode 100644 index 38591dc53fd6..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/BeginStatementTests.scala +++ /dev/null @@ -1,40 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class BeginStatementTests extends RubyParserAbstractTest { - - "BEGIN statement" should { - - "be parsed as a statement" when { - - "defined in a single line" in { - val code = "BEGIN { 1 }" - printAst(_.statement(), code) shouldEqual - """BeginStatement - | BEGIN - | { - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | }""".stripMargin - } - - "empty (single-line)" in { - val code = "BEGIN {}" - printAst(_.statement(), code) shouldEqual - """BeginStatement - | BEGIN - | { - | CompoundStatement - | }""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/CaseConditionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/CaseConditionTests.scala deleted file mode 100644 index 994946a46ad7..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/CaseConditionTests.scala +++ /dev/null @@ -1,194 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class CaseConditionTests extends RubyParserAbstractTest { - - "A case expression" should { - - "be parsed as a primary expression" when { - - "it contains just one `when` branch" in { - val code = - """case something - | when 1 - | puts 2 - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """CaseExpressionPrimary - | CaseExpression - | case - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | something - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ThenClause - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | end""".stripMargin - } - - "it contains both an empty `when` and `else` branch" in { - val code = - """case something - | when 1 - | else - | end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """CaseExpressionPrimary - | CaseExpression - | case - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | something - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ThenClause - | CompoundStatement - | ElseClause - | else - | CompoundStatement - | end""".stripMargin - } - - "it uses `then` as separator for `when`" in { - val code = - """case something - | when 1 then - | end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """CaseExpressionPrimary - | CaseExpression - | case - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | something - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ThenClause - | then - | CompoundStatement - | end""".stripMargin - } - - "it contains two single-line `when-then` branches" in { - val code = - """case x - | when 1 then 2 - | when 2 then 3 - | end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """CaseExpressionPrimary - | CaseExpression - | case - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ThenClause - | then - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | ThenClause - | then - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 3 - | end""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ClassDefinitionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ClassDefinitionTests.scala deleted file mode 100644 index 37ddb665eed7..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ClassDefinitionTests.scala +++ /dev/null @@ -1,112 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class ClassDefinitionTests extends RubyParserAbstractTest { - - "A one-line singleton class definition" should { - - "be parsed as a primary expression" when { - - "it contains no members" in { - val code = "class << self ; end" - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | << - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | PseudoVariableIdentifierVariableReference - | SelfPseudoVariableIdentifier - | self - | ; - | BodyStatement - | CompoundStatement - | end""".stripMargin - } - - "it contains a single numeric literal in its body" in { - val code = "class X 1 end" - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | ClassOrModuleReference - | X - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | end""".stripMargin - } - } - } - - "A multi-line singleton class definition" should { - - "be parsed as a primary expression" when { - - "it contains a single method definition" in { - val code = - """class << x - | def show; puts self; end - |end""".stripMargin - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | << - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | show - | MethodParameterPart - | BodyStatement - | CompoundStatement - | ; - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | PseudoVariableIdentifierVariableReference - | SelfPseudoVariableIdentifier - | self - | ; - | end - | end""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/EnsureClauseTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/EnsureClauseTests.scala deleted file mode 100644 index 4610a18f801c..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/EnsureClauseTests.scala +++ /dev/null @@ -1,58 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class EnsureClauseTests extends RubyParserAbstractTest { - - "An ensure statement" should { - - "be parsed as a standalone statement" when { - - "in the immediate scope of a `def` block" in { - val code = - """def refund - | ensure - | redirect_to paddle_charge_path(@charge) - |end""".stripMargin - printAst(_.methodDefinition(), code) shouldEqual - """MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | refund - | MethodParameterPart - | BodyStatement - | CompoundStatement - | EnsureClause - | ensure - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | redirect_to - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | paddle_charge_path - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | @charge - | ) - | end""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/HashLiteralTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/HashLiteralTests.scala deleted file mode 100644 index f6e0c6d68fee..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/HashLiteralTests.scala +++ /dev/null @@ -1,129 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class HashLiteralTests extends RubyParserAbstractTest { - - "A standalone hash literal" should { - - "be parsed as a primary expression" when { - - "it contains no elements" in { - val code = "{ }" - printAst(_.primary(), code) shouldEqual - """HashConstructorPrimary - | HashConstructor - | { - | }""".stripMargin - } - - "it contains a single splatting identifier" in { - val code = "{ **x }" - printAst(_.primary(), code) shouldEqual - """HashConstructorPrimary - | HashConstructor - | { - | HashConstructorElements - | HashConstructorElement - | ** - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | }""".stripMargin - } - - "it contains two consecutive splatting identifiers" in { - val code = "{**x, **y}" - printAst(_.primary(), code) shouldEqual - """HashConstructorPrimary - | HashConstructor - | { - | HashConstructorElements - | HashConstructorElement - | ** - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | , - | HashConstructorElement - | ** - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | }""".stripMargin - - } - - "it contains an association between two splatting identifiers" in { - val code = "{**x, y => 1, **z}" - printAst(_.primary(), code) shouldEqual - """HashConstructorPrimary - | HashConstructor - | { - | HashConstructorElements - | HashConstructorElement - | ** - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | , - | HashConstructorElement - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | => - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | , - | HashConstructorElement - | ** - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | z - | }""".stripMargin - } - - "it contains a single splatting method invocation" in { - val code = "{**group_by_type(some)}" - printAst(_.primary(), code) shouldEqual - """HashConstructorPrimary - | HashConstructor - | { - | HashConstructorElements - | HashConstructorElement - | ** - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | group_by_type - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | some - | ) - | }""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithParenthesesTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithParenthesesTests.scala deleted file mode 100644 index baaded7a1f47..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithParenthesesTests.scala +++ /dev/null @@ -1,311 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class InvocationWithParenthesesTests extends RubyParserAbstractTest { - - "A method invocation with parentheses" should { - - "be parsed as a primary expression" when { - - "it contains no arguments" in { - val code = "foo()" - - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | BlankArgsArgumentsWithParentheses - | ( - | )""".stripMargin - } - - "it contains no arguments but has newline in between" in { - val code = - """foo( - |) - |""".stripMargin - - printAst(_.primary(), code) shouldEqual - s"""InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | BlankArgsArgumentsWithParentheses - | ( - | )""".stripMargin - } - - "it contains a single numeric literal positional argument" in { - val code = "foo(1)" - - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | )""".stripMargin - } - - "it contains a single numeric literal keyword argument" in { - val code = "foo(region: 1)" - - printAst(_.primary(), code) shouldEqual - s"""InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | AssociationArgument - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | region - | : - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | )""".stripMargin - } - - "it contains an identifier keyword argument" in { - val code = "foo(region:region)" - - printAst(_.primary(), code) shouldEqual - s"""InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | AssociationArgument - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | region - | : - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | region - | )""".stripMargin - } - - "it contains a non-empty regex literal keyword argument" in { - val code = "foo(id: /.*/)" - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | AssociationArgument - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | id - | : - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | .* - | / - | )""".stripMargin - } - - "it contains a single symbol literal positional argument" in { - val code = "foo(:region)" - - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | SymbolLiteral - | Symbol - | :region - | )""".stripMargin - } - - "it contains a single symbol literal positional argument and trailing comma" in { - val code = "foo(:region,)" - - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | SymbolLiteral - | Symbol - | :region - | , - | )""".stripMargin - } - - "it contains a splatting expression before a keyword argument" in { - val code = "foo(*x, y: 1)" - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | SplattingArgumentArgument - | SplattingArgument - | * - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | , - | AssociationArgument - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | : - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | )""".stripMargin - } - - "it contains a keyword-named keyword argument" in { - val code = "foo(if: true)" - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | AssociationArgument - | Association - | Keyword - | if - | : - | PrimaryExpression - | VariableReferencePrimary - | PseudoVariableIdentifierVariableReference - | TruePseudoVariableIdentifier - | true - | )""".stripMargin - } - - "it contains a safe navigation operator with no parameters" in { - val code = "foo&.bar()" - printAst(_.primary(), code) shouldEqual - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | &. - | MethodName - | MethodIdentifier - | bar - | BlankArgsArgumentsWithParentheses - | ( - | )""".stripMargin - } - - "it contains a safe navigation operator with non-zero parameters" in { - val code = "foo&.bar(1, 2)" - printAst(_.primary(), code) shouldEqual - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | &. - | MethodName - | MethodIdentifier - | bar - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | , - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | )""".stripMargin - } - - "it spans two lines, with the second line starting with `.`" in { - val code = "foo\n .bar" - printAst(_.primary(), code) shouldEqual - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | . - | MethodName - | MethodIdentifier - | bar""".stripMargin - } - - "it spans two lines, with the first line ending with `.`" in { - val code = "foo.\n bar" - printAst(_.primary(), code) shouldEqual - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | . - | MethodName - | MethodIdentifier - | bar""".stripMargin - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithoutParenthesesTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithoutParenthesesTests.scala deleted file mode 100644 index b5d3c7b03776..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/InvocationWithoutParenthesesTests.scala +++ /dev/null @@ -1,176 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class InvocationWithoutParenthesesTests extends RubyParserAbstractTest { - - "A method invocation without parentheses" should { - - "be parsed as a primary expression" when { - - "it contains a keyword?-named member" in { - val code = "task.nil?" - - printAst(_.primary(), code) shouldBe - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | task - | . - | MethodName - | MethodIdentifier - | MethodOnlyIdentifier - | Keyword - | nil - | ?""".stripMargin - } - - "it is keyword?-named" in { - val code = "do?" - - printAst(_.primary(), code) shouldBe - """MethodOnlyIdentifierPrimary - | MethodOnlyIdentifier - | Keyword - | do - | ?""".stripMargin - } - - "it is keyword!-named" in { - val code = "return!" - - printAst(_.primary(), code) shouldBe - """MethodOnlyIdentifierPrimary - | MethodOnlyIdentifier - | Keyword - | return - | !""".stripMargin - } - } - } - - "A command with do block" should { - - "be parsed as a statement" when { - - "it contains only one argument" in { - val code = """it 'should print 1' do - | puts 1 - |end - |""".stripMargin - - printAst(_.statement(), code) shouldBe - """ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | ChainedCommandDoBlockInvocationWithoutParentheses - | ChainedCommandWithDoBlock - | ArgsAndDoBlockAndMethodIdCommandWithDoBlock - | MethodIdentifier - | it - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | StringExpressionPrimary - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'should print 1' - | DoBlock - | do - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | end""".stripMargin - - } - - "it contains a safe navigation operator with no parameters" in { - val code = "foo&.bar" - printAst(_.primary(), code) shouldEqual - """ChainedInvocationPrimary - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | &. - | MethodName - | MethodIdentifier - | bar""".stripMargin - } - - "it contains a safe navigation operator with non-zero parameters" in { - val code = "foo&.bar 1,2" - printAst(_.command(), code) shouldEqual - """MemberAccessCommand - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | &. - | MethodName - | MethodIdentifier - | bar - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | , - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2""".stripMargin - } - - } - } - - "invocation with association arguments" should { - "have correct structure for association arguments" in { - val code = """foo bar:""" - printAst(_.program(), code) shouldBe - """Program - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | foo - | ArgumentsWithoutParentheses - | Arguments - | AssociationArgument - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | bar - | : - | EOF""".stripMargin - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/MethodDefinitionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/MethodDefinitionTests.scala deleted file mode 100644 index d055e207d37f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/MethodDefinitionTests.scala +++ /dev/null @@ -1,933 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class MethodDefinitionTests extends RubyParserAbstractTest { - - "A one-line empty method definition" should { - - "be parsed as a primary expression" when { - - "it contains no parameters" in { - val code = "def foo; end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains a mandatory parameter" in { - val code = "def foo(x);end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains an optional numeric parameter" in { - val code = "def foo(x=1);end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | OptionalParameter - | x - | = - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains two parameters, the last of which a &-parameter" in { - val code = "def foo(x, &y); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | , - | Parameter - | ProcParameter - | & - | y - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains a named (array) splatting argument" in { - val code = "def foo(*arr); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | ArrayParameter - | * - | arr - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains a named (hash) splatting argument" in { - val code = "def foo(**hash); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | HashParameter - | ** - | hash - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains both a named array and hash splatting argument" in { - val code = "def foo(*arr, **hash); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | ArrayParameter - | * - | arr - | , - | Parameter - | HashParameter - | ** - | hash - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains an optional parameter before a mandatory one" in { - val code = "def foo(x=1,y); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | OptionalParameter - | x - | = - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | , - | Parameter - | MandatoryParameter - | y - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains a keyword parameter" in { - val code = "def foo(x: 1); end" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | x - | : - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains a mandatory keyword parameter" in { - val code = "def foo(x:) ; end" - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | x - | : - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains two mandatory keyword parameters" in { - val code = "def foo(name:, surname:) ; end" - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | name - | : - | , - | Parameter - | KeywordParameter - | surname - | : - | ) - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - } - } - - "A multi-line method definition" should { - - "be parsed as a primary expression" when { - - "it contains a `rescue` clause" in { - val code = """def foo - | 1/0 - | rescue ZeroDivisionError => e - |end""".stripMargin - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultiplicativeExpression - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | / - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | RescueClause - | rescue - | ExceptionClass - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | ZeroDivisionError - | ExceptionVariableAssignment - | => - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | e - | ThenClause - | CompoundStatement - | end""".stripMargin - - } - } - - } - - "An endless method definition" should { - - "be parsed as a primary expression" when { - - "it contains no arguments" in { - val code = "def foo = x" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | MethodIdentifier - | foo - | MethodParameterPart - | = - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x""".stripMargin - } - - "it contains a line break right after `=`" in { - val code = "def foo =\n x" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | MethodIdentifier - | foo - | MethodParameterPart - | = - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x""".stripMargin - } - - "it contains no arguments and a string literal on the RHS" in { - val code = """def foo = "something"""" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | MethodIdentifier - | foo - | MethodParameterPart - | = - | PrimaryExpression - | StringExpressionPrimary - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | something - | """".stripMargin - } - - "it contains a single mandatory argument" in { - val code = "def id(x) = x" - printAst(_.primary(), code) shouldEqual - """MethodDefinitionPrimary - | MethodDefinition - | def - | MethodIdentifier - | id - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | ) - | = - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x""".stripMargin - } - } - - "not be recognized" when { - - // This test exists to make sure that `foo2=` is not parsed as an endless method, as - // endless methods cannot end in `=`. - "its name ends in `=`" in { - val code = - """def foo1 - |end - |def foo2=(arg) - |end - |""".stripMargin - - printAst(_.compoundStatement(), code) shouldEqual - """CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo1 - | MethodParameterPart - | BodyStatement - | CompoundStatement - | end - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | AssignmentLikeMethodIdentifier - | foo2= - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | arg - | ) - | BodyStatement - | CompoundStatement - | end""".stripMargin - - } - - // This test makes sure that the `end` after `def foo2=` is not parsed as part of its definition, - // which could happen if `foo2=` was parsed as two separate tokens (LOCAL_VARIABLE_IDENTIFIER, EQ) - // instead of just ASSIGNMENT_LIKE_METHOD_IDENTIFIER. - // Issue report: https://github.com/joernio/joern/issues/3270 - "its name ends in `=` and the next keyword is `end`" in { - val code = - """module SomeModule - |def foo1 - | return unless true - |end - |def foo2=(arg) - |end - |end - |""".stripMargin - printAst(_.compoundStatement(), code) shouldEqual - """CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | ModuleDefinitionPrimary - | ModuleDefinition - | module - | ClassOrModuleReference - | SomeModule - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo1 - | MethodParameterPart - | BodyStatement - | CompoundStatement - | Statements - | ModifierStatement - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | ReturnArgsInvocationWithoutParentheses - | return - | unless - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | PseudoVariableIdentifierVariableReference - | TruePseudoVariableIdentifier - | true - | end - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | AssignmentLikeMethodIdentifier - | foo2= - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | arg - | ) - | BodyStatement - | CompoundStatement - | end - | end""".stripMargin - } - } - } - - "method definition with proc parameters" should { - "have correct structure for proc parameters with name" in { - val code = - """def foo(&block) - | yield - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | ProcParameter - | & - | block - | ) - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | YieldWithOptionalArgumentPrimary - | YieldWithOptionalArgument - | yield - | end""".stripMargin - } - - "have correct structure for proc parameters with no name" in { - val code = - """def foo(&) - | yield - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | ProcParameter - | & - | ) - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | YieldWithOptionalArgumentPrimary - | YieldWithOptionalArgument - | yield - | end""".stripMargin - } - } - - "method definition for mandatory parameters" should { - "have correct structure for mandatory parameters" in { - val code = "def foo(bar:) end" - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | bar - | : - | ) - | BodyStatement - | CompoundStatement - | end""".stripMargin - } - - "have correct structure for a hash created using a method" in { - val code = - """def data - | { - | first_link:, - | action_link_group:, - | } - |end""".stripMargin - - printAst(_.primary(), code) shouldBe - """MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | data - | MethodParameterPart - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | HashConstructorPrimary - | HashConstructor - | { - | HashConstructorElements - | HashConstructorElement - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | first_link - | : - | , - | HashConstructorElement - | Association - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | action_link_group - | : - | , - | } - | end""".stripMargin - } - - "have correct structure when a method parameter is defined using whitespace" in { - val code = - """class SampleClass - | def sample_method( first_param:, second_param:) - | end - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | ClassOrModuleReference - | SampleClass - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | sample_method - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | first_param - | : - | , - | Parameter - | KeywordParameter - | second_param - | : - | ) - | BodyStatement - | CompoundStatement - | end - | end""".stripMargin - } - - "have correct structure when method parameters are defined using new line" in { - val code = - """class SomeClass - | def initialize( - | name, age) - | end - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | ClassOrModuleReference - | SomeClass - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | initialize - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | name - | , - | Parameter - | MandatoryParameter - | age - | ) - | BodyStatement - | CompoundStatement - | end - | end""".stripMargin - } - - "have correct structure when method parameters are defined using wsOrNL" in { - val code = - """class SomeClass - | def initialize( - | name, age - | ) - | end - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | ClassOrModuleReference - | SomeClass - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | initialize - | MethodParameterPart - | ( - | Parameters - | Parameter - | MandatoryParameter - | name - | , - | Parameter - | MandatoryParameter - | age - | ) - | BodyStatement - | CompoundStatement - | end - | end""".stripMargin - } - - "have correct structure when keyword parameters are defined using wsOrNL" in { - val code = - """class SomeClass - | def initialize( - | name: nil, age - | ) - | end - |end - |""".stripMargin - - printAst(_.primary(), code) shouldBe - """ClassDefinitionPrimary - | ClassDefinition - | class - | ClassOrModuleReference - | SomeClass - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | MethodDefinitionPrimary - | MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | initialize - | MethodParameterPart - | ( - | Parameters - | Parameter - | KeywordParameter - | name - | : - | PrimaryExpression - | VariableReferencePrimary - | PseudoVariableIdentifierVariableReference - | NilPseudoVariableIdentifier - | nil - | , - | Parameter - | MandatoryParameter - | age - | ) - | BodyStatement - | CompoundStatement - | end - | end""".stripMargin - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ModuleTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ModuleTests.scala deleted file mode 100644 index 8b8f7e64cfbf..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ModuleTests.scala +++ /dev/null @@ -1,24 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class ModuleTests extends RubyParserAbstractTest { - - "Empty module definition" should { - - "be parsed as a definition" when { - - "defined in a single line" in { - val code = """module Bar; end""" - printAst(_.moduleDefinition(), code) shouldEqual - """ModuleDefinition - | module - | ClassOrModuleReference - | Bar - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ProcDefinitionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ProcDefinitionTests.scala deleted file mode 100644 index 5466e3bed5f4..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ProcDefinitionTests.scala +++ /dev/null @@ -1,214 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class ProcDefinitionTests extends RubyParserAbstractTest { - - "A one-line proc definition" should { - - "be parsed as a primary expression" when { - - "it contains no parameters and no statements in a brace block" in { - val code = "-> {}" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | BraceBlockBlock - | BraceBlock - | { - | BodyStatement - | CompoundStatement - | }""".stripMargin - } - - "it contains no parameters and no statements in a do block" in { - val code = "-> do ; end" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | DoBlockBlock - | DoBlock - | do - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains no parameters and returns a literal in a do block" in { - val code = "-> do 1 end" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | DoBlockBlock - | DoBlock - | do - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | end""".stripMargin - } - - "it contains a mandatory parameter and no statements in a brace block" in { - val code = "-> (x) {}" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | ) - | BraceBlockBlock - | BraceBlock - | { - | BodyStatement - | CompoundStatement - | }""".stripMargin - } - - "it contains a mandatory parameter and no statements in a do block" in { - val code = "-> (x) do ; end" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | ) - | DoBlockBlock - | DoBlock - | do - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains an optional numeric parameter and no statements in a brace block" in { - val code = "->(x = 1) {}" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | ( - | Parameters - | Parameter - | OptionalParameter - | x - | = - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ) - | BraceBlockBlock - | BraceBlock - | { - | BodyStatement - | CompoundStatement - | }""".stripMargin - } - - "it contains a keyword parameter and no statements in a do block" in { - val code = "-> (foo: 1) do ; end" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | ( - | Parameters - | Parameter - | KeywordParameter - | foo - | : - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | ) - | DoBlockBlock - | DoBlock - | do - | BodyStatement - | CompoundStatement - | ; - | end""".stripMargin - } - - "it contains two mandatory parameters and two puts statements in a brace block" in { - val code = "->(x, y) {puts x; puts y}" - printAst(_.primary(), code) shouldBe - """ProcDefinitionPrimary - | ProcDefinition - | -> - | ( - | Parameters - | Parameter - | MandatoryParameter - | x - | , - | Parameter - | MandatoryParameter - | y - | ) - | BraceBlockBlock - | BraceBlock - | { - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | ; - | ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | }""".stripMargin - } - - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexTests.scala deleted file mode 100644 index e62be02ad99f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RegexTests.scala +++ /dev/null @@ -1,497 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class RegexTests extends RubyParserAbstractTest { - - "An empty regex literal" when { - - "by itself" should { - val code = "//" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """LiteralPrimary - | RegularExpressionLiteral - | / - | /""".stripMargin - } - } - - "on the RHS of an assignment" should { - val code = "x = //" - - "be parsed as a single assignment to a regex literal" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | SingleAssignmentExpression - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | x - | = - | MultipleRightHandSide - | ExpressionOrCommands - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | /""".stripMargin - } - } - - "as the argument to a `puts` command" should { - val code = "puts //" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | /""".stripMargin - } - } - - "as the sole argument to a parenthesized invocation" should { - val code = "puts(//)" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | puts - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | / - | )""".stripMargin - } - } - - "as the second argument to a parenthesized invocation" should { - val code = "puts(1, //)" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | puts - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | , - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | / - | )""".stripMargin - } - } - - "used in a `when` clause" should { - val code = - """case foo - | when /^ch_/ - | bar - |end""".stripMargin - - "be parsed as such" in { - printAst(_.primary(), code) shouldEqual - """CaseExpressionPrimary - | CaseExpression - | case - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | WhenClause - | when - | WhenArgument - | Expressions - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | ^ch_ - | / - | ThenClause - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | bar - | end""".stripMargin - } - - "used in a `unless` clause" should { - val code = - """unless /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i.match?(value) - |end""".stripMargin - - "be parsed as such" in { - printAst(_.primary(), code) shouldEqual - """UnlessExpressionPrimary - | UnlessExpression - | unless - | ExpressionExpressionOrCommand - | PrimaryExpression - | ChainedInvocationPrimary - | LiteralPrimary - | RegularExpressionLiteral - | / - | \A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z - | /i - | . - | MethodName - | MethodIdentifier - | MethodOnlyIdentifier - | match - | ? - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | value - | ) - | ThenClause - | CompoundStatement - | end""".stripMargin - } - } - } - } - - "A non-interpolated regex literal" when { - - "by itself" should { - val code = "/(eu|us)/" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """LiteralPrimary - | RegularExpressionLiteral - | / - | (eu|us) - | /""".stripMargin - } - } - - "on the RHS of an assignment" should { - val code = "x = /(eu|us)/" - - "be parsed as a single assignment to a regex literal" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | SingleAssignmentExpression - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | x - | = - | MultipleRightHandSide - | ExpressionOrCommands - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | (eu|us) - | /""".stripMargin - } - } - - "as the argument to a `puts` command" should { - val code = "puts /(eu|us)/" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | (eu|us) - | /""".stripMargin - } - } - - "as the argument to an parenthesized invocation" should { - val code = "puts(/(eu|us)/)" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | puts - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | RegularExpressionLiteral - | / - | (eu|us) - | / - | )""".stripMargin - } - } - } - - "A quoted non-interpolated (`%r`) regex literal" when { - - "by itself and using the `{`-`}` delimiters" should { - - "be parsed as a primary expression" in { - val code = "%r{a-z}" - printAst(_.primary(), code) shouldEqual - """QuotedRegexInterpolationPrimary - | QuotedRegexInterpolation - | %r{ - | a-z - | }""".stripMargin - } - } - - "by itself and using the `<`-`>` delimiters" should { - - "be parsed as a primary expression" in { - val code = "%r" - printAst(_.primary(), code) shouldEqual - """QuotedRegexInterpolationPrimary - | QuotedRegexInterpolation - | %r< - | eu|us - | >""".stripMargin - } - } - - "by itself, empty and using the `[`-`]` delimiters" should { - - "be parsed as a primary expression" in { - val code = "%r[]" - printAst(_.primary(), code) shouldEqual - """QuotedRegexInterpolationPrimary - | QuotedRegexInterpolation - | %r[ - | ]""".stripMargin - } - } - - } - - "A (numeric literal)-interpolated regex literal" when { - - "by itself" should { - val code = "/x#{1}y/" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """RegexInterpolationPrimary - | RegexInterpolation - | / - | x - | InterpolatedRegexSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | y - | /""".stripMargin - } - } - - "on the RHS of an assignment" should { - val code = "x = /x#{1}y/" - - "be parsed as a single assignment to a regex literal" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | SingleAssignmentExpression - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | x - | = - | MultipleRightHandSide - | ExpressionOrCommands - | ExpressionExpressionOrCommand - | PrimaryExpression - | RegexInterpolationPrimary - | RegexInterpolation - | / - | x - | InterpolatedRegexSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | y - | /""".stripMargin - } - } - - "as the argument to a `puts` command" should { - val code = "puts /x#{1}y/" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | RegexInterpolationPrimary - | RegexInterpolation - | / - | x - | InterpolatedRegexSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | y - | /""".stripMargin - } - } - - "as the argument to an parenthesized invocation" should { - val code = "puts(/x#{1}y/)" - - "be parsed as such" in { - printAst(_.expressionOrCommand(), code) shouldEqual - """ExpressionExpressionOrCommand - | PrimaryExpression - | InvocationWithParenthesesPrimary - | MethodIdentifier - | puts - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | RegexInterpolationPrimary - | RegexInterpolation - | / - | x - | InterpolatedRegexSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | y - | / - | )""".stripMargin - } - } - } - - "An interpolated quoted (`%r`) regex" when { - - "by itself, containing a numeric literal interpolation and text" should { - - "be parsed as a primary expression" in { - val code = """%r{x#{0}|y}""" - printAst(_.primary(), code) shouldEqual - """QuotedRegexInterpolationPrimary - | QuotedRegexInterpolation - | %r{ - | x - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | } - | |y - | }""".stripMargin - - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RescueClauseTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RescueClauseTests.scala deleted file mode 100644 index fd40cf0ed07a..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RescueClauseTests.scala +++ /dev/null @@ -1,181 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class RescueClauseTests extends RubyParserAbstractTest { - - "A rescue statement" should { - - "be parsed as a standalone statement" when { - - "in the immediate scope of a `begin` block" in { - val code = - """begin - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin - - printAst(_.beginExpression(), code) shouldEqual - """BeginExpression - | begin - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultiplicativeExpression - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | / - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | RescueClause - | rescue - | ExceptionClass - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | ZeroDivisionError - | ExceptionVariableAssignment - | => - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | e - | ThenClause - | CompoundStatement - | end""".stripMargin - } - - "in the immediate scope of a `def` block" in { - val code = - """def foo; - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin - - printAst(_.methodDefinition(), code) shouldEqual - """MethodDefinition - | def - | SimpleMethodNamePart - | DefinedMethodName - | MethodName - | MethodIdentifier - | foo - | MethodParameterPart - | BodyStatement - | CompoundStatement - | ; - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultiplicativeExpression - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | / - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | RescueClause - | rescue - | ExceptionClass - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | ZeroDivisionError - | ExceptionVariableAssignment - | => - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | e - | ThenClause - | CompoundStatement - | end""".stripMargin - } - - "in the immediate scope of a `do` block" in { - val code = - """foo x do |y| - |y/0 - |rescue ZeroDivisionError => e - |end""".stripMargin - - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | ChainedCommandDoBlockInvocationWithoutParentheses - | ChainedCommandWithDoBlock - | ArgsAndDoBlockAndMethodIdCommandWithDoBlock - | MethodIdentifier - | foo - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | DoBlock - | do - | BlockParameter - | | - | BlockParameters - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | y - | | - | BodyStatement - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | MultiplicativeExpression - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | / - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | RescueClause - | rescue - | ExceptionClass - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | ZeroDivisionError - | ExceptionVariableAssignment - | => - | VariableIdentifierOnlySingleLeftHandSide - | VariableIdentifier - | e - | ThenClause - | CompoundStatement - | end""".stripMargin - } - } - - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ReturnTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ReturnTests.scala deleted file mode 100644 index f2df5706fa48..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/ReturnTests.scala +++ /dev/null @@ -1,65 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class ReturnTests extends RubyParserAbstractTest { - - "A standalone return statement" should { - - "be parsed as statement" when { - - "it contains no arguments" in { - val code = "return" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | ReturnArgsInvocationWithoutParentheses - | return""".stripMargin - - } - - "it contains a scoped chain invocation" in { - val code = "return ::X.y()" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | ReturnArgsInvocationWithoutParentheses - | return - | Arguments - | ExpressionArgument - | PrimaryExpression - | ChainedInvocationPrimary - | SimpleScopedConstantReferencePrimary - | :: - | X - | . - | MethodName - | MethodIdentifier - | y - | BlankArgsArgumentsWithParentheses - | ( - | )""".stripMargin - } - - "it contains arguments in parentheses" in { - val code = "return(0)" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | ReturnWithParenthesesPrimary - | return - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 0 - | )""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyLexerTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyLexerTests.scala deleted file mode 100644 index 51f5364d4ef7..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyLexerTests.scala +++ /dev/null @@ -1,1315 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyLexer.* -import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyLexerPostProcessor -import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.Token.EOF -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -class RubyLexerTests extends AnyFlatSpec with Matchers { - - class RubySyntaxErrorListener extends BaseErrorListener { - var errors = 0 - override def syntaxError( - recognizer: Recognizer[?, ?], - offendingSymbol: Any, - line: Int, - charPositionInLine: Int, - msg: String, - e: RecognitionException - ): Unit = - errors += 1 - } - - private def tokenizer(code: String, postProcessor: TokenSource => TokenSource): Iterable[Int] = { - val lexer = new DeprecatedRubyLexer(CharStreams.fromString(code)) - val syntaxErrorListener = new RubySyntaxErrorListener - lexer.addErrorListener(syntaxErrorListener) - val stream = new CommonTokenStream(postProcessor(lexer)) - stream.fill() // Run the lexer - if (syntaxErrorListener.errors > 0) { - Seq() - } else { - import scala.jdk.CollectionConverters.CollectionHasAsScala - stream.getTokens.asScala.map(_.getType) - } - } - - def tokenize(code: String): Iterable[Int] = tokenizer(code, identity) - - def tokenizeOpt(code: String): Iterable[Int] = tokenizer(code, DeprecatedRubyLexerPostProcessor.apply) - - "Single-line comments" should "be discarded" in { - val code = - """ - |# comment 1 - | #comment 2 - |## comment 3 - |""".stripMargin - - tokenize(code) shouldBe Seq(NL, NL, WS, NL, NL, EOF) - } - - "Multi-line comments" should "only be allowed if they start and end on the first column" in { - val code = - """ - |=begin Everything delimited by this =begin..=end - |block is ignored. - | =end - |=end This is the real closing delimiter - |""".stripMargin - - tokenize(code) shouldBe Seq(NL, EOF) - } - - "End-of-program marker (`__END__`)" should "only be recognized if it's on a line of its own" in { - val code = - """ - |# not valid: - |__END__ # comment - | __END__ - |# valid: - |__END__ - |This is now part of the data section and thus removed from the - |main lexer channel. Even __END__ is removed from the main channel. - |""".stripMargin - - tokenize(code) shouldBe Seq(NL, NL, LOCAL_VARIABLE_IDENTIFIER, WS, NL, WS, LOCAL_VARIABLE_IDENTIFIER, NL, NL, EOF) - } - - "Prefixed decimal integer literals" should "be recognized as such" in { - val eg = Seq("0d123456789", "0d1_2_3", "0d0") - all(eg.map(tokenize)) shouldBe Seq(DECIMAL_INTEGER_LITERAL, EOF) - } - - "Non-prefixed decimal integer literals" should "be recognized as such" in { - val eg = Seq("123456789", "1_2_3", "0") - all(eg.map(tokenize)) shouldBe Seq(DECIMAL_INTEGER_LITERAL, EOF) - } - - "Non-prefixed octal integer literals" should "be not be mistaken for decimal integer literals" in { - val eg = Seq("07", "01_2", "01", "0_123", "00") - all(eg.map(tokenize)) shouldBe Seq(OCTAL_INTEGER_LITERAL, EOF) - } - - "Prefixed octal integer literals" should "be recognized as such" in { - val eg = Seq("0o0", "0o1_7", "0o1_2_3") - all(eg.map(tokenize)) shouldBe Seq(OCTAL_INTEGER_LITERAL, EOF) - } - - "Binary integer literals" should "be recognized as such" in { - val eg = Seq("0b0", "0b1", "0b11", "0b1_0", "0b0_1_0") - all(eg.map(tokenize)) shouldBe Seq(BINARY_INTEGER_LITERAL, EOF) - } - - "Hexadecimal integer literals" should "be recognized as such" in { - val eg = Seq("0xA", "0x0_f1", "0x0abcFF_8") - all(eg.map(tokenize)) shouldBe Seq(HEXADECIMAL_INTEGER_LITERAL, EOF) - } - - "Floating-point literals without exponent" should "be recognized as such" in { - val eg = Seq("0.0", "1_2.2_1") - all(eg.map(tokenize)) shouldBe Seq(FLOAT_LITERAL_WITHOUT_EXPONENT, EOF) - } - - "Floating-point literals with exponent" should "be recognized as such" in { - val eg = Seq("0e0", "1E+10", "12e-10", "1.2e4") - all(eg.map(tokenize)) shouldBe Seq(FLOAT_LITERAL_WITH_EXPONENT, EOF) - } - - "Keyword-named symbols" should "be recognized as such" in { - val eg = Seq(":while", ":def", ":if") - all(eg.map(tokenize)) shouldBe Seq(SYMBOL_LITERAL, EOF) - } - - "Operator-named symbols" should "be recognized as such" in { - val eg = Seq(":^", ":==", ":[]", ":[]=", ":+", ":%", ":**", ":>>", ":+@") - all(eg.map(tokenize)) shouldBe Seq(SYMBOL_LITERAL, EOF) - } - - "Assignment-like-named symbols" should "be recognized as such" in { - val eg = Seq(":X=", ":xyz=") - all(eg.map(tokenize)) shouldBe Seq(SYMBOL_LITERAL, EOF) - } - - "Local variable identifiers" should "be recognized as such" in { - val eg = Seq("i", "x1", "old_value", "_internal", "_while") - all(eg.map(tokenize)) shouldBe Seq(LOCAL_VARIABLE_IDENTIFIER, EOF) - } - - "Constant identifiers" should "be recognized as such" in { - val eg = Seq("PI", "Const") - all(eg.map(tokenize)) shouldBe Seq(CONSTANT_IDENTIFIER, EOF) - } - - "Global variable identifiers" should "be recognized as such" in { - val eg = Seq("$foo", "$Foo", "$_", "$__foo") - all(eg.map(tokenize)) shouldBe Seq(GLOBAL_VARIABLE_IDENTIFIER, EOF) - } - - "Instance variable identifiers" should "be recognized as such" in { - val eg = Seq("@x", "@_int", "@if", "@_", "@X0") - all(eg.map(tokenize)) shouldBe Seq(INSTANCE_VARIABLE_IDENTIFIER, EOF) - } - - "Class variable identifiers" should "be recognized as such" in { - val eg = Seq("@@counter", "@@if", "@@While_0") - all(eg.map(tokenize)) shouldBe Seq(CLASS_VARIABLE_IDENTIFIER, EOF) - } - - "Single-quoted string literals" should "be recognized as such" in { - val eg = Seq("''", "'\nx'", "'\\''", "'\\'\n\\\''") - all(eg.map(tokenize)) shouldBe Seq(SINGLE_QUOTED_STRING_LITERAL, EOF) - } - - "Non-interpolated, non-escaped double-quoted string literals" should "be recognized as such" in { - val eg = Seq("\"something\"", "\"x\n\"") - all(eg.map(tokenize)) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Double-quoted string literals containing identifier interpolations" should "be recognized as such" in { - val eg = Seq("\"$x = #$x\"", "\"@xyz = #@xyz\"", "\"@@counter = #@@counter\"") - all(eg.map(tokenize)) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - INTERPOLATED_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Double-quoted string literals containing escaped `#` characters" should "not be mistaken for interpolations" in { - val code = "\"x = \\#$x\"" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Double-quoted string literals containing `#`" should "not be mistaken for interpolations" in { - val code = "\"x = #\"" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Double-quoted string literals containing `\\u` character sequences" should "be recognized as such" in { - val code = """"AB\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000!\u0000file"""" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Interpolated double-quoted string literal" should "be recognized as such" in { - val code = "\"x is #{1+1}\"" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - STRING_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - PLUS, - DECIMAL_INTEGER_LITERAL, - STRING_INTERPOLATION_END, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Recursively interpolated double-quoted string literal" should "be recognized as such" in { - val code = "\"x is #{\"#{1+1}\"}!\"" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - STRING_INTERPOLATION_BEGIN, - DOUBLE_QUOTED_STRING_START, - STRING_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - PLUS, - DECIMAL_INTEGER_LITERAL, - STRING_INTERPOLATION_END, - DOUBLE_QUOTED_STRING_END, - STRING_INTERPOLATION_END, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Escaped `\"` in double-quoted string literal" should "not be mistaken for end of string" in { - val code = "\"x is \\\"4\\\"\"" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Escaped `\\)` in double-quoted string literal" should "be recognized as a single character" in { - val code = """"\)"""" - tokenize(code) shouldBe Seq( - DOUBLE_QUOTED_STRING_START, - DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE, - DOUBLE_QUOTED_STRING_END, - EOF - ) - } - - "Escaped `\\)` in `%Q` string literal" should "be recognized as a single character" in { - val code = """%Q(\))""" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "Empty regex literal" should "be recognized as such" in { - val code = "//" - tokenize(code) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_END, EOF) - } - - "Empty regex literal on the RHS of an assignment" should "be recognized as such" in { - // This test exists to check if RubyLexer properly decided between SLASH and REGULAR_EXPRESSION_START - val code = "x = //" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - WS, - EQ, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Empty regex literal on the RHS of an association" should "be recognized as such" in { - val code = "{x: //}" - tokenize(code) shouldBe Seq( - LCURLY, - LOCAL_VARIABLE_IDENTIFIER, - COLON, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_END, - RCURLY, - EOF - ) - } - - "Non-empty regex literal on the RHS of a keyword argument" should "be recognized as such" in { - val code = "foo(x: /.*/)" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - LPAREN, - LOCAL_VARIABLE_IDENTIFIER, - COLON, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - RPAREN, - EOF - ) - } - - "Non-empty regex literal on the RHS of an assignment" should "be recognized as such" in { - val code = """NAME_REGEX = /\A[^0-9!\``@#\$%\^&*+_=]+\z/""" - tokenize(code) shouldBe Seq( - CONSTANT_IDENTIFIER, - WS, - EQ, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Non-empty regex literal on the RHS of an regex matching operation" should "be recognized as such" in { - val code = """content_filename =~ /filename="(.*)"/""" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - WS, - EQTILDE, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Non-empty regex literal after `when`" should "be recognized as such" in { - val code = "when /^ch_/" - tokenize(code) shouldBe Seq( - WHEN, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Non-empty regex literal after `unless`" should "be recognized as such" in { - val code = "unless /^ch_/" - tokenize(code) shouldBe Seq( - UNLESS, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Regex literals without metacharacters" should "be recognized as such" in { - val eg = Seq("/regexp/", "/a regexp/") - all(eg.map(tokenize)) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_BODY, REGULAR_EXPRESSION_END, EOF) - } - - "Regex literals with metacharacters" should "be recognized as such" in { - val eg = Seq("/(us|eu)/", "/[a-z]/", "/[A-Z]*/", "/(us|eu)?/", "/[0-9]+/") - all(eg.map(tokenize)) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_BODY, REGULAR_EXPRESSION_END, EOF) - } - - "Regex literals with character classes" should "be recognized as such" in { - val eg = Seq("/\\w/", "/\\W/", "/\\S/") - all(eg.map(tokenize)) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_BODY, REGULAR_EXPRESSION_END, EOF) - } - - "Regex literals with groups" should "be recognized as such" in { - val eg = Seq("/[aeiou]\\w{2}/", "/(\\d{2}:\\d{2}) (\\w+) (.*)/", "/(?\\w+) (?\\d+)/") - all(eg.map(tokenize)) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_BODY, REGULAR_EXPRESSION_END, EOF) - } - - "Regex literals with options" should "be recognized as such" in { - val eg = Seq("/./m", "/./i", "/./x", "/./o") - all(eg.map(tokenize)) shouldBe Seq(REGULAR_EXPRESSION_START, REGULAR_EXPRESSION_BODY, REGULAR_EXPRESSION_END, EOF) - } - - "Interpolated (with a local variable) regex literal" should "be recognized as such" in { - val code = "/#{foo}/" - tokenize(code) shouldBe Seq( - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_INTERPOLATION_BEGIN, - LOCAL_VARIABLE_IDENTIFIER, - REGULAR_EXPRESSION_INTERPOLATION_END, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Interpolated (with a numeric expression) regex literal" should "be recognized as such" in { - val code = "/#{1+1}/" - tokenize(code) shouldBe Seq( - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - PLUS, - DECIMAL_INTEGER_LITERAL, - REGULAR_EXPRESSION_INTERPOLATION_END, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Interpolated (with a local variable) regex literal containing also textual body elements" should "be recognized as such" in { - val code = "/x\\.#{foo}\\./" - tokenize(code) shouldBe Seq( - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_INTERPOLATION_BEGIN, - LOCAL_VARIABLE_IDENTIFIER, - REGULAR_EXPRESSION_INTERPOLATION_END, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Vacuously interpolated regex literal" should "be recognized as such" in { - val code = "/#{}/" - tokenize(code) shouldBe Seq( - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_INTERPOLATION_BEGIN, - REGULAR_EXPRESSION_INTERPOLATION_END, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Division operator between identifiers" should "not be confused with regex start" in { - val code = "x / y" - tokenize(code) shouldBe Seq(LOCAL_VARIABLE_IDENTIFIER, WS, SLASH, WS, LOCAL_VARIABLE_IDENTIFIER, EOF) - } - - "Addition between class fields" should "not be confused with +@ token" in { - // This test exists to check if RubyLexer properly decided between PLUS and PLUSAT - val code = "x+@y" - tokenize(code) shouldBe Seq(LOCAL_VARIABLE_IDENTIFIER, PLUS, INSTANCE_VARIABLE_IDENTIFIER, EOF) - } - - "Subtraction between class fields" should "not be confused with -@ token" in { - // This test exists to check if RubyLexer properly decided between MINUS and MINUSAT - val code = "x-@y" - tokenize(code) shouldBe Seq(LOCAL_VARIABLE_IDENTIFIER, MINUS, INSTANCE_VARIABLE_IDENTIFIER, EOF) - } - - "Invocation of command with regex literal" should "not be confused with binary division" in { - val code = "puts /x/" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - WS, - REGULAR_EXPRESSION_START, - REGULAR_EXPRESSION_BODY, - REGULAR_EXPRESSION_END, - EOF - ) - } - - "Multi-line string literal concatenation" should "be recognized as two string literals separated by whitespace" in { - val code = - """'abc' \ - |'cde'""".stripMargin - tokenize(code) shouldBe Seq(SINGLE_QUOTED_STRING_LITERAL, WS, SINGLE_QUOTED_STRING_LITERAL, EOF) - } - - "Multi-line string literal concatenation" should "be optimized as two consecutive string literals" in { - val code = - """'abc' \ - |'cde'""".stripMargin - tokenizeOpt(code) shouldBe Seq(SINGLE_QUOTED_STRING_LITERAL, SINGLE_QUOTED_STRING_LITERAL, EOF) - } - - "empty `%q` string literals" should "be recognized as such" in { - val eg = Seq("%q()", "%q[]", "%q{}", "%q<>", "%q##", "%q!!", "%q--", "%q@@", "%q++", "%q**", "%q//", "%q&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "single-character `%q` string literals" should "be recognized as such" in { - val eg = - Seq("%q(x)", "%q[y]", "%q{z}", "%q", "%q#a#", "%q!b!", "%q-_-", "%q@c@", "%q+d+", "%q*e*", "%q/#/", "%q&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - NON_EXPANDED_LITERAL_CHARACTER, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "delimiter-escaped-single-character `%q` string literals" should "be recognized as such" in { - val eg = Seq( - "%q(\\))", - "%q[\\]]", - "%q{\\}}", - "%q<\\>>", - "%q#\\##", - "%q!\\!!", - "%q-\\--", - "%q@\\@@", - "%q+\\++", - "%q*\\**", - "%q/\\//", - "%q&\\&&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - NON_EXPANDED_LITERAL_CHARACTER, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "nested `%q` string literals" should "be recognized as such" in { - val eg = Seq("%q(()())", "%q[[][]]", "%q{{}{}}", "%q<<><>>") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - NON_EXPANDED_LITERAL_CHARACTER, - NON_EXPANDED_LITERAL_CHARACTER, - NON_EXPANDED_LITERAL_CHARACTER, - NON_EXPANDED_LITERAL_CHARACTER, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "empty `%Q` string literals" should "be recognized as such" in { - val eg = Seq("%Q()", "%Q[]", "%Q{}", "%Q<>", "%Q##", "%Q!!", "%Q--", "%Q@@", "%Q++", "%Q**", "%Q//", "%Q&&") - all(eg.map(tokenize)) shouldBe Seq(QUOTED_EXPANDED_STRING_LITERAL_START, QUOTED_EXPANDED_STRING_LITERAL_END, EOF) - } - - "single-character `%Q` string literals" should "be recognized as such" in { - val eg = - Seq("%Q(x)", "%Q[y]", "%Q{z}", "%Q", "%Q#a#", "%Q!b!", "%Q-_-", "%Q@c@", "%Q+d+", "%Q*e*", "%Q/#/", "%Q&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "delimiter-escaped-single-character `%Q` string literals" should "be recognized as such" in { - val eg = Seq( - "%Q(\\))", - "%Q[\\]]", - "%Q{\\}}", - "%Q<\\>>", - "%Q#\\##", - "%Q!\\!!", - "%Q-\\--", - "%Q@\\@@", - "%Q+\\++", - "%Q*\\**", - "%Q/\\//", - "%Q&\\&&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "nested `%Q` string literals" should "be recognized as such" in { - val eg = Seq("%Q(()())", "%Q[[][]]", "%Q{{}{}}", "%Q<<><>>") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "interpolated (with a numeric expression) `%Q` string literals" should "be recognized as such" in { - val code = "%Q(#{1})" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - DELIMITED_STRING_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_STRING_INTERPOLATION_END, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%Q` string literals containing identifier interpolations" should "be recognized as such" in { - val eg = Seq("%Q[x = #$x]", "%Q{x = #@xyz}", "%Q") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_VARIABLE_CHARACTER_SEQUENCE, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%Q` string literals containing escaped `#` characters" should "not be mistaken for interpolations" in { - val code = """%Q(\#$x)""" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%Q` string literals containing `#`" should "not be mistaken for interpolations" in { - val code = "%Q[#]" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "empty `%(` string literals" should "be recognized as such" in { - val code = "%()" - tokenize(code) shouldBe Seq(QUOTED_EXPANDED_STRING_LITERAL_START, QUOTED_EXPANDED_STRING_LITERAL_END, EOF) - } - - "single-character `%(` string literals" should "be recognized as such" in { - val code = "%(-)" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "delimiter-escaped-single-character `%(` string literals" should "be recognized as such" in { - val code = "%(\\))" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "nested `%(` string literals" should "be recognized as such" in { - val code = "%(()())" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "interpolated (with a numeric expression) `%(` string literals" should "be recognized as such" in { - val code = "%(#{1})" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - DELIMITED_STRING_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_STRING_INTERPOLATION_END, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%(` string literals containing identifier interpolations" should "be recognized as such" in { - val eg = Seq("%(x = #$x)", "%(x = #@xyz)", "%(x = #@@counter)") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_VARIABLE_CHARACTER_SEQUENCE, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%(` after a decimal literal" should "not be mistaken for an expanded string literal" in { - val code = "100%(x+1)" - tokenize(code) shouldBe Seq( - DECIMAL_INTEGER_LITERAL, - PERCENT, - LPAREN, - LOCAL_VARIABLE_IDENTIFIER, - PLUS, - DECIMAL_INTEGER_LITERAL, - RPAREN, - EOF - ) - } - - "`%(` in a `puts` argument" should "be recognized as such" in { - val code = "puts %()" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - WS, - QUOTED_EXPANDED_STRING_LITERAL_START, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%(` string literals containing escaped `#` characters" should "not be mistaken for interpolations" in { - val code = """%(\#$x)""" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "`%(` string literals containing `#`" should "not be mistaken for interpolations" in { - val code = "%(#)" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "Empty `%x` literals" should "be recognized as such" in { - val eg = Seq("%x()", "%x[]", "%x{}", "%x<>", "%x##", "%x!!", "%x--", "%x@@", "%x++", "%x**", "%x//", "%x&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - EOF - ) - } - - "`%x` literals containing `#`" should "not be mistaken for interpolations" in { - val code = "%x[#]" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - EOF - ) - } - - "`%x` literals containing escaped `#` characters" should "not be mistaken for interpolations" in { - val code = """%x(\#$x)""" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - EOF - ) - } - - "`%x` literals containing identifier interpolations" should "be recognized as such" in { - val eg = Seq("%x[#$x]", "%x{#@xyz}", "%x<#@@counter>") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START, - EXPANDED_VARIABLE_CHARACTER_SEQUENCE, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - EOF - ) - } - - "Interpolated (with a local variable) `%x` literals" should "be recognized as such" in { - val code = "%x(#{ls})" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_START, - DELIMITED_STRING_INTERPOLATION_BEGIN, - LOCAL_VARIABLE_IDENTIFIER, - DELIMITED_STRING_INTERPOLATION_END, - QUOTED_EXPANDED_EXTERNAL_COMMAND_LITERAL_END, - EOF - ) - } - - "empty `%r` regex literals" should "be recognized as such" in { - val eg = Seq("%r()", "%r[]", "%r{}", "%r<>", "%r##", "%r!!", "%r--", "%r@@", "%r++", "%r**", "%r//", "%r&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_REGULAR_EXPRESSION_START, - QUOTED_EXPANDED_REGULAR_EXPRESSION_END, - EOF - ) - } - - "single-character `%r` regex literals" should "be recognized as such" in { - val eg = - Seq("%r(x)", "%r[y]", "%r{z}", "%r", "%r#a#", "%r!b!", "%r-_-", "%r@c@", "%r+d+", "%r*e*", "%r/#/", "%r&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_REGULAR_EXPRESSION_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_REGULAR_EXPRESSION_END, - EOF - ) - } - - "delimiter-escaped-single-character `%r` regex literals" should "be recognized as such" in { - val eg = Seq( - "%r(\\))", - "%r[\\]]", - "%r{\\}}", - "%r<\\>>", - "%r#\\##", - "%r!\\!!", - "%r-\\--", - "%r@\\@@", - "%r+\\++", - "%r*\\**", - "%r/\\//", - "%r&\\&&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_REGULAR_EXPRESSION_START, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_REGULAR_EXPRESSION_END, - EOF - ) - } - - "nested `%r` regex literals" should "be recognized as such" in { - val eg = Seq("%r(()())", "%r[[][]]", "%r{{}{}}", "%r<<><>>") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_REGULAR_EXPRESSION_START, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - EXPANDED_LITERAL_CHARACTER, - QUOTED_EXPANDED_REGULAR_EXPRESSION_END, - EOF - ) - } - - "empty `%w` string array literals" should "be recognized as such" in { - val eg = Seq("%w()", "%w[]", "%w{}", "%w<>", "%w##", "%w!!", "%w--", "%w@@", "%w++", "%w**", "%w//", "%w&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single-character `%w` string array literals" should "be recognized as such" in { - val eg = - Seq("%w(x)", "%w[y]", "%w{z}", "%w", "%w#a#", "%w!b!", "%w-_-", "%w@c@", "%w+d+", "%w*e*", "%w/#/", "%w&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "two-word `%w` string array literals" should "be recognized as such" in { - val eg = Seq( - "%w(xx y)", - "%w[yy z]", - "%w{z0 w}", - "%w", - "%w#a& ?#", - "%w!b_ c!", - "%w-_= +-", - "%w@c\" d@", - "%w+d/ *+", - "%w*ef <*", - "%w/#< >/", - "%w&!! %&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single word `%w` string array literal containing an escaped whitespace" should "be recognized as such" in { - val code = """%w[x\ y]""" - tokenize(code) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "multi-line `%w` string array literal" should "be recognized as such" in { - val code = - """%w( - | bob - | cod - | dod)""".stripMargin - tokenize(code) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "empty `%W` string array literals" should "be recognized as such" in { - val eg = Seq("%W()", "%W[]", "%W{}", "%W<>", "%W##", "%W!!", "%W--", "%W@@", "%W++", "%W**", "%W//", "%W&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single-character `%W` string array literals" should "be recognized as such" in { - val eg = - Seq("%W(x)", "%W[y]", "%W{z}", "%W", "%W#a#", "%W!b!", "%W-_-", "%W@c@", "%W+d+", "%W*e*", "%W/#/", "%W&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "two-word `%W` string array literals" should "be recognized as such" in { - val eg = Seq( - "%W(xx y)", - "%W[yy z]", - "%W{z0 w}", - "%W", - "%W#a& ?#", - "%W!b_ c!", - "%W-_= +-", - "%W@c\" d@", - "%W+d/ *+", - "%W*ef <*", - "%W/#< >/", - "%W&!! %&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single interpolated word `%W` string array literal" should "be recognized as such" in { - val code = "%W{#{0}}" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single word `%W` string array literal containing text and an interpolated numeric" should "be recognized as such" in { - val code = "%W" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "two-word `%W` string array literal containing text and interpolated numerics" should "be recognized as such" in { - val code = "%W(x#{0} x#{1})" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "single word `%W` string array literal containing an escaped whitespace" should "be recognized as such" in { - val code = """%W[x\ y]""" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "multi-line `%W` string array literal" should "be recognized as such" in { - val code = - """%W( - | bob - | cod - | dod)""".stripMargin - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_EXPANDED_STRING_ARRAY_LITERAL_END, - EOF - ) - } - - "empty `%i` symbol array literals" should "be recognized as such" in { - val eg = Seq("%i()", "%i[]", "%i{}", "%i<>", "%i##", "%i!!", "%i--", "%i@@", "%i++", "%i**", "%i//", "%i&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "single-character `%i` symbol array literals" should "be recognized as such" in { - val eg = - Seq("%i(x)", "%i[y]", "%i{z}", "%i", "%i#a#", "%i!b!", "%i-_-", "%i@c@", "%i+d+", "%i*e*", "%i/#/", "%i&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "two-word `%i` symbol array literals" should "be recognized as such" in { - val eg = Seq( - "%i(xx y)", - "%i[yy z]", - "%i{z0 w}", - "%i", - "%i#a& ?#", - "%i!b_ c!", - "%i-_= +-", - "%i@c\" d@", - "%i+d/ *+", - "%i*ef <*", - "%i/#< >/", - "%i&!! %&" - ) - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "multi-line two-word `%i` symbol array literals" should "be recognized as such" in { - val code = - """%i( - |x - |y - |)""".stripMargin - tokenize(code) shouldBe Seq( - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - NON_EXPANDED_ARRAY_ITEM_CHARACTER, - NON_EXPANDED_ARRAY_ITEM_SEPARATOR, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "empty `%I` symbol array literals" should "be recognized as such" in { - val eg = Seq("%I()", "%I[]", "%I{}", "%I<>", "%I##", "%I!!", "%I--", "%I@@", "%I++", "%I**", "%I//", "%I&&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "single-character `%I` symbol array literals" should "be recognized as such" in { - val eg = - Seq("%I(x)", "%I[y]", "%I{z}", "%I", "%I#a#", "%I!b!", "%I-_-", "%I@c@", "%I+d+", "%I*e*", "%I/#/", "%I&!&") - all(eg.map(tokenize)) shouldBe Seq( - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "two-word `%I` symbol array literal containing text and interpolated numerics" should "be recognized as such" in { - val code = "%I(x#{0} x#{1})" - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_CHARACTER, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - DELIMITED_ARRAY_ITEM_INTERPOLATION_BEGIN, - DECIMAL_INTEGER_LITERAL, - DELIMITED_ARRAY_ITEM_INTERPOLATION_END, - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "multi-line two-word `%I` symbol array literals" should "be recognized as such" in { - val code = - """%I( - |x - |y - |)""".stripMargin - tokenize(code) shouldBe Seq( - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_SEPARATOR, - EXPANDED_ARRAY_ITEM_CHARACTER, - EXPANDED_ARRAY_ITEM_SEPARATOR, - QUOTED_EXPANDED_SYMBOL_ARRAY_LITERAL_END, - EOF - ) - } - - "identifier used in a keyword argument" should "not be mistaken for a symbol literal" in { - // This test exists to check if RubyLexer properly decided between COLON and SYMBOL_LITERAL - val code = "foo(x:y)" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - LPAREN, - LOCAL_VARIABLE_IDENTIFIER, - COLON, - LOCAL_VARIABLE_IDENTIFIER, - RPAREN, - EOF - ) - } - - "instance variable used in a keyword argument" should "not be mistaken for a symbol literal" in { - // This test exists to check if RubyLexer properly decided between COLON and SYMBOL_LITERAL - val code = "foo(x:@y)" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - LPAREN, - LOCAL_VARIABLE_IDENTIFIER, - COLON, - INSTANCE_VARIABLE_IDENTIFIER, - RPAREN, - EOF - ) - } - - "operator-named symbol used in a whitespace-free `=>` association" should "not be include `=` as part of its name" in { - // This test exists to check if RubyLexer properly recognizes EQGT - val code = "{:x=>1}" - tokenize(code) shouldBe Seq(LCURLY, SYMBOL_LITERAL, EQGT, DECIMAL_INTEGER_LITERAL, RCURLY, EOF) - } - - "class variable used in a keyword argument" should "not be mistaken for a symbol literal" in { - // This test exists to check if RubyLexer properly decided between COLON and SYMBOL_LITERAL - val code = "foo(x:@@y)" - tokenize(code) shouldBe Seq( - LOCAL_VARIABLE_IDENTIFIER, - LPAREN, - LOCAL_VARIABLE_IDENTIFIER, - COLON, - CLASS_VARIABLE_IDENTIFIER, - RPAREN, - EOF - ) - } - - "Regex match global variables" should "be recognized as such" in { - val eg = Seq("$0", "$10", "$2", "$3") - all(eg.map(tokenize)) shouldBe Seq(GLOBAL_VARIABLE_IDENTIFIER, EOF) - } - - "Assignment-like method identifiers" should "be recognized as such" in { - val eg = Seq("def x=", "def X=") - all(eg.map(tokenize)) shouldBe Seq(DEF, WS, ASSIGNMENT_LIKE_METHOD_IDENTIFIER, EOF) - } - - "Unrecognized escape character" should "emit an UNRECOGNIZED token" in { - val code = "\\!" - tokenize(code) shouldBe Seq(UNRECOGNIZED, EMARK, EOF) - } - - "Single NON_EXPANDED_LITERAL_CHARACTER token" should "be rewritten into a single NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE token" in { - val code = "%q{ }" - tokenizeOpt(code) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "Consecutive NON_EXPANDED_LITERAL_CHARACTER tokens" should "be rewritten into a single NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE token" in { - val code = "%q(1 2 3 4)" - tokenizeOpt(code) shouldBe Seq( - QUOTED_NON_EXPANDED_STRING_LITERAL_START, - NON_EXPANDED_LITERAL_CHARACTER_SEQUENCE, - QUOTED_NON_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "Single EXPANDED_LITERAL_CHARACTER token" should "be rewritten into a single EXPANDED_LITERAL_CHARACTER_SEQUENCE token" in { - val code = "%Q( )" - tokenizeOpt(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER_SEQUENCE, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } - - "Consecutive EXPANDED_LITERAL_CHARACTER tokens" should "be rewritten into a single EXPANDED_LITERAL_CHARACTER_SEQUENCE token" in { - val code = "%Q{1 2 3 4 5}" - tokenizeOpt(code) shouldBe Seq( - QUOTED_EXPANDED_STRING_LITERAL_START, - EXPANDED_LITERAL_CHARACTER_SEQUENCE, - QUOTED_EXPANDED_STRING_LITERAL_END, - EOF - ) - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyParserAbstractTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyParserAbstractTest.scala deleted file mode 100644 index b9754656ad84..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/RubyParserAbstractTest.scala +++ /dev/null @@ -1,31 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -import io.joern.rubysrc2cpg.parser.AstPrinter -import org.antlr.v4.runtime.{CharStreams, CommonTokenStream, ParserRuleContext} -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import java.util.stream.Collectors - -// TODO: Should share the same lexer/token stream/parser as the frontend itself. -// See `io.joern.rubysrc2cpg.astcreation.AntlrParser` -abstract class RubyParserAbstractTest extends AnyWordSpec with Matchers { - - def rubyStream(code: String): CommonTokenStream = - new CommonTokenStream(DeprecatedRubyLexerPostProcessor(new DeprecatedRubyLexer(CharStreams.fromString(code)))) - - def rubyParser(code: String): DeprecatedRubyParser = - new DeprecatedRubyParser(rubyStream(code)) - - def printAst(withContext: DeprecatedRubyParser => ParserRuleContext, input: String): String = - omitWhitespaceLines(AstPrinter.print(withContext(rubyParser(input)))) - - private def omitWhitespaceLines(text: String): String = - text.lines().filter(_.strip().nonEmpty).collect(Collectors.joining("\n")) - - def accepts(withContext: DeprecatedRubyParser => ParserRuleContext, input: String): Boolean = { - val parser = rubyParser(input) - withContext(parser) - parser.getNumberOfSyntaxErrors == 0 - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/StringTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/StringTests.scala deleted file mode 100644 index 16f0df7d4db2..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/StringTests.scala +++ /dev/null @@ -1,671 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class StringTests extends RubyParserAbstractTest { - - "A single-quoted string literal" when { - - "empty" should { - val code = "''" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | SimpleStringExpression - | SingleQuotedStringLiteral - | ''""".stripMargin - } - } - - "separated by whitespace" should { - val code = "'x' 'y'" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'x' - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'y'""".stripMargin - } - } - - "separated by '\\\\n' " should { - val code = """'x' \ - | 'y'""".stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'x' - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'y'""".stripMargin - } - } - - "separated by '\\\\n' twice" should { - val code = - """'x' \ - | 'y' \ - | 'z'""".stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'x' - | ConcatenatedStringExpression - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'y' - | SimpleStringExpression - | SingleQuotedStringLiteral - | 'z'""".stripMargin - } - } - } - - "A non-expanded `%q` string literal" should { - - "be parsed as a primary expression" when { - - "it is empty and uses the `(`-`)` delimiters" in { - val code = "%q()" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | )""".stripMargin - } - - "it is empty and uses the `[`-`]` delimiters" in { - val code = "%q[]" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q[ - | ]""".stripMargin - } - - "it is empty and uses the `{`-`}` delimiters" in { - val code = "%q{}" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q{ - | }""".stripMargin - } - - "it is empty and uses the `<`-`>` delimiters" in { - val code = "%q<>" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q< - | >""".stripMargin - } - - "it is empty and uses the `#` delimiters" in { - val code = "%q##" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q# - | #""".stripMargin - } - - "it contains a single non-escaped character and uses the `(`-`)` delimiters" in { - val code = "%q(x)" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | x - | )""".stripMargin - } - - "it contains a single non-escaped character and uses the `[`-`]` delimiters" in { - val code = "%q[x]" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q[ - | x - | ]""".stripMargin - } - - "it contains a single non-escaped character and uses the `#` delimiters" in { - val code = "%q#x#" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q# - | x - | #""".stripMargin - } - - "it contains a single escaped character and uses the `(`-`)` delimiters" in { - val code = "%q(\\()" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | \( - | )""".stripMargin - } - - "it contains a single escaped character and uses the `[`-`]` delimiters" in { - val code = "%q[\\]]" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q[ - | \] - | ]""".stripMargin - } - - "it contains a single escaped character and uses the `#` delimiters" in { - val code = "%q#\\##" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q# - | \# - | #""".stripMargin - } - - "it contains a word and uses the `(`-`)` delimiters" in { - val code = "%q(foo)" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | foo - | )""".stripMargin - } - - "it contains an empty nested string using the `(`-`)` delimiters" in { - val code = "%q( () )" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | () - | )""".stripMargin - } - - "it contains an escaped single-character nested string using the `(`-`)` delimiters" in { - val code = "%q( (\\)) )" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q( - | (\)) - | )""".stripMargin - } - - "it contains an escaped single-character nested string using the `<`-`>` delimiters" in { - val code = "%q< <\\>> >" - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | NonExpandedQuotedStringLiteral - | %q< - | <\>> - | >""".stripMargin - } - } - } - - "An expanded `%Q` string literal" when { - - "empty" should { - val code = "%Q()" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %Q( - | )""".stripMargin - } - } - - "containing text and a numeric literal interpolation" should { - val code = "%Q{text=#{1}}" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %Q{ - | text= - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | }""".stripMargin - } - } - - "containing two consecutive numeric literal interpolations" should { - val code = "%Q[#{1}#{2}]" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %Q[ - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | } - | ]""".stripMargin - } - } - - } - - "An expanded `%(` string literal" when { - - "empty" should { - val code = "%()" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %( - | )""".stripMargin - } - } - - "containing text and a numeric literal interpolation" should { - val code = "%(text=#{1})" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %( - | text= - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | )""".stripMargin - } - } - - "containing two consecutive numeric literal interpolations" should { - val code = "%(#{1}#{2})" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %( - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | } - | )""".stripMargin - } - } - - "used as the argument to a `puts` command" should { - - "be parsed as a statement" in { - val code = "puts %()" - printAst(_.statement(), code) shouldEqual - """ExpressionOrCommandStatement - | InvocationExpressionOrCommand - | SingleCommandOnlyInvocationWithoutParentheses - | SimpleMethodCommand - | MethodIdentifier - | puts - | ArgumentsWithoutParentheses - | Arguments - | ExpressionArgument - | PrimaryExpression - | QuotedStringExpressionPrimary - | ExpandedQuotedStringLiteral - | %( - | )""".stripMargin - } - } - } - - "A double-quoted string literal" when { - - "empty" should { - val code = "\"\"" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | """".stripMargin - } - - "separated by whitespace" should { - val code = "\"x\" \"y\"" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | x - | " - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | y - | """".stripMargin - } - } - - "separated by '\\\\n'" should { - val code = - """"x" \ - | "y" """.stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | x - | " - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | y - | """".stripMargin - } - } - } - - "containing text and a numeric literal interpolation" should { - val code = """"text=#{1}"""" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | InterpolatedStringExpression - | StringInterpolation - | " - | text= - | InterpolatedStringSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | """".stripMargin - } - } - - "containing two numeric literal interpolations" should { - val code = """"#{1}#{2}"""" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | InterpolatedStringExpression - | StringInterpolation - | " - | InterpolatedStringSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 1 - | } - | InterpolatedStringSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 2 - | } - | """".stripMargin - } - } - - "separated by '\\\\n'" should { - val code = """"x" \ - | "y" """.stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | x - | " - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | y - | """".stripMargin - } - - "separated by '\\\\n' and containing a numeric interpolation" should { - val code = """"#{10}" \ - | "is a number."""".stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """StringExpressionPrimary - | ConcatenatedStringExpression - | InterpolatedStringExpression - | StringInterpolation - | " - | InterpolatedStringSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 10 - | } - | " - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | is a number. - | """".stripMargin - } - } - } - } - - "An expanded `%x` external command literal" when { - - "empty" should { - val code = "%x//" - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedExternalCommandLiteral - | %x/ - | /""".stripMargin - } - } - - "containing text and a string literal interpolation" should { - val code = "%x{l#{'s'}}" - - "be parsed as primary expression" in { - printAst(_.primary(), code) shouldEqual - """QuotedStringExpressionPrimary - | ExpandedExternalCommandLiteral - | %x{ - | l - | DelimitedStringInterpolation - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | StringExpressionPrimary - | SimpleStringExpression - | SingleQuotedStringLiteral - | 's' - | } - | }""".stripMargin - } - } - } - - "A HERE_DOCs expression" when { - - "used to generate a single string" should { - val code = - """<<-SQL - |SELECT * FROM food - |WHERE healthy = true - |SQL - |""".stripMargin - - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """LiteralPrimary - | HereDocLiteral - | <<-SQL - |SELECT * FROM food - |WHERE healthy = true - |SQL""".stripMargin - } - - } - - "used to generate a single string parameter for a function call" should { - val code = - """foo(<<-SQL) - |SELECT * FROM food - |WHERE healthy = true - |SQL - |""".stripMargin - - // TODO: The rest of the HERE_DOC should probably be parsed somehow - "be parsed as a primary expression" in { - printAst(_.primary(), code) shouldEqual - """InvocationWithParenthesesPrimary - | MethodIdentifier - | foo - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | HereDocArgument - | <<-SQL - | )""".stripMargin - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/SymbolTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/SymbolTests.scala deleted file mode 100644 index a48bb91d8167..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/SymbolTests.scala +++ /dev/null @@ -1,132 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class SymbolTests extends RubyParserAbstractTest { - - "Symbol literals" should { - - "be parsed as primary expressions" when { - - def symbolLiteralParseTreeText(symbolName: String): String = - s"""LiteralPrimary - | SymbolLiteral - | Symbol - | $symbolName""".stripMargin - - "they are named after keywords" in { - val eg = Seq( - ":__LINE__", - ":__ENCODING__", - ":__FILE__", - ":BEGIN", - ":END", - ":alias", - ":begin", - ":break", - ":case", - ":class", - ":def", - ":defined?", - ":do", - ":else", - ":elsif", - ":end", - ":ensure", - ":for", - ":false", - ":if", - ":in", - ":module", - ":next", - ":nil", - ":not", - ":or", - ":redo", - ":rescue", - ":retry", - ":self", - ":super", - ":then", - ":true", - ":undef", - ":unless", - ":until", - ":when", - ":while", - ":yield" - ) - eg.map(code => printAst(_.primary(), code)) shouldEqual eg.map(symbolLiteralParseTreeText) - } - - "they are named after operators" in { - val eg = Seq( - ":^", - ":&", - ":|", - ":<=>", - ":==", - ":===", - ":=~", - ":>", - ":>=", - ":<", - ":<=", - ":<<", - ":>>", - ":+", - ":-", - ":*", - ":/", - ":%", - ":**", - ":~", - ":+@", - ":-@", - ":[]", - ":[]=" - ) - eg.map(code => printAst(_.primary(), code)) shouldEqual eg.map(symbolLiteralParseTreeText) - } - - "they are given by a non-interpolated double-quoted string literal" in { - val code = """:"x y z"""" - printAst(_.primary(), code) shouldEqual - """LiteralPrimary - | SymbolLiteral - | Symbol - | : - | SimpleStringExpression - | DoubleQuotedStringLiteral - | " - | x y z - | """".stripMargin - } - - "they are given by an interpolated double-quoted string literal" in { - val code = """:"#{10}"""" - printAst(_.primary(), code) shouldEqual - """LiteralPrimary - | SymbolLiteral - | Symbol - | : - | InterpolatedStringExpression - | StringInterpolation - | " - | InterpolatedStringSequence - | #{ - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | LiteralPrimary - | NumericLiteralLiteral - | NumericLiteral - | UnsignedNumericLiteral - | 10 - | } - | """".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/TernaryConditionalTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/TernaryConditionalTests.scala deleted file mode 100644 index af86bb1d5f5d..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/TernaryConditionalTests.scala +++ /dev/null @@ -1,62 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class TernaryConditionalTests extends RubyParserAbstractTest { - - "Ternary conditional expressions" should { - - "be parsed as expressions" when { - - "they are a standalone one-line expression" in { - val code = "x ? y : z" - printAst(_.expression(), code) shouldEqual - """ConditionalOperatorExpression - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | ? - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | : - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | z""".stripMargin - - } - - "they are a standalone multi-line expression" in { - val code = - """x ? - | y - |: z - |""".stripMargin - printAst(_.expression(), code) shouldEqual - """ConditionalOperatorExpression - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | x - | ? - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | y - | : - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | z""".stripMargin - } - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/UnlessConditionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/UnlessConditionTests.scala deleted file mode 100644 index bc307bf80af3..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/parser/UnlessConditionTests.scala +++ /dev/null @@ -1,136 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.parser - -class UnlessConditionTests extends RubyParserAbstractTest { - - "An unless expression" should { - "be parsed as a primary expression" when { - - "it uses a newline instead of the keyword then" in { - val code = - """unless foo - | bar - |end - |""".stripMargin - - printAst(_.primary(), code) shouldEqual - """UnlessExpressionPrimary - | UnlessExpression - | unless - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | ThenClause - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | bar - | end""".stripMargin - } - - "it uses a semicolon instead of the keyword then" in { - val code = - """unless foo; bar - |end - |""".stripMargin - - printAst(_.primary(), code) shouldEqual - """UnlessExpressionPrimary - | UnlessExpression - | unless - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | ThenClause - | ; - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | bar - | end""".stripMargin - } - - "it uses the keyword then" in { - val code = - """unless foo then - | bar - |end - |""".stripMargin - - printAst(_.primary(), code) shouldEqual - """UnlessExpressionPrimary - | UnlessExpression - | unless - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | foo - | ThenClause - | then - | CompoundStatement - | Statements - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | bar - | end""".stripMargin - } - } - } - - "An unless (modifier) statement" should { - - "be parsed as a statement" when { - - "it explicitly returns an identifier out of a method" in { - val code = "return(value) unless item" - - printAst(_.statement(), code) shouldEqual - """ModifierStatement - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | ReturnWithParenthesesPrimary - | return - | ArgsOnlyArgumentsWithParentheses - | ( - | Arguments - | ExpressionArgument - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | value - | ) - | unless - | ExpressionOrCommandStatement - | ExpressionExpressionOrCommand - | PrimaryExpression - | VariableReferencePrimary - | VariableIdentifierVariableReference - | VariableIdentifier - | item""".stripMargin - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ConfigFileCreationPassTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ConfigFileCreationPassTest.scala deleted file mode 100644 index ecd048bbb4a4..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ConfigFileCreationPassTest.scala +++ /dev/null @@ -1,62 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import better.files.File -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.passes.frontend.MetaDataPass -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class ConfigFileCreationPassTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "ConfigFileCreationPass for Gemfile files" should { - - "generate a ConfigFile accordingly" in { - val gemFileContents = - """ - |source 'https://rubygems.org' - |gem 'json' - |""".stripMargin - val cpg = code(gemFileContents, "Gemfile") - - val List(configFile) = cpg.configFile.l - configFile.name shouldBe "Gemfile" - configFile.content shouldBe gemFileContents - } - - "ignore non-root Gemfile files" in { - val cpg = code("# ignore me", Seq("subdir", "Gemfile").mkString(java.io.File.pathSeparator)) - cpg.configFile.size shouldBe 0 - } - } - - "ConfigFileCreationPass for Gemfile.lock files" should { - - "generate a ConfigFile accordingly" in { - val gemFileContents = - """ - |GEM - | remote: https://rubygems.org/ - | specs: - | CFPropertyList (3.0.1) - | - |PLATFORMS - | ruby - | - |BUNDLED WITH - | 2.1.4 - |""".stripMargin - val cpg = code(gemFileContents, "Gemfile.lock") - val List(configFile) = cpg.configFile.l - configFile.name shouldBe "Gemfile.lock" - configFile.content shouldBe gemFileContents - } - - "ignore non-root Gemfile.lock files" in { - val cpg = code("# ignore me", Seq("subdir", "Gemfile.lock").mkString(java.io.File.pathSeparator)) - cpg.configFile.size shouldBe 0 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala deleted file mode 100644 index eeeb970917c1..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/MetaDataPassTests.scala +++ /dev/null @@ -1,22 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import better.files.File -import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} -import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language.* -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class MetaDataPassTests extends AnyWordSpec with Matchers { - - "MetaDataPass" should { - - "create a metadata node with correct language" in { - File.usingTemporaryDirectory("rubysrc2cpgTest") { dir => - val config = Config().withInputPath(dir.pathAsString).withOutputPath(dir.pathAsString) - val cpg = new RubySrc2Cpg().createCpg(config).get - cpg.metaData.language.l shouldBe List(Languages.RUBYSRC) - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryTests.scala deleted file mode 100644 index 12a034b4ae94..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/RubyTypeRecoveryTests.scala +++ /dev/null @@ -1,258 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.rubysrc2cpg.deprecated.utils.PackageTable -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.Defines as XDefines -import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.shiftleft.semanticcpg.language.importresolver.* -import io.shiftleft.semanticcpg.language.* - -import scala.collection.immutable.List - -object RubyTypeRecoveryTests { - def getPackageTable: PackageTable = { - val packageTable = PackageTable() - packageTable.addTypeDecl("sendgrid-ruby", "API", "SendGrid.API") - packageTable.addModule("dbi", "DBI", "DBI") - packageTable.addTypeDecl("logger", "Logger", "Logger") - packageTable.addModule("stripe", "Customer", "Stripe.Customer") - packageTable - } - -} -class RubyTypeRecoveryTests - extends RubyCode2CpgFixture( - withPostProcessing = true, - packageTable = Some(RubyTypeRecoveryTests.getPackageTable), - useDeprecatedFrontend = true - ) { - - "Type information for nodes with external dependency" should { - - val cpg = code( - """ - |require "sendgrid-ruby" - | - |def func - | sg = SendGrid::API.new(api_key: ENV['SENDGRID_API_KEY']) - | response = sg.client.mail._('send').post(request_body: data) - |end - |""".stripMargin, - "main.rb" - ) - - "be present in (Case 1)" ignore { - cpg.identifier("sg").lineNumber(5).typeFullName.l shouldBe List("sendgrid-ruby::program.SendGrid.API") - cpg.call("client").dispatchType.l shouldBe List(DispatchTypes.DYNAMIC_DISPATCH) - cpg.call("client").methodFullName.l shouldBe List("sendgrid-ruby::program.SendGrid.API.client") - } - - "be present in (Case 2)" ignore { - cpg.call("post").methodFullName.l shouldBe List( - "sendgrid-ruby::program.SendGrid.API.client.mail.anonymous.post" - ) - } - } - - "literals declared from built-in types" should { - val cpg = code( - """ - |x = 123 - | - |def newfunc - | x = "foo" - |end - |module MyNamespace - | MY_CONSTANT = 42 - |end - |""".stripMargin, - "main.rb" - ) - "resolve 'x' identifier types despite shadowing" in { - val List(xOuterScope, xInnerScope) = cpg.identifier("x").take(2).l - xOuterScope.dynamicTypeHintFullName shouldBe Seq("__builtin.Integer", "__builtin.String") - xInnerScope.dynamicTypeHintFullName shouldBe Seq("__builtin.Integer", "__builtin.String") - } - - "resolve module constant type" in { - cpg.typeDecl("MyNamespace").size shouldBe 1 - val List(typeDecl) = cpg.typeDecl("MyNamespace").l - val List(myconst) = typeDecl.member.l - myconst.typeFullName shouldBe "__builtin.Integer" - } - } - - "recovering paths for built-in calls" should { - lazy val cpg = code( - """ - |print("Hello world") - |puts "Hello" - | - |def sleep(input) - |end - | - |sleep(2) - |""".stripMargin, - "main.rb" - ).cpg - - "resolve 'print' and 'puts' calls" in { - val List(printCall) = cpg.call("print").l - printCall.methodFullName shouldBe "__builtin.print" - val List(maxCall) = cpg.call("puts").l - maxCall.methodFullName shouldBe "__builtin.puts" - } - - "present the declared method name when a built-in with the same name is used in the same compilation unit" in { - val List(absCall) = cpg.call("sleep").l - absCall.methodFullName shouldBe "main.rb::program.sleep" - } - } - - "recovering module members across modules" should { - lazy val cpg = code( - """ - |require "dbi" - | - |module FooModule - | x = 1 - | y = "test" - | db = DBI.connect("DBI:Mysql:TESTDB:localhost", "testuser", "test123") - |end - | - |""".stripMargin, - "foo.rb" - ).moreCode( - """ - |require_relative "./foo.rb" - | - |z = FooModule::x - |z = FooModule::y - | - |d = FooModule::db - | - |row = d.select_one("SELECT VERSION()") - | - |""".stripMargin, - "bar.rb" - ).cpg - - // TODO Waiting for Module modelling to be done - "resolve correct imports via tag nodes" ignore { - val List(foo: ResolvedTypeDecl) = - cpg.file(".*foo.rb").ast.isCall.where(_.referencedImports).tag._toEvaluatedImport.toList: @unchecked - foo.fullName shouldBe "dbi::program.DBI" - val List(bar: ResolvedTypeDecl) = - cpg.file(".*bar.rb").ast.isCall.where(_.referencedImports).tag._toEvaluatedImport.toList: @unchecked - bar.fullName shouldBe "foo.rb::program.FooModule" - } - - "resolve 'x' and 'y' locally under foo.rb" in { - val Some(x) = cpg.identifier("x").where(_.file.name(".*foo.*")).headOption: @unchecked - x.typeFullName shouldBe "__builtin.Integer" - val Some(y) = cpg.identifier("y").where(_.file.name(".*foo.*")).headOption: @unchecked - y.typeFullName shouldBe "__builtin.String" - } - - "resolve 'FooModule.x' and 'FooModule.y' field access primitive types correctly" ignore { - val List(z1, z2) = cpg.file - .name(".*bar.*") - .ast - .isIdentifier - .name("z") - .l - z1.typeFullName shouldBe "ANY" - z1.dynamicTypeHintFullName shouldBe Seq("__builtin.Integer", "__builtin.String") - z2.typeFullName shouldBe "ANY" - z2.dynamicTypeHintFullName shouldBe Seq("__builtin.Integer", "__builtin.String") - } - - "resolve 'FooModule.d' field access object types correctly" ignore { - val Some(d) = cpg.file - .name(".*bar.*") - .ast - .isIdentifier - .name("d") - .headOption: @unchecked - d.typeFullName shouldBe "dbi::program.DBI.connect." - d.dynamicTypeHintFullName shouldBe Seq() - } - - "resolve a 'select_one' call indirectly from 'FooModule.d' field access correctly" ignore { - val List(d) = cpg.file - .name(".*bar.*") - .ast - .isCall - .name("select_one") - .l - d.methodFullName shouldBe "dbi::program.DBI.connect..select_one" - d.dynamicTypeHintFullName shouldBe Seq() - d.callee(NoResolve).isExternal.headOption shouldBe Some(true) - } - - } - - "assignment from a call to a identifier inside an imported module using new" should { - lazy val cpg = code(""" - |require 'logger' - | - |log = Logger.new(STDOUT) - |log.error("foo") - | - |""".stripMargin).cpg - - "resolve correct imports via tag nodes" in { - val List(logging: ResolvedMethod, _) = - cpg.call.where(_.referencedImports).tag._toEvaluatedImport.toList: @unchecked - logging.fullName shouldBe s"logger::program.Logger.${XDefines.ConstructorMethodName}" - } - - "provide a dummy type" ignore { - val List(error) = cpg.call("error").l: @unchecked - error.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH - val Some(log) = cpg.identifier("log").headOption: @unchecked - log.typeFullName shouldBe "logger::program.Logger" - val List(errorCall) = cpg.call("error").l - errorCall.methodFullName shouldBe "logger::program.Logger.error" - } - } - - "assignment from a call to a identifier inside an imported module using methodCall" should { - lazy val cpg = code(""" - |require 'stripe' - | - |customer = Stripe::Customer.create - | - |""".stripMargin).cpg - - "resolved the type of call" in { - val Some(create) = cpg.call("create").headOption: @unchecked - create.methodFullName shouldBe "stripe::program.Stripe.Customer.create" - } - - "resolved the type of identifier" in { - val Some(customer) = cpg.identifier("customer").headOption: @unchecked - customer.typeFullName shouldBe "stripe::program.Stripe.Customer.create." - } - } - - "recovery of type for call having a method with same name" should { - lazy val cpg = code(""" - |require "dbi" - | - |def connect - | puts "I am here" - |end - | - |d = DBI.connect("DBI:Mysql:TESTDB:localhost", "testuser", "test123") - |""".stripMargin) - - "have a correct type for call `connect`" in { - cpg.call("connect").methodFullName.l shouldBe List("dbi::program.DBI.connect") - } - - "have a correct type for identifier `d`" in { - cpg.identifier("d").typeFullName.l shouldBe List("dbi::program.DBI.connect.") - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/UnknownConstructPass.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/UnknownConstructPass.scala deleted file mode 100644 index 84a233ed8b2b..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/UnknownConstructPass.scala +++ /dev/null @@ -1,89 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.utils.Environment -import io.joern.x2cpg.utils.Environment.OperatingSystemType -import io.shiftleft.codepropertygraph.generated.nodes.Method -import io.shiftleft.semanticcpg.language.* - -class UnknownConstructPass extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "invalid assignment" ignore { - val cpg = code(""" - |a = 1 - |b = [[] - |c = 2 - |""".stripMargin) - - "be ignored" in { - val List(a, c) = cpg.assignment.l - a.target.code shouldBe "a" - a.source.code shouldBe "1" - - c.target.code shouldBe "c" - c.source.code shouldBe "2" - } - } - - "invalid method body" ignore { - val cpg = code(""" - |x = 1 - |def random(a) - | b = 3 + 2) - | y = 2 - |end - |z = 2 - |""".stripMargin) - - "preserve code around it and show the rest of the method body" in { - val List(x, y, z, random) = cpg.assignment.l - x.target.code shouldBe "x" - x.source.code shouldBe "1" - - y.target.code shouldBe "y" - y.source.code shouldBe "2" - - z.target.code shouldBe "z" - z.source.code shouldBe "2" - - random.target.code shouldBe "random" - random.source.code shouldBe "def random(...)" - - val List(m: Method) = cpg.method.nameExact("random").l - val List(_y) = m.assignment.l - y.id() shouldBe _y.id() - } - } - - "unrecognized token in the RHS of an assignment" ignore { - val cpg = code(""" - |x = \! - |y = 1 - |""".stripMargin) - - "be ignored" in { - val List(y) = cpg.assignment.l - - y.target.code shouldBe "y" - y.source.code shouldBe "1" - } - } - - "an attempted fix" ignore { - val cpg = code(""" - |class DerivedClass < BaseClass - | KEYS = %w( - | id1 - | id2 - | id3 - | ).freeze - |end - |""".stripMargin) - - "not cause an infinite loop once the last line is blanked out, at the cost of the structure (in Unix)" in { - cpg.typeDecl("DerivedClass").size shouldBe - (if (Environment.operatingSystem == OperatingSystemType.Windows) 1 else 0) - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AssignCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AssignCpgTests.scala deleted file mode 100644 index 95d098a62ae8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AssignCpgTests.scala +++ /dev/null @@ -1,193 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{DifferentInNewFrontend, RubyCode2CpgFixture, SameInNewFrontend} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} -import io.shiftleft.semanticcpg.language.* -import org.scalatest.Tag - -class AssignCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "single target assign" should { - val cpg = code("""x = 2""".stripMargin) - - "test local and identifier nodes" taggedAs SameInNewFrontend in { - val localX = cpg.local.head - localX.name shouldBe "x" - val List(idX) = localX.referencingIdentifiers.l: @unchecked - idX.name shouldBe "x" - } - - "test assignment node properties" taggedAs SameInNewFrontend in { - val assignCall = cpg.call.methodFullName(Operators.assignment).head - assignCall.code shouldBe "x = 2" - assignCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - assignCall.lineNumber shouldBe Some(1) - assignCall.columnNumber shouldBe Some(2) - } - - "test assignment node ast children" taggedAs SameInNewFrontend in { - cpg.call - .methodFullName(Operators.assignment) - .astChildren - .order(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.assignment) - .astChildren - .order(2) - .isLiteral - .head - .code shouldBe "2" - } - - "test assignment node arguments" taggedAs SameInNewFrontend in { - cpg.call - .methodFullName(Operators.assignment) - .argument - .argumentIndex(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.assignment) - .argument - .argumentIndex(2) - .isLiteral - .head - .code shouldBe "2" - } - } - - "nested decomposing assign" should { - val cpg = code("""x, (y, z) = [1, [2, 3]]""".stripMargin) - - def getSurroundingBlock: nodes.Block = { - cpg.all.collect { case block: nodes.Block if block.code != "" => block }.head - } - - "test block exists" in { - // Throws if block does not exist. - getSurroundingBlock - } - - // TODO: .code property need to be fixed - "test block node properties" ignore { - val block = getSurroundingBlock - block.code shouldBe - """tmp0 = list - |x = tmp0[0] - |y = tmp0[1][0] - |z = tmp0[1][1]""".stripMargin - block.lineNumber shouldBe Some(1) - } - - // TODO: Need to fix the local variables - "test local node" ignore { - cpg.method.name("Test0.rb::program").local.name("tmp0").headOption should not be empty - } - - "test tmp variable assignment" in { - val block = getSurroundingBlock - val tmpAssignNode = block.astChildren.isCall.sortBy(_.order).head - // tmpAssignNode.code shouldBe "tmp0 = list" - tmpAssignNode.methodFullName shouldBe Operators.assignment - tmpAssignNode.lineNumber shouldBe Some(1) - } - - // TODO: Fix the code property of the Block node & the order too - "test assignments to targets" ignore { - val block = getSurroundingBlock - val assignNodes = block.astChildren.isCall.sortBy(_.order).tail - assignNodes.map(_.code) should contain theSameElementsInOrderAs List( - "x = tmp0[0]", - "y = tmp0[1][0]", - "z = tmp0[1][1]" - ) - assignNodes.map(_.lineNumber.get) should contain theSameElementsInOrderAs List(1, 1, 1) - } - - } - - "array destructuring assign" should { - val cpg = code("""x, *, y = [1, 2, 3, 5]""".stripMargin) - - def getSurroundingBlock: nodes.Block = { - cpg.all.collect { case block: nodes.Block if block.code != "" => block }.head - } - - "test block exists" in { - // Throws if block does not exist. - getSurroundingBlock - } - - // TODO: .code property need to be fixed - "test block node properties" ignore { - val block = getSurroundingBlock - block.code shouldBe - """tmp0 = list - |x = tmp0[0] - |y = tmp0[1][0] - |z = tmp0[1][1]""".stripMargin - block.astChildren.length shouldBe 4 - cpg.identifier("x").isEmpty shouldBe false - cpg.identifier("y").isEmpty shouldBe false - cpg.identifier("z").isEmpty shouldBe false - - } - - // TODO: Need to fix the local variables - "test local node" ignore { - cpg.method.name("Test0.rb::program").local.name("tmp0").headOption should not be empty - } - - } - - "multi target assign" should { - val cpg = code("""x = y = "abcd"""".stripMargin) - - def getSurroundingBlock: nodes.Block = { - cpg.all.collect { case block: nodes.Block if block.code != "" => block }.head - } - - "test block exists" taggedAs DifferentInNewFrontend in { - // Throws if block does not exist. - getSurroundingBlock - } - - // TODO: Fix the code property of the Block node - "test block node properties" taggedAs DifferentInNewFrontend ignore { - val block = getSurroundingBlock - block.code shouldBe - """tmp0 = list - |x = tmp0 - |y = tmp0""".stripMargin - block.lineNumber shouldBe Some(1) - } - - // TODO: Need to fix the local variables - "test local node" taggedAs DifferentInNewFrontend ignore { - cpg.method.name("Test0.rb::program").local.name("tmp0").headOption should not be empty - } - - // TODO: Need to fix the code property - "test tmp variable assignment" taggedAs DifferentInNewFrontend ignore { - val block = getSurroundingBlock - val tmpAssignNode = block.astChildren.isCall.sortBy(_.order).head - tmpAssignNode.code shouldBe "tmp0 = list" - tmpAssignNode.methodFullName shouldBe Operators.assignment - tmpAssignNode.lineNumber shouldBe Some(1) - } - } - - "empty array assignment" should { - val cpg = code("""x.y = []""".stripMargin) - - "have an empty assignment" taggedAs DifferentInNewFrontend in { - val List(assignment) = cpg.call.name(Operators.assignment).l - assignment.argument.where(_.argumentIndex(2)).isCall.name.l shouldBe List(Operators.arrayInitializer) - assignment.argument.where(_.argumentIndex(2)).isCall.argument.l shouldBe List() - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AttributeCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AttributeCpgTests.scala deleted file mode 100644 index 847ce13f137e..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/AttributeCpgTests.scala +++ /dev/null @@ -1,53 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{RubyCode2CpgFixture, SameInNewFrontend} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language.* - -class AttributeCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code("""x.y""".stripMargin) - - // TODO: Class Modeling testcase - "test field access call node properties" taggedAs SameInNewFrontend ignore { - val callNode = cpg.call.methodFullName(Operators.fieldAccess).head - callNode.code shouldBe "x.y" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(1) - } - - // TODO: Class Modeling testcase - "test field access call ast children" ignore { - cpg.call - .methodFullName(Operators.fieldAccess) - .astChildren - .order(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.fieldAccess) - .astChildren - .order(2) - .isFieldIdentifier - .head - .code shouldBe "y" - } - - // TODO: Class Modeling testcase - "test field access call arguments" ignore { - cpg.call - .methodFullName(Operators.fieldAccess) - .argument - .argumentIndex(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.fieldAccess) - .argument - .argumentIndex(2) - .isFieldIdentifier - .head - .code shouldBe "y" - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BinOpCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BinOpCpgTests.scala deleted file mode 100644 index 6b28930d82c2..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BinOpCpgTests.scala +++ /dev/null @@ -1,53 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes, DispatchTypes, Operators, nodes} -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal - -class BinOpCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code("""1 + 2""".stripMargin) - - "test binOp 'add' call node properties" in { - val additionCall = cpg.call.methodFullName(Operators.addition).head - additionCall.code shouldBe "1 + 2" - additionCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - additionCall.lineNumber shouldBe Some(1) - // TODO additionCall.columnNumber shouldBe Some(1) - } - - "test binOp 'add' ast children" in { - cpg.call - .methodFullName(Operators.addition) - .astChildren - .order(1) - .isLiteral - .head - .code shouldBe "1" - cpg.call - .methodFullName(Operators.addition) - .astChildren - .order(2) - .isLiteral - .head - .code shouldBe "2" - } - - "test binOp 'add' arguments" in { - cpg.call - .methodFullName(Operators.addition) - .argument - .argumentIndex(1) - .isLiteral - .head - .code shouldBe "1" - cpg.call - .methodFullName(Operators.addition) - .argument - .argumentIndex(2) - .isLiteral - .head - .code shouldBe "2" - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BoolOpCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BoolOpCpgTests.scala deleted file mode 100644 index e3e5690a2aed..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/BoolOpCpgTests.scala +++ /dev/null @@ -1,68 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{DifferentInNewFrontend, RubyCode2CpgFixture, SameInNewFrontend} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language.* - -class BoolOpCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code("""x or y or z""".stripMargin) - - "test boolOp 'or' call node properties" taggedAs SameInNewFrontend in { - val orCall = cpg.call.head -// val orCall = cpg.call.methodFullName(Operators.logicalOr).head - orCall.code shouldBe "x or y or z" - orCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - orCall.lineNumber shouldBe Some(1) - // TODO orCall.columnNumber shouldBe Some(3) - } - - // TODO: Fix this multi logicalOr operation - "test boolOp 'or' ast children" taggedAs DifferentInNewFrontend ignore { - cpg.call - .methodFullName(Operators.logicalOr) - .astChildren - .order(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.logicalOr) - .astChildren - .order(2) - .isIdentifier - .head - .code shouldBe "y" - cpg.call - .methodFullName(Operators.logicalOr) - .astChildren - .order(3) - .isIdentifier - .head - .code shouldBe "z" - } - - // TODO: Fix this multi logicalOr operation arguments - "test boolOp 'or' arguments" taggedAs DifferentInNewFrontend ignore { - cpg.call - .methodFullName(Operators.logicalOr) - .argument - .argumentIndex(1) - .isIdentifier - .head - .code shouldBe "x" - cpg.call - .methodFullName(Operators.logicalOr) - .argument - .argumentIndex(2) - .isIdentifier - .head - .code shouldBe "y" - cpg.call - .methodFullName(Operators.logicalOr) - .argument - .argumentIndex(3) - .isIdentifier - .head - .code shouldBe "z" - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala deleted file mode 100644 index b2c499a6cd93..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CallCpgTests.scala +++ /dev/null @@ -1,279 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.testfixtures.{RubyCode2CpgFixture, SameInNewFrontend} -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, MethodRef} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, nodes} -import io.shiftleft.semanticcpg.language.* - -class CallCpgTests extends RubyCode2CpgFixture(withPostProcessing = true, useDeprecatedFrontend = true) { - "simple call method" should { - val cpg = code("""foo("a", b)""".stripMargin) - - "test call node properties" taggedAs SameInNewFrontend in { - val callNode = cpg.call.name("foo").head - callNode.code shouldBe """foo("a", b)""" - callNode.signature shouldBe "" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(1) - } - - "test call arguments" taggedAs SameInNewFrontend in { - val callNode = cpg.call.name("foo").head - val arg1 = callNode.argument(1) - arg1.code shouldBe "\"a\"" - - val arg2 = callNode.argument(2) - arg2.code shouldBe "b" - } - - "test astChildren" taggedAs SameInNewFrontend in { - val callNode = cpg.call.name("foo").head - val children = callNode.astChildren - children.size shouldBe 2 - - val firstChild = children.head - val secondChild = children.last - - firstChild.code shouldBe "\"a\"" - secondChild.code shouldBe "b" - } - } - - "call on identifier with named argument" should { - val cpg = code("""x.foo("a", b)""".stripMargin) - - "test call node properties" in { - val callNode = cpg.call.name("foo").head - callNode.code shouldBe """x.foo("a", b)""" - callNode.signature shouldBe "" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(1) - } - - "test call arguments" in { - val callNode = cpg.call.name("foo").head - val arg1 = callNode.argument(1) - arg1.code shouldBe "\"a\"" - - val arg2 = callNode.argument(2) - arg2.code shouldBe "b" - } - - "test astChildren" in { - val callNode = cpg.call.name("foo").head - val children = callNode.astChildren - children.size shouldBe 3 - - val firstChild = children.head - val lastChild = children.last - - firstChild.code shouldBe "x" - lastChild.code shouldBe "b" - } - } - - "call following a definition within the same module" should { - val cpg = code(""" - |def func(a, b) - | return a + b - |end - |x = func(a, b) - |""".stripMargin) - - "test call node properties" in { - val callNode = cpg.call.name("func").head - callNode.name shouldBe "func" - callNode.signature shouldBe "" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(5) - } - } - - "call with the splat operator" should { - val cpg = code(""" - |def print_list_of(**books_and_articles) - | books_and_articles.each do |book, article| - | puts book - | puts article - | end - |end - |# As an argument, we define a hash in which we will write books and articles. - |books_and_articles_we_love = { - | "Ruby on Rails 4": "What is webpack?", - | "Ruby essentials": "What is Ruby Object Model?", - | "Javascript essentials": "What is Object?" - |} - |print_list_of(books_and_articles_we_love) - |""".stripMargin) - - "test call node properties with children & argument" in { - val callNode = cpg.call.name("print_list_of").head - callNode.code shouldBe "print_list_of(books_and_articles_we_love)" - callNode.signature shouldBe "" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(14) - callNode.astChildren.last.code shouldBe "books_and_articles_we_love" - callNode.argument.last.code shouldBe "books_and_articles_we_love" - } - } - - "call with a heredoc parameter" should { - val cpg = code("""foo(<<~SQL) - |SELECT * FROM food - |WHERE healthy = true - |SQL - |""".stripMargin) - - "take note of the here doc location and construct a literal from the following statements" in { - val List(sql) = cpg.call.nameExact("foo").argument.isLiteral.l: @unchecked - sql.code shouldBe - """SELECT * FROM food - |WHERE healthy = true - |""".stripMargin.trim - sql.lineNumber shouldBe Option(1) - sql.columnNumber shouldBe Option(4) - sql.typeFullName shouldBe Defines.String - } - } - - // TODO: Handle multiple heredoc parameters - "call with multiple heredoc parameters" ignore { - val cpg = code("""puts(<<-ONE, <<-TWO) - |content for heredoc one - |ONE - |content for heredoc two - |TWO - |""".stripMargin) - - "take note of the here doc locations and construct the literals respectively from the following statements" in { - val List(one, two) = cpg.call.nameExact("puts").argument.isLiteral.l: @unchecked - one.code shouldBe "content for heredoc one" - one.lineNumber shouldBe Option(1) - one.columnNumber shouldBe Option(5) - one.typeFullName shouldBe Defines.String - two.code shouldBe "content for heredoc two" - two.lineNumber shouldBe Option(1) - two.columnNumber shouldBe Option(13) - two.typeFullName shouldBe Defines.String - } - } - - "a call with a normal and a do block argument" should { - val cpg = code(""" - |def client - | Faraday.new(API_HOST) do |builder| - | builder.request :json - | builder.options[:timeout] = READ_TIMEOUT - | builder.options[:open_timeout] = OPEN_TIMEOUT - | end - |end - |""".stripMargin) - - "have the correct arguments in the correct ordering" in { - val List(n) = cpg.call.nameExact("new").l: @unchecked - val List(faraday: Identifier, apiHost: Identifier, doRef: MethodRef) = n.argument.l: @unchecked - faraday.name shouldBe "Faraday" - faraday.argumentIndex shouldBe 0 - apiHost.name shouldBe "API_HOST" - apiHost.argumentIndex shouldBe 1 - doRef.methodFullName shouldBe "Test0.rb::program.new3" - doRef.argumentIndex shouldBe 2 - } - } - - "a call without parenthesis before the method definition is seen/resolved" should { - val cpg = code( - """ - |require "foo.rb" - | - |def event_params - | @event_params ||= device_params - | .merge(params) - | .merge(encoded_partner_params) - | .merge( - | s2s: 1, - | created_at_unix: Time.current.to_i, - | app_token: app_token, - | event_token: event_token, - | install_source: install_source - | ) - |end - |""".stripMargin, - "bar.rb" - ) - .moreCode( - """ - |def device_params - | case platform - | when :android - | { adid: adid, gps_adid: gps_adid } - | when :ios - | { adid: adid, idfa: idfa } - | else - | {} - | end - |end - |""".stripMargin, - "foo.rb" - ) - - "have its call node correctly identified and created" in { - val List(deviceParams) = cpg.call.nameExact("device_params").l: @unchecked - deviceParams.name shouldBe "device_params" - deviceParams.code shouldBe "device_params" - deviceParams.methodFullName shouldBe "foo.rb::program.device_params" - deviceParams.typeFullName shouldBe Defines.Any - deviceParams.lineNumber shouldBe Option(5) - deviceParams.columnNumber shouldBe Option(22) - deviceParams.argumentIndex shouldBe 0 - } - } - - "a parenthesis-less call (defined later in the module) in a call's argument" should { - val cpg = code(""" - |module Pay - | module Webhooks - | class BraintreeController < Pay::ApplicationController - | if Rails.application.config.action_controller.default_protect_from_forgery - | skip_before_action :verify_authenticity_token - | end - | - | def create - | queue_event(verified_event) # <------ verified event is a call here - | head :ok - | rescue ::Braintree::InvalidSignature - | head :bad_request - | end - | - | private - | - | def queue_event(event) - | return unless Pay::Webhooks.delegator.listening?("braintree.#{event.kind}") - | - | record = Pay::Webhook.create!( - | processor: :braintree, - | event_type: event.kind, - | event: {bt_signature: params[:bt_signature], bt_payload: params[:bt_payload]} - | ) - | Pay::Webhooks::ProcessJob.perform_later(record) - | end - | - | def verified_event - | Pay.braintree_gateway.webhook_notification.parse(params[:bt_signature], params[:bt_payload]) - | end - | end - | end - |end - |""".stripMargin) - - "be a call node instead of an identifier" in { - inside(cpg.call("queue_event").argument.l) { - case (verifiedEvent: Call) :: Nil => - verifiedEvent.name shouldBe "verified_event" - case xs => - fail(s"Expected a single call argument, received [${xs.map(x => x.label -> x.code).mkString(", ")}] instead!") - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CustomAssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CustomAssignmentTests.scala deleted file mode 100644 index 429a39c47961..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/CustomAssignmentTests.scala +++ /dev/null @@ -1,57 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{DifferentInNewFrontend, RubyCode2CpgFixture} -import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, MethodRef, TypeRef} -import io.shiftleft.semanticcpg.language.* - -class CustomAssignmentTests extends RubyCode2CpgFixture(withPostProcessing = true, useDeprecatedFrontend = true) { - - "custom assignment for builtIn" should { - val cpg = code(""" - |puts "This is ruby" - |""".stripMargin) - "be created for builtin presence" taggedAs DifferentInNewFrontend in { - val List(putsAssignmentCall, _) = cpg.call.l - putsAssignmentCall.name shouldBe ".assignment" - - val List(putsIdentifier: Identifier, putsBuiltInTypeRef: TypeRef) = putsAssignmentCall.argument.l: @unchecked - - putsIdentifier.name shouldBe "puts" - putsBuiltInTypeRef.code shouldBe "__builtin.puts" - putsBuiltInTypeRef.typeFullName shouldBe "__builtin.puts" - } - - "resolve type for `puts`" in { - val List(_, putsCall) = cpg.call.l - putsCall.name shouldBe "puts" - putsCall.methodFullName shouldBe "__builtin.puts" - } - } - - "custom assignment for user defined function" should { - val cpg = code(""" - |def foo() - | return "This is my foo" - |end - | - |foo() - |""".stripMargin) - "be created" in { - val List(fooAssignmentCall, _) = cpg.call.l - fooAssignmentCall.name shouldBe ".assignment" - - val List(fooIdentifier: Identifier, fooMethodRef: MethodRef) = fooAssignmentCall.argument.l: @unchecked - - fooIdentifier.name shouldBe "foo" - fooMethodRef.methodFullName shouldBe "Test0.rb::program.foo" - fooMethodRef.referencedMethod.name shouldBe "foo" - } - - "resolve type for `foo`" in { - val List(_, fooCall) = cpg.call.l - fooCall.name shouldBe "foo" - fooCall.methodFullName shouldBe "Test0.rb::program.foo" - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/DoBlockTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/DoBlockTest.scala deleted file mode 100644 index 060d15d2d2a1..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/DoBlockTest.scala +++ /dev/null @@ -1,34 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class DoBlockTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "defining a method using metaprogramming and a do-block function" should { - val cpg = code(s""" - |define_method foo do |name, age| - | value = public_send("#{name}_value") - | unit = public_send("#{name}_unit") - | - | puts "My name is #{name} and age is #{age}" - | - | next unless value.present? && unit.present? - | value.public_send(unit) - |end - |""".stripMargin) - - "create a do-block method called `foo`" in { - val nameMethod :: _ = cpg.method.nameExact("foo").l: @unchecked - - val List(name, age) = nameMethod.parameter.l - name.name shouldBe "name" - age.name shouldBe "age" - - val List(value, unit) = nameMethod.local.l - value.name shouldBe "value" - unit.name shouldBe "unit" - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FileTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FileTests.scala deleted file mode 100644 index d10d44769cdd..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FileTests.scala +++ /dev/null @@ -1,62 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.language.types.structure.FileTraversal -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal - -class FileTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code(""" - |def foo() - |end - |def bar() - |end - |class MyClass - |end - |""".stripMargin) - - // TODO: Fix this unit test - "should contain two file nodes in total, both with order=0" ignore { - cpg.file.order.l shouldBe List(0, 0) - cpg.file.name(FileTraversal.UNKNOWN).size shouldBe 1 - cpg.file.nameNot(FileTraversal.UNKNOWN).size shouldBe 1 - } - - "should contain exactly one placeholder file node with `name=\"\"/order=0`" in { - cpg.file(FileTraversal.UNKNOWN).order.l shouldBe List(0) - cpg.file(FileTraversal.UNKNOWN).hash.l shouldBe List() - } - - "should allow traversing from file to its namespace blocks" in { - cpg.file.nameNot(FileTraversal.UNKNOWN).namespaceBlock.name.toSetMutable shouldBe Set( - NamespaceTraversal.globalNamespaceName - ) - } - - "should allow traversing from file to its methods via namespace block" in { - cpg.file.nameNot(FileTraversal.UNKNOWN).method.name.toSetMutable shouldBe Set("foo", "bar", "", ":program") - } - - // TODO: TypeDecl fix this unit test - "should allow traversing from file to its type declarations via namespace block" ignore { - cpg.file - .nameNot(FileTraversal.UNKNOWN) - .typeDecl - .nameNot(NamespaceTraversal.globalNamespaceName) - .name - .l - .sorted shouldBe List("MyClass") - } - - // TODO: Need to fix this test. - "should allow traversing to namespaces" ignore { - val List(ns1, ns2) = cpg.file.namespaceBlock.l - // At present it returning full file system path. It should return relative path - ns1.filename shouldBe "Test0.rb" - // At present it returning full file system path. It should return relative path - ns1.fullName shouldBe "Test0.rb:" - ns2.filename shouldBe "" - ns2.fullName shouldBe "" - cpg.file.namespace.name(NamespaceTraversal.globalNamespaceName).l.size shouldBe 2 - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FormatStringCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FormatStringCpgTests.scala deleted file mode 100644 index 3b58d7b5ec54..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/FormatStringCpgTests.scala +++ /dev/null @@ -1,59 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes, DispatchTypes, Operators, nodes} -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal - -class FormatStringCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - "#string interpolation" should { - val cpg = code("""puts "pre#{x}post"""".stripMargin) - "test formatValue operator node" in { - val callNode = cpg.call.methodFullName(".formatValue").head - callNode.code shouldBe "#{x}" - callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - callNode.lineNumber shouldBe Some(1) - } - - "test formatString operator node arguments" ignore { - val callNode = cpg.call.methodFullName(".formatValue").head - - val child1 = callNode.astChildren.order(1).isLiteral.head - child1.code shouldBe "pre" - child1.argumentIndex shouldBe 1 - - val child2 = callNode.astChildren.order(2).isCall.head - child2.code shouldBe "#{x}" - child2.argumentIndex shouldBe 2 - child2.methodFullName shouldBe ".formatValue" - child2.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - - val child3 = callNode.astChildren.order(3).isLiteral.head - child3.code shouldBe "post" - child3.argumentIndex shouldBe 3 - } - - "test formattedValue operator child" ignore { - val callNode = cpg.call.methodFullName(".formatValue").head - - val child1 = callNode.astChildren.order(1).isIdentifier.head - child1.code shouldBe "x" - child1.argumentIndex shouldBe 1 - } - } - - "test format string with multiple replacement fields" in { - val cpg = code("""puts "The number #{a} is less than #{b}"""".stripMargin) - val callNodeA = cpg.call.methodFullName(".formatValue").head - val callNodeB = cpg.call.methodFullName(".formatValue").last - callNodeA.code shouldBe "#{a}" - callNodeB.code shouldBe "#{b}" - } - - "test format string with only single replacement field" in { - val cpg = code("""puts "#{a}"""".stripMargin) - val callNode = cpg.call.methodFullName(".formatValue").head - callNode.code shouldBe "#{a}" - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/IdentifierLocalTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/IdentifierLocalTests.scala deleted file mode 100644 index af1d8a6f45e4..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/IdentifierLocalTests.scala +++ /dev/null @@ -1,85 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language._ - -class IdentifierLocalTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - val cpg = code(""" - |def method1() - | x = 1 - | x = 2 - |end - | - |def method2(x) - | x = 2 - |end - | - |def method3(x) - | y = 0 - | - | if true - | innerx = 0 - | innery = 0 - | - | innerx = 1 - | innery = 1 - | end - | - | x = 1 - | y = 1 - |end - | - |""".stripMargin) - - // TODO: Need to be fixed. - "be correct for local x in method1" ignore { - val List(method) = cpg.method.nameExact("method1").l - method.block.ast.isIdentifier.l.size shouldBe 2 - val List(identifierX, _) = method.block.ast.isIdentifier.l - identifierX.name shouldBe "x" - - val localX = identifierX._localViaRefOut.get - localX.name shouldBe "x" - } - - "be correct for parameter x in method2" in { - val List(method) = cpg.method.nameExact("method2").l - val List(identifierX) = method.block.ast.isIdentifier.l - identifierX.name shouldBe "x" - - identifierX.refsTo.l.size shouldBe 1 - val List(paramx) = identifierX.refsTo.l - paramx.name shouldBe "x" - - val parameterX = identifierX._methodParameterInViaRefOut.get - parameterX.name shouldBe "x" - } - - "Reach parameter from last identifier" in { - val List(method) = cpg.method.nameExact("method3").l - val List(outerIdentifierX) = method.ast.isIdentifier.lineNumber(22).l - val parameterX = outerIdentifierX._methodParameterInViaRefOut.get - parameterX.name shouldBe "x" - } - - // TODO: Need to be fixed. - "inner block test" ignore { - val List(method) = cpg.method.nameExact("method3").l - method.block.astChildren.isBlock.l.size shouldBe 1 - val List(nestedBlock) = method.block.astChildren.isBlock.l - nestedBlock.ast.isIdentifier.nameExact("innerx").l.size shouldBe 2 - } - - // TODO: Need to be fixed. - "nested block identifier to local traversal" ignore { - val List(method) = cpg.method.nameExact("method3").l - method.block.astChildren.isBlock.l.size shouldBe 1 - val List(nestedBlock) = method.block.astChildren.isBlock.l - nestedBlock.ast.isIdentifier.nameExact("innerx").l.size shouldBe 2 - val List(nestedIdentifierX, _) = nestedBlock.ast.isIdentifier.nameExact("innerx").l - - val nestedLocalX = nestedIdentifierX._localViaRefOut.get - nestedLocalX.name shouldBe "innerx" - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ImportAstCreationTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ImportAstCreationTest.scala deleted file mode 100644 index 3149bcdbf85d..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ImportAstCreationTest.scala +++ /dev/null @@ -1,33 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class ImportAstCreationTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "Ast creation for import node" should { - val cpg = code(""" - |require "dummy_logger" - |require_relative "util/help.rb" - |load "mymodule.rb" - |""".stripMargin) - val imports = cpg.imports.l - val calls = cpg.call("require|require_relative|load").l - "have a valid import node" in { - imports.importedEntity.l shouldBe List("dummy_logger", "util/help.rb", "mymodule.rb") - imports.importedAs.l shouldBe List("dummy_logger", "util/help.rb", "mymodule.rb") - } - - "have a valid call node" in { - calls.code.l shouldBe List( - "require \"dummy_logger\"", - "require_relative \"util/help.rb\"", - "load \"mymodule.rb\"" - ) - } - - "have a valid linking" in { - calls.referencedImports.importedEntity.l shouldBe List("dummy_logger", "util/help.rb", "mymodule.rb") - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/LiteralCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/LiteralCpgTests.scala deleted file mode 100644 index 35b1b4a561c8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/LiteralCpgTests.scala +++ /dev/null @@ -1,27 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* -class LiteralCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "A here doc string literal" should { - val cpg = code("""<<-SQL - |SELECT * FROM food - |WHERE healthy = true - |SQL - |""".stripMargin) - - "be interpreted as a single literal string" in { - val List(sql) = cpg.literal.l: @unchecked - sql.code shouldBe - """SELECT * FROM food - |WHERE healthy = true - |""".stripMargin.trim - sql.lineNumber shouldBe Option(1) - sql.columnNumber shouldBe Option(0) - sql.typeFullName shouldBe Defines.String - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MetaDataTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MetaDataTests.scala deleted file mode 100644 index 4ff668b72a19..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MetaDataTests.scala +++ /dev/null @@ -1,28 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} -import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ -class MetaDataTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code("""puts 123""") - - "should contain exactly one node with all mandatory fields set" in { - val List(x) = cpg.metaData.l - x.language shouldBe Languages.RUBYSRC - x.version shouldBe "0.1" - x.overlays shouldBe List( - Base.overlayName, - ControlFlow.overlayName, - TypeRelations.overlayName, - CallGraph.overlayName - ) - x.hash shouldBe None - } - - "should not have any incoming or outgoing edges" in { - cpg.metaData.size shouldBe 1 - cpg.metaData.in.l shouldBe List() - cpg.metaData.out.l shouldBe List() - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodOneTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodOneTests.scala deleted file mode 100644 index ad3c1d9d3fb9..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodOneTests.scala +++ /dev/null @@ -1,185 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{DifferentInNewFrontend, RubyCode2CpgFixture, SameInNewFrontend} -import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language.* - -class MethodOneTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "Method test with regular keyword def and end " should { - val cpg = code(""" - |def foo(a, b) - | return "" - |end - |""".stripMargin) - - "should contain exactly one method node with correct fields" in { - inside(cpg.method.name("foo").l) { case List(x) => - x.name shouldBe "foo" - x.isExternal shouldBe false - x.fullName shouldBe "Test0.rb::program.foo" - x.code should startWith("def foo(a, b)") - x.isExternal shouldBe false - x.order shouldBe 3 - x.filename.endsWith("Test0.rb") - x.lineNumber shouldBe Option(2) - x.lineNumberEnd shouldBe Option(4) - } - } - - "should return correct number of lines" taggedAs SameInNewFrontend in { - cpg.method.name("foo").numberOfLines.l shouldBe List(3) - } - - "should allow traversing to parameters" in { - cpg.method.name("foo").parameter.name.toSetMutable shouldBe Set("a", "b") - } - - "should allow traversing to methodReturn" ignore { - cpg.method.name("foo").methodReturn.l.size shouldBe 1 - cpg.method.name("foo").methodReturn.typeFullName.head shouldBe "String" - } - - "should allow traversing to method" taggedAs DifferentInNewFrontend in { - cpg.methodReturn.method.name.l shouldBe List("foo", ":program", ".assignment") - } - - "should allow traversing to file" in { - cpg.method.name("foo").file.name.l should not be empty - } - - // TODO: need to be fixed. - "test corresponding type, typeDecl and binding" ignore { - cpg.method.fullName("Test0.rb::program.foo").referencingBinding.bindingTypeDecl.l should not be empty - val bindingTypeDecl = - cpg.method.fullName("Test0.rb::program.foo").referencingBinding.bindingTypeDecl.head - - bindingTypeDecl.name shouldBe "foo" - bindingTypeDecl.fullName shouldBe "Test0.rb::program.foo" - bindingTypeDecl.referencingType.name.head shouldBe "foo" - bindingTypeDecl.referencingType.fullName.head shouldBe "Test0.rb::program.foo" - } - - "test method parameter nodes" in { - cpg.method.name("foo").parameter.name.l.size shouldBe 2 - val parameter1 = cpg.method.fullName("Test0.rb::program.foo").parameter.order(1).head - parameter1.name shouldBe "a" - parameter1.index shouldBe 1 - parameter1.typeFullName shouldBe "ANY" - - val parameter2 = cpg.method.fullName("Test0.rb::program.foo").parameter.order(2).head - parameter2.name shouldBe "b" - parameter2.index shouldBe 2 - parameter2.typeFullName shouldBe "ANY" - } - - "should allow traversing from parameter to method" in { - cpg.parameter.name("a").method.name.l shouldBe List("foo") - cpg.parameter.name("b").method.name.l shouldBe List("foo") - } - } - - "Method with variable arguments" should { - val cpg = code(""" - |def foo(*names) - | return "" - |end - |""".stripMargin) - - "Variable argument properties should be rightly set" in { - cpg.parameter.name("names").l.size shouldBe 1 - val param = cpg.parameter.name("names").l.head - param.isVariadic shouldBe true - } - } - - "Multiple Return tests" should { - val cpg = code(""" - |def foo(names) - | if names == "Alice" - | return 1 - | else - | return 2 - | end - |end - |""".stripMargin) - - "be correct for multiple returns" in { - cpg.method("foo").methodReturn.l.size shouldBe 1 - cpg.method("foo").ast.isReturn.l.size shouldBe 2 - inside(cpg.method("foo").methodReturn.l) { case List(fooReturn) => - fooReturn.typeFullName shouldBe "ANY" - } - val astReturns = cpg.method("foo").ast.isReturn.l - inside(astReturns) { case List(ret1, ret2) => - ret1.code shouldBe "return 1" - ret1.lineNumber shouldBe Option(4) - ret2.code shouldBe "return 2" - ret2.lineNumber shouldBe Option(6) - } - } - } - - "Function with empty array in block" should { - val cpg = code(""" - |def foo - | [] - |end - |""".stripMargin) - - "contain empty array" taggedAs SameInNewFrontend in { - cpg.method.name("foo").size shouldBe 1 - cpg.method.name("foo").block.containsCallTo(Operators.arrayInitializer).size shouldBe 1 - } - } - - "Function as a list element in accessor" should { - val cpg = code(""" - |class Bar - | attr_accessor :a, - | :b, - | def self.c - | 1 - | end - |end - |""".stripMargin) - - "contain empty array" taggedAs DifferentInNewFrontend in { - cpg.identifier("c").astParent.isCallTo("attr_accessor").size shouldBe 1 - } - } - - "Function for private_class_method" should { - val cpg = code(""" - |private_class_method def foo(a) - | b - |end - |""".stripMargin) - - "have function identifier as argument and function definition" in { - // one from the METHOD_REF node and one on line `def self.c` - cpg.identifier("foo").astParent.isCallTo("private_class_method").size shouldBe 1 - cpg.method.nameExact("foo").size shouldBe 1 - } - - "Function for multiple function prefixes" should { - val cpg = code(""" - |class Foo - | private attr_reader :bar - | - | def bar - | x - | end - |end - |""".stripMargin) - - "have function identifier as argument and function definition" taggedAs DifferentInNewFrontend ignore { - /* FIXME: We are capturing the prefixes but order in ast is private -> attr_reader -> LITERAL(bar) - * We should duplicate the bar node and set parent as both methods */ - cpg.identifier("bar").astParent.isCallTo("private").size shouldBe 1 - cpg.identifier("bar").astParent.isCallTo("attr_reader").size shouldBe 1 - cpg.method.nameExact("bar").size shouldBe 1 - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala deleted file mode 100644 index 4037cebde09f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/MethodTwoTests.scala +++ /dev/null @@ -1,103 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes} -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal - -class MethodTwoTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "Method test with define_method" should { - val cpg = code(""" - |define_method(:foo) do |a, b| - | return "" - |end - |""".stripMargin) - - // TODO: This test cases needs to be fixed. - "should contain exactly one method node with correct fields" ignore { - inside(cpg.method.name("foo").l) { case List(x) => - x.name shouldBe "foo" - x.isExternal shouldBe false - x.fullName shouldBe "Test0.rb::program:foo" - x.code should startWith("def foo(a, b)") - x.isExternal shouldBe false - x.order shouldBe 1 - x.filename.endsWith("Test0.rb") - x.lineNumber shouldBe Option(2) - x.lineNumberEnd shouldBe Option(4) - } - } - - // TODO: This test cases needs to be fixed. - "should return correct number of lines" ignore { - cpg.method.name("foo").numberOfLines.l shouldBe List(3) - } - - // TODO: This test cases needs to be fixed. - "should allow traversing to parameters" ignore { - cpg.method.name("foo").parameter.name.toSetMutable shouldBe Set("a", "b") - } - - // TODO: This test cases needs to be fixed. - "should allow traversing to methodReturn" ignore { - cpg.method.name("foo").methodReturn.l.size shouldBe 1 - cpg.method.name("foo").methodReturn.typeFullName.head shouldBe "ANY" - } - - // TODO: This test cases needs to be fixed. - "should allow traversing to method" ignore { - cpg.methodReturn.method.name.l shouldBe List("foo", ":program") - } - - // TODO: This test cases needs to be fixed. - "should allow traversing to file" ignore { - cpg.method.name("foo").file.name.l should not be empty - } - - // TODO: Need to be fixed - "test function method ref" ignore { - cpg.methodRef("foo").referencedMethod.fullName.l should not be empty - cpg.methodRef("foo").referencedMethod.fullName.head shouldBe - "Test0.rb::program:foo" - } - - // TODO: Need to be fixed. - "test existence of local variable in module function" ignore { - cpg.method.fullName("Test0.rb::program").local.name.l should contain("foo") - } - - // TODO: need to be fixed. - "test corresponding type, typeDecl and binding" ignore { - cpg.method.fullName("Test0.rb::program:foo").referencingBinding.bindingTypeDecl.l should not be empty - val bindingTypeDecl = - cpg.method.fullName("Test0.rb::program:foo").referencingBinding.bindingTypeDecl.head - - bindingTypeDecl.name shouldBe "foo" - bindingTypeDecl.fullName shouldBe "Test0.rb::program:foo" - bindingTypeDecl.referencingType.name.head shouldBe "foo" - bindingTypeDecl.referencingType.fullName.head shouldBe "Test0.rb::program:foo" - } - - // TODO: Need to be fixed - "test method parameter nodes" ignore { - - cpg.method.name("foo").parameter.name.l.size shouldBe 2 - val parameter1 = cpg.method.fullName("Test0.rb::program:foo").parameter.order(1).head - parameter1.name shouldBe "a" - parameter1.index shouldBe 1 - parameter1.typeFullName shouldBe "ANY" - - val parameter2 = cpg.method.fullName("Test0.rb::program:foo").parameter.order(2).head - parameter2.name shouldBe "b" - parameter2.index shouldBe 2 - parameter2.typeFullName shouldBe "ANY" - } - - // TODO: Need to be fixed - "should allow traversing from parameter to method" ignore { - cpg.parameter.name("a").method.name.l shouldBe List("foo") - cpg.parameter.name("b").method.name.l shouldBe List("foo") - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ModuleTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ModuleTests.scala deleted file mode 100644 index 9ac50dd70fc8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ModuleTests.scala +++ /dev/null @@ -1,210 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.types.structure.FileTraversal -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal -import io.joern.x2cpg.Defines as XDefines -import io.shiftleft.codepropertygraph.generated.Operators -class ModuleTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "Simple module checks" should { - val cpg = code(""" - |module MyNamespace - | MY_CONSTANT = 20 - |end - |""".stripMargin) - "Check namespace basic block structure" in { - cpg.namespaceBlock - .nameNot(NamespaceTraversal.globalNamespaceName) - .filenameNot(FileTraversal.UNKNOWN) - .l - .size shouldBe 1 - val List(x) = cpg.namespaceBlock - .nameNot(NamespaceTraversal.globalNamespaceName) - .filenameNot(FileTraversal.UNKNOWN) - .l - x.name shouldBe "MyNamespace" - x.fullName shouldBe "Test0.rb::program.MyNamespace" - } - - "Respective dummy Method in place" in { - cpg.method(XDefines.StaticInitMethodName).l.size shouldBe 1 - val List(x) = cpg.method(XDefines.StaticInitMethodName).l - x.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" - } - - "Respective dummy TypeDecl in place" in { - cpg.typeDecl("MyNamespace").l.size shouldBe 1 - val List(x) = cpg.typeDecl("MyNamespace").l - x.fullName shouldBe s"Test0.rb::program.MyNamespace" - } - } - - "Hierarchical module checks" should { - val cpg = code(""" - |module MyNamespaceParent - | module MyNamespaceChild - | SOME_CONSTATN = 10 - | end - |end - |""".stripMargin) - "Check namespace basic block structure" in { - cpg.namespaceBlock - .nameNot(NamespaceTraversal.globalNamespaceName) - .filenameNot(FileTraversal.UNKNOWN) - .l - .size shouldBe 2 - val List(x, x1) = cpg.namespaceBlock - .nameNot(NamespaceTraversal.globalNamespaceName) - .filenameNot(FileTraversal.UNKNOWN) - .l - x.name shouldBe "MyNamespaceParent" - x.fullName shouldBe s"Test0.rb::program.MyNamespaceParent" - - x1.name shouldBe "MyNamespaceChild" - x1.fullName shouldBe s"Test0.rb::program.MyNamespaceParent.MyNamespaceChild" - } - - "Respective dummy Method in place" in { - cpg.method(XDefines.StaticInitMethodName).l.size shouldBe 2 - cpg.method(XDefines.StaticInitMethodName).fullName.l shouldBe List( - s"Test0.rb::program.MyNamespaceParent.${XDefines.StaticInitMethodName}", - s"Test0.rb::program.MyNamespaceParent.MyNamespaceChild.${XDefines.StaticInitMethodName}" - ) - } - - "Respective dummy TypeDecl in place" in { - cpg.typeDecl("MyNamespaceChild").l.size shouldBe 1 - val List(x) = cpg.typeDecl("MyNamespaceChild").l - x.fullName shouldBe s"Test0.rb::program.MyNamespaceParent.MyNamespaceChild" - } - } - - "Module Internal structure checks with member variable" should { - val cpg = code(""" - |module MyNamespace - | @@plays = 0 - | class MyClass - | def method1 - | puts "Method 1" - | end - | end - |end - |""".stripMargin) - "Class structure in plcae" in { - cpg.typeDecl("MyClass").l.size shouldBe 1 - val List(x) = cpg.typeDecl("MyClass").l - x.fullName shouldBe s"Test0.rb::program.MyNamespace.MyClass" - } - - "Class Method structure in place" in { - cpg.method("method1").l.size shouldBe 1 - val List(x) = cpg.method("method1").l - x.fullName shouldBe s"Test0.rb::program.MyNamespace.MyClass.method1" - } - - "member variables structure in place" in { - val List(classInit) = cpg.method(XDefines.StaticInitMethodName).l - classInit.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" - val List(playsDef) = classInit.call.nameExact(Operators.fieldAccess).fieldAccess.l - playsDef.fieldIdentifier.canonicalName.headOption shouldBe Option("plays") - - val List(myclassTd) = cpg.typeDecl("MyNamespace").l - val List(plays) = myclassTd.member.l - plays.name shouldBe "plays" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("plays") - } - } - - "Module internal structure checks with Constant defined in module" should { - val cpg = code(""" - |module MyNamespace - | MY_CONSTANT = 0 - |end - |""".stripMargin) - // TODO Ignoring below test case and the function where this is implemented treats every UpperCase node as Constant which is incorrect and causing conflicts elsewhere - "member variables structure in place" ignore { - val List(moduleInit) = cpg.method(XDefines.StaticInitMethodName).l - moduleInit.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" - val List(myconstant) = moduleInit.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstant.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - - val List(myclassTd) = cpg.typeDecl("MyNamespace").l - val List(myConstant) = myclassTd.member.l - myConstant.name shouldBe "MY_CONSTANT" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("MY_CONSTANT") - } - } - - "Hierarchical module checks with constants" ignore { - val cpg = code(""" - |module MyNamespace - | MY_CONSTANT = 0 - | @@plays = 0 - | module ChildModule - | @@name = 0 - | MY_CONSTANT = 0 - | end - |end - |""".stripMargin) - - "member variables structure in place" in { - val List(modInit1, modInit2) = cpg.method(XDefines.StaticInitMethodName).l - modInit1.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" - val List(myconstantfa, playsfa) = modInit1.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstantfa.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - playsfa.fieldIdentifier.canonicalName.headOption shouldBe Option("plays") - - modInit2.fullName shouldBe s"Test0.rb::program.MyNamespace.ChildModule.${XDefines.StaticInitMethodName}" - val List(namefa, myconstant2fa) = modInit2.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstant2fa.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - namefa.fieldIdentifier.canonicalName.headOption shouldBe Option("name") - - val List(myclassTd2) = cpg.typeDecl("ChildModule").l - val List(namem, myConstant2m) = myclassTd2.member.l - myConstant2m.name shouldBe "MY_CONSTANT" - namem.name shouldBe "name" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("name", "MY_CONSTANT") - - val List(myclassTd) = cpg.typeDecl("MyNamespace").l - val List(myconstantm, playsm) = myclassTd.member.l - myconstantm.name shouldBe "MY_CONSTANT" - playsm.name shouldBe "plays" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("MY_CONSTANT", "plays") - - } - } - - "Class inside module checks with constants" ignore { - val cpg = code(""" - |module MyNamespace - | MY_CONSTANT = 0 - | class ChildCls - | MY_CONSTANT = 0 - | end - |end - |""".stripMargin) - - "member variables structure in place" in { - val List(modInit1, modInit2) = cpg.method(XDefines.StaticInitMethodName).l - modInit1.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" - val List(myconstant) = modInit1.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstant.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - - modInit2.fullName shouldBe s"Test0.rb::program.MyNamespace.ChildCls.${XDefines.StaticInitMethodName}" - val List(myconstant2) = modInit2.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstant2.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - - val List(myclassTd) = cpg.typeDecl("MyNamespace").l - val List(myConstant) = myclassTd.member.l - myConstant.name shouldBe "MY_CONSTANT" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("MY_CONSTANT") - - val List(myclassTd2) = cpg.typeDecl("ChildCls").l - val List(myConstant2) = myclassTd2.member.l - myConstant2.name shouldBe "MY_CONSTANT" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("MY_CONSTANT") - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/NamespaceBlockTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/NamespaceBlockTest.scala deleted file mode 100644 index d79061ac44e5..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/NamespaceBlockTest.scala +++ /dev/null @@ -1,55 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.language.types.structure.FileTraversal -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal -import io.joern.x2cpg.Defines - -class NamespaceBlockTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - val cpg = code("""puts 123 - |def foo() - |end - |class MyClass - |end - |""".stripMargin) - - "should contain a correct global namespace block for the `` file" in { - val List(x) = cpg.namespaceBlock.filename(FileTraversal.UNKNOWN).l - x.name shouldBe NamespaceTraversal.globalNamespaceName - x.fullName shouldBe NamespaceTraversal.globalNamespaceName - x.order shouldBe 1 - } - - "should contain correct namespace block for known file" in { - val List(x) = cpg.namespaceBlock.filenameNot(FileTraversal.UNKNOWN).l - x.name shouldBe NamespaceTraversal.globalNamespaceName - x.filename should not be empty - x.fullName shouldBe s"${x.filename}:${NamespaceTraversal.globalNamespaceName}" - x.order shouldBe 1 - } - - "should allow traversing from namespace block to method" in { - cpg.namespaceBlock.filenameNot(FileTraversal.UNKNOWN).ast.isMethod.name.l shouldBe List( - ":program", - "foo", - Defines.ConstructorMethodName - ) - } - - "should allow traversing from namespace block to type declaration" in { - cpg.namespaceBlock - .filenameNot(FileTraversal.UNKNOWN) - .ast - .isTypeDecl - .nameNot(NamespaceTraversal.globalNamespaceName) - .name - .l shouldBe List("MyClass") - } - - "should allow traversing from namespace block to namespace" in { - cpg.namespaceBlock.filenameNot(FileTraversal.UNKNOWN).namespace.name.l shouldBe List( - NamespaceTraversal.globalNamespaceName - ) - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/RescueKeywordCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/RescueKeywordCpgTests.scala deleted file mode 100644 index 950d9c752226..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/RescueKeywordCpgTests.scala +++ /dev/null @@ -1,50 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes, DispatchTypes, Operators, nodes} -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal - -class RescueKeywordCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - "rescue in the immediate scope of a `def` block" in { - val cpg = code("""def foo - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin) - - val methodNode = cpg.method.name("foo").head - methodNode.name shouldBe "foo" - methodNode.numberOfLines shouldBe 4 - methodNode.astChildren.isBlock.astChildren.code.contains("try") shouldBe true - - val zeroDivisionErrorIdentifier = cpg.identifier("ZeroDivisionError").head - zeroDivisionErrorIdentifier.code shouldBe "ZeroDivisionError" - zeroDivisionErrorIdentifier.astSiblings.isIdentifier.head.name shouldBe "e" - zeroDivisionErrorIdentifier.astParent.isBlock shouldBe true - } - - "rescue in the immediate scope of a `do` block" ignore { - val cpg = code("""foo x do |y| - |y/0 - |rescue ZeroDivisionError => e - |end""".stripMargin) - - val zeroDivisionErrorIdentifier = cpg.identifier("ZeroDivisionError").head - zeroDivisionErrorIdentifier.code shouldBe "ZeroDivisionError" - zeroDivisionErrorIdentifier.astSiblings.isIdentifier.head.name shouldBe "e" - zeroDivisionErrorIdentifier.astParent.isBlock shouldBe true - } - - "rescue in the immediate scope of a `begin` block" in { - val cpg = code("""begin - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin) - - val zeroDivisionErrorIdentifier = cpg.identifier("ZeroDivisionError").head - zeroDivisionErrorIdentifier.code shouldBe "ZeroDivisionError" - zeroDivisionErrorIdentifier.astSiblings.isIdentifier.head.name shouldBe "e" - zeroDivisionErrorIdentifier.astParent.isBlock shouldBe true - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ReturnTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ReturnTests.scala deleted file mode 100644 index 9a85b3d1a26f..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/ReturnTests.scala +++ /dev/null @@ -1,22 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.{DifferentInNewFrontend, RubyCode2CpgFixture} -import io.shiftleft.codepropertygraph.generated.nodes.MethodRef -import io.shiftleft.semanticcpg.language.* - -class ReturnTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "a method, where the last statement is a method" should { - val cpg = code(""" - |Row = Struct.new(:cancel_date) do - | def end_date = cancel_date - |end - |""".stripMargin) - - "return a method ref" taggedAs DifferentInNewFrontend in { - val List(mRef: MethodRef) = cpg.method("new2").ast.isReturn.astChildren.l: @unchecked - mRef.methodFullName shouldBe "Test0.rb::program.end_date" - } - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/SimpleAstCreationPassTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/SimpleAstCreationPassTest.scala deleted file mode 100644 index fba69fcc69bf..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/SimpleAstCreationPassTest.scala +++ /dev/null @@ -1,1486 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.deprecated.astcreation.AstCreator -import io.joern.rubysrc2cpg.deprecated.passes.Defines -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Literal, NewIdentifier} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.semanticcpg.language.* - -class SimpleAstCreationPassTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "AST generation for simple fragments" should { - - "have correct structure for a single command call" in { - val cpg = code("""puts 123""") - - val List(assign, puts) = cpg.call.l - val List(arg) = puts.argument.isLiteral.l - - puts.code shouldBe "puts 123" - puts.lineNumber shouldBe Some(1) - - arg.code shouldBe "123" - arg.lineNumber shouldBe Some(1) - arg.columnNumber shouldBe Some(5) - - assign.name shouldBe ".assignment" // call node for builtin typeRef assignment - } - - "have correct structure for an unsigned, decimal integer literal" in { - val cpg = code("123") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "123" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a +integer, decimal literal" in { - val cpg = code("+1") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "+1" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a -integer, decimal literal" in { - val cpg = code("-1") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "-1" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an unsigned, decimal float literal" in { - val cpg = code("3.14") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Float" - literal.code shouldBe "3.14" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a +float, decimal literal" in { - val cpg = code("+3.14") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Float" - literal.code shouldBe "+3.14" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a -float, decimal literal" in { - val cpg = code("-3.14") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Float" - literal.code shouldBe "-3.14" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an unsigned, decimal float literal with unsigned exponent" in { - val cpg = code("3e10") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Float" - literal.code shouldBe "3e10" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an unsigned, decimal float literal with -exponent" in { - val cpg = code("12e-10") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Float" - literal.code shouldBe "12e-10" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an unsigned, binary integer literal" in { - val cpg = code("0b01") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "0b01" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a -integer, binary literal" in { - val cpg = code("-0b01") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "-0b01" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a +integer, binary literal" in { - val cpg = code("+0b01") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "+0b01" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an unsigned, hexadecimal integer literal" in { - val cpg = code("0xabc") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "0xabc" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a -integer, hexadecimal literal" in { - val cpg = code("-0xa") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "-0xa" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a +integer, hexadecimal literal" in { - val cpg = code("+0xa") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.Integer" - literal.code shouldBe "+0xa" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for `nil` literal" in { - val cpg = code("puts nil") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.NilClass - literal.code shouldBe "nil" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(5) - } - - "have correct structure for `true` literal" in { - val cpg = code("puts true") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.TrueClass - literal.code shouldBe "true" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(5) - } - - "have correct structure for `false` literal" in { - val cpg = code("puts false") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.FalseClass - literal.code shouldBe "false" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(5) - } - - "have correct structure for `self` identifier" in { - val cpg = code("puts self") - val List(self, _) = cpg.identifier.l - self.typeFullName shouldBe Defines.Object - self.code shouldBe "self" - self.lineNumber shouldBe Some(1) - self.columnNumber shouldBe Some(5) - } - - "have correct structure for `__FILE__` identifier" in { - val cpg = code("puts __FILE__") - val List(file, _) = cpg.identifier.l - file.typeFullName shouldBe "__builtin.String" - file.code shouldBe "__FILE__" - file.lineNumber shouldBe Some(1) - file.columnNumber shouldBe Some(5) - } - - "have correct structure for `__LINE__` identifier" in { - val cpg = code("puts __LINE__") - val List(line, _) = cpg.identifier.l - line.typeFullName shouldBe "__builtin.Integer" - line.code shouldBe "__LINE__" - line.lineNumber shouldBe Some(1) - line.columnNumber shouldBe Some(5) - } - - "have correct structure for `__ENCODING__` identifier" in { - val cpg = code("puts __ENCODING__") - val List(encoding, _) = cpg.identifier.l - encoding.typeFullName shouldBe Defines.Encoding - encoding.code shouldBe "__ENCODING__" - encoding.lineNumber shouldBe Some(1) - encoding.columnNumber shouldBe Some(5) - } - - "have correct structure for a single-line double-quoted string literal" in { - val cpg = code("\"hello\"") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.String" - literal.code shouldBe "\"hello\"" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a single-line single-quoted string literal" in { - val cpg = code("'hello'") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.String" - literal.code shouldBe "'hello'" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a single-line quoted non-expanded string literal" in { - val cpg = code("%q(hello)") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.String" - literal.code shouldBe "%q(hello)" - literal.lineNumber shouldBe Some(1) - } - - "have correct structure for a multi-line quoted non-expanded string literal" in { - val cpg = code("""%q< - |xyz - |123 - |>""".stripMargin) - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe "__builtin.String" - literal.code shouldBe - """%q< - |xyz - |123 - |>""".stripMargin - literal.lineNumber shouldBe Some(1) - } - - "have correct structure for an identifier symbol literal" in { - val cpg = code(":someSymbolName") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Symbol - literal.code shouldBe ":someSymbolName" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a single-quoted-string symbol literal" in { - val cpg = code(":'someSymbolName'") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Symbol - literal.code shouldBe ":'someSymbolName'" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for an identifier symbol literal used in an `undef` statement" in { - val cpg = code("undef :symbolName") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Symbol - literal.code shouldBe ":symbolName" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(6) - } - - "have correct structure for a single-line regular expression literal" in { - val cpg = code("/(eu|us)/") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Regexp - literal.code shouldBe "/(eu|us)/" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(0) - } - - "have correct structure for a single-line quoted (%r) regular expression literal" in { - val cpg = code("%r{eu|us}") - val List(literalNode) = cpg.literal.l - literalNode.typeFullName shouldBe Defines.Regexp - literalNode.code shouldBe "%r{eu|us}" - literalNode.lineNumber shouldBe Some(1) - literalNode.columnNumber shouldBe Some(0) - } - - "have correct structure for an empty regular expression literal used as the second argument to a call" in { - val cpg = code("puts(x, //)") - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Regexp - literal.code shouldBe "//" - literal.lineNumber shouldBe Some(1) - literal.columnNumber shouldBe Some(8) - } - - "have correct structure for a single-line regular expression literal passed as argument to a command" in { - val cpg = code("puts /x/") - - val List(_, callNode) = cpg.call.l - callNode.code shouldBe "puts /x/" - callNode.name shouldBe "puts" - callNode.lineNumber shouldBe Some(1) - - val List(literalArg) = callNode.argument.isLiteral.l - literalArg.argumentIndex shouldBe 1 - literalArg.typeFullName shouldBe Defines.Regexp - literalArg.code shouldBe "/x/" - literalArg.lineNumber shouldBe Some(1) - } - - "have correct structure for a single left had side call" in { - val cpg = code("array[n] = 10") - val List(callNode) = cpg.call.name(Operators.indexAccess).l - callNode.code shouldBe "array[n]" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(5) - } - - "have correct structure for a binary expression" in { - val cpg = code("x+y") - val List(callNode) = cpg.call.name(Operators.addition).l - callNode.code shouldBe "x+y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a not expression" in { - val cpg = code("not y") - val List(callNode) = cpg.call.name(Operators.not).l - callNode.code shouldBe "not y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a power expression" in { - val cpg = code("x**y") - val List(callNode) = cpg.call.name(Operators.exponentiation).l - callNode.code shouldBe "x**y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a inclusive range expression" in { - val cpg = code("1..10") - val List(callNode) = cpg.call.name(Operators.range).l - callNode.code shouldBe "1..10" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a non-inclusive range expression" in { - val cpg = code("1...10") - val List(callNode) = cpg.call.name(Operators.range).l - callNode.code shouldBe "1...10" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a relational expression" in { - val cpg = code("x> y") - val List(callNode) = cpg.call.name(Operators.logicalShiftRight).l - callNode.code shouldBe "x >> y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a shift left expression" in { - val cpg = code("x << y") - val List(callNode) = cpg.call.name(Operators.shiftLeft).l - callNode.code shouldBe "x << y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a compare expression" in { - val cpg = code("x <=> y") - val List(callNode) = cpg.call.name(Operators.compare).l - callNode.code shouldBe "x <=> y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a indexing expression" in { - val cpg = code("def some_method(index)\n some_map[index]\nend") - val List(callNode) = cpg.call.name(Operators.indexAccess).l - callNode.code shouldBe "some_map[index]" - callNode.lineNumber shouldBe Some(2) - callNode.columnNumber shouldBe Some(9) - } - - "have correct structure for overloaded index operator method" in { - val cpg = code(""" - |class MyClass - |def [](key) - | @member_hash[key] - |end - |end - |""".stripMargin) - - val List(methodNode) = cpg.method.name("\\[]").l - methodNode.fullName shouldBe "Test0.rb::program.MyClass.[]" - methodNode.code shouldBe "def [](key)\n @member_hash[key]\nend" - methodNode.lineNumber shouldBe Some(3) - methodNode.lineNumberEnd shouldBe Some(5) - methodNode.columnNumber shouldBe Some(4) - } - - "have correct structure for overloaded equality operator method" in { - val cpg = code(""" - |class MyClass - |def ==(other) - | @my_member==other - |end - |end - |""".stripMargin) - - val List(methodNode) = cpg.method.name("==").l - methodNode.fullName shouldBe "Test0.rb::program.MyClass.==" - methodNode.code shouldBe "def ==(other)\n @my_member==other\nend" - methodNode.lineNumber shouldBe Some(3) - methodNode.lineNumberEnd shouldBe Some(5) - methodNode.columnNumber shouldBe Some(4) - } - - "have correct structure for class method" in { - val cpg = code(""" - |class MyClass - |def some_method(param) - |end - |end - |""".stripMargin) - - val List(methodNode) = cpg.method.name("some_method").l - methodNode.fullName shouldBe "Test0.rb::program.MyClass.some_method" - methodNode.code shouldBe "def some_method(param)\nend" - methodNode.lineNumber shouldBe Some(3) - methodNode.lineNumberEnd shouldBe Some(4) - methodNode.columnNumber shouldBe Some(4) - } - - "have correct structure for scope resolution operator call" in { - val cpg = code(""" - |def foo(param) - |::SomeConstant = param - |end - |""".stripMargin) - - val List(identifierNode) = cpg.identifier.name("SomeConstant").l - identifierNode.code shouldBe "SomeConstant" - identifierNode.lineNumber shouldBe Some(3) - identifierNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a addition expression with space before addition" in { - val cpg = code("x + y") - val List(callNode) = cpg.call.name(Operators.addition).l - callNode.code shouldBe "x + y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for a addition expression with space before subtraction" in { - val cpg = code("x - y") - val List(callNode) = cpg.call.name(Operators.subtraction).l - callNode.code shouldBe "x - y" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for object's method access (chainedInvocationPrimary)" in { - val cpg = code("object.some_method(arg1,arg2)") - val List(callNode) = cpg.call.name("some_method").l - callNode.code shouldBe "object.some_method(arg1,arg2)" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(6) - - val List(identifierNode1) = cpg.identifier.name("arg1").l - identifierNode1.code shouldBe "arg1" - identifierNode1.lineNumber shouldBe Some(1) - identifierNode1.columnNumber shouldBe Some(19) - - val List(identifierNode2) = cpg.identifier.name("arg2").l - identifierNode2.code shouldBe "arg2" - identifierNode2.lineNumber shouldBe Some(1) - identifierNode2.columnNumber shouldBe Some(24) - } - - "have correct structure for object's method.member access (chainedInvocationPrimary)" ignore { - val cpg = code("object.some_member") - val List(identifierNode) = cpg.identifier.name("some_member").l - identifierNode.code shouldBe "some_member" - identifierNode.lineNumber shouldBe Some(1) - identifierNode.columnNumber shouldBe Some(0) - } - - "have correct structure for negation before block (invocationExpressionOrCommand)" in { - val cpg = code("!foo arg do\nputs arg\nend") - - val List(callNode1) = cpg.call.name(Operators.not).l - callNode1.code shouldBe "!foo arg do\nputs arg\nend" - callNode1.lineNumber shouldBe Some(1) - callNode1.columnNumber shouldBe Some(0) - - val List(callNode2) = cpg.call.name("foo").l - callNode2.code shouldBe "foo arg do\nputs arg\nend" - callNode2.lineNumber shouldBe Some(1) - callNode2.columnNumber shouldBe Some(1) - - val List(callNode3) = cpg.call.name("puts").l - callNode3.code shouldBe "puts arg" - callNode3.lineNumber shouldBe Some(2) - callNode3.columnNumber shouldBe Some(0) - - val List(argArgumentOfPuts, argArgumentOfFoo) = cpg.identifier.name("arg").l - argArgumentOfFoo.code shouldBe "arg" - argArgumentOfFoo.lineNumber shouldBe Some(2) - argArgumentOfFoo.columnNumber shouldBe Some(5) - - argArgumentOfPuts.code shouldBe "arg" - argArgumentOfPuts.lineNumber shouldBe Some(1) - } - - "have correct structure for a hash initialisation" in { - val cpg = code("hashMap = {\"k1\" => 1, \"k2\" => 2}") - val callNodes = cpg.call.name(".keyValueAssociation").l - callNodes.size shouldBe 2 - callNodes.head.code shouldBe "\"k1\" => 1" - callNodes.head.lineNumber shouldBe Some(1) - callNodes.head.columnNumber shouldBe Some(16) - } - - "have correct structure for defined? command" in { - val cpg = code("defined? x") - - val List(callNode) = cpg.call.name(".defined").l - callNode.code shouldBe "defined? x" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - - val List(identifierNode) = cpg.identifier.name("x").l - identifierNode.code shouldBe "x" - identifierNode.lineNumber shouldBe Some(1) - identifierNode.columnNumber shouldBe Some(9) - } - - "have correct structure for defined? call" in { - val cpg = code("defined?(x)") - - val List(callNode) = cpg.call.name(".defined").l - callNode.code shouldBe "defined?(x)" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - - val List(identifierNode) = cpg.identifier.name("x").l - identifierNode.code shouldBe "x" - identifierNode.lineNumber shouldBe Some(1) - identifierNode.columnNumber shouldBe Some(9) - } - - "have correct structure for chainedInvocationWithoutArgumentsPrimary" in { - val cpg = code("object::foo do\nputs \"right here\"\nend") - - val List(callNode1) = cpg.call.name("foo").l - callNode1.code shouldBe "puts \"right here\"" - callNode1.lineNumber shouldBe Some(1) - callNode1.columnNumber shouldBe Some(3) - - val List(callNode2) = cpg.call.name("puts").l - callNode2.code shouldBe "puts \"right here\"" - callNode2.lineNumber shouldBe Some(2) - callNode2.columnNumber shouldBe Some(0) - } - - "have correct structure for require with an expression" in { - val cpg = code("Dir[Rails.root.join('a', 'b', '**', '*.rb')].each { |f| require f }") - - val List(callNode) = cpg.call.name("require").l - callNode.code shouldBe "require f" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(56) - } - - "have correct structure for undef" in { - val cpg = code("undef method1,method2") - - val List(callNode) = cpg.call.name(".undef").l - callNode.code shouldBe "undef method1,method2" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(0) - } - - "have correct structure for ternary if expression" in { - val cpg = code("a ? b : c") - val List(controlNode) = cpg.controlStructure.l - - controlNode.controlStructureType shouldBe ControlStructureTypes.IF - controlNode.code shouldBe "a ? b : c" - controlNode.lineNumber shouldBe Some(1) - controlNode.columnNumber shouldBe Some(0) - - val List(a) = controlNode.condition.isIdentifier.l - a.code shouldBe "a" - a.name shouldBe "a" - a.lineNumber shouldBe Some(1) - a.columnNumber shouldBe Some(0) - - val List(_, b, c) = controlNode.astChildren.isIdentifier.l - b.code shouldBe "b" - b.name shouldBe "b" - b.lineNumber shouldBe Some(1) - b.columnNumber shouldBe Some(4) - - c.code shouldBe "c" - c.name shouldBe "c" - c.lineNumber shouldBe Some(1) - c.columnNumber shouldBe Some(8) - } - - "have correct structure for if statement" in { - val cpg = code("""if x == 0 then - | puts 1 - |end - |""".stripMargin) - - val List(ifNode) = cpg.controlStructure.l - ifNode.controlStructureType shouldBe ControlStructureTypes.IF - ifNode.lineNumber shouldBe Some(1) - - val List(ifCondition, ifBlock) = ifNode.astChildren.l - ifCondition.code shouldBe "x == 0" - ifCondition.lineNumber shouldBe Some(1) - - val List(puts) = ifBlock.astChildren.l - puts.code shouldBe "puts 1" - puts.lineNumber shouldBe Some(2) - } - - "have correct structure for if-else statement" in { - val cpg = code("""if x == 0 then - | puts 1 - |else - | puts 2 - |end - |""".stripMargin) - - val List(ifNode) = cpg.controlStructure.l - ifNode.controlStructureType shouldBe ControlStructureTypes.IF - ifNode.lineNumber shouldBe Some(1) - - val List(ifCondition, ifBlock, elseBlock) = ifNode.astChildren.l - ifCondition.code shouldBe "x == 0" - ifCondition.lineNumber shouldBe Some(1) - - val List(puts1) = ifBlock.astChildren.l - puts1.code shouldBe "puts 1" - puts1.lineNumber shouldBe Some(2) - - val List(puts2) = elseBlock.astChildren.l - puts2.code shouldBe "puts 2" - puts2.lineNumber shouldBe Some(4) - } - - "have correct structure for class definition with body having only identifiers" in { - val cpg = code("class MyClass\nidentifier1\nidentifier2\nend") - - val List(identifierNode1) = cpg.identifier.name("identifier1").l - identifierNode1.code shouldBe "identifier1" - identifierNode1.lineNumber shouldBe Some(2) - identifierNode1.columnNumber shouldBe Some(0) - - val List(identifierNode2) = cpg.identifier.name("identifier2").l - identifierNode2.code shouldBe "identifier2" - identifierNode2.lineNumber shouldBe Some(3) - identifierNode2.columnNumber shouldBe Some(0) - } - - // NOTE: The representation for `super` may change, in order to accommodate its meaning. - // But until then, modelling it as a call seems the appropriate thing to do. - "have correct structure for `super` expression call without block" in { - val cpg = code("super(1)") - - val List(callNode) = cpg.call.l - callNode.code shouldBe "super(1)" - callNode.name shouldBe ".super" - callNode.lineNumber shouldBe Some(1) - - val List(literalArg) = callNode.argument.isLiteral.l - literalArg.argumentIndex shouldBe 1 - literalArg.code shouldBe "1" - literalArg.lineNumber shouldBe Some(1) - } - - "have correct structure for `super` command call without block" in { - val cpg = code("super 1") - - val List(callNode) = cpg.call.l - callNode.code shouldBe "super 1" - callNode.name shouldBe ".super" - callNode.lineNumber shouldBe Some(1) - - val List(literalArg) = callNode.argument.isLiteral.l - literalArg.argumentIndex shouldBe 1 - literalArg.code shouldBe "1" - literalArg.lineNumber shouldBe Some(1) - } - - "have generated call nodes for regex interpolation" in { - val cpg = code("/x#{Regexp.quote(foo)}b#{x+'z'}a/") - val List(literalNode) = cpg.literal.l - cpg.call.size shouldBe 2 - literalNode.code shouldBe "'z'" - } - - "have correct structure for keyword? named method usage usage" in { - val cpg = code("x = 1.nil?") - - val List(callNode) = cpg.call.nameExact("nil?").l - callNode.code shouldBe "1.nil?" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(5) - - val List(arg) = callNode.argument.isLiteral.l - arg.code shouldBe "1" - } - - "have correct structure for keyword usage inside association" in { - val cpg = code("foo if: x.nil?") - - val List(callNode) = cpg.call.nameExact("nil?").l - callNode.code shouldBe "x.nil?" - callNode.lineNumber shouldBe Some(1) - callNode.columnNumber shouldBe Some(9) - - val List(arg) = callNode.argument.isIdentifier.l - arg.code shouldBe "x" - - val List(assocCallNode) = cpg.call.nameExact(".activeRecordAssociation").l - assocCallNode.code shouldBe "if: x.nil?" - assocCallNode.lineNumber shouldBe Some(1) - assocCallNode.columnNumber shouldBe Some(6) - - assocCallNode.argument.size shouldBe 2 - assocCallNode.argument.argumentIndex(1).head.code shouldBe "if" - assocCallNode.argument.argumentIndex(2).head.code shouldBe "x.nil?" - } - - "have correct structure for proc definiton with procParameters and empty block" in { - val cpg = - code("-> (x,y) {}") - cpg.parameter.size shouldBe 2 - } - - "have correct structure for proc definiton with procParameters and non-empty block" in { - val cpg = - code("""-> (x,y) { - |if (x) - | y - |else - | b - |end - |}""".stripMargin) - cpg.parameter.size shouldBe 2 - val List(paramOne, paramTwo) = cpg.parameter.l - paramOne.name shouldBe "x" - paramTwo.name shouldBe "y" - cpg.ifBlock.size shouldBe 1 - } - - "have correct structure for proc definition with no parameters and empty block" in { - val cpg = code("-> {}") - cpg.parameter.size shouldBe 0 - } - - "have correct structure for proc definition with additional context" in { - val cpg = code( - "scope :get_all_doctors, -> { (select('id, first_name').where('role = :user_role', user_role: User.roles[:doctor])) }" - ) - cpg.parameter.size shouldBe 6 - cpg.call.name("proc_2").size shouldBe 1 - cpg.call.name("scope").size shouldBe 1 - cpg.call.name("where").size shouldBe 1 - cpg.call.name("select").size shouldBe 1 - cpg.call.name("roles").size shouldBe 1 - cpg.call.name(".activeRecordAssociation").size shouldBe 1 - cpg.call.name(".indexAccess").size shouldBe 1 - } - - "have correct structure when method called with safe navigation without parameters" in { - val cpg = code("foo&.bar") - cpg.call.size shouldBe 1 - } - - "have correct structure when method called with safe navigation with parameters with parantheses" in { - val cpg = code("foo&.bar(1)") - - val List(callNode) = cpg.call.l - val List(actualArg) = callNode.argument.argumentIndex(1).l - actualArg.code shouldBe "1" - cpg.argument.size shouldBe 2 - cpg.call.size shouldBe 1 - } - - "have correct structure when method called with safe navigation with parameters without parantheses" in { - val cpg = code("foo&.bar 1,2") - - val List(callNode) = cpg.call.l - val List(actualArg1) = callNode.argument.argumentIndex(1).l - actualArg1.code shouldBe "1" - val List(actualArg2) = callNode.argument.argumentIndex(2).l - actualArg2.code shouldBe "2" - cpg.argument.size shouldBe 3 - cpg.call.size shouldBe 1 - } - - "have correct structure when method call present in next line, with the second line starting with `.`" in { - val cpg = code("foo\n .bar(1)") - - val List(callNode) = cpg.call.l - cpg.call.size shouldBe 1 - callNode.code shouldBe ("foo\n .bar(1)") - callNode.name shouldBe "bar" - callNode.lineNumber shouldBe Some(2) - val List(actualArg) = callNode.argument.argumentIndex(1).l - actualArg.code shouldBe "1" - } - - "have correct structure when method call present in next line, with the first line ending with `.`" in { - val cpg = code("foo.\n bar(1)") - - val List(callNode) = cpg.call.l - cpg.call.size shouldBe 1 - callNode.code shouldBe ("foo.\n bar(1)") - callNode.name shouldBe "bar" - callNode.lineNumber shouldBe Some(1) - val List(actualArg) = callNode.argument.argumentIndex(1).l - actualArg.code shouldBe "1" - } - - "have correct structure for proc parameter with name" in { - val cpg = code("def foo(&block) end") - val List(actualParameter) = cpg.method("foo").parameter.l - actualParameter.name shouldBe "block" - } - - "have correct structure for proc parameter with no name" in { - val cpg = code("def foo(&) end") - val List(actualParameter) = cpg.method("foo").parameter.l - actualParameter.name shouldBe "param_0" - } - - "have correct structure when regular expression literal passed after `when`" in { - val cpg = code(""" - |case foo - | when /^ch/ - | bar - |end - |""".stripMargin) - - val List(literalArg) = cpg.literal.l - literalArg.typeFullName shouldBe Defines.Regexp - literalArg.code shouldBe "/^ch/" - literalArg.lineNumber shouldBe Some(3) - } - - "have correct structure when have interpolated double-quoted string literal" in { - val cpg = code(""" - |v = :"w x #{y} z" - |""".stripMargin) - - cpg.call.size shouldBe 4 - cpg.call.name(".formatString").head.code shouldBe """:"w x #{y} z"""" - cpg.call.name(".formatValue").head.code shouldBe "#{y}" - - cpg.literal.size shouldBe 2 - cpg.literal.code("w x ").size shouldBe 1 - cpg.literal.code(" z").size shouldBe 1 - - cpg.identifier.name("y").size shouldBe 1 - cpg.identifier.name("v").size shouldBe 1 - } - - "have correct structure when have non-interpolated double-quoted string literal" in { - val cpg = code(""" - |x = :"y z" - |""".stripMargin) - - cpg.call.size shouldBe 1 - val List(literal) = cpg.literal.l - literal.code shouldBe ":\"y z\"" - literal.typeFullName shouldBe Defines.Symbol - } - - "have correct structure when have symbol " in { - val cpg = code(s""" - |x = :"${10}" - |""".stripMargin) - - cpg.call.size shouldBe 1 - val List(literal) = cpg.literal.l - literal.typeFullName shouldBe Defines.Symbol - literal.code shouldBe ":\"10\"" - } - } - - "have correct structure when no RHS for a mandatory parameter is provided" in { - val cpg = code(""" - |def foo(bar:) - |end - |""".stripMargin) - - val List(parameterNode) = cpg.method("foo").parameter.l - parameterNode.name shouldBe "bar" - parameterNode.lineNumber shouldBe Some(2) - } - - "have correct structure when RHS for a mandatory parameter is provided" in { - val cpg = code(""" - |def foo(bar: world) - |end - |""".stripMargin) - - val List(parameterNode) = cpg.method("foo").parameter.l - parameterNode.name shouldBe "bar" - parameterNode.lineNumber shouldBe Some(2) - } - - // Change below test cases to focus on the argument of call `foo` - "have correct structure when a association is passed as an argument with parantheses" in { - val cpg = code("""foo(bar:)""".stripMargin) - - cpg.argument.size shouldBe 2 - cpg.argument.l(0).code shouldBe "bar:" - cpg.call.size shouldBe 2 - val List(callNode, operatorNode) = cpg.call.l - callNode.name shouldBe "foo" - operatorNode.name shouldBe ".activeRecordAssociation" - } - - "have correct structure when a association is passed as an argument without parantheses" in { - val cpg = code("""foo bar:""".stripMargin) - - cpg.argument.size shouldBe 2 - cpg.argument.l.head.code shouldBe "bar:" - - cpg.call.size shouldBe 2 - val List(callNode, operatorNode) = cpg.call.l - callNode.name shouldBe "foo" - operatorNode.name shouldBe ".activeRecordAssociation" - } - - "have correct structure with ternary operator with multiple line" in { - val cpg = code("""x = a ? - | b - |: c""".stripMargin) - - val List(controlNode) = cpg.controlStructure.l - controlNode.controlStructureType shouldBe ControlStructureTypes.IF - controlNode.code shouldBe "a ?\n b\n: c" - controlNode.lineNumber shouldBe Some(1) - controlNode.columnNumber shouldBe Some(4) - - val List(a) = controlNode.condition.isIdentifier.l - a.code shouldBe "a" - a.name shouldBe "a" - a.lineNumber shouldBe Some(1) - a.columnNumber shouldBe Some(4) - - val List(_, b, c) = controlNode.astChildren.isIdentifier.l - b.code shouldBe "b" - b.name shouldBe "b" - b.lineNumber shouldBe Some(2) - b.columnNumber shouldBe Some(1) - - c.code shouldBe "c" - c.name shouldBe "c" - c.lineNumber shouldBe Some(3) - c.columnNumber shouldBe Some(2) - } - - "have correct structure for blank indexing arguments" in { - val cpg = code(""" - |bar = Set[] - |""".stripMargin) - - val List(callNode) = cpg.call.name(".indexAccess").l - callNode.lineNumber shouldBe Some(2) - callNode.columnNumber shouldBe Some(9) - } - - "method defined inside a class using << operator" in { - val cpg = code(""" - class MyClass - | - | class << self - | def print - | puts "log #{self}" - | end - | end - | class << self - | end - |end - | - |MyClass.print""".stripMargin) - - val List(callNode) = cpg.call.name("print").l - callNode.lineNumber shouldBe Some(13) - callNode.columnNumber shouldBe Some(7) - callNode.name shouldBe "print" - } - - "have correct structure for body statements inside a do block" in { - val cpg = code(""" - |def foo - |1/0 - |rescue ZeroDivisionError => e - |end""".stripMargin) - - val List(methodNode) = cpg.method.code(".*foo.*").l - methodNode.name shouldBe "foo" - methodNode.lineNumber shouldBe Some(2) - - val List(assignmentOperator, divisionOperator) = cpg.method.name(".*operator.*").l - divisionOperator.name shouldBe ".division" - assignmentOperator.name shouldBe ".assignment" - } - - "have correct structure when regex literal is used on RHS of association" in { - val cpg = code(""" - |books = [ - | { - | id: /.*/ - | } - |] - |""".stripMargin) - - val List(assocOperator) = cpg.call(".*activeRecordAssociation.*").l - assocOperator.code shouldBe "id: /.*/" - assocOperator.astChildren.code.l(1) shouldBe "/.*/" - assocOperator.lineNumber shouldBe Some(4) - } - - "have double-quoted string literals containing \\u character" in { - val cpg = code(""" - |val fileName = "AB\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000!\u0000file" - |""".stripMargin) - - cpg.identifier.size shouldBe 1 - cpg.identifier.name.head shouldBe "fileName" - cpg.literal.head.code - .stripPrefix("\"") - .stripSuffix("\"") - .trim shouldBe """AB\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000!\u0000file""" - - } - - "have correct structure for a endless method" in { - val cpg = code(""" - |def foo(a,b) = a*b - |""".stripMargin) - - val List(methodNode) = cpg.method.name("foo").l - methodNode.lineNumber shouldBe Some(2) - methodNode.columnNumber shouldBe Some(4) - } - - "have correct structure for symbol literal defined using \\:" in { - val cpg = code(""" - |foo = {:bar=>zoo} - |""".stripMargin) - - val List(keyValueAssocOperator) = cpg.call(".*keyValueAssociation.*").l - keyValueAssocOperator.code shouldBe ":bar=>zoo" - keyValueAssocOperator.astChildren.l.head.code shouldBe ":bar" - keyValueAssocOperator.astChildren.l(1).code shouldBe "zoo" - } - - "having a binary expression includes + and @" in { - val cpg = code(""" - |class MyClass - | def initialize(a) - | @a = a - | end - | - | def calculate_x(b) - | x = b+@a - | return x - | end - |end - |""".stripMargin) - cpg.identifier("a").dedup.size shouldBe 1 - cpg.identifier("b").dedup.size shouldBe 1 - cpg.identifier("x").name.dedup.size shouldBe 1 - cpg.method("calculate_x").size shouldBe 1 - } - - "have correct structure for empty %w array" in { - val cpg = code(""" - |a = %w[] - |""".stripMargin) - - val List(assignmentCallNode) = cpg.call.name(Operators.assignment).l - assignmentCallNode.size shouldBe 1 - val List(arrayCallNode) = cpg.call.name(Operators.arrayInitializer).l - arrayCallNode.size shouldBe 1 - arrayCallNode.argument.size shouldBe 0 - } - - "have correct structure for %w array with %w()" in { - val cpg = code(""" - |a = %w(b c d) - |""".stripMargin) - - val List(assignmentCallNode) = cpg.call.name(Operators.assignment).l - assignmentCallNode.size shouldBe 1 - val List(arrayCallNode) = cpg.call.name(Operators.arrayInitializer).l - arrayCallNode.size shouldBe 1 - arrayCallNode.argument - .where(_.argumentIndex(1)) - .code - .l shouldBe List("b") - arrayCallNode.argument - .where(_.argumentIndex(2)) - .code - .l shouldBe List("c") - arrayCallNode.argument - .where(_.argumentIndex(3)) - .code - .l shouldBe List("d") - } - - "have correct structure for %w array with %w() with entries separated by whitespace" in { - val cpg = code(""" - |a = %w( - | bob - | cod - | dod - |) - |""".stripMargin) - - val List(assignmentCallNode) = cpg.call.name(Operators.assignment).l - assignmentCallNode.size shouldBe 1 - val List(arrayCallNode) = cpg.call.name(Operators.arrayInitializer).l - arrayCallNode.size shouldBe 1 - arrayCallNode.argument - .where(_.argumentIndex(1)) - .code - .l shouldBe List("bob") - arrayCallNode.argument - .where(_.argumentIndex(2)) - .code - .l shouldBe List("cod") - arrayCallNode.argument - .where(_.argumentIndex(3)) - .code - .l shouldBe List("dod") - } - - "have correct structure for %w array with %w- -" in { - val cpg = code(""" - |a = %w-b c- - |""".stripMargin) - - val List(assignmentCallNode) = cpg.call.name(Operators.assignment).l - assignmentCallNode.size shouldBe 1 - val List(arrayCallNode) = cpg.call.name(Operators.arrayInitializer).l - arrayCallNode.size shouldBe 1 - arrayCallNode.argument - .where(_.argumentIndex(1)) - .code - .l shouldBe List("b") - arrayCallNode.argument - .where(_.argumentIndex(2)) - .code - .l shouldBe List("c") - } - - "have correct structure for %i() array with two elements" in { - val cpg = code("x = %i(yy zz)") - - val List(arrayInit) = cpg.call.name(Operators.arrayInitializer).l - val List(yyNode: Literal, zzNode: Literal) = arrayInit.argument.isLiteral.l - - yyNode.code shouldBe "yy" - yyNode.argumentIndex shouldBe 1 - yyNode.typeFullName shouldBe Defines.Symbol - - zzNode.code shouldBe "zz" - zzNode.argumentIndex shouldBe 2 - zzNode.typeFullName shouldBe Defines.Symbol - } - - "have correct structure parenthesised arguments in a return jump" in { - val cpg = code("""return(value) unless item""".stripMargin) - - cpg.identifier.size shouldBe 2 - cpg.identifier.name("value").size shouldBe 1 - cpg.identifier.name("item").size shouldBe 1 - - val List(methodReturn) = cpg.ret.l - methodReturn.code shouldBe "return(value)" - methodReturn.lineNumber shouldBe Some(1) - methodReturn.columnNumber shouldBe Some(0) - } - - "have correct structure for a hash containing splatting elements" in { - val cpg = code(""" - |bar={:x=>1} - |foo = { - |**bar - |} - |""".stripMargin) - - val List(keyValueAssocOperator) = cpg.call(".*keyValueAssociation.*").l - keyValueAssocOperator.code shouldBe ":x=>1" - keyValueAssocOperator.astChildren.l(1).code shouldBe "1" - - val List(pseudoIdentifier, actualIdentifier) = cpg.identifier("bar").l - pseudoIdentifier.lineNumber shouldBe Some(2) - pseudoIdentifier.columnNumber shouldBe Some(0) - } - - "have correct structure for regex match global variables" in { - val cpg = code(""" - |content_filename =~ /filename="(.*)"/ - |value = $1 - |""".stripMargin) - - cpg.call.size shouldBe 2 - cpg.call.code(".*filename.*").head.methodFullName shouldBe ".patternMatch" - - cpg.identifier.code("value").size shouldBe 1 - cpg.identifier.name("\\$1").size shouldBe 1 - cpg.identifier.name("\\$1").head.typeFullName shouldBe Defines.String - - cpg.literal.code("/filename=\"(.*)\"/").head.typeFullName shouldBe Defines.Regexp - } - - "have correct structure of unless keyword and regex statement" in { - val cpg = code("""def contains_numbers?(string) - | # Define a regular expression pattern to match any digit - | regex_pattern = /\d/ - | - | # Check if the string contains any numbers using the 'unless' keyword - | unless string.match(regex_pattern).nil? - | return true - | end - | - | return false - |end""".stripMargin) - - cpg.identifier.code("regex_pattern").name.dedup.size shouldBe 1 - cpg.method("contains_numbers\\?").name.size shouldBe 1 - cpg.call(".assignment").name.size shouldBe 2 - cpg.call("match").name.size shouldBe 1 - } - - "have correct structure for association identifier" in { - val cpg = code(""" - |foo(a:b) - |""".stripMargin) - - cpg.call.size shouldBe 2 - cpg.call.name(".activeRecordAssociation").size shouldBe 1 - - cpg.identifier.size shouldBe 2 - cpg.identifier.name("a").size shouldBe 1 - cpg.identifier.name("b").size shouldBe 1 - } - - "have correct structure for multiline %i" in { - val cpg = code(""" - |w = %i(x y - | z) - |""".stripMargin) - - val List(arrayInit) = cpg.call.name(Operators.arrayInitializer).l - val List(xNode: Literal, yNode: Literal, zNode: Literal) = arrayInit.argument.isLiteral.l - - xNode.code shouldBe "x" - xNode.argumentIndex shouldBe 1 - xNode.typeFullName shouldBe Defines.Symbol - - yNode.code shouldBe "y" - yNode.argumentIndex shouldBe 2 - yNode.typeFullName shouldBe Defines.Symbol - - zNode.code shouldBe "z" - zNode.argumentIndex shouldBe 3 - zNode.typeFullName shouldBe Defines.Symbol - } - - "have correct structure for packing LHS in multiple assignment" in { - val cpg = code(""" - |some_lhs,*pack_lhs = some_rhs,pack_rhs1, pack_rhs2 - |""".stripMargin) - - cpg.call.name(".assignment").size shouldBe 2 - val callNode1 = cpg.call.code("some_lhs = some_rhs").l.head - callNode1.lineNumber shouldBe Some(2) - callNode1.columnNumber shouldBe Some(19) - val args1 = callNode1.argument.l - args1.size shouldBe 2 - args1.head.code shouldBe "some_lhs" - args1.tail.code.l.head shouldBe "some_rhs" - - val callNode2 = cpg.call.code("pack_lhs = pack_rhs1, pack_rhs2").l.head - callNode2.lineNumber shouldBe Some(2) - callNode2.columnNumber shouldBe Some(19) - val args2 = callNode2.argument.l - args2.size shouldBe 2 - args2.head.code shouldBe "pack_lhs" - args2.tail.code.l.head shouldBe "pack_rhs1, pack_rhs2" - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/TypeDeclAstCreationPassTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/TypeDeclAstCreationPassTest.scala deleted file mode 100644 index 0eea15e938c8..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/TypeDeclAstCreationPassTest.scala +++ /dev/null @@ -1,290 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.semanticcpg.language.* -import io.joern.x2cpg.Defines as XDefines -import io.shiftleft.codepropertygraph.generated.Operators - -class TypeDeclAstCreationPassTest extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "AST generation for simple classes declarations" should { - - "generate a basic type declaration node for an empty class" in { - val cpg = code(""" - |class MyClass - |end - |""".stripMargin) - val List(myClass) = cpg.typeDecl.nameExact("MyClass").l - myClass.name shouldBe "MyClass" - myClass.fullName shouldBe "Test0.rb::program.MyClass" - } - - // TODO: Need to be fixed. - "generate a basic type declaration node for an empty class with Class.new" ignore { - val cpg = code(""" - |MyClass = Class.new do - |end - |""".stripMargin) - val List(myClass) = cpg.typeDecl.nameExact("MyClass").l - myClass.name shouldBe "MyClass" - myClass.fullName shouldBe "Test0.rb::program.MyClass" - } - - // TODO: Need to be fixed. - "populate class name correctly for a derived class" in { - val cpg = code(""" - |module ApplicationCable - | class Channel < ActionCable::Channel::Base - | end - |end - |""".stripMargin) - val List(myClass) = cpg.typeDecl.nameExact("Channel").l - myClass.name shouldBe "Channel" - myClass.fullName shouldBe "Test0.rb::program.ApplicationCable.Channel" - } - - "generate methods under type declarations" in { - val cpg = code(""" - |class Vehicle - | - | def self.speeding - | "Hello, from a class method" - | end - | - | def Vehicle.halting - | "Hello, from another class method" - | end - | - | def driving - | "Hello, from an instance method" - | end - | - |end - |""".stripMargin) - val List(vehicle) = cpg.typeDecl.nameExact("Vehicle").l - vehicle.name shouldBe "Vehicle" - vehicle.fullName shouldBe "Test0.rb::program.Vehicle" - - val List(_, speeding, halting, driving) = vehicle.method.l - speeding.name shouldBe "speeding" - halting.name shouldBe "halting" - driving.name shouldBe "driving" - - speeding.fullName shouldBe "Test0.rb::program.Vehicle.speeding" - halting.fullName shouldBe "Test0.rb::program.Vehicle.halting" - driving.fullName shouldBe "Test0.rb::program.Vehicle.driving" - } - - "generate members for various class members under the respective type declaration" in { - val cpg = code(""" - |class Song - | @@plays = 0 - | def initialize(name, artist, duration) - | @name = name - | @artist = artist - | @duration = duration - | end - |end - |""".stripMargin) - val List(song) = cpg.typeDecl.nameExact("Song").l - song.name shouldBe "Song" - song.fullName shouldBe "Test0.rb::program.Song" - - val List(classInit) = song.method.name(XDefines.StaticInitMethodName).l - classInit.fullName shouldBe s"Test0.rb::program.Song.${XDefines.StaticInitMethodName}" - val List(playsDef) = classInit.call.nameExact(Operators.fieldAccess).fieldAccess.l - playsDef.fieldIdentifier.canonicalName.headOption shouldBe Option("plays") - - val List(artist, duration, name, plays) = song.member.l - - plays.name shouldBe "plays" - name.name shouldBe "name" - artist.name shouldBe "artist" - duration.name shouldBe "duration" - - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("plays", "name", "artist", "duration") - } - - "generate members for various class members when using the `attr_reader` and `attr_writer` idioms" ignore { - val cpg = code(""" - |class Song - | attr_reader :name, :artist, :duration - | attr_writer :album - |end - |""".stripMargin) - val List(song) = cpg.typeDecl.nameExact("Song").l - song.name shouldBe "Song" - song.fullName shouldBe "Test0.rb::program.Song" - - val List(name, artist, duration, album) = song.member.l - name.name shouldBe "name" - artist.name shouldBe "artist" - duration.name shouldBe "duration" - album.name shouldBe "album" - } - - "generate methods with the correct access control modifiers case 1" in { - val cpg = code(""" - |class MyClass - | - | def method1 # default is 'public' - | #... - | end - | - | protected # subsequent methods will be 'protected' - | - | def method2 # will be 'protected' - | #... - | end - | - | private # subsequent methods will be 'private' - | - | def method3 # will be 'private' - | #... - | end - | - | public # subsequent methods will be 'public' - | - | def method4 # and this will be 'public' - | #... - | end - |end - |""".stripMargin) - val List(myClass) = cpg.typeDecl.nameExact("MyClass").l - myClass.name shouldBe "MyClass" - myClass.fullName shouldBe "Test0.rb::program.MyClass" - - val List(_, _, m1, m2, m3, m4) = myClass.method.l - m1.name shouldBe "method1" - m2.name shouldBe "method2" - m3.name shouldBe "method3" - m4.name shouldBe "method4" - - m1.fullName shouldBe "Test0.rb::program.MyClass.method1" - m2.fullName shouldBe "Test0.rb::program.MyClass.method2" - m3.fullName shouldBe "Test0.rb::program.MyClass.method3" - m4.fullName shouldBe "Test0.rb::program.MyClass.method4" - - m1.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) - m2.modifier.modifierType.l shouldBe List(ModifierTypes.PROTECTED) - m3.modifier.modifierType.l shouldBe List(ModifierTypes.PRIVATE) - m4.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) - } - - "generate methods with the correct access control modifiers case 2" ignore { - val cpg = code(""" - |class MyClass - | - | def method1 - | end - | - | def method2 - | end - | - | def method3 - | end - | - | def method4 - | end - | - | public :method1, :method4 - | protected :method2 - | private :method3 - |end - |""".stripMargin) - val List(myClass) = cpg.typeDecl.nameExact("MyClass").l - myClass.name shouldBe "MyClass" - myClass.fullName shouldBe "Test0.rb::program.MyClass" - - val List(m1, m2, m3, m4) = myClass.method.l - m1.name shouldBe "method1" - m2.name shouldBe "method2" - m3.name shouldBe "method3" - m4.name shouldBe "method4" - - m1.fullName shouldBe "Test0.rb::program.MyClass.method1" - m2.fullName shouldBe "Test0.rb::program.MyClass.method2" - m3.fullName shouldBe "Test0.rb::program.MyClass.method3" - m4.fullName shouldBe "Test0.rb::program.MyClass.method4" - - m1.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) - m2.modifier.modifierType.l shouldBe List(ModifierTypes.PROTECTED) - m3.modifier.modifierType.l shouldBe List(ModifierTypes.PRIVATE) - m4.modifier.modifierType.l shouldBe List(ModifierTypes.PUBLIC) - } - - } - - "Polymorphism in classes" should { - - "correctly contain the inherited base type name in the super type" ignore { - val cpg = code(""" - |class GeeksforGeeks - | def initialize - | puts "This is Superclass" - | end - | - | def super_method - | puts "Method of superclass" - | end - |end - | - |class Sudo_Placement < GeeksforGeeks - | def initialize - | puts "This is Subclass" - | end - |end - |""".stripMargin) - - val List(baseType) = cpg.typeDecl.nameExact("GeeksforGeeks").l - baseType.name shouldBe "GeeksforGeeks" - baseType.fullName shouldBe "Test0.rb::program.GeeksforGeeks" - - val List(subType) = cpg.typeDecl.nameExact("Sudo_Placement").l - subType.name shouldBe "Sudo_Placement" - subType.fullName shouldBe "Test0.rb::program.Sudo_Placement" - subType.inheritsFromTypeFullName shouldBe Seq("Test0.rb::program.GeeksforGeeks") - } - - } - - "Hierarchical class checks with constants" ignore { - val cpg = code(""" - |class MyClass - | MY_CONSTANT = 0 - | @@plays = 0 - | class ChildCls - | @@name = 0 - | MY_CONSTANT = 0 - | end - |end - |""".stripMargin) - - "member variables structure in place" in { - val List(clsInit1, clsInit2) = cpg.method(XDefines.StaticInitMethodName).l - clsInit1.fullName shouldBe s"Test0.rb::program.MyClass.${XDefines.StaticInitMethodName}" - val List(myconstantfa, playsfa) = clsInit1.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstantfa.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - playsfa.fieldIdentifier.canonicalName.headOption shouldBe Option("plays") - - clsInit2.fullName shouldBe s"Test0.rb::program.MyClass.ChildCls.${XDefines.StaticInitMethodName}" - val List(namefa, myconstant2fa) = clsInit2.call.nameExact(Operators.fieldAccess).fieldAccess.l - myconstant2fa.fieldIdentifier.canonicalName.headOption shouldBe Option("MY_CONSTANT") - namefa.fieldIdentifier.canonicalName.headOption shouldBe Option("name") - - val List(myclassTd2) = cpg.typeDecl("ChildCls").l - val List(namem, myConstant2m) = myclassTd2.member.l - myConstant2m.name shouldBe "MY_CONSTANT" - namem.name shouldBe "name" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("name", "MY_CONSTANT") - - val List(myclassTd) = cpg.typeDecl("MyClass").l - val List(myconstantm, playsm) = myclassTd.member.l - myconstantm.name shouldBe "MY_CONSTANT" - playsm.name shouldBe "plays" - cpg.fieldAccess.fieldIdentifier.canonicalName.l shouldBe List("MY_CONSTANT", "plays") - - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/UnaryOpCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/UnaryOpCpgTests.scala deleted file mode 100644 index 8cf301a99371..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/ast/UnaryOpCpgTests.scala +++ /dev/null @@ -1,47 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.ast - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.shiftleft.semanticcpg.language.* - -class UnaryOpCpgTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - "#unaryOp not" should { - val cpg = code("""!true""".stripMargin) - "test unaryOp 'not' call node properties" in { - val plusCall = cpg.call.methodFullName(Operators.not).head - plusCall.code shouldBe "!true" - plusCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - plusCall.lineNumber shouldBe Some(1) - } - - "test unaryOp 'not' arguments" in { - cpg.call - .methodFullName(Operators.not) - .argument - .argumentIndex(1) - .isLiteral - .head - .code shouldBe "true" - } - } - - "#unaryOp invert" should { - val cpg = code("""~2""".stripMargin) - "test unaryOp 'invert' call node properties" in { - val plusCall = cpg.call.methodFullName(Operators.not).head - plusCall.code shouldBe "~2" - plusCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH - plusCall.lineNumber shouldBe Some(1) - } - - "test unaryOp 'invert' arguments" in { - cpg.call - .methodFullName(Operators.not) - .argument - .argumentIndex(1) - .isLiteral - .head - .code shouldBe "2" - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/cfg/SimpleCfgCreationPassTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/cfg/SimpleCfgCreationPassTest.scala deleted file mode 100644 index 4e72b25ef4bd..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/passes/cfg/SimpleCfgCreationPassTest.scala +++ /dev/null @@ -1,132 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.passes.cfg - -import io.joern.rubysrc2cpg.testfixtures.RubyCfgTestCpg -import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.AlwaysEdge -import io.joern.x2cpg.testfixtures.CfgTestFixture -import io.shiftleft.codepropertygraph.generated.Cpg - -class SimpleCfgCreationPassTest extends CfgTestFixture(() => new RubyCfgTestCpg(useDeprecatedFrontend = true)) { - - "CFG generation for simple fragments" should { - "have correct structure for empty array literal" ignore { - implicit val cpg: Cpg = code("x = []") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x") shouldBe expected(("x = []", AlwaysEdge)) - succOf("x = []") shouldBe expected(("RET", AlwaysEdge)) - } - - "have correct structure for array literal with values" in { - implicit val cpg: Cpg = code("x = [1, 2]") - succOf("1") shouldBe expected(("2", AlwaysEdge)) - succOf("x = [1, 2]") shouldBe expected(("RET", AlwaysEdge)) - } - - "assigning a literal value" in { - implicit val cpg: Cpg = code("x = 1") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x = 1") shouldBe expected(("RET", AlwaysEdge)) - } - - "assigning a string literal value" in { - implicit val cpg: Cpg = code("x = 'some literal'") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x = 'some literal'") shouldBe expected(("RET", AlwaysEdge)) - } - - "addition of two numbers" in { - implicit val cpg: Cpg = code("x = 1 + 2") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x = 1 + 2") shouldBe expected(("RET", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("2") shouldBe expected(("1 + 2", AlwaysEdge)) - succOf("1") shouldBe expected(("2", AlwaysEdge)) - succOf("1 + 2") shouldBe expected(("x = 1 + 2", AlwaysEdge)) - } - - "addition of two string" in { - implicit val cpg: Cpg = code("x = 1 + 2") - succOf(":program") shouldBe expected(("x", AlwaysEdge)) - succOf("x = 1 + 2") shouldBe expected(("RET", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("2") shouldBe expected(("1 + 2", AlwaysEdge)) - succOf("1") shouldBe expected(("2", AlwaysEdge)) - succOf("1 + 2") shouldBe expected(("x = 1 + 2", AlwaysEdge)) - } - - "addition of multiple string" in { - implicit val cpg: Cpg = code(""" - |a = "Nice to meet you" - |b = ", " - |c = "do you like blueberries?" - |a+b+c - |""".stripMargin) - succOf(":program") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("\"Nice to meet you\"", AlwaysEdge)) - succOf("b") shouldBe expected(("\", \"", AlwaysEdge)) - succOf("c") shouldBe expected(("\"do you like blueberries?\"", AlwaysEdge)) - succOf("a+b+c") shouldBe expected(("RET", AlwaysEdge)) - succOf("a+b") shouldBe expected(("c", AlwaysEdge)) - succOf("\"Nice to meet you\"") shouldBe expected(("a = \"Nice to meet you\"", AlwaysEdge)) - succOf("\", \"") shouldBe expected(("b = \", \"", AlwaysEdge)) - succOf("\"do you like blueberries?\"") shouldBe expected(("c = \"do you like blueberries?\"", AlwaysEdge)) - } - - "addition of multiple string and assign to variable" in { - implicit val cpg: Cpg = code(""" - |a = "Nice to meet you" - |b = ", " - |c = "do you like blueberries?" - |x = a+b+c - |""".stripMargin) - succOf(":program") shouldBe expected(("a", AlwaysEdge)) - succOf("a") shouldBe expected(("\"Nice to meet you\"", AlwaysEdge)) - succOf("b") shouldBe expected(("\", \"", AlwaysEdge)) - succOf("c") shouldBe expected(("\"do you like blueberries?\"", AlwaysEdge)) - succOf("a+b+c") shouldBe expected(("x = a+b+c", AlwaysEdge)) - succOf("a+b") shouldBe expected(("c", AlwaysEdge)) - succOf("\"Nice to meet you\"") shouldBe expected(("a = \"Nice to meet you\"", AlwaysEdge)) - succOf("\", \"") shouldBe expected(("b = \", \"", AlwaysEdge)) - succOf("\"do you like blueberries?\"") shouldBe expected(("c = \"do you like blueberries?\"", AlwaysEdge)) - succOf("x") shouldBe expected(("a", AlwaysEdge)) - } - - "single hierarchy of if else statement" in { - implicit val cpg: Cpg = code(""" - |x = 1 - |if x > 2 - | puts "x is greater than 2" - |end - |""".stripMargin) - succOf(":program") shouldBe expected(("puts", AlwaysEdge)) - succOf("puts") shouldBe expected(("__builtin.puts", AlwaysEdge)) - succOf("__builtin.puts") shouldBe expected(("puts = __builtin.puts", AlwaysEdge)) - succOf("puts = __builtin.puts") shouldBe expected(("x", AlwaysEdge)) - succOf("1") shouldBe expected(("x = 1", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("2") shouldBe expected(("x > 2", AlwaysEdge)) - } - - "multiple hierarchy of if else statement" in { - implicit val cpg: Cpg = code(""" - |x = 1 - |if x > 2 - | puts "x is greater than 2" - |elsif x <= 2 and x!=0 - | puts "x is 1" - |else - | puts "I can't guess the number" - |end - |""".stripMargin) - succOf(":program") shouldBe expected(("puts", AlwaysEdge)) - succOf("puts") shouldBe expected(("__builtin.puts", AlwaysEdge)) - succOf("__builtin.puts") shouldBe expected(("puts = __builtin.puts", AlwaysEdge)) - succOf("puts = __builtin.puts") shouldBe expected(("x", AlwaysEdge)) - succOf("1") shouldBe expected(("x = 1", AlwaysEdge)) - succOf("x") shouldBe expected(("1", AlwaysEdge)) - succOf("2") shouldBe expected(("x > 2", AlwaysEdge)) - succOf("x <= 2 and x!=0") subsetOf expected(("\"x is 1\"", AlwaysEdge)) - succOf("x <= 2 and x!=0") subsetOf expected(("RET", AlwaysEdge)) - } - - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/AssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/AssignmentTests.scala deleted file mode 100644 index 7ad9ad37ad09..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/AssignmentTests.scala +++ /dev/null @@ -1,87 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language.* - -class AssignmentTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "CPG for code with identifiers and literals in simple assignments" should { - val cpg = code(""" - |# call instance methods - |a = 1 - |b = 2 - |a = 3 - |b = 4 - |c = a*b - |puts "Multiplication is : #{c}" - |""".stripMargin) - - "recognize all assignment nodes" in { - cpg.assignment.size shouldBe 6 // One assignment is for `puts = typeRef(__builtin.puts)` - } - - "have call nodes for .assignment as method name" in { - cpg.assignment.foreach { assignment => - assignment.name shouldBe Operators.assignment - assignment.methodFullName shouldBe Operators.assignment - } - } - - "should have identifiers as LHS for each assignment node" in { - cpg.call.nameExact(Operators.assignment).argument.where(_.argumentIndex(1)).foreach { idx => - idx.isIdentifier shouldBe true - } - } - - "recognise all identifier nodes" in { - cpg.identifier.name("a").size shouldBe 3 - cpg.identifier.name("b").size shouldBe 3 - cpg.identifier.name("c").size shouldBe 2 - } - - "recognise all literal nodes" in { - cpg.literal.code("1").size shouldBe 1 - cpg.literal.code("2").size shouldBe 1 - cpg.literal.code("3").size shouldBe 1 - cpg.literal.code("4").size shouldBe 1 - } - } - - "CPG for code with multiple assignments" should { - val cpg = code(""" - |a, b, c = [1, 2, 3] - |a, b, c = b, c, a - |str1, str2 = ["hello", "world"] - |p, q = [foo(), bar()] - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("a").size shouldBe 3 - cpg.identifier.name("b").size shouldBe 3 - cpg.identifier.name("c").size shouldBe 3 - cpg.identifier.name("str1").size shouldBe 1 - cpg.identifier.name("str2").size shouldBe 1 - cpg.identifier.name("p").size shouldBe 1 - cpg.identifier.name("q").size shouldBe 1 - } - - "recognise all literal nodes" in { - cpg.literal.code("1").size shouldBe 1 - cpg.literal.code("2").size shouldBe 1 - cpg.literal.code("3").size shouldBe 1 - cpg.literal.code("\"hello\"").size shouldBe 1 - cpg.literal.code("\"world\"").size shouldBe 1 - } - - "recognize call nodes in RHS" in { - cpg.call.codeExact("foo()").size shouldBe 1 - cpg.call.codeExact("bar()").size shouldBe 1 - } - - "recognise all assignment call nodes" in { - /* here we are also checking the synthetic assignment nodes for each element on both sides */ - cpg.call.name(Operators.assignment).size shouldBe 10 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/CallGraphTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/CallGraphTests.scala deleted file mode 100644 index ff46a8e689eb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/CallGraphTests.scala +++ /dev/null @@ -1,29 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.nodes.Method -import io.shiftleft.semanticcpg.language.* - -class CallGraphTests extends RubyCode2CpgFixture(withPostProcessing = true, useDeprecatedFrontend = true) { - - val cpg = code(""" - |def bar(content) - |puts content - |end - | - |def foo - |bar( 1 ) - |end - |""".stripMargin) - - "should identify call from `foo` to `bar`" in { - val List(callToBar) = cpg.call("bar").l - callToBar.name shouldBe "bar" - callToBar.methodFullName shouldBe "Test0.rb::program.bar" - callToBar.lineNumber shouldBe Some(7) - val List(bar: Method) = cpg.method("bar").internal.l - bar.fullName shouldBe callToBar.methodFullName - bar.caller.name.l shouldBe List("foo") - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/ControlStructureTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/ControlStructureTests.scala deleted file mode 100644 index 12074751fcc6..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/ControlStructureTests.scala +++ /dev/null @@ -1,396 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.ControlStructureTypes -import io.shiftleft.codepropertygraph.generated.nodes.{Block, ControlStructure} -import io.shiftleft.semanticcpg.language.* -class ControlStructureTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "CPG for code with doBlock iterating over a constant array" should { - val cpg = code(""" - |[1, 2, "three"].each do |n| - | puts n - |end - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("n").size shouldBe 1 - cpg.identifier.size shouldBe 2 // 1 identifier node is for `puts = typeDef(__builtin.puts)` - } - - "recognize all call nodes" in { - cpg.call.name("each").size shouldBe 1 - cpg.call.name("puts").size shouldBe 1 - } - } - - "CPG for code iterating over hash discarding key using _" should { - val cpg = code(""" - |x.each do |_, y| - | puts y - |end - |""".stripMargin) - - "have a valid each call and method" in { - cpg.call("each").size shouldBe 1 - cpg.call("each").argument.where(_.isIdentifier).code.l shouldBe List("x") - } - - "have valid identifiers" in { - cpg.identifier.name("x").size shouldBe 1 - cpg.identifier.name("y").size shouldBe 1 - } - } - - "CPG for code with doBlock iterating over a constant array and multiple params" should { - val cpg = code(""" - |[1, 2, "three"].each do |n, m| - | expect { - | someObject.someMethod(n) - | someObject.someMethod(m) - | }.to otherMethod(n).by(1) - |end - | - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("n").size shouldBe 2 - cpg.identifier.name("m").size shouldBe 1 - cpg.identifier.size shouldBe 5 - cpg.method.name("fakeName").dotAst.l - } - - "recognize all call nodes" in { - cpg.call.name("each").size shouldBe 1 - cpg.call.name("someMethod").size shouldBe 2 - cpg.call.name("expect").size shouldBe 1 - cpg.call.name("to").size shouldBe 1 - cpg.call.name("otherMethod").size shouldBe 1 - cpg.call.name("by").size shouldBe 1 - } - } - - "CPG for code with return having an if statement" should { - val cpg = code(""" - |def some_method - | return if some_var - |end - | - |""".stripMargin) - - /* - * This code used jumpExpression. This validated t - */ - "recognise identifier nodes in the jump statement" in { - cpg.identifier.name("some_var").size shouldBe 1 - } - - "identify the control structure code" in { - cpg.controlStructure.code("return if some_var").size shouldBe 1 - } - } - - "CPG for code with yield" should { - val cpg = code(""" - |def yield_with_args_method - | yield 2*3 - | yield 100 - | yield - |end - | - |yield_with_args_method {|i| puts "arg is #{i}"} - | - |""".stripMargin) - - "recognise all method nodes" in { - cpg.method.name("yield_with_args_method").size shouldBe 1 - cpg.method.name("yield_with_args_method_yield").size shouldBe 1 - } - } - - "CPG for code with if/else condition" should { - val cpg = code(""" - |x = 1 - |if x > 2 - | puts "x is greater than 2" - |elsif x <= 2 and x!=0 - | puts "x is 1" - |else - | puts "I can't guess the number" - |end - | - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("x").size shouldBe 4 - } - - "recognize all literal nodes" in { - cpg.literal.code("1").size shouldBe 1 - cpg.literal.code("2").size shouldBe 2 - cpg.literal.code("0").size shouldBe 1 - cpg.literal.code("\"x is 1\"").size shouldBe 1 - cpg.literal.code("\"I can't guess the number\"").size shouldBe 1 - } - } - - "CPG for code with conditional operator" should { - val cpg = code(""" - |y = ( x > 2 ) ? x : x + 1 - |""".stripMargin) - - "recognise all literal and identifier nodes" in { - cpg.identifier.name("x").size shouldBe 3 - cpg.identifier.name("y").size shouldBe 1 - cpg.literal.code("1").size shouldBe 1 - } - } - - "CPG for code with unless condition" should { - val cpg = code(""" - |x = 1 - |unless x > 2 - | puts "x is less than or equal to 2" - |else - | puts "x is greater than 2" - |end - | - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.identifier.name("x").size shouldBe 2 - cpg.literal.code("2").size shouldBe 1 - cpg.literal.code("\"x is less than or equal to 2\"").size shouldBe 1 - cpg.literal.code("\"x is greater than 2\"").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 2 - } - } - - "CPG for code with case statement and case argument" should { - val cpg = code(""" - |choice = "5" - |case choice - |when "1","2" - | puts "1 or 2" - |when "3","4" - | puts "3 or 4" - |when "5","6" - | puts "5 or 6" - |when "7","8" - | puts "7 or 8" - |else - | "No match" - |end - | - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.identifier.name("choice").size shouldBe 2 - cpg.literal.code("\"1\"").size shouldBe 1 - cpg.literal.code("\"2\"").size shouldBe 1 - cpg.literal.code("\"3\"").size shouldBe 1 - cpg.literal.code("\"4\"").size shouldBe 1 - cpg.literal.code("\"5\"").size shouldBe 2 - cpg.literal.code("\"6\"").size shouldBe 1 - cpg.literal.code("\"7\"").size shouldBe 1 - cpg.literal.code("\"8\"").size shouldBe 1 - cpg.literal.code("\"1 or 2\"").size shouldBe 1 - cpg.literal.code("\"3 or 4\"").size shouldBe 1 - cpg.literal.code("\"5 or 6\"").size shouldBe 1 - cpg.literal.code("\"7 or 8\"").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 4 - } - } - - "CPG for code with case statement and no case" should { - val cpg = code(""" - |str = "some_string" - | - |case - |when str.match('/\d/') - | puts 'String contains numbers' - |when str.match('/[a-zA-Z]/') - | puts 'String contains letters' - |else - | puts 'String does not contain numbers & letters' - |end - | - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.identifier.name("str").size shouldBe 3 - cpg.literal.code("\"some_string\"").size shouldBe 1 - cpg.literal.code("'String contains numbers'").size shouldBe 1 - cpg.literal.code("'String contains letters'").size shouldBe 1 - cpg.literal.code("'String does not contain numbers & letters'").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 3 - } - } - - "CPG for code with a while loop" should { - val cpg = code(""" - |x = 10 - |while x >= 1 - | x = x - 1 - | puts "In the loop" - |end - |""".stripMargin) - - "recognise all method nodes" in { - cpg.identifier - .name("x") - .size shouldBe 4 // FIXME this shows as 3 when the puts is the first loop statemnt. Find why - cpg.literal.code("\"In the loop\"").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 1 - } - } - - "CPG for code with a until loop" should { - val cpg = code(""" - |x = 10 - |until x == 0 - | puts "In the loop" - | x = x - 1 - |end - |""".stripMargin) - - "recognise all method nodes" in { - cpg.identifier.name("x").size shouldBe 4 - cpg.literal.code("\"In the loop\"").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 1 - } - - "recognise `until` as a `while` control structure" in { - val List(controlStructure) = cpg.whileBlock.l - controlStructure.lineNumber shouldBe Some(3) - - val List(condition) = controlStructure.astChildren.isCall.l - condition.code shouldBe "x == 0" - condition.lineNumber shouldBe Some(3) - - val List(body) = controlStructure.astChildren.isBlock.l - val List(puts, assignment) = body.astChildren.l - puts.code shouldBe "puts \"In the loop\"" - puts.lineNumber shouldBe Some(4) - assignment.lineNumber shouldBe Some(5) - assignment.assignment.size shouldBe 1 - } - - } - - "CPG for code with a for loop" should { - val cpg = code(""" - |for x in 1..10 do - | puts x - |end - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.identifier.name("x").size shouldBe 2 - cpg.literal.code("1").size shouldBe 1 - cpg.literal.code("10").size shouldBe 1 - - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 1 - } - } - - "CPG for code with modifier statements" should { - val cpg = code(""" - |for i in 1..10 - | next if i % 2 == 0 - | redo if i > 8 - | retry if i > 7 - | puts i if i == 9 - | i += 4 unless i > 5 - | - | value1 = 0 - | value1 += 1 while value1 < 100 - | - | value2 = 0 - | value2 += 1 until value2 >= 100 - |end - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("i").size shouldBe 8 - cpg.identifier.name("value1").size shouldBe 3 - cpg.identifier.name("value2").size shouldBe 3 - } - - "recognize all literal nodes" in { - cpg.literal.code("1").size shouldBe 3 - cpg.literal.code("2").size shouldBe 1 - cpg.literal.code("8").size shouldBe 1 - cpg.literal.code("7").size shouldBe 1 - cpg.literal.code("9").size shouldBe 1 - cpg.literal.code("5").size shouldBe 1 - cpg.literal.code("0").size shouldBe 3 - cpg.literal.code("1").size shouldBe 3 - cpg.literal.code("10").size shouldBe 1 - cpg.literal.code("100").size shouldBe 2 - } - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 1 - } - } - - "Next statements used as a conditional return for literals" should { - val cpg = code(""" - |grouped_currencies = Money::Currency.all.group_by do |currency| - | next "Major" if MAJOR_CURRENCY_CODES.include?(currency.iso_code) - | "Exotic" - |end - |""".stripMargin) - - "convert the CONTINUE to a RETURN" in { - cpg.controlStructure.controlStructureType(ControlStructureTypes.CONTINUE).size shouldBe 0 - cpg.controlStructure.controlStructureType(ControlStructureTypes.IF).size shouldBe 1 - } - - "return `Major` under the if-statement but return `Exotic` otherwise" in { - val List(ifStmt) = cpg.controlStructure.controlStructureType(ControlStructureTypes.IF).l: @unchecked - val List(ifReturn) = ifStmt.astChildren.isReturn.l: @unchecked - val List(majorLiteral) = ifReturn.astChildren.isLiteral.l: @unchecked - majorLiteral.code shouldBe "\"Major\"" - val List(blockReturn) = ifStmt.astSiblings.isReturn.l: @unchecked - val List(exoticLiteral) = blockReturn.astChildren.isLiteral.l: @unchecked - exoticLiteral.code shouldBe "\"Exotic\"" - } - } - - "Next statements used as a conditional continue for calls" should { - val cpg = code(""" - |for i in 1..10 - | next if i % 2 == 0 - | puts i - |end - |""".stripMargin) - - "retain the CONTINUE under the `next` with no return value" in { - val List(cont: ControlStructure) = - cpg.controlStructure.controlStructureType(ControlStructureTypes.CONTINUE).l: @unchecked - val ifStmt = cont.astParent.asInstanceOf[ControlStructure]: @unchecked - ifStmt.controlStructureType shouldBe ControlStructureTypes.IF - } - - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FieldAccessTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FieldAccessTests.scala deleted file mode 100644 index c348f6e07edb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FieldAccessTests.scala +++ /dev/null @@ -1,50 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class FieldAccessTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "Test class field access" should { - val cpg = code(""" - |class Person - | attr_reader :name, :age - | - | def initialize(name, age) - | @name = name - | @age = age - | end - |end - | - |p = Person.new("name", 66) - |p.age - |""".stripMargin) - - "be correct for field access" ignore { - cpg.call.name("age").size shouldBe 1 - val List(program) = cpg.method.nameExact(":program").l - val List(programBlock) = program.astChildren.isBlock.l - val List(call) = programBlock.astChildren.isCall.codeExact("p.age").l - call.astChildren.isFieldIdentifier.canonicalNameExact("age").size shouldBe 1 - call.astChildren.isIdentifier.nameExact("p").size shouldBe 1 - } - } - - "Test array access" should { - val cpg = code("result = persons[1].age") - - "be correct for filed access" ignore { - cpg.call.name(":program").l - val List(program) = cpg.method.nameExact(":program").l - val List(programBlock) = program.astChildren.isBlock.l - val List(call) = programBlock.astChildren.isCall.l - val List(rowsCall) = call.astChildren.isCall.l - rowsCall.astChildren.isFieldIdentifier.canonicalNameExact("age").size shouldBe 1 - - val List(rowsCallLeft) = rowsCall.astChildren.isCall.l - rowsCallLeft.astChildren.isLiteral.codeExact("1").size shouldBe 1 - rowsCallLeft.astChildren.isIdentifier.nameExact("persons").size shouldBe 1 - call.astChildren.isIdentifier.nameExact("result").size shouldBe 1 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FunctionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FunctionTests.scala deleted file mode 100644 index c4ee925b88b2..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/FunctionTests.scala +++ /dev/null @@ -1,214 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.Defines -import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language.* - -class FunctionTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "CPG for code with class methods, members and locals in methods" should { - val cpg = code(""" - |class Person - | attr_accessor :name, :age - | - | def initialize(name, age) - | @name = name - | @age = age - | end - | - | def greet - | puts "Hello, my name is #{@name} and I am #{@age} years old." - | end - | - | def have_birthday - | @age += 1 - | puts "Happy birthday! You are now #{@age} years old." - | end - |end - | - |p = Person.new - |p.greet - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("name").size shouldBe 1 - cpg.identifier.name("age").size shouldBe 1 - cpg.fieldAccess.fieldIdentifier.canonicalName("name").size shouldBe 2 - cpg.fieldAccess.fieldIdentifier.canonicalName("age").size shouldBe 4 - cpg.identifier.size shouldBe 13 // 4 identifier node is for `puts = typeDef(__builtin.puts)` 1 node for class Person = typeDef - } - - "recognize all call nodes" in { - cpg.call.name("greet").size shouldBe 1 - cpg.call.name("puts").size shouldBe 2 - } - - "recognize all method nodes" in { - // Initialize => - cpg.method.name("initialize").size shouldBe 0 - cpg.method.name(Defines.ConstructorMethodName).size shouldBe 1 - cpg.method.name("greet").size shouldBe 1 - cpg.method.name("have_birthday").size shouldBe 1 - } - } - - "CPG for code with square brackets as methods" should { - val cpg = code(""" - |class MyClass < MyBaseClass - | def initialize - | @my_hash = {} - | end - | - | def [](key) - | @my_hash[key.to_s] - | end - | - | def []=(key, value) - | @my_hash[key.to_s] = value - | end - |end - | - |my_object = MyClass.new - | - |""".stripMargin) - - "recognise all method nodes" in { - cpg.method.name("\\[]").size shouldBe 1 - cpg.method.name("\\[]=").size shouldBe 1 - cpg.method.name("initialize").size shouldBe 0 - cpg.method.name(Defines.ConstructorMethodName).size shouldBe 1 - } - - "recognize all call nodes" in { - cpg.call - .name(Operators.assignment) - .size shouldBe 4 // +1 identifier node for TypeRef's assignment - cpg.call.name("to_s").size shouldBe 2 - cpg.call.name(Defines.ConstructorMethodName).size shouldBe 1 - cpg.call.size shouldBe 12 // 1 identifier node for TypeRef's assignment - } - - "recognize all identifier nodes" in { - cpg.fieldAccess.fieldIdentifier.canonicalName("my_hash").size shouldBe 3 - cpg.identifier.name("key").size shouldBe 2 - cpg.identifier.name("value").size shouldBe 1 - cpg.identifier.name("my_object").size shouldBe 1 - /* - * FIXME - * def []=(key, value) gets parsed incorrectly with parser error "no viable alternative at input 'def []=(key, value)'" - * This needs a fix in the parser and update to this UT after the fix - * FIXME - * MyClass is identified as a variableIdentifier and so an identifier. This needs to be fixed - */ - } - } - - "CPG for code with modules" should { - val cpg = code(""" - |module Module1 - | def method1_1 - | end - | def method1_2 - | end - |end - | - |module Module2 - | def method2_1 - | end - | def method2_2 - | end - |end - |""".stripMargin) - - "recognise all method nodes defined in modules" in { - cpg.method.name("method1_1").l.size shouldBe 1 - cpg.method.name("method1_2").l.size shouldBe 1 - cpg.method.name("method2_1").l.size shouldBe 1 - cpg.method.name("method2_2").l.size shouldBe 1 - } - } - - "CPG for code with private/protected/public" should { - val cpg = code(""" - |class SomeClass - | private - | def method1 - | end - | - | protected - | def method2 - | end - | - | public - | def method3 - | end - |end - | - |""".stripMargin) - - "recognise all method nodes" in { - cpg.method - .name("method1") - .size shouldBe 1 - cpg.method - .name("method1") - .size shouldBe 1 - cpg.method - .name("method3") - .size shouldBe 1 - - } - } - - "CPG for code with multiple yields" should { - val cpg = code(""" - |def yield_with_arguments - | x = "something" - | y = "something_else" - | yield(x,y) - |end - | - |yield_with_arguments { |arg1, arg2| puts "Yield block 1 #{arg1} and #{arg2}" } - |yield_with_arguments { |arg1, arg2| puts "Yield block 2 #{arg2} and #{arg1}" } - | - |""".stripMargin) - - "recognise all method nodes" in { - cpg.method - .name("yield_with_arguments") - .size shouldBe 1 - cpg.method - .name("yield_with_arguments_yield") - .size shouldBe 2 - } - - "recognise all call nodes" in { - cpg.call - .name("yield_with_arguments_yield") - .size shouldBe 1 - - cpg.call - .name("puts") - .size shouldBe 2 - } - - "recognise all identifier nodes" in { - cpg.identifier - .name("arg1") - .size shouldBe 2 - - cpg.identifier - .name("arg2") - .size shouldBe 2 - - cpg.identifier - .name("x") - .size shouldBe 2 - - cpg.identifier - .name("y") - .size shouldBe 2 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/IdentifierTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/IdentifierTests.scala deleted file mode 100644 index e4b41ec4c9f6..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/IdentifierTests.scala +++ /dev/null @@ -1,122 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class IdentifierTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "CPG for code with a function call, arguments and function called from function " should { - val cpg = code(""" - | - |def extrareturn() - | ret = 6 - | return ret - |end - | - |def add_three_numbers(num1, num2, num3) - | sum = num1 + num2 + num3 + extrareturn() - | return sum - |end - | - |a = 1 - |b = 2 - |c = 3 - | - |sumOfThree = add_three_numbers( a, b, c ) - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("a").size shouldBe 2 - cpg.identifier.name("b").size shouldBe 2 - cpg.identifier.name("c").size shouldBe 2 - cpg.identifier.name("sumOfThree").size shouldBe 1 - cpg.identifier.name("num1").size shouldBe 1 - cpg.identifier.name("num2").size shouldBe 1 - cpg.identifier.name("num3").size shouldBe 1 - cpg.identifier.name("sum").size shouldBe 2 - cpg.identifier.name("ret").size shouldBe 2 - cpg.identifier.size shouldBe 16 // 2 identifier node is for methodRef's assigment - } - - "identify a single call node" in { - cpg.call.name("add_three_numbers").size shouldBe 1 - } - } - - "CPG for code with expressions of various types" should { - val cpg = code(""" - |a = 1 - |b = 2 if a > 1 - |b = !a - |c = ~a - |e = +a - |f = b**a - |g = a*b - |h = a+b - |i = a >> b - |j = a | b - |k = a & b - |l = a && b - |m = a || b - |n = a .. b - |o = a ... b - |p = ( a > b ) ? c : e - |q = not p - |r = p and q - |s = p or q - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("a").size shouldBe 16 - cpg.identifier.name("b").size shouldBe 13 // unaryExpression - cpg.identifier.name("c").size shouldBe 2 // unaryExpression - cpg.identifier.name("e").size shouldBe 2 // unaryExpression - cpg.identifier.name("f").size shouldBe 1 // powerExpression - cpg.identifier.name("g").size shouldBe 1 // multiplicative Expression - cpg.identifier.name("h").size shouldBe 1 // additive Expression - cpg.identifier.name("i").size shouldBe 1 // bitwise shift Expression - cpg.identifier.name("j").size shouldBe 1 // bitwise or Expression - cpg.identifier.name("k").size shouldBe 1 // bitwise and Expression - cpg.identifier.name("l").size shouldBe 1 // operator and Expression - cpg.identifier.name("m").size shouldBe 1 // operator or Expression - cpg.identifier.name("n").size shouldBe 1 // inclusive range Expression - cpg.identifier.name("o").size shouldBe 1 // exclusive range Expression - cpg.identifier.name("p").size shouldBe 4 // conditionalOperatorExpression - cpg.identifier.name("q").size shouldBe 3 // notExpressionOrCommand - cpg.identifier.name("r").size shouldBe 1 // orAndExpressionOrCommand and part - cpg.identifier.name("s").size shouldBe 1 // orAndExpressionOrCommand or part - cpg.identifier.size shouldBe 52 - } - } - - "CPG for code with identifier and method name conflicts" should { - val cpg = code(""" - |def create_conflict(id) - | puts id - |end - | - |create_conflict = 123 - | - |puts create_conflict - |puts create_conflict + 1 - |puts create_conflict(1) - | - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier - .name("create_conflict") - .size shouldBe 4 // 1 identifier node is for methodRef's assignment - } - - "recognise all call nodes" in { - cpg.call - .name("puts") - .size shouldBe 4 - - cpg.call - .name("create_conflict") - .size shouldBe 1 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/MiscTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/MiscTests.scala deleted file mode 100644 index c40c6ddd2a2c..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/MiscTests.scala +++ /dev/null @@ -1,332 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.joern.x2cpg.Defines -import io.shiftleft.semanticcpg.language.* - -class MiscTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) { - - "CPG for code with BEGIN and END blocks" should { - val cpg = code(""" - |#!/usr/bin/env ruby - | - |# This code block will be executed before the program begins - |BEGIN { - | beginvar = 5 - | beginbool = beginvar > 21 - |} - | - |# This is the main logic of the program - |puts "Hello, world!" - | - |# This code block will be executed after the program finishes - |END { - | endvar = 67 - | endbool = endvar > 23 - |} - |""".stripMargin) - - "recognise all identifier and call nodes" in { - cpg.identifier.name("beginvar").size shouldBe 2 - cpg.identifier.name("endvar").size shouldBe 2 - cpg.identifier.name("beginbool").size shouldBe 1 - cpg.identifier.name("endbool").size shouldBe 1 - cpg.call.name("puts").size shouldBe 1 - cpg.identifier.size shouldBe 7 // 1 identifier node is for `puts = typeDef(__builtin.puts)` - } - } - - "CPG for code with namespace resolution being used" should { - val cpg = code(""" - |Rails.application.configure do - | config.log_formatter = ::Logger::Formatter.new - |end - | - |""".stripMargin) - - "recognise all identifier and call nodes" in { - cpg.call.name("application").size shouldBe 1 - cpg.call.name("configure").size shouldBe 1 - cpg.call.name(Defines.ConstructorMethodName).size shouldBe 1 - cpg.call.name(".scopeResolution").size shouldBe 2 - cpg.identifier.name("Rails").size shouldBe 1 - cpg.identifier.name("config").size shouldBe 1 - cpg.identifier.name("Formatter").size shouldBe 1 - cpg.identifier.name("Logger").size shouldBe 1 - cpg.identifier.name("log_formatter").size shouldBe 1 - cpg.identifier.size shouldBe 5 - } - } - - "CPG for code with defined? keyword" should { - val cpg = code(""" - |radius = 2 - | - |area = 3.14 * radius * radius - | - |# Checking if the variable is defined or not - |# Using defined? keyword - |res1 = defined? radius - |res2 = defined? height - |res3 = defined? area - |res4 = defined? Math::PI - | - |# Displaying results - |puts "Result 1: #{res1}" - |puts "Result 2: #{res2}" - |puts "Result 3: #{res3}" - |puts "Result 4: #{res4}" - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier.name("radius").size shouldBe 4 - cpg.identifier.name("area").size shouldBe 2 - cpg.identifier.name("height").size shouldBe 1 - cpg.identifier.name("res1").size shouldBe 2 - cpg.identifier.name("res2").size shouldBe 2 - cpg.identifier.name("res3").size shouldBe 2 - cpg.identifier.name("res4").size shouldBe 2 - cpg.identifier.name("Math").size shouldBe 1 - cpg.identifier.name("PI").size shouldBe 1 - } - - "recognise all literal nodes" in { - cpg.literal.code("3.14").size shouldBe 1 - cpg.literal.code("2").size shouldBe 1 - cpg.literal.code("Result 1: ").size shouldBe 1 - cpg.literal.code("Result 2: ").size shouldBe 1 - cpg.literal.code("Result 3: ").size shouldBe 1 - cpg.literal.code("Result 4: ").size shouldBe 1 - } - } - - "CPG for code with association statements" should { - val cpg = code(""" - |class Employee < EmployeeBase - | has_many :teams, foreign_key: "team_id", class_name: "Team" - | has_many :worklocations, foreign_key: "location_id", class_name: "WorkLocation" - |end - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.literal - .code("\"team_id\"") - .size shouldBe 1 - cpg.literal - .code("\"location_id\"") - .size shouldBe 1 - } - - "recognise all activeRecordAssociation operator calls" in { - cpg.call - .name(".activeRecordAssociation") - .size shouldBe 4 - } - } - - "CPG for code with class having a scoped constant reference" should { - val cpg = code(""" - |class ModuleName::ClassName - | def some_method - | puts "Inside the method" - | end - |end - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.literal - .code("\"Inside the method\"") - .size shouldBe 1 - } - - "recognise all method nodes" in { - cpg.method - .name("some_method") - .size shouldBe 1 - } - } - - "CPG for code with alias" should { - val cpg = code(""" - |def some_method(arg) - |puts arg - |end - |alias :alias_name :some_method - |alias_name("some param") - |""".stripMargin) - - "recognise all call nodes" in { - cpg.call - .name("puts") - .size shouldBe 1 - cpg.call - .name("some_method") - .size shouldBe 1 - } - } - - "CPG for code with rescue clause" should { - val cpg = code(""" - |begin - | puts "In begin" - |rescue SomeException - | puts "SomeException occurred" - |rescue => exceptionVar - | puts "Caught exception in variable #{exceptionVar}" - |rescue - | puts "Catch-all block" - |end - | - |""".stripMargin) - - "recognise all literal nodes" in { - cpg.literal - .code("\"In begin\"") - .size shouldBe 1 - cpg.literal - .code("\"SomeException occurred\"") - .size shouldBe 1 - cpg.literal - .code("\"Catch-all block\"") - .size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call - .name("puts") - .size shouldBe 4 - } - - "recognise all identifier nodes" in { - cpg.identifier - .name("exceptionVar") - .size shouldBe 2 - } - } - - "CPG for code with addition of method returns" should { - val cpg = code(""" - |def num1; 1; end - |def num2; 2; end - |def num3; 3; end - |x = num1 + num2 + num3 - |puts x - |""".stripMargin) - - "recognise all identifier nodes" in { - cpg.identifier - .name("x") - .size shouldBe 2 - } - - "recognise all call nodes" in { - cpg.call - .name("num1") - .size shouldBe 1 - - cpg.call - .name("num2") - .size shouldBe 1 - - cpg.call - .name("num3") - .size shouldBe 1 - } - } - - "CPG for code with chained constants as argument" should { - val cpg = code(""" - |SomeFramework.someMethod SomeModule::SomeSubModule::submoduleMethod do - |puts "nothing important" - |end - |""".stripMargin) - - "recognise all method nodes" ignore { - cpg.method.name("submoduleMethod2").size shouldBe 1 - } - - "recognise all call nodes" in { - cpg.call - .name("submoduleMethod") - .size shouldBe 1 - - cpg.call - .name("puts") - .size shouldBe 1 - - cpg.call - .name(".scopeResolution") - .size shouldBe 1 - } - } - - "CPG for code with singleton object of some class" ignore { - val cpg = code(""" - |class << "some_class" - |end - |""".stripMargin) - - "recognise all typedecl nodes" in { - cpg.typeDecl.name("some_class").size shouldBe 1 - } - } - - // TODO obj.foo="arg" should be interpreted as obj.foo("arg"). code change pending - "CPG for code with method ending with =" should { - val cpg = code(""" - |class MyClass - | def foo=(value) - | puts value - | end - |end - | - |obj = MyClass.new - |obj.foo="arg" - |""".stripMargin) - - "recognise all call nodes" in { - cpg.call.name("puts").size shouldBe 1 - cpg.call.name(".fieldAccess").size shouldBe 1 - } - - "recognise all method nodes" in { - cpg.method.name("foo=").size shouldBe 1 - } - } - - // expectation is that this should not cause a crash - "CPG for code with method having a singleton class" should { - val cpg = code(""" - |module SomeModule - | def self.someMethod(arg) - | class << arg - | end - | end - |end - |""".stripMargin) - - "recognise all namespace nodes" in { - cpg.namespace.name("SomeModule").size shouldBe 1 - } - } - - "CPG for code with super without arguments" should { - val cpg = code(""" - |class Parent - | def foo(arg) - | end - |end - | - |class Child < Parent - | def foo(arg) - | super - | end - |end - |""".stripMargin) - - "recognise all call nodes" in { - cpg.call.name(".super").size shouldBe 1 - cpg.method.name("foo").size shouldBe 2 - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/RubyMethodFullNameTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/RubyMethodFullNameTests.scala deleted file mode 100644 index 4ce813878f75..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/deprecated/querying/RubyMethodFullNameTests.scala +++ /dev/null @@ -1,88 +0,0 @@ -package io.joern.rubysrc2cpg.deprecated.querying - -import io.joern.rubysrc2cpg.Config -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* -import org.scalatest.BeforeAndAfterAll - -class RubyMethodFullNameTests extends RubyCode2CpgFixture(useDeprecatedFrontend = true) with BeforeAndAfterAll { - - private val config = Config().withDownloadDependencies(true) - - "Code for method full name when method present in module" should { - val cpg = code( - """ - |require "dummy_logger" - | - |v = Main_module::Main_outer_class.new - |v.first_fun("value") - | - |g = Help.new - |g.help_print() - | - |""".stripMargin, - "main.rb" - ) - .moreCode( - """ - |source 'https://rubygems.org' - |gem 'dummy_logger' - | - |""".stripMargin, - "Gemfile" - ) - .withConfig(config) - "recognise call node" in { - cpg.call.name("first_fun").l.size shouldBe 1 - } - - "recognise methodFullName for call Node" ignore { - if (!scala.util.Properties.isWin) { - cpg.call.name("first_fun").head.methodFullName should equal( - "dummy_logger::program:Main_module:Main_outer_class:first_fun" - ) - cpg.call - .name("help_print") - .head - .methodFullName shouldBe "dummy_logger::program:Help:help_print" - } - } - } - - "Code for method full name when method present in other file" should { - val cpg = code( - """ - |require_relative "util/help.rb" - | - |v = Outer.new - |v.printValue() - | - |""".stripMargin, - "main.rb" - ) - .moreCode( - """ - |class Outer - | def printValue() - | puts "print" - | end - |end - |""".stripMargin, - Seq("util", "help.rb").mkString(java.io.File.separator) - ) - .withConfig(config) - - "recognise call node" in { - cpg.call.name("printValue").size shouldBe 1 - } - - "recognise method full name for call node" ignore { - if (!scala.util.Properties.isWin) { - cpg.call - .name("printValue") - .head - .methodFullName shouldBe "util/help.rb::program:Outer:printValue" - } - } - } -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..5538dd30b360 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala @@ -0,0 +1,83 @@ +package io.joern.rubysrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class RubySrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("rubysrc2cpgTestsHttpTest") + val file = dir / "main.rb" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse(""""Hello, World!"""") + file.writeText(s""" + |def main + | puts $indexStr + |end + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.rubysrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.rubysrc2cpg.Main.stop() + } + + "Using rubysrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("rubysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("""puts "Hello, World!"""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("rubysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"puts $index") + } + } + } + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala new file mode 100644 index 000000000000..bf1e9ad220f2 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala @@ -0,0 +1,56 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.parser +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ArrayParserTests extends RubyParserFixture with Matchers { + "array structures" in { + test("[1, 2 => 3]", "[1,2=> 3]") + test("[]") + test("%w[]") + test("%i[]") + test("%w[x y z]") + test("%w(x y z)") + test("%w{x y z}") + test("%w") + test("%w-x y z-") + test( + """%w( + | bob + | cod + | dod + |)""".stripMargin, + """%w(bob + |cod + |dod)""".stripMargin + ) + test("%W(x#{1})") + test( + """%W[ + | x#{0} + |]""".stripMargin, + "%W[x#{0}]" + ) + test("%W()") + test("%i") + test("%i{x\\ y}") + test("%i[x [y]]") + test("%i[x [y]]") + test( + """%i( + |x y + |z + |)""".stripMargin, + """%i(x + |y + |z)""".stripMargin + ) + test("%I{}") + test("%I(x#{0} x1)") + } + + "array params" in { + test("[1 => 2]", "[1=> 2]") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala new file mode 100644 index 000000000000..4b48440153df --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala @@ -0,0 +1,77 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class AssignmentParserTests extends RubyParserFixture with Matchers { + "Single assignment" in { + test("x=1", "x = 1") + test("hash[:sym] = s[:sym]") + test("a = 1, 2, 3, 4") + test("a = b.c 1") + test("a ||= b") + test("a &&= b") + test("a += 1") + test("a /= 1") + test("a[[1, 2]] = 3", "a[[1,2]] = 3") + test("a[] += b") + test("@a = 42") + test("a&.b = 1", "a&.b= 1") + test("c = a&.b") + test("a.b ||= c 1", "if a.b.nil? then a.b = c 1 end") + test("A.B ||= c 1", "if A.B.nil? then A.B = c 1 end") + test("A::b += 1") + test("A::b *= c d") + test("a[:b] ||= c 1, 2", "if a[:b].nil? then a[:b] = c 1, 2 end") + } + + "Multiple assignment" in { + test("p, q = [foo(), bar()]") + test("a, b::c = d") + test("a, b.C = d") + test("::A, ::B = 1, 2") + test("[1,2,3,4][from..to] = [\"a\",\"b\",\"c\"]") + test("a, = b.c 1") + test("(a, b) = c.d") + test("a ||= b.c 2") + test("a, b, c, * = f") + test("a, b, c, *s = f") + test("*s, x, y, z = f") + test("a = b 1 rescue 2") + test("*, a = b") + test("*, x, y, z = f") + } + + "Destructured Assignment" in { + test("a, b, c = 1, 2, 3") + test("a, b, c, d = 1, 2, 3") + test("a, b, *c = 1, 2, 3, 4") + test("a, *b, c = 1, 2, 3") + test("*a, b, c = 1, 2, 3, 4") + test("a, b, c = 1, 2, *list") + test("a, b, c = 1, *list") + test("a = *c, b, d") + } + + "Class Constant Assign" in { + test("A::b = 1") + test("a.B = 1") + } + + "Assignment with block" in { + test( + """h[k]=begin + |42 + |end + |""".stripMargin, + """h[k] = begin + |42 + |end""".stripMargin + ) + } + + "Assignment with rescue" in { + test("a = 1 rescue 2") + test("a = b(1) rescue 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginExpressionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginExpressionParserTests.scala new file mode 100644 index 000000000000..c995154c32e9 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginExpressionParserTests.scala @@ -0,0 +1,19 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BeginExpressionParserTests extends RubyParserFixture with Matchers { + "Begin expression" in { + test( + """begin + |1/0 + |rescue ZeroDivisionError => e + |end""".stripMargin, + """begin + |1 / 0 + |rescue ZeroDivisionError => e + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala new file mode 100644 index 000000000000..4cbc4bb4b626 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala @@ -0,0 +1,18 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BeginStatementParserTests extends RubyParserFixture with Matchers { + // TODO: Syntax Errors + "BEGIN statement" ignore { + test("BEGIN { 1 }") + test("BEGIN {}") + } + + // TODO: Syntax errors + "END statement" ignore { + test("END { 1 }") + test("END {}") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala new file mode 100644 index 000000000000..c1cf500c720e --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala @@ -0,0 +1,15 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BitwiseOperatorParserTests extends RubyParserFixture with Matchers { + "Bitwise operators" in { + test("1 & 3") + test("2 & 4 & 3") + test("1 | 9") + test("1 ^ 20") + test("1 >> 2") + test("1 << 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala new file mode 100644 index 000000000000..eb81eaa3596d --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala @@ -0,0 +1,33 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BlockParserTests extends RubyParserFixture with Matchers { + "Blocks" in { + test("""a = 42 + |a""".stripMargin) + + test( + """a + |b # comment + |c + |""".stripMargin, + """a + |b + |c""".stripMargin + ) + + test( + """a + |b # comment + |# another comment + |c + |""".stripMargin, + """a + |b + |c""".stripMargin + ) + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala new file mode 100644 index 000000000000..74fe984dc429 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala @@ -0,0 +1,33 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BooleanParserTests extends RubyParserFixture with Matchers { + "Boolean word operators" in { + test("1 or 2") + test("1 and 2") + test("not 1") + test("not 1 and 2") + test("1 and not 2") + test("1 or 2 or 3") + test("1 and 2 and 3") + } + + "Boolean sign operators" in { + test("1 || 2") + test("1 && 2") + test("!1") + test("!1 && 2") + test("1 && !2") + test("1 || 2 || 3") + test("1 && 2 && 3") + test("1 != 2") + test("1 == [:b, :c]", "1 == [:b,:c]") + test("! foo 1", "!foo 1") + } + + "Spaceship" in { + test("a <=> b") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala new file mode 100644 index 000000000000..d1bd016b77f6 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala @@ -0,0 +1,60 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class CaseConditionParserTests extends RubyParserFixture with Matchers { + "A case expression" in { + test( + """case something + |when 1 + | puts 2 + |end + |""".stripMargin, + """case something + |when 1 + |puts 2 + |end""".stripMargin + ) + + test( + """case something + |when 1 + |else + |end + |""".stripMargin, + """case something + |when 1 + |else + |end""".stripMargin + ) + + test( + """case something + |when 1 then + |end + |""".stripMargin, + """case something + |when 1 then + |end""".stripMargin + ) + + test( + """case x + | when 1 then 2 + | when 2 then 3 + | end + |""".stripMargin, + """case x + |when 1 then + |2 + |when 2 then + |3 + |end""".stripMargin + ) + + test("""case a + |when *b then + |end""".stripMargin) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala new file mode 100644 index 000000000000..572a5d59dae6 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala @@ -0,0 +1,69 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ClassDefinitionParserTests extends RubyParserFixture with Matchers { + "class definitions" in { + test( + "class << self ; end", + """class << self. + |end""".stripMargin + ) + test( + "class X 1 end", + """class X + |def + |1 + |end + |end""".stripMargin + ) + test( + """class << x + | def show; puts self; end + |end + |""".stripMargin, + """class << x + |def show + |puts self + |end + |end""".stripMargin + ) + test( + """class Foo + | def self.show + | end + |end + |""".stripMargin, + """class Foo + |def + | + |end + |def self.show + | end + |end""".stripMargin + ) + } + + "class definitions with comments" in { + test( + """#blah 1 + |#blah 2 + |class X + |#blah 3 + |def blah + |#blah4 + |end + |end + |""".stripMargin, + """class X + |def + | + |end + |def blah + |#blah4 + |end + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala new file mode 100644 index 000000000000..1e32c1a34116 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala @@ -0,0 +1,104 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ControlStructureParserTests extends RubyParserFixture with Matchers { + "while" in { + test( + """while x > 0 do + |end + |""".stripMargin, + """while x > 0 + |end""".stripMargin + ) + } + + "until" in { + test("""until not var.nil? + |'foo' + |end""".stripMargin) + } + + "if" in { + test( + """if __LINE__ > 1 then + |end + |""".stripMargin, + """if __LINE__ > 1 + |end""".stripMargin + ) + + test( + """if __LINE__ > 1 then + |else + |end + |""".stripMargin, + """if __LINE__ > 1 + |else + |end""".stripMargin + ) + + test( + """if __LINE__ > 1 then + |elsif __LINE__ > 0 then + |end + |""".stripMargin, + """if __LINE__ > 1 + |elsif __LINE__ > 0 + |end""".stripMargin + ) + + test( + "a = if (y > 3) then 123 elsif(y < 6) then 2003 elsif(y < 10) then 982 else 456 end", + """a = if y > 3 + |123 + |elsif y < 6 + |2003 + |elsif y < 10 + |982 + |else + |456 + |end""".stripMargin + ) + + test( + "if a..b then end", + """if a..b + |end""".stripMargin + ) + + test( + "if :x; end", + """if :x + |end""".stripMargin + ) + + test( + "if not var.nil? then 'foo' else 'bar'\nend", + """if not var.nil? + |'foo' + |else + |'bar' + |end""".stripMargin + ) + } + + "for loops" in { + test( + """for i in 1..10 do + |end + |""".stripMargin, + """for i in 1..10 + |end""".stripMargin + ) + + test( + """for i in 1..x do + |end + |""".stripMargin, + """for i in 1..x + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala new file mode 100644 index 000000000000..33f099b77640 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala @@ -0,0 +1,192 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class DoBlockParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test( + "break foo arg do |bar| end" + ) // syntax error - possibly false syntax error due to just having a code sample starting with break which our parser doesn't allow + test("yield foo arg do |bar| end") // syntax error + test("a.b do | ; c | end") // syntax error + } + + "Some block" in { + test( + "f { |a, (b, *, c)| }", + """f { + |{|a,(b, c, *)|} + |}""".stripMargin + ) + + test( + "a { |b, c=1, *d, e, &f| }", + """a { + |{|b,c=1,*d,&f,e|} + |}""".stripMargin + ) + + test( + "f { |a, (b, c), d| }", + """f { + |{|a,(b, c),d|} + |}""".stripMargin + ) + + test( + "a { |b, *c, d| }", + """a { + |{|b,*c,d|} + |}""".stripMargin + ) + + test( + "f { |(*a)| }", + """f { + |{|(*a)|} + |}""".stripMargin + ) + + // Order on params is out because we sort by line/col in RubyNode creator but we don't have access to that + // in the AstPrinter + test( + "f { |(*, a)| }", + """f { + |{|(a, *)|} + |}""".stripMargin + ) + + test( + "f { |(a, *b, c)| }", + """f { + |{|(a, c, *b)|} + |}""".stripMargin + ) + + test( + "def foo █end", + """def foo(&block) + |end""".stripMargin + ) + test("""arr.each { |item| }""", """arr.each {|item|}""") + test( + """hash.each do |key, value| + |end + |""".stripMargin, + """hash.each do |key,value| + |end""".stripMargin + ) + test( + s"x = proc { \"Hello #{myValue}\" }", + """x = proc { + |"Hello #{myValue}" + |}""".stripMargin + ) + test( + "Array.new(x) { |i| i += 1 }", + """Array.new(x) {|i| + |i += 1 + |}""".stripMargin + ) + test( + "test_name 'Foo' do;end", + """test_name 'Foo' do + |end""".stripMargin + ) + test( + "f{ |a, b| }", + """f { + |{|a,b|} + |}""".stripMargin + ) + test( + """f do |x, y| + |x + y + |end + |""".stripMargin, + """f do |x,y| + |x + y + |end""".stripMargin + ) + + test("""f(a) do |x,y| + |x + y + |end""".stripMargin) + + test( + """a.b do | | end""".stripMargin, + """a.b do + |end""".stripMargin + ) + + test( + "f { |(*, a)| }", + """f { + |{|(a, *)|} + |}""".stripMargin + ) + } + + "Block arguments" in { + test( + "f { |a, b| }", + """f { + |{|a,b|} + |}""".stripMargin + ) + + test( + "a { |b, c=1, d, &e| }", + """a { + |{|b,c=1,d,&e|} + |}""".stripMargin + ) + + test( + "a { |b, c=1, *d| }", + """a { + |{|b,c=1,*d|} + |}""".stripMargin + ) + + test( + "f { |a, b = 42| [a, b] }", + """f { + |{|a,b=42| + |[a,b] + |} + |}""".stripMargin + ) + + test( + "a { | b=1, c=2 | }", + """a { + |{|b=1,c=2|} + |}""".stripMargin + ) + + test( + "a { |**b | }", + """a { + |{|**b|} + |}""".stripMargin + ) + + test( + "bl { |kw: :val| kw }", + """bl { + |{|kw::val| + |kw + |} + |}""".stripMargin + ) + + test( + "a { |b, *c, d| }", + """a { + |{|b,*c,d|} + |}""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/EnsureClauseParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/EnsureClauseParserTests.scala new file mode 100644 index 000000000000..43a505befb72 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/EnsureClauseParserTests.scala @@ -0,0 +1,20 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class EnsureClauseParserTests extends RubyParserFixture with Matchers { + "ensure statement" in { + test( + """def refund + | ensure + | redirect_to paddle_charge_path(@charge) + |end + |""".stripMargin, + """def refund + |ensure + |redirect_to paddle_charge_path(@charge) + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala new file mode 100644 index 000000000000..b6e9ad677d0b --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala @@ -0,0 +1,30 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class FieldAccessParserTests extends RubyParserFixture with Matchers { + "Normal field access" in { + test("x.y") + test("self.x") + test( + """a + |.b + |.c""".stripMargin, + "a.b.c" + ) + test( + """a + |.b + |#.c + |.d + |""".stripMargin, + "a.b.d" + ) + test( + """a. + |b""".stripMargin, + "a.b" + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala new file mode 100644 index 000000000000..25e4a0e59608 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala @@ -0,0 +1,20 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class HashLiteralParserTests extends RubyParserFixture with Matchers { + "hash-literal" in { + test("{ }", "{}") + test("{**x}") + test("{**x, **y}", "{**x,**y}") + test("{**x, y => 1, **z}", "{**x,y=> 1,**z}") + test("{**group_by_type(some)}") + test( + """{ + |:s1 => 1, + |}""".stripMargin, + "{:s1=> 1}" + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala new file mode 100644 index 000000000000..77c377aa7a6a --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala @@ -0,0 +1,13 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class IndexAccessParserTests extends RubyParserFixture with Matchers { + "Index access" in { + test("a[1]") + test("a[1,2]") + test("a[2,]", "a[2]") + test("a[2=>3,]", "a[2=> 3]") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala new file mode 100644 index 000000000000..61c3ca4c7963 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala @@ -0,0 +1,68 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class InvocationWithParenthesisParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("foo.()") // syntax error + test("x(&)") // Syntax error + } + + "method invocation with parenthesis" in { + test("defined?(42)") + test("foo()") + test("foo([:c => 1, :d])", "foo([:c=> 1,:d])") + test( + """foo( + |) + |""".stripMargin, + "foo()" + ) + test("foo(1)") + test("foo(region: 1)") + test("foo(region:region)", "foo(region: region)") + test("foo(id: /.*/)") + test("foo(*x, y)", "foo(*x,y)") + test("foo(:region)") + test("foo(:region,)", "foo(:region)") + test("foo(if: true)") + test("foo(1, 2=>3)", "foo(1,2=> 3)") + test("foo(1, 2=>3,)", "foo(1,2=> 3)") + test("foo(1=> 2,)", "foo(1=> 2)") + test("foo(1, kw: 2, **3)", "foo(1,kw: 2,**3)") + test("foo(b, **1)", "foo(b,**1)") + test("""foo(b: if :c + |1 + |else + |2 + |end)""".stripMargin) + test("foo&.bar()") + test("foo&.bar(1, 2)", "foo&.bar(1,2)") + test( + """foo + |.bar + |""".stripMargin, + "foo.bar" + ) + test( + """foo. + |bar + |""".stripMargin, + "foo.bar" + ) + + test("f(1, kw:2, **3)", "f(1,kw: 2,**3)") + } + + "Method with comments" in { + test( + """# blah 1 + |# blah 2 + |def blah + |end""".stripMargin, + """def blah + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala new file mode 100644 index 000000000000..8773fd6963be --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala @@ -0,0 +1,33 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class InvocationWithoutParenthesesParserTests extends RubyParserFixture with Matchers { + "method invocation without parenthesis" in { + test("task.nil?") + test("foo?") + test("foo!") + test("foo a.b 1") + } + + "command with do block" in { + test( + """it 'should print 1' do + | puts 1 + |end + |""".stripMargin, + """it 'should print 1' do + |puts 1 + |end""".stripMargin + ) + + test("foo&.bar") + test("foo&.bar 1, 2") + } + + "method invocation without parenthesis with reserved keywords" in { + test("batch.retry!") + test("batch.retry") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala new file mode 100644 index 000000000000..e04eaaf9bcb9 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala @@ -0,0 +1,15 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class MathOperatorParserTests extends RubyParserFixture with Matchers { + "Math operators" in { + test("1 + 2") + test("1 - 2") + test("1 * 2") + test("1 / 2") + test("1 ** 2") + test("1 + 2 - 3 * 4 / 5 ** 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala new file mode 100644 index 000000000000..3e8ad4791574 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala @@ -0,0 +1,252 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class MethodDefinitionParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("""def a(...) + |b(...) + |end""".stripMargin) // Syntax error + } + + "single line method definition" in { + test( + "def f(a=1, *b, c) end", + """def f(a=1,*b,c) + |end""".stripMargin + ) + + test( + "def foo; end", + """def foo + |end""".stripMargin + ) + + test( + """def foo arg = false + |end""".stripMargin, + """def foo(arg=false) + |end""".stripMargin + ) + + test( + "def foo(x); end", + """def foo(x) + |end""".stripMargin + ) + + test( + "def f(a=nil, b) end", + """def f(a=nil,b) + |end""".stripMargin + ) + + test( + "def f(a, b = :c, d) end", + """def f(a,b=:c,d) + |end""".stripMargin + ) + + test( + "def foo(x=1); end", + """def foo(x=1) + |end""".stripMargin + ) + + test( + "def foo(x, &y); end", + """def foo(x,&y) + |end""".stripMargin + ) + + test( + "def foo(*arr); end", + """def foo(*arr) + |end""".stripMargin + ) + + test( + "def foo(**hash); end", + """def foo(**hash) + |end""".stripMargin + ) + + test( + "def foo(*arr, **hash); end", + """def foo(*arr,**hash) + |end""".stripMargin + ) + + test( + "def foo(x=1, y); end", + """def foo(x=1,y) + |end""".stripMargin + ) + + test( + "def foo(x: 1); end", + """def foo(x:1) + |end""".stripMargin + ) + + test( + "def foo(x:); end", + """def foo(x:) + |end""".stripMargin + ) + + test( + "def foo(name:, surname:); end", + """def foo(name:,surname:) + |end""".stripMargin + ) + + test( + "def f(*,a) end", + """def f(*,a) + |end""".stripMargin + ) + } + + "multi-line method definition" in { + test( + """def foo + | 1/0 + | rescue ZeroDivisionError => e + |end + |""".stripMargin, + """def foo + |1 / 0 + |rescue ZeroDivisionError => e + |end""".stripMargin + ) + + test("""def x(y) + |p(y) + |y *= 2 + |return y + |end""".stripMargin) + + test("""def test(**testing) + |test_splat(**testing) + |end""".stripMargin) + + test( + """def fun(kw: :val) + |kw + |end""".stripMargin, + """def fun(kw::val) + |kw + |end""".stripMargin + ) + + test( + """def x a:, b: + |end""".stripMargin, + """def x(a:,b:) + |end""".stripMargin + ) + + test( + """def exec(cmd) + |system(cmd) + |rescue + |nil + |end""".stripMargin, + """def exec(cmd) + |system(cmd) + |rescue + |nil + |end""".stripMargin + ) + } + + "endless method definition" in { + test("def foo = x") + test("def foo =\n x", "def foo = x") + test("def foo = \"something\"") + test("def id(x) = x") + test("def foo = bar 42") + } + + "method def with proc params" in { + test( + """def foo(&block) + | yield + |end + |""".stripMargin, + """def foo(&block) + |yield + |end""".stripMargin + ) + } + + "method def for mandatory parameters" in { + test( + "def foo(bar:) end", + """def foo(bar:) + |end""".stripMargin + ) + + test( + """ + |class SampleClass + | def sample_method (first_param:, second_param:) + | end + |end + |""".stripMargin, + """class SampleClass + |def + | + |end + |def sample_method (first_param:, second_param:) + | end + |end""".stripMargin + ) + } + + "method defs in classes" in { + test( + """ + |class SomeClass + | def initialize( + | name, age) + | end + |end + |""".stripMargin, + """class SomeClass + |def + | + |end + |def initialize( + | name, age) + | end + |end""".stripMargin + ) + + test( + """ + |class SomeClass + | def initialize( + | name: nil, age + | ) + | end + |end + |""".stripMargin, + """class SomeClass + |def + | + |end + |def initialize( + | name: nil, age + | ) + | end + |end""".stripMargin + ) + } + + "alias method" in { + test("alias :start :on") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ModuleParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ModuleParserTests.scala new file mode 100644 index 000000000000..018f7c4b8f36 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ModuleParserTests.scala @@ -0,0 +1,17 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ModuleParserTests extends RubyParserFixture with Matchers { + "Module Definition" in { + test( + "module Bar; end", + """module Bar + |def + | + |end + |end""".stripMargin + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala new file mode 100644 index 000000000000..39265281bf1f --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala @@ -0,0 +1,73 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ProcDefinitionParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("-> (a;b) {}") // Syntax error + } + + "one-line proc definition" in { + test("->a{}", "->(a) {}") + test("->(b, c=1, *d, e, &f){}", "->(b,c=1,*d,&f,e) {}") + + test("-> {}") + + test( + "-> do ; end", + """-> do + |end""".stripMargin + ) + + test( + "-> do 1 end", + """-> do + |1 + |end""".stripMargin + ) + + test("-> (x) {}", "->(x) {}") + + test( + "-> (x) do ; end", + """->(x) do + |end""".stripMargin + ) + + test("->(x = 1) {}", "->(x=1) {}") + + test( + "-> (foo: 1) do ; end", + """->(foo:1) do + |end""".stripMargin + ) + + test( + "->(x, y) {puts x; puts y}", + """->(x,y) { + |puts x + |puts y + |}""".stripMargin + ) + + test( + """a -> do 1 end do 2 end""", + """a -> do + |1 + |end do + |2 + |end""".stripMargin + ) + + test( + """a ->() { g do end }""", + """a -> { + |g do + |end + |}""".stripMargin + ) + + test("""-> (k:) { }""", """->(k:) {}""") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala new file mode 100644 index 000000000000..e946a0a3f1ef --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala @@ -0,0 +1,32 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class RangeParserTests extends RubyParserFixture with Matchers { + "Range Operator" in { + test("0..", "0..Float::INFINITY") + test("1..2") + test( + """0.. + |4 + |a.. + |b + |c + |""".stripMargin, + """0..4 + |a..b + |c""".stripMargin + ) + test( + """0.. + |; + |2.. + |""".stripMargin, + """0..Float::INFINITY + |2..Float::INFINITY""".stripMargin + ) + test("..2", "-Float::INFINITY..2") + test("...3", "-Float::INFINITY...3") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala new file mode 100644 index 000000000000..0bfe4eacc3ac --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala @@ -0,0 +1,43 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class RegexParserTests extends RubyParserFixture with Matchers { + "Regex" in { + test("//") + test("x = //") + test("puts //") + test("puts(//)") + test("puts(1, //)", "puts(1,//)") + test("puts /x#{1}y/") + test("puts(/x#{1}y/)") + + test( + """case foo + | when /^ch_/ + | bar + |end""".stripMargin, + """case foo + |when /^ch_/ + |bar + |end""".stripMargin + ) + + test("/(eu|us)/") + test("/x#{1}y/") + test("x = /(eu|us)/") + test("x = /x#{1}y/") + test("puts /(eu|us)/") + test("puts(/eu|us/)") + test("%r{a-z}") + test("%r") + test("%r[]") + test("%r{x#{0}|y}") + + test("""unless /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i.match?(value) + |end""".stripMargin) + + test("1 !~ 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RequireParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RequireParserTests.scala new file mode 100644 index 000000000000..e70cb58fe5b1 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RequireParserTests.scala @@ -0,0 +1,12 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class RequireParserTests extends RubyParserFixture with Matchers { + "require" in { + test("require 'sendgrid-ruby'") + test("require_all './dir'") + test("require_relative 'util/help/dir/'") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala new file mode 100644 index 000000000000..d6c0d5e5f272 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala @@ -0,0 +1,47 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class RescueClauseParserTests extends RubyParserFixture with Matchers { + "resuce statement" in { + test( + """begin + |1/0 + |rescue ZeroDivisionError => e + |end + |""".stripMargin, + """begin + |1 / 0 + |rescue ZeroDivisionError => e + |end""".stripMargin + ) + + test( + """def foo; + |1/0 + |rescue ZeroDivisionError => e + |end + |""".stripMargin, + """def foo + |1 / 0 + |rescue ZeroDivisionError => e + |end""".stripMargin + ) + + test( + """foo x do |y| + |y/0 + |rescue ZeroDivisionError => e + |end + |""".stripMargin, + """foo x do |y| + |y / 0 + |rescue ZeroDivisionError => e + |end""".stripMargin + ) + + test("a (b rescue c)", "a b rescue c") + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala new file mode 100644 index 000000000000..37b7765e0944 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala @@ -0,0 +1,16 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class ReturnParserTests extends RubyParserFixture with Matchers { + "Standalone return statement" in { + test("return") + test("return ::X.y()", "return self::X.y()") + test("return(0)", "return 0") + test("return y(z:1)", "return y(z: 1)") + test("return y(z=> 1)") + test("return 1, :z => 1", "return 1,:z=> 1") + test("return render 1,2,3") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala new file mode 100644 index 000000000000..6f7009c64f76 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala @@ -0,0 +1,83 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class StringParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("%{ { #{ \"#{1}\" } } }") // syntax error + } + + "single quoted literal" in { + test("''") + test("'x' 'y'", "'x''y'") + test( + """'x' \ + | 'y' + |""".stripMargin, + "'x''y'" + ) + test( + """'x' \ + | 'y' \ + | 'z'""".stripMargin, + "'x''y''z'" + ) + } + + "non expanded `%q` literal" in { + test("%q()") + test("%q[]") + test("%q{}") + test("%q<>") + test("%q##") + test("%q(x)") + test("%q[x]") + test("%q#x#") + test("%q(\\()") + test("%q[\\]]") + test("%q#\\##") + test("%q(foo)") + test("%q( () )") + test("%q( (\\)) )") + test("%q< <\\>> >") + } + + "expanded `%Q` literal" in { + test("%Q()") + test("%Q{text=#{1}}") + test("%Q[#{1}#{2}]") + test("%Q[before [#{nest}] after]") + } + + "expanded `%(` string literal" in { + test("%()") + test("%(text=#{1})") + test("%(#{1}#{2})") + test("puts %()") + } + + "double quoted string literal" in { + test("\"\"") + test("\"x\" \"y\"", "\"x\"\"y\"") + test( + """ + |"x" \ + | "y"""".stripMargin, + "\"x\"\"y\"" + ) + } + + "double quoted string interpolation" in { + test("\"#{1}#{2}\"") + test(""""#{10} \ + | is a number."""".stripMargin) + test(""""{a.b? ? ""+a+"" : ""}"""") + } + + "Expanded `%x` external command literal" in { + test("%x//") + test("%x{l#{'s'}}") + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/TernaryConditionalParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/TernaryConditionalParserTests.scala new file mode 100644 index 000000000000..230b444c67a9 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/TernaryConditionalParserTests.scala @@ -0,0 +1,17 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class TernaryConditionalParserTests extends RubyParserFixture with Matchers { + "ternary conditional expressions" in { + test("x ? y : z") + test( + """x ? + | y + |: z + |""".stripMargin, + "x ? y : z" + ) + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/UnlessConditionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/UnlessConditionParserTests.scala new file mode 100644 index 000000000000..e2e44f3f2bd5 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/UnlessConditionParserTests.scala @@ -0,0 +1,49 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class UnlessConditionParserTests extends RubyParserFixture with Matchers { + "Unless expression" in { + test( + """unless foo + | bar + |end + |""".stripMargin, + """unless foo + |bar + |end""".stripMargin + ) + + test( + """unless foo; bar + |end + |""".stripMargin, + """unless foo + |bar + |end""".stripMargin + ) + + test( + """unless foo then + | bar + |end + |""".stripMargin, + """unless foo + |bar + |end""".stripMargin + ) + + test( + """unless __LINE__ == 0 then + |else + |end + |""".stripMargin, + """unless __LINE__ == 0 + |else + |end""".stripMargin + ) + + test("return(value) unless item", "return value unless item") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala new file mode 100644 index 000000000000..87d621534433 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala @@ -0,0 +1,11 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class YieldParserTests extends RubyParserFixture with Matchers { + "Yield tests" in { + test("yield y(z: 1)") + test("yield 1, :z => 1", "yield 1,:z=> 1") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPassTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPassTests.scala new file mode 100644 index 000000000000..7ac475dfd55d --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ConfigFileCreationPassTests.scala @@ -0,0 +1,60 @@ +package io.joern.rubysrc2cpg.passes + +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.semanticcpg.language.* + +class ConfigFileCreationPassTests extends RubyCode2CpgFixture { + + "yaml files should be included" in { + val cpg = code( + """ + |foo: + | bar + |""".stripMargin, + "config.yaml" + ) + + val config = cpg.configFile.name("config.yaml").head + config.content should include("foo:") + } + + "yml files should be included" in { + val cpg = code( + """ + |foo: + | bar + |""".stripMargin, + "config.yml" + ) + + val config = cpg.configFile.name("config.yml").head + config.content should include("foo:") + } + + "xml files should be included" in { + val cpg = code( + """ + | + |

bar

+ | + |""".stripMargin, + "config.xml" + ) + + val config = cpg.configFile.name("config.xml").head + config.content should include("

bar

") + } + + "erb files should be included" in { + val cpg = code( + """ + |<%= 1 + 2 %> + |""".stripMargin, + "foo.erb" + ) + + val config = cpg.configFile.name("foo.erb").head + config.content should include("1 + 2") + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala index bb06adf658b3..fd780666c55a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.passes +import io.joern.rubysrc2cpg.passes.Defines.Main import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines as XDefines import io.joern.rubysrc2cpg.passes.GlobalTypes.kernelPrefix @@ -59,14 +60,14 @@ class RubyInternalTypeRecoveryTests extends RubyCode2CpgFixture(withPostProcessi "resolve 'print' and 'puts' StubbedRubyType calls" in { val List(printCall) = cpg.call("print").l - printCall.methodFullName shouldBe s"$kernelPrefix:print" + printCall.methodFullName shouldBe s"$kernelPrefix.print" val List(maxCall) = cpg.call("puts").l - maxCall.methodFullName shouldBe s"$kernelPrefix:puts" + maxCall.methodFullName shouldBe s"$kernelPrefix.puts" } "present the declared method name when a built-in with the same name is used in the same compilation unit" in { val List(absCall) = cpg.call("sleep").l - absCall.methodFullName shouldBe "main.rb:::program:sleep" + absCall.methodFullName shouldBe s"main.rb:$Main.sleep" } } @@ -141,7 +142,7 @@ class RubyInternalTypeRecoveryTests extends RubyCode2CpgFixture(withPostProcessi inside(constructAssignment.argument.l) { case (lhs: Identifier) :: rhs :: Nil => - lhs.typeFullName shouldBe "test2.rb:::program.Test2A" + lhs.typeFullName shouldBe s"test2.rb:$Main.Test2A" case xs => fail(s"Expected lhs and rhs, got [${xs.code.mkString(",")}]") } case xs => fail(s"Expected lhs and rhs, got [${xs.code.mkString(",")}]") @@ -167,8 +168,8 @@ class RubyInternalTypeRecoveryTests extends RubyCode2CpgFixture(withPostProcessi "propagate to identifier" ignore { inside(cpg.identifier.name("(a|b)").l) { case aIdent :: bIdent :: Nil => - aIdent.typeFullName shouldBe "Test0.rb:::program.A" - bIdent.typeFullName shouldBe "Test0.rb:::program.A" + aIdent.typeFullName shouldBe s"Test0.rb:$Main.A" + bIdent.typeFullName shouldBe s"Test0.rb:$Main.A" case xs => fail(s"Expected one identifier, got [${xs.name.mkString(",")}]") } } @@ -195,7 +196,7 @@ class RubyExternalTypeRecoveryTests // TODO: Revisit "be present in (Case 1)" ignore { cpg.identifier("sg").lineNumber(5).typeFullName.l shouldBe List("sendgrid-ruby.SendGrid.API") - cpg.call("client").methodFullName.headOption shouldBe Option("sendgrid-ruby.SendGrid.API:client") + cpg.call("client").methodFullName.headOption shouldBe Option("sendgrid-ruby.SendGrid.API.client") } "resolve correct imports via tag nodes" in { @@ -219,7 +220,7 @@ class RubyExternalTypeRecoveryTests "be present in (Case 2)" ignore { cpg.call("post").methodFullName.l shouldBe List( - "sendgrid-ruby::program.SendGrid.API.client.mail.anonymous.post" + s"sendgrid-ruby.$Main.SendGrid.API.client.mail.anonymous.post" ) } } @@ -280,7 +281,7 @@ class RubyExternalTypeRecoveryTests .isIdentifier .name("d") .headOption: @unchecked - d.typeFullName shouldBe "dbi::program.DBI.connect." + d.typeFullName shouldBe "dbi.$Main.DBI.connect." d.dynamicTypeHintFullName shouldBe Seq() } @@ -291,7 +292,7 @@ class RubyExternalTypeRecoveryTests .isCall .name("select_one") .l - d.methodFullName shouldBe "dbi::program.DBI.connect..select_one" + d.methodFullName shouldBe "dbi.$Main.DBI.connect..select_one" d.dynamicTypeHintFullName shouldBe Seq() d.callee(NoResolve).isExternal.headOption shouldBe Some(true) } @@ -300,10 +301,10 @@ class RubyExternalTypeRecoveryTests "resolve correct imports via tag nodes" ignore { val List(foo: ResolvedTypeDecl) = cpg.file(".*foo.rb").ast.isCall.where(_.referencedImports).tag._toEvaluatedImport.toList: @unchecked - foo.fullName shouldBe "dbi::program.DBI" + foo.fullName shouldBe s"dbi.$Main.DBI" val List(bar: ResolvedTypeDecl) = cpg.file(".*bar.rb").ast.isCall.where(_.referencedImports).tag._toEvaluatedImport.toList: @unchecked - bar.fullName shouldBe "foo.rb::program.FooModule" + bar.fullName shouldBe s"foo.rb.$Main.FooModule" } } @@ -341,7 +342,7 @@ class RubyExternalTypeRecoveryTests val Some(log) = cpg.identifier("log").headOption: @unchecked log.typeFullName shouldBe "logger.Logger" val List(errorCall) = cpg.call("error").l - errorCall.methodFullName shouldBe "logger.Logger:error" + errorCall.methodFullName shouldBe "logger.Logger.error" } } @@ -360,12 +361,12 @@ class RubyExternalTypeRecoveryTests "resolved the type of call" in { val Some(create) = cpg.call("create").headOption: @unchecked - create.methodFullName shouldBe "stripe.rb:::program.Stripe.Customer:create" + create.methodFullName shouldBe s"stripe.rb:$Main.Stripe.Customer.create" } "resolved the type of identifier" in { val Some(customer) = cpg.identifier("customer").headOption: @unchecked - customer.typeFullName shouldBe "stripe::program.Stripe.Customer.create." + customer.typeFullName shouldBe s"stripe.$Main.Stripe.Customer.create." } } @@ -384,11 +385,11 @@ class RubyExternalTypeRecoveryTests .moreCode(RubyExternalTypeRecoveryTests.LOGGER_GEMFILE, "Gemfile") "have a correct type for call `connect`" in { - cpg.call("error").methodFullName.l shouldBe List("logger.Logger:error") + cpg.call("error").methodFullName.l shouldBe List("logger.Logger.error") } "have a correct type for identifier `d`" in { - cpg.identifier("e").typeFullName.l shouldBe List("logger.Logger:error.") + cpg.identifier("e").typeFullName.l shouldBe List("logger.Logger.error.") } } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AccessModifierTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AccessModifierTests.scala new file mode 100644 index 000000000000..88a5a31e5429 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AccessModifierTests.scala @@ -0,0 +1,107 @@ +package io.joern.rubysrc2cpg.querying + +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.joern.rubysrc2cpg.passes.Defines +import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.nodes.Call +import io.shiftleft.semanticcpg.language.* + +class AccessModifierTests extends RubyCode2CpgFixture { + + "methods defined on the

level are private" in { + val cpg = code(""" + |def foo + |end + |""".stripMargin) + + cpg.method("foo").head.isPrivate.size shouldBe 1 + } + + "a method should be public by default, with the `initialize` default constructor private" in { + val cpg = code(""" + |class Foo + | def bar + | end + |end + |""".stripMargin) + + cpg.method("bar").head.isPublic.size shouldBe 1 + cpg.method(Defines.Initialize).head.isPrivate.size shouldBe 1 + cpg.method(Defines.TypeDeclBody).head.isPrivate.size shouldBe 1 + } + + "an access modifier should affect the visibility of subsequent method definitions" in { + val cpg = code(""" + |class Foo + | def bar + | end + | + | private + | + | def baz + | end + | + | def faz + | end + | + |end + | + |class Baz + | def test1 + | end + | + | protected + | + | def test2 + | end + |end + |""".stripMargin) + + cpg.method("bar").head.isPublic.size shouldBe 1 + + cpg.method("baz").head.isPrivate.size shouldBe 1 + cpg.method("faz").head.isPrivate.size shouldBe 1 + + cpg.method("test1").head.isPublic.size shouldBe 1 + cpg.method("test2").head.isProtected.size shouldBe 1 + } + + "nested types should 'remember' their access modifier mode according to scope" in { + val cpg = code(""" + |class Foo + | private + | + | class Bar + | + | public + | def baz + | end + | + | end + | + | def test + | end + |end + |""".stripMargin) + + cpg.method("baz").isPublic.size shouldBe 1 + cpg.method("test").isPrivate.size shouldBe 1 + } + + "an identifier sharing the same name as an access modifier in an unambiguous spot should not be confused" in { + val cpg = code(""" + | def message_params + | { + | private: @private + | } + | end + |""".stripMargin) + + val privateKey = cpg.literal(":private").head + val indexAccess = privateKey.astParent.asInstanceOf[Call] + indexAccess.name shouldBe Operators.indexAccess + indexAccess.methodFullName shouldBe Operators.indexAccess + indexAccess.code shouldBe "[:private]" + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ArrayTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ArrayTests.scala index 86d9008a8a13..80ec3bf775f6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ArrayTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ArrayTests.scala @@ -2,8 +2,12 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.GlobalTypes.{builtinPrefix, kernelPrefix} import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal} import io.shiftleft.semanticcpg.language.* +import io.joern.rubysrc2cpg.passes.Defines +import io.joern.rubysrc2cpg.passes.Defines.RubyOperators +import io.joern.x2cpg.Defines as XDefines class ArrayTests extends RubyCode2CpgFixture { @@ -98,6 +102,43 @@ class ArrayTests extends RubyCode2CpgFixture { y.typeFullName shouldBe s"$kernelPrefix.Symbol" } + "%W is represented an `arrayInitializer` operator call" in { + val cpg = code("""%W(x#{1 + 3} y#{23} z) + |""".stripMargin) + + val List(arrayCall) = cpg.call.name(Operators.arrayInitializer).l + + arrayCall.code shouldBe "%W(x#{1 + 3} y#{23} z)" + arrayCall.lineNumber shouldBe Some(1) + + val List(xFmt, yFmt) = arrayCall.argument.isCall.l + xFmt.name shouldBe Operators.formatString + xFmt.typeFullName shouldBe Defines.getBuiltInType(Defines.String) + + yFmt.name shouldBe Operators.formatString + yFmt.typeFullName shouldBe Defines.getBuiltInType(Defines.String) + + val List(xFmtStr) = xFmt.astChildren.isCall.l + xFmtStr.name shouldBe Operators.formattedValue + + val List(xFmtStrAdd) = xFmtStr.astChildren.isCall.l + xFmtStrAdd.name shouldBe Operators.addition + + val List(lhs, rhs) = xFmtStrAdd.argument.l + lhs.code shouldBe "1" + rhs.code shouldBe "3" + + val List(yFmtStr) = yFmt.astChildren.isCall.l + yFmtStr.name shouldBe Operators.formattedValue + + val List(yFmtStrLit: Literal) = yFmtStr.argument.l: @unchecked + yFmtStrLit.code shouldBe "23" + + val List(zLit) = arrayCall.argument.isLiteral.l + zLit.code shouldBe "z" + zLit.typeFullName shouldBe s"$kernelPrefix.String" + } + "an implicit array constructor (Array::[]) should be lowered to an array initializer" in { val cpg = code(""" |x = Array [1, 2, 3] @@ -121,4 +162,95 @@ class ArrayTests extends RubyCode2CpgFixture { } + "%I array" in { + val cpg = code("%I(test_#{1} test_2)") + + val List(arrayCall) = cpg.call.name(Operators.arrayInitializer).l + arrayCall.lineNumber shouldBe Some(1) + arrayCall.code shouldBe "%I(test_#{1} test_2)" + + val List(test1Fmt) = arrayCall.argument.isCall.l + test1Fmt.name shouldBe Operators.formatString + test1Fmt.typeFullName shouldBe Defines.getBuiltInType(Defines.Symbol) + test1Fmt.code shouldBe "test_#{1}" + + val List(test1FmtSymbol) = test1Fmt.astChildren.isCall.l + test1FmtSymbol.name shouldBe Operators.formattedValue + test1FmtSymbol.typeFullName shouldBe Defines.getBuiltInType(Defines.Symbol) + + val List(test1FmtFinal: Literal) = test1FmtSymbol.argument.l: @unchecked + test1FmtFinal.code shouldBe "1" + + val List(test2) = arrayCall.argument.isLiteral.l + test2.code shouldBe "test_2" + test2.typeFullName shouldBe Defines.getBuiltInType(Defines.Symbol) + } + + "shift-left operator interpreted as a call (append)" in { + val cpg = code("[1, 2, 3] << 4") + + inside(cpg.call("<<").headOption) { + case Some(append) => + append.name shouldBe "<<" + append.methodFullName shouldBe XDefines.DynamicCallUnknownFullName + append.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + append.argument(0).code shouldBe "[1, 2, 3]" + append.argument(1).code shouldBe "4" + case None => fail(s"Expected call `<<`") + } + } + + "Array bodies with mixed elements" in { + val cpg = code("[1, 2 => 1]") + + inside(cpg.call.name(Operators.arrayInitializer).argument.l) { + case (argLit: Literal) :: (argAssoc: Call) :: Nil => + argLit.code shouldBe "1" + + argAssoc.code shouldBe "2 => 1" + argAssoc.methodFullName shouldBe Defines.RubyOperators.association + case xs => fail(s"Expected two elements for array init, got ${xs.code.mkString(",")}") + } + } + + "Array with mixed elements" in { + val cpg = code(""" + |[ + | *::ApplicationSettingsHelper.visible_attributes, + | { default_branch_protection_defaults: [ + | :allow_force_push, + | :developer_can_initial_push, + | { + | allowed_to_merge: [:access_level], + | allowed_to_push: [:access_level] + | } + | ] }, + | :can_create_organization, + | *::ApplicationSettingsHelper.some_other_attributes, + |] + |""".stripMargin) + + cpg.call.name(Operators.arrayInitializer).headOption match { + case Some(arrayInit) => + inside(arrayInit.argument.l) { + case (splatArgOne: Call) :: (hashLiteralArg: Block) :: (symbolArg: Literal) :: (splatArgTwo: Call) :: Nil => + splatArgOne.methodFullName shouldBe RubyOperators.splat + splatArgOne.code shouldBe "*::ApplicationSettingsHelper.visible_attributes" + + symbolArg.code shouldBe ":can_create_organization" + symbolArg.typeFullName shouldBe Defines.getBuiltInType(Defines.Symbol) + + splatArgTwo.methodFullName shouldBe RubyOperators.splat + splatArgTwo.code shouldBe "*::ApplicationSettingsHelper.some_other_attributes" + + val List(hashInitAssignment: Call, _) = + hashLiteralArg.astChildren.isCall.name(Operators.assignment).l: @unchecked + val List(_: Identifier, hashInitCall: Call) = hashInitAssignment.argument.l: @unchecked + hashInitCall.methodFullName shouldBe RubyOperators.hashInitializer + + case xs => fail(s"Expected 4 arguments, got [${xs.code.mkString(",")}]") + } + case None => fail("Expected one call for head arrayInit") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AttributeAccessorTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AttributeAccessorTests.scala new file mode 100644 index 000000000000..873ca54aa735 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AttributeAccessorTests.scala @@ -0,0 +1,62 @@ +package io.joern.rubysrc2cpg.querying + +import io.joern.x2cpg.Defines +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.{Operators, DispatchTypes} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier} +import io.shiftleft.semanticcpg.language.* + +class AttributeAccessorTests extends RubyCode2CpgFixture { + + "`x.y=1` is approximated by a `x.y =` assignment with argument `1`" in { + val cpg = code("""x = Foo.new + |x.y = 1 + |""".stripMargin) + inside(cpg.assignment.where(_.source.isLiteral.codeExact("1")).l) { + case xyAssign :: Nil => + xyAssign.lineNumber shouldBe Some(2) + xyAssign.code shouldBe "x.y = 1" + + val fieldTarget = xyAssign.target.asInstanceOf[Call] + fieldTarget.code shouldBe "x.y" + fieldTarget.name shouldBe Operators.fieldAccess + fieldTarget.methodFullName shouldBe Operators.fieldAccess + fieldTarget.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + + inside(fieldTarget.argument.l) { + case (base: Identifier) :: (field: FieldIdentifier) :: Nil => + base.name shouldBe "x" + field.canonicalName shouldBe "@y" + field.code shouldBe "y" + case xs => fail("Expected field access to have two targets") + } + case xs => fail("Expected a single assignment to the literal `1`") + } + } + + "`x.y` is represented by a field access `x.y`" in { + val cpg = code("""x = Foo.new + |a = x.y + |b = x.z() + |""".stripMargin) + // Test the field access + inside(cpg.fieldAccess.lineNumber(2).codeExact("x.y").l) { + case xyCall :: Nil => + xyCall.lineNumber shouldBe Some(2) + xyCall.code shouldBe "x.y" + xyCall.methodFullName shouldBe Operators.fieldAccess + xyCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + case xs => fail("Expected a single field access for `x.y`") + } + // Test an explicit call with parenthesis + inside(cpg.call("z").lineNumber(3).l) { + case xzCall :: Nil => + xzCall.lineNumber shouldBe Some(3) + xzCall.code shouldBe "x.z()" + xzCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName + xzCall.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + case xs => fail("Expected a single call for `x.z()`") + } + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CallTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CallTests.scala index 7286d66ef50a..04344aac9eec 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CallTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CallTests.scala @@ -1,12 +1,12 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.{GlobalTypes, Defines as RubyDefines} -import io.joern.rubysrc2cpg.passes.Defines.RubyOperators +import io.joern.rubysrc2cpg.passes.Defines.{Main, RubyOperators} import io.joern.rubysrc2cpg.passes.GlobalTypes.kernelPrefix import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, NodeTypes, Operators} import io.shiftleft.semanticcpg.language.* class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { @@ -19,7 +19,7 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { val List(puts) = cpg.call.name("puts").l puts.lineNumber shouldBe Some(2) puts.code shouldBe "puts 'hello'" - puts.methodFullName shouldBe s"$kernelPrefix:puts" + puts.methodFullName shouldBe s"$kernelPrefix.puts" puts.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH val List(selfReceiver: Identifier, hello: Literal) = puts.argument.l: @unchecked @@ -53,7 +53,7 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { val List(puts) = cpg.call.name("puts").l puts.lineNumber shouldBe Some(2) puts.code shouldBe "Kernel.puts 'hello'" - puts.methodFullName shouldBe s"$kernelPrefix:puts" + puts.methodFullName shouldBe s"$kernelPrefix.puts" puts.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH val List(kernelRec: Call) = puts.receiver.l: @unchecked @@ -71,7 +71,7 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { val List(atan2) = cpg.call.name("atan2").l atan2.lineNumber shouldBe Some(3) atan2.code shouldBe "Math.atan2(1, 1)" - atan2.methodFullName shouldBe s"${GlobalTypes.builtinPrefix}.Math:atan2" + atan2.methodFullName shouldBe s"${GlobalTypes.builtinPrefix}.Math.atan2" atan2.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH val List(mathRec: Call) = atan2.receiver.l: @unchecked @@ -155,35 +155,38 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { "a simple object instantiation" should { val cpg = code("""class A + | def initialize(a, b) + | end |end | - |a = A.new + |a = A.new 1, 2 |""".stripMargin) - "create an assignment from `a` to an invocation block" in { - inside(cpg.method(":program").assignment.where(_.target.isIdentifier.name("a")).l) { + "create an assignment from `a` to an alloc lowering invocation block" in { + inside(cpg.method.isModule.assignment.and(_.target.isIdentifier.name("a"), _.source.isBlock).l) { case assignment :: Nil => assignment.code shouldBe "a = A.new" inside(assignment.argument.l) { case (a: Identifier) :: (_: Block) :: Nil => a.name shouldBe "a" - a.dynamicTypeHintFullName should contain("Test0.rb:::program.A") + a.dynamicTypeHintFullName should contain(s"Test0.rb:$Main.A") case xs => fail(s"Expected one identifier and one call argument, got [${xs.code.mkString(",")}]") } case xs => fail(s"Expected a single assignment, got [${xs.code.mkString(",")}]") } } - "create an assignment from a temp variable to the call" in { - inside(cpg.method(":program").assignment.where(_.target.isIdentifier.name("")).l) { + "create an assignment from a temp variable to the alloc call" in { + inside(cpg.method.isModule.assignment.where(_.target.isIdentifier.name("")).l) { case assignment :: Nil => inside(assignment.argument.l) { case (a: Identifier) :: (alloc: Call) :: Nil => - a.name shouldBe "" + a.name shouldBe "" alloc.name shouldBe Operators.alloc alloc.methodFullName shouldBe Operators.alloc alloc.code shouldBe "A.new" + alloc.argument.size shouldBe 0 case xs => fail(s"Expected one identifier and one call argument, got [${xs.code.mkString(",")}]") } case xs => fail(s"Expected a single assignment, got [${xs.code.mkString(",")}]") @@ -191,15 +194,71 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { } "create a call to the object's constructor, with the temp variable receiver" in { - inside(cpg.call.nameExact("new").l) { + inside(cpg.call.nameExact(RubyDefines.Initialize).l) { + case constructor :: Nil => + inside(constructor.argument.l) { + case (a: Identifier) :: (one: Literal) :: (two: Literal) :: Nil => + a.name shouldBe "" + a.typeFullName shouldBe s"Test0.rb:$Main.A" + a.argumentIndex shouldBe 0 + + one.code shouldBe "1" + two.code shouldBe "2" + case xs => fail(s"Expected one identifier and one call argument, got [${xs.code.mkString(",")}]") + } + + val recv = constructor.receiver.head.asInstanceOf[Call] + recv.methodFullName shouldBe Operators.fieldAccess + recv.name shouldBe Operators.fieldAccess + recv.code shouldBe s"A.${RubyDefines.Initialize}" + + recv.argument(1).label shouldBe NodeTypes.CALL + recv.argument(1).code shouldBe "self.A" + recv.argument(2).label shouldBe NodeTypes.FIELD_IDENTIFIER + recv.argument(2).code shouldBe RubyDefines.Initialize + case xs => fail(s"Expected a single alloc, got [${xs.code.mkString(",")}]") + } + } + } + + "an object instantiation from some expression" should { + val cpg = code("""def foo + | params[:type].constantize.new(path) + |end + |""".stripMargin) + + "create a call node on the receiver end of the constructor lowering" in { + inside(cpg.call.nameExact(RubyDefines.Initialize).l) { case constructor :: Nil => inside(constructor.argument.l) { - case (a: Identifier) :: Nil => + case (a: Identifier) :: (selfPath: Call) :: Nil => a.name shouldBe "" - a.typeFullName shouldBe "Test0.rb:::program.A" + a.typeFullName shouldBe Defines.Any a.argumentIndex shouldBe 0 + + selfPath.code shouldBe "self.path" case xs => fail(s"Expected one identifier and one call argument, got [${xs.code.mkString(",")}]") } + + val recv = constructor.receiver.head.asInstanceOf[Call] + recv.methodFullName shouldBe Operators.fieldAccess + recv.name shouldBe Operators.fieldAccess + recv.code shouldBe s"( = params[:type].constantize).${RubyDefines.Initialize}" + + recv.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe RubyDefines.Initialize + + inside(recv.argument(1).start.isCall.argument(2).isCall.argument.l) { + case (paramsAssign: Call) :: (constantize: FieldIdentifier) :: Nil => + paramsAssign.code shouldBe " = params[:type]" + inside(paramsAssign.argument.l) { case (tmpIdent: Identifier) :: (indexAccess: Call) :: Nil => + tmpIdent.name shouldBe "" + + indexAccess.name shouldBe Operators.indexAccess + indexAccess.code shouldBe "params[:type]" + } + + constantize.canonicalName shouldBe "constantize" + } case xs => fail(s"Expected a single alloc, got [${xs.code.mkString(",")}]") } } @@ -218,7 +277,26 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { inside(cpg.call("src").l) { case src :: Nil => src.name shouldBe "src" - src.methodFullName shouldBe "Test0.rb:::program:src" + src.methodFullName shouldBe s"Test0.rb:$Main.src" + case xs => fail(s"Expected exactly one `src` call, instead got [${xs.code.mkString(",")}]") + } + } + } + + "a parenthesis-less call as the base of a member access" should { + val cpg = code(""" + |def f(p) + | src.join(",") + |end + | + |def src = [1, 2] + |""".stripMargin) + + "correctly create a `src` call instead of identifier" in { + inside(cpg.call("src").l) { + case src :: Nil => + src.name shouldBe "src" + src.methodFullName shouldBe s"Test0.rb:$Main.src" case xs => fail(s"Expected exactly one `src` call, instead got [${xs.code.mkString(",")}]") } } @@ -260,8 +338,24 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { inArg.argumentName shouldBe Option("in") } + "Calls with named arguments using symbols and hash rocket syntax" in { + val cpg = code("render :foo => \"bar\"") + val List(_, barArg: Literal) = cpg.call.nameExact("render").argument.l: @unchecked + barArg.code shouldBe "\"bar\"" + barArg.argumentName shouldBe Option("foo") + } + + "named parameters in parenthesis-less call with a known keyword as the association key should shadow the keyword" in { + val cpg = code(""" + |foo retry: 3 + |""".stripMargin) + val List(_, retry) = cpg.call.nameExact("foo").argument.l: @unchecked + retry.code shouldBe "3" + retry.argumentName shouldBe Some("retry") + } + "a call with a quoted regex literal should have a literal receiver" in { - val cpg = code("%r{^/}.freeze") + val cpg = code("%r{^/}.freeze()") val regexLiteral = cpg.call.nameExact("freeze").receiver.fieldAccess.argument(1).head.asInstanceOf[Literal] regexLiteral.typeFullName shouldBe s"$kernelPrefix.Regexp" regexLiteral.code shouldBe "%r{^/}" @@ -271,14 +365,167 @@ class CallTests extends RubyCode2CpgFixture(withPostProcessing = true) { val cpg = code("::Augeas.open { |aug| aug.get('/augeas/version') }") val augeasReceiv = cpg.call.nameExact("open").receiver.head.asInstanceOf[Call] augeasReceiv.methodFullName shouldBe Operators.fieldAccess - augeasReceiv.code shouldBe "::Augeas.open" + augeasReceiv.code shouldBe "( = ::Augeas).open" val selfAugeas = augeasReceiv.argument(1).asInstanceOf[Call] - selfAugeas.argument(1).asInstanceOf[Identifier].name shouldBe RubyDefines.Self - selfAugeas.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "Augeas" + selfAugeas.argument(1).asInstanceOf[Identifier].name shouldBe "" + selfAugeas.argument(2).asInstanceOf[Call].code shouldBe "self::Augeas" augeasReceiv.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "open" } + "`nil` keyword as a member access should be a literal" in { + val cpg = code("nil.to_json") + val toJson = cpg.fieldAccess.codeExact("nil.to_json").head + val nilRec = toJson.argument(1).asInstanceOf[Literal] + + nilRec.code shouldBe "nil" + nilRec.lineNumber shouldBe Option(1) + } + + "Object initialize calls should be DynamicUnknown" in { + val cpg = code("""Date.new(2013, 19, 20)""") + + inside(cpg.call.name(RubyDefines.Initialize).l) { + case initCall :: Nil => + initCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName + case xs => fail(s"Expected one call to initialize, got ${xs.code.mkString}") + } + } + + "Member calls where the LHS is a call" should { + + "assign the first call to a temp variable to avoid a second invocation at arg 0" in { + val cpg = code("a().b()") + + val bCall = cpg.call("b").head + bCall.code shouldBe "( = a()).b()" + + // Check receiver + val bAccess = bCall.receiver.isCall.head + bAccess.name shouldBe Operators.fieldAccess + bAccess.methodFullName shouldBe Operators.fieldAccess + bAccess.code shouldBe "( = a()).b" + + bAccess.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "b" + + val aAssign = bAccess.argument(1).asInstanceOf[Call] + aAssign.name shouldBe Operators.assignment + aAssign.methodFullName shouldBe Operators.assignment + aAssign.code shouldBe " = a()" + + aAssign.argument(1).asInstanceOf[Identifier].name shouldBe "" + aAssign.argument(2).asInstanceOf[Call].name shouldBe "a" + + // Check (cached) base + val base = bCall.argument(0).asInstanceOf[Identifier] + base.name shouldBe "" + } + } + + "Call with Array Argument" in { + val cpg = code(""" + |def foo(a) + | puts a + |end + | + |foo([:b, :c => 1]) + |""".stripMargin) + + inside(cpg.call.name("foo").l) { + case fooCall :: Nil => + inside(fooCall.argument.l) { + case _ :: (arrayArg: Call) :: Nil => + arrayArg.code shouldBe "[:b, :c => 1]" + arrayArg.methodFullName shouldBe Operators.arrayInitializer + + inside(arrayArg.argument.l) { + case (elem1: Literal) :: (elem2: Call) :: Nil => + elem1.code shouldBe ":b" + elem2.code shouldBe ":c => 1" + + elem2.methodFullName shouldBe RubyDefines.RubyOperators.association + case xs => fail(s"Expected two args for elements, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected two args, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected one call for foo, got ${xs.code.mkString}") + } + } + + "Calls separated by `tmp` should render correct `code` properties" in { + val cpg = code(""" + |User.find_by(auth_token: cookies[:auth_token].to_s) + |""".stripMargin) + + cpg.call("find_by").code.head shouldBe "( = User).find_by(auth_token: cookies[:auth_token].to_s)" + cpg.call(Operators.indexAccess).code.head shouldBe "cookies[:auth_token]" + cpg.fieldAccess + .where(_.fieldIdentifier.canonicalNameExact("@to_s")) + .code + .head shouldBe "( = cookies[:auth_token]).to_s" + } + + "Calls with multiple splat args" in { + val cpg = code(""" + | doorkeeper_application&.includes_scope?( + | *::Gitlab::Auth::API_SCOPE, *::Gitlab::Auth::READ_API_SCOPE, + | *::Gitlab::Auth::ADMIN_SCOPES, *::Gitlab::Auth::REPOSITORY_SCOPES, + | *::Gitlab::Auth::REGISTRY_SCOPES + | ) + |""".stripMargin) + + inside(cpg.call.name("includes_scope\\?").argument.l) { + case _ :: (apiScopeSplat: Call) :: (readScopeSplat: Call) :: (adminScopeSplat: Call) :: (repoScopeSplat: Call) :: (registryScopeSplat: Call) :: Nil => + apiScopeSplat.code shouldBe "*::Gitlab::Auth::API_SCOPE" + apiScopeSplat.methodFullName shouldBe RubyOperators.splat + + readScopeSplat.code shouldBe "*::Gitlab::Auth::READ_API_SCOPE" + readScopeSplat.methodFullName shouldBe RubyOperators.splat + + adminScopeSplat.code shouldBe "*::Gitlab::Auth::ADMIN_SCOPES" + adminScopeSplat.methodFullName shouldBe RubyOperators.splat + + repoScopeSplat.code shouldBe "*::Gitlab::Auth::REPOSITORY_SCOPES" + repoScopeSplat.methodFullName shouldBe RubyOperators.splat + + registryScopeSplat.code shouldBe "*::Gitlab::Auth::REGISTRY_SCOPES" + registryScopeSplat.methodFullName shouldBe RubyOperators.splat + + case xs => fail(s"Expected 5 arguments for call, got [${xs.code.mkString(",")}]") + } + } + + "Multiple different arg types in a call" in { + val cpg = code(""" + |params.require(:issue).permit( + | *issue_params_attributes, + | sentry_issue_attributes: [:sentry_issue_identifier], + | *some_other_splat, + | "1234", + | 10 + | ) + | + |""".stripMargin) + + inside(cpg.call.name("permit").argument.l) { + case _ :: (issueSplat: Call) :: (sentryAssoc: Call) :: (someOtherSplat: Call) :: (strLiteral: Literal) :: (numericLiteral: Literal) :: Nil => + issueSplat.code shouldBe "*issue_params_attributes" + issueSplat.methodFullName shouldBe RubyOperators.splat + + sentryAssoc.code shouldBe "[:sentry_issue_identifier]" + sentryAssoc.methodFullName shouldBe Operators.arrayInitializer + + someOtherSplat.code shouldBe "*some_other_splat" + someOtherSplat.methodFullName shouldBe RubyOperators.splat + + strLiteral.code shouldBe "\"1234\"" + strLiteral.typeFullName shouldBe RubyDefines.getBuiltInType(RubyDefines.String) + + numericLiteral.code shouldBe "10" + numericLiteral.typeFullName shouldBe RubyDefines.getBuiltInType(RubyDefines.Integer) + case xs => fail(s"Expected 6 parameters for call, got [${xs.code.mkString(", ")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CaseTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CaseTests.scala index 2d9ae6491306..f46d99664045 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CaseTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/CaseTests.scala @@ -1,10 +1,9 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.Operators - +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class CaseTests extends RubyCode2CpgFixture { "`case x ... end` should be represented with if-else chain and multiple match expressions should be or-ed together" in { @@ -20,7 +19,7 @@ class CaseTests extends RubyCode2CpgFixture { |""".stripMargin val cpg = code(caseCode) - val block @ List(_) = cpg.method(":program").block.astChildren.isBlock.l + val block @ List(_) = cpg.method.isModule.block.astChildren.isBlock.l val List(assign) = block.astChildren.assignment.l; val List(lhs, rhs) = assign.argument.l @@ -68,7 +67,7 @@ class CaseTests extends RubyCode2CpgFixture { |end |""".stripMargin) - val block @ List(_) = cpg.method(":program").block.astChildren.isBlock.l + val block @ List(_) = cpg.method.isModule.block.astChildren.isBlock.l val headIf @ List(_) = block.astChildren.isControlStructure.l val ifStmts @ List(_, _, _, _) = headIf.repeat(_.astChildren.order(3).astChildren.isControlStructure)(_.emit).l; diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ClassTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ClassTests.scala index 9c03d330146d..2cacd6d43a71 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ClassTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ClassTests.scala @@ -3,9 +3,12 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.{GlobalTypes, Defines as RubyDefines} import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines -import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* +import io.joern.rubysrc2cpg.passes.Defines.{Initialize, Main, TypeDeclBody} +import io.joern.rubysrc2cpg.passes.GlobalTypes +import io.shiftleft.codepropertygraph.generated.NodeTypes class ClassTests extends RubyCode2CpgFixture { @@ -17,7 +20,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(classC) = cpg.typeDecl.name("C").l classC.inheritsFromTypeFullName shouldBe List() - classC.fullName shouldBe "Test0.rb:::program.C" + classC.fullName shouldBe s"Test0.rb:$Main.C" classC.lineNumber shouldBe Some(2) classC.baseType.l shouldBe List() classC.member.name.l shouldBe List(RubyDefines.TypeDeclBody, RubyDefines.Initialize) @@ -25,7 +28,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(singletonC) = cpg.typeDecl.nameExact("C").l singletonC.inheritsFromTypeFullName shouldBe List() - singletonC.fullName shouldBe "Test0.rb:::program.C" + singletonC.fullName shouldBe s"Test0.rb:$Main.C" singletonC.lineNumber shouldBe Some(2) singletonC.baseType.l shouldBe List() singletonC.member.name.l shouldBe List() @@ -42,7 +45,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(classC) = cpg.typeDecl.name("C").l classC.inheritsFromTypeFullName shouldBe List("D") - classC.fullName shouldBe "Test0.rb:::program.C" + classC.fullName shouldBe s"Test0.rb:$Main.C" classC.lineNumber shouldBe Some(2) classC.member.name.l shouldBe List(RubyDefines.TypeDeclBody, RubyDefines.Initialize) classC.method.name.l shouldBe List(RubyDefines.TypeDeclBody, RubyDefines.Initialize) @@ -53,7 +56,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(singletonC) = cpg.typeDecl.nameExact("C").l singletonC.inheritsFromTypeFullName shouldBe List("D") - singletonC.fullName shouldBe "Test0.rb:::program.C" + singletonC.fullName shouldBe s"Test0.rb:$Main.C" singletonC.lineNumber shouldBe Some(2) singletonC.member.name.l shouldBe List() singletonC.method.name.l shouldBe List() @@ -74,6 +77,9 @@ class ClassTests extends RubyCode2CpgFixture { val List(singletonC) = cpg.typeDecl.name("C").l singletonC.member.nameExact("@a").isEmpty shouldBe true + + val List(aGetterMember) = classC.member.nameExact("a").l + aGetterMember.dynamicTypeHintFullName should contain("Test0.rb:
.C.a") } "`attr_reader :'abc'` is represented by a `@abc` MEMBER node" in { @@ -88,6 +94,9 @@ class ClassTests extends RubyCode2CpgFixture { abcMember.code shouldBe "attr_reader :'abc'" abcMember.lineNumber shouldBe Some(3) + + val List(aMember) = classC.member.nameExact("abc").l + aMember.dynamicTypeHintFullName should contain("Test0.rb:
.C.abc") } "`attr_reader :'abc' creates an `abc` METHOD node" in { @@ -102,14 +111,17 @@ class ClassTests extends RubyCode2CpgFixture { methodAbc.code shouldBe "def abc (...)" methodAbc.lineNumber shouldBe Some(3) - methodAbc.parameter.isEmpty shouldBe true - methodAbc.fullName shouldBe "Test0.rb:::program.C:abc" + methodAbc.parameter.indexGt(0).isEmpty shouldBe true + methodAbc.fullName shouldBe s"Test0.rb:$Main.C.abc" - // TODO: Make sure that @abc in this return is the actual field val List(ret: Return) = methodAbc.methodReturn.cfgIn.l: @unchecked - val List(abcField: Identifier) = ret.astChildren.l: @unchecked - ret.code shouldBe "return @abc" - abcField.name shouldBe "@abc" + val List(abcFieldAccess: Call) = ret.astChildren.l: @unchecked + ret.code shouldBe "@abc" + abcFieldAccess.name shouldBe Operators.fieldAccess + abcFieldAccess.code shouldBe "self.@abc" + + val List(aMember) = classC.member.nameExact("abc").l + aMember.dynamicTypeHintFullName should contain("Test0.rb:
.C.abc") } "`attr_reader :a, :b` is represented by `@a`, `@b` MEMBER nodes" in { @@ -152,15 +164,19 @@ class ClassTests extends RubyCode2CpgFixture { methodA.code shouldBe "def a= (...)" methodA.lineNumber shouldBe Some(3) - methodA.fullName shouldBe "Test0.rb:::program.C:a=" + methodA.fullName shouldBe s"Test0.rb:$Main.C.a=" // TODO: there's probably a better way for testing this - val List(param) = methodA.parameter.l - val List(assignment) = methodA.assignment.l - val List(lhs: Identifier, rhs: Identifier) = assignment.argument.l: @unchecked + val List(_, param) = methodA.parameter.l + val List(assignment) = methodA.assignment.l + val List(lhs: Call, rhs: Identifier) = assignment.argument.l: @unchecked param.name shouldBe rhs.name - lhs.name shouldBe "@a" + lhs.name shouldBe Operators.fieldAccess + lhs.code shouldBe "self.@a" + + val List(aMember) = classC.member.nameExact("a=").l + aMember.dynamicTypeHintFullName should contain("Test0.rb:
.C.a=") } "`attr_accessor :a` is represented by a `@a` MEMBER node" in { @@ -189,7 +205,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(classC) = cpg.typeDecl.name("C").l val List(methodF) = classC.method.name("f").l - methodF.fullName shouldBe "Test0.rb:::program.C:f" + methodF.fullName shouldBe s"Test0.rb:$Main.C.f" val List(memberF) = classC.member.nameExact("f").l memberF.dynamicTypeHintFullName.toSet should contain(methodF.fullName) @@ -261,7 +277,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(classC) = cpg.typeDecl.name("C").l val List(methodInit) = classC.method.name(RubyDefines.Initialize).l - methodInit.fullName shouldBe s"Test0.rb:::program.C:${RubyDefines.Initialize}" + methodInit.fullName shouldBe s"Test0.rb:$Main.C.${RubyDefines.Initialize}" methodInit.isConstructor.isEmpty shouldBe false } @@ -274,7 +290,7 @@ class ClassTests extends RubyCode2CpgFixture { val List(classC) = cpg.typeDecl.name("C").l val List(methodInit) = classC.method.name(RubyDefines.Initialize).l - methodInit.fullName shouldBe s"Test0.rb:::program.C:${RubyDefines.Initialize}" + methodInit.fullName shouldBe s"Test0.rb:$Main.C.${RubyDefines.Initialize}" } "only `def initialize() ... end` directly under class has the constructor modifier" in { @@ -312,8 +328,8 @@ class ClassTests extends RubyCode2CpgFixture { | |""".stripMargin) - cpg.member("MConst").typeDecl.fullName.head shouldBe "Test0.rb:::program.MMM" - cpg.member("NConst").typeDecl.fullName.head shouldBe "Test0.rb:::program.MMM.Nested" + cpg.member("MConst").typeDecl.fullName.head shouldBe s"Test0.rb:$Main.MMM" + cpg.member("NConst").typeDecl.fullName.head shouldBe s"Test0.rb:$Main.MMM.Nested" } "a basic anonymous class" should { @@ -329,14 +345,14 @@ class ClassTests extends RubyCode2CpgFixture { inside(cpg.typeDecl.nameExact("").l) { case anonClass :: Nil => anonClass.name shouldBe "" - anonClass.fullName shouldBe "Test0.rb:::program." + anonClass.fullName shouldBe s"Test0.rb:$Main." inside(anonClass.method.l) { case hello :: defaultConstructor :: Nil => defaultConstructor.name shouldBe RubyDefines.Initialize - defaultConstructor.fullName shouldBe s"Test0.rb:::program.:${RubyDefines.Initialize}" + defaultConstructor.fullName shouldBe s"Test0.rb:$Main..${RubyDefines.Initialize}" hello.name shouldBe "hello" - hello.fullName shouldBe "Test0.rb:::program.:hello" + hello.fullName shouldBe s"Test0.rb:$Main..hello" case xs => fail(s"Expected a single method, but got [${xs.map(x => x.label -> x.code).mkString(",")}]") } case xs => fail(s"Expected a single anonymous class, but got [${xs.map(x => x.label -> x.code).mkString(",")}]") @@ -344,18 +360,20 @@ class ClassTests extends RubyCode2CpgFixture { } "generate an assignment to the variable `a` with the source being a constructor invocation of the class" in { - inside(cpg.method(":program").assignment.l) { - case aAssignment :: Nil => + inside(cpg.method.isModule.assignment.l) { + case aAssignment :: tmpAssign :: Nil => aAssignment.target.code shouldBe "a" - aAssignment.source.code shouldBe "Class.new (...)" + aAssignment.source.code shouldBe "( = Class.new (...)).new" + + tmpAssign.target.code shouldBe "" + tmpAssign.source.code shouldBe "self.Class.new (...)" case xs => fail(s"Expected a single assignment, but got [${xs.map(x => x.label -> x.code).mkString(",")}]") } } } - // TODO: This should be remodelled as a property access `animal.bark = METHOD_REF` - "a basic singleton class" ignore { + "a basic singleton class extending an object instance" should { val cpg = code("""class Animal; end |animal = Animal.new | @@ -363,35 +381,49 @@ class ClassTests extends RubyCode2CpgFixture { | def bark | 'Woof' | end + | + | def legs + | 4 + | end |end | |animal.bark # => 'Woof' |""".stripMargin) - "generate a type decl with the associated members" in { - inside(cpg.typeDecl.nameExact("").l) { - case anonClass :: Nil => - anonClass.name shouldBe "" - anonClass.fullName shouldBe "Test0.rb:::program." - // TODO: Attempt to resolve the below with the `scope` class once we're handling constructors - anonClass.inheritsFromTypeFullName shouldBe Seq("animal") - inside(anonClass.method.l) { - case defaultConstructor :: bark :: Nil => - defaultConstructor.name shouldBe Defines.ConstructorMethodName - defaultConstructor.fullName shouldBe s"Test0.rb:::program.:${Defines.ConstructorMethodName}" + "Create assignments to method refs for methods on singleton object" in { + inside(cpg.method.isModule.block.assignment.l) { + case _ :: _ :: _ :: barkAssignment :: legsAssignment :: Nil => + inside(barkAssignment.argument.l) { + case (lhs: Call) :: (rhs: TypeRef) :: Nil => + val List(identifier, fieldIdentifier) = lhs.argument.l: @unchecked + identifier.code shouldBe "animal" + fieldIdentifier.code shouldBe "bark" + + rhs.typeFullName shouldBe s"Test0.rb:$Main.class< fail(s"Expected two arguments for assignment, got [${xs.code.mkString(",")}]") + } - bark.name shouldBe "bark" - bark.fullName shouldBe "Test0.rb:::program.:bark" - case xs => fail(s"Expected a single method, but got [${xs.map(x => x.label -> x.code).mkString(",")}]") + inside(legsAssignment.argument.l) { + case (lhs: Call) :: (rhs: TypeRef) :: Nil => + val List(identifier, fieldIdentifier) = lhs.argument.l: @unchecked + identifier.code shouldBe "animal" + fieldIdentifier.code shouldBe "legs" + + rhs.typeFullName shouldBe s"Test0.rb:$Main.class< fail(s"Expected two arguments for assignment, got [${xs.code.mkString(",")}]") } - case xs => fail(s"Expected a single anonymous class, but got [${xs.map(x => x.label -> x.code).mkString(",")}]") + case xs => fail(s"Expected five assignments, got [${xs.code.mkString(",")}]") } } - "register that `animal` may possibly be an instantiation of the singleton type" in { - cpg.local("animal").possibleTypes.l should contain("Test0.rb:::program.") + "Create TYPE_DECL nodes for two singleton methods" in { + inside(cpg.typeDecl.name("(bark|legs)").l) { + case barkTypeDecl :: legsTypeDecl :: Nil => + barkTypeDecl.fullName shouldBe s"Test0.rb:$Main.class< fail(s"Expected two type_decls, got [${xs.code.mkString(",")}]") + } } - } "if: as function param" should { @@ -466,12 +498,14 @@ class ClassTests extends RubyCode2CpgFixture { } } - "fully qualified base types" should { + "base types names extending a class in the definition" should { val cpg = code("""require "rails/all" | |module Bar - | class Baz + | module Baz + | class Boz + | end | end |end | @@ -479,12 +513,12 @@ class ClassTests extends RubyCode2CpgFixture { | class Application < Rails::Application | end | - | class Foo < Bar::Baz + | class Foo < Bar::Baz::Boz | end |end |""".stripMargin) - "not confuse the internal `Application` with `Rails::Application` and leave the type unresolved" in { + "handle a qualified base type from an external type correctly" in { inside(cpg.typeDecl("Application").headOption) { case Some(app) => app.inheritsFromTypeFullName.head shouldBe "Rails.Application" @@ -492,10 +526,10 @@ class ClassTests extends RubyCode2CpgFixture { } } - "resolve the internal type being referenced" in { + "handle a deeply qualified internal base type correctly" in { inside(cpg.typeDecl("Foo").headOption) { case Some(app) => - app.inheritsFromTypeFullName.head shouldBe "Test0.rb:::program.Bar.Baz" + app.inheritsFromTypeFullName.head shouldBe "Bar.Baz.Boz" case None => fail("Expected a type decl for 'Foo', instead got nothing") } } @@ -576,6 +610,18 @@ class ClassTests extends RubyCode2CpgFixture { case xs => fail(s"Expected TypeDecl for Foo, instead got ${xs.name.mkString(", ")}") } } + + "call the body method" in { + inside(cpg.call.nameExact(RubyDefines.TypeDeclBody).headOption) { + case Some(bodyCall) => + bodyCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + bodyCall.methodFullName shouldBe s"Test0.rb:$Main.Foo.${RubyDefines.TypeDeclBody}" + bodyCall.code shouldBe "( = self::Foo)::()" + bodyCall.receiver.isEmpty shouldBe true + bodyCall.argument(0).code shouldBe "" + case None => fail("Expected call") + } + } } "Class Variables in Class and Methods" should { @@ -680,7 +726,7 @@ class ClassTests extends RubyCode2CpgFixture { "create the `StandardError` local variable" in { cpg.local.nameExact("some_variable").dynamicTypeHintFullName.toList shouldBe List( - s"<${GlobalTypes.builtinPrefix}.StandardError>" + s"${GlobalTypes.builtinPrefix}.StandardError" ) } @@ -744,12 +790,19 @@ class ClassTests extends RubyCode2CpgFixture { inside(methodBlock.astChildren.l) { case methodCall :: Nil => inside(methodCall.astChildren.l) { - case (base: Call) :: (self: Identifier) :: (literal: Literal) :: (methodRef: MethodRef) :: Nil => + case (base: Call) :: (self: Identifier) :: (literal: Literal) :: (typeRef: TypeRef) :: Nil => base.code shouldBe "self.scope" self.name shouldBe "self" literal.code shouldBe ":hits_by_ip" - methodRef.methodFullName shouldBe s"Test0.rb:::program.Foo:${RubyDefines.TypeDeclBody}:0" - methodRef.referencedMethod.parameter.indexGt(0).name.l shouldBe List("ip", "col") + typeRef.typeFullName shouldBe s"Test0.rb:$Main.Foo.${RubyDefines.TypeDeclBody}.0&Proc" + cpg.method + .fullNameExact( + typeRef.typ.referencedTypeDecl.member.name("call").dynamicTypeHintFullName.toSeq* + ) + .parameter + .indexGt(0) + .name + .l shouldBe List("ip", "col") case xs => fail(s"Expected three children, got ${xs.code.mkString(", ")} instead") } case xs => fail(s"Expected one call, got ${xs.code.mkString(", ")} instead") @@ -776,11 +829,192 @@ class ClassTests extends RubyCode2CpgFixture { case assignCall :: Nil => inside(assignCall.argument.l) { case lhs :: (rhs: Call) :: Nil => - rhs.typeFullName shouldBe "<__builtin.Encoding.Converter>:asciicompat_encoding" + rhs.typeFullName shouldBe "__builtin.Encoding.Converter.asciicompat_encoding" case xs => fail(s"Expected lhs and rhs for assignment call, got [${xs.code.mkString(",")}]") } case xs => fail(s"Expected one call for assignment, got [${xs.code.mkString(",")}]") } } } + + "Class definition on one line" should { + val cpg = code(""" + |class X 1 end + |""".stripMargin) + + "create TYPE_DECL" in { + inside(cpg.typeDecl.name("X").l) { + case xClass :: Nil => + inside(xClass.astChildren.isMethod.l) { + case bodyMethod :: initMethod :: Nil => + inside(bodyMethod.block.astChildren.l) { + case (literal: Literal) :: Nil => + literal.code shouldBe "1" + case xs => fail(s"Expected literal for body method, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected body and init method, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one class, got [${xs.code.mkString(",")}]") + } + } + } + + "A call to super" should { + val cpg = code(""" + |class A + | def foo(a) + | end + |end + |class B < A + | def foo(a) + | super(a) + | end + |end + |""".stripMargin) + + "create a simple call" in { + val superCall = cpg.call.nameExact("super").head + superCall.code shouldBe "super(a)" + superCall.name shouldBe "super" + superCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName + } + } + + "a class that is redefined should have a counter suffixed to ensure uniqueness" in { + val cpg = code(""" + |class Foo + | def foo;end + |end + |class Bar;end + |class Foo + | def foo;end + |end + |class Foo;end + |""".stripMargin) + + cpg.typeDecl.name("(Foo|Bar).*").filterNot(_.name.endsWith("")).name.l shouldBe List( + "Foo", + "Bar", + "Foo", + "Foo" + ) + cpg.typeDecl.name("(Foo|Bar).*").filterNot(_.name.endsWith("")).fullName.l shouldBe List( + s"Test0.rb:$Main.Foo", + s"Test0.rb:$Main.Bar", + s"Test0.rb:$Main.Foo0", + s"Test0.rb:$Main.Foo1" + ) + + cpg.method.nameExact("foo").fullName.l shouldBe List(s"Test0.rb:$Main.Foo.foo", s"Test0.rb:$Main.Foo0.foo") + + } + + "Class with nonAllowedTypeDeclChildren and explicit init" should { + val cpg = code(""" + |class Foo + | 1 + | def initialize(bar) + | puts bar + | end + |end + |""".stripMargin) + + "have an explicit init method" in { + inside(cpg.typeDecl.nameExact("Foo").method.l) { + case initMethod :: bodyMethod :: Nil => + bodyMethod.name shouldBe TypeDeclBody + + initMethod.name shouldBe Initialize + inside(initMethod.parameter.l) { + case selfParam :: barParam :: Nil => + selfParam.name shouldBe "self" + barParam.name shouldBe "bar" + case xs => fail(s"Expected two params, got [${xs.code.mkString(",")}]") + } + + inside(initMethod.block.astChildren.l) { + case (putsCall: Call) :: Nil => + putsCall.name shouldBe "puts" + case xs => fail(s"Expected one call, got [${xs.code.mkString(",")}]") + } + + inside(bodyMethod.block.astChildren.l) { + case (one: Literal) :: Nil => + one.code shouldBe "1" + one.typeFullName shouldBe s"${GlobalTypes.kernelPrefix}.Integer" + case xs => fail(s"Expected one literal, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected body method and init method, got [${xs.code.mkString(",")}]") + } + } + } + + "Class defined in Namespace" in { + val cpg = code(""" + |class Api::V1::MobileController + |end + |""".stripMargin) + + inside(cpg.namespaceBlock.fullNameExact("Api.V1").typeDecl.l) { + case mobileNamespace :: mobileClassNamespace :: Nil => + mobileNamespace.name shouldBe "MobileController" + mobileNamespace.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + + mobileClassNamespace.name shouldBe "MobileController" + mobileClassNamespace.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + case xs => fail(s"Expected two namespace blocks, got ${xs.code.mkString(",")}") + } + + inside(cpg.typeDecl.name("MobileController").l) { + case mobileTypeDecl :: Nil => + mobileTypeDecl.name shouldBe "MobileController" + mobileTypeDecl.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + mobileTypeDecl.astParentFullName shouldBe "Api.V1" + mobileTypeDecl.astParentType shouldBe NodeTypes.NAMESPACE_BLOCK + + mobileTypeDecl.astParent.isNamespaceBlock shouldBe true + + val namespaceDecl = mobileTypeDecl.astParent.asInstanceOf[NamespaceBlock] + namespaceDecl.name shouldBe "Api.V1" + namespaceDecl.filename shouldBe "Test0.rb" + + namespaceDecl.astParent.isFile shouldBe true + val parentFileDecl = namespaceDecl.astParent.asInstanceOf[File] + parentFileDecl.name shouldBe "Test0.rb" + + case xs => fail(s"Expected one class decl, got [${xs.code.mkString(",")}]") + } + } + + "Namespace scope is popping properly" in { + val cpg = code(""" + |class Foo::Bar + |end + | + |class Baz + |end + |""".stripMargin) + + inside(cpg.typeDecl.name("Baz").l) { + case bazTypeDecl :: Nil => + bazTypeDecl.fullName shouldBe "Test0.rb:
.Baz" + case xs => fail(s"Expected one type decl, got [${xs.code.mkString(",")}]") + } + } + + "Self param in static method" in { + val cpg = code(""" + |class Benefits < ApplicationRecord + |def self.save(file, backup = false) + | data_path = Rails.root.join("public", "data") + | full_file_name = "#{data_path}/#{file.original_filename}" + | f = File.open(full_file_name, "wb+") + | f.write file.read + | f.close + | make_backup(file, data_path, full_file_name) if backup == "true" + |end + |end + |""".stripMargin) + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ConditionalTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ConditionalTests.scala index 3ed14c936954..20e8b8c6f4a4 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ConditionalTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ConditionalTests.scala @@ -1,7 +1,7 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.Operators +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Local} import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.Call @@ -47,10 +47,9 @@ class ConditionalTests extends RubyCode2CpgFixture { inside(cpg.call(Operators.conditional).l) { case cond :: Nil => inside(cond.argument.l) { - case x :: y :: z :: Nil => { + case x :: y :: z :: Nil => x.code shouldBe "x" List(y, z).isBlock.astChildren.isIdentifier.code.l shouldBe List("y", "z") - } case xs => fail(s"Expected exactly three arguments to conditional, got [${xs.code.mkString(",")}]") } case xs => fail(s"Expected exactly one conditional, got [${xs.code.mkString(",")}]") @@ -61,15 +60,11 @@ class ConditionalTests extends RubyCode2CpgFixture { val cpg = code("""x, y, z = false, true, false |f(unless x then y else z end) |""".stripMargin) - inside(cpg.call(Operators.conditional).l) { - case cond :: Nil => - inside(cond.argument.l) { - case x :: y :: z :: Nil => { - List(x).isCall.name(Operators.logicalNot).argument.code.l shouldBe List("x") - List(y, z).isBlock.astChildren.isIdentifier.code.l shouldBe List("y", "z") - } - case xs => fail(s"Expected exactly three arguments to conditional, got [${xs.code.mkString(",")}]") - } + inside(cpg.controlStructure.controlStructureTypeExact(ControlStructureTypes.IF).l) { + case ifNode :: Nil => + ifNode.whenTrue.astChildren.code.l shouldBe List("y") + ifNode.whenFalse.astChildren.code.l shouldBe List("z") + ifNode.condition.isCall.name(Operators.logicalNot).argument.code.l shouldBe List("x") case xs => fail(s"Expected exactly one conditional, got [${xs.code.mkString(",")}]") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ContentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ContentTests.scala new file mode 100644 index 000000000000..85a8db6c92a9 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ContentTests.scala @@ -0,0 +1,77 @@ +package io.joern.rubysrc2cpg.querying + +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.semanticcpg.language.* + +class ContentTests extends RubyCode2CpgFixture(disableFileContent = false) { + "Content of file" in { + val fileContent = + """ + |class Animal + |end + | + |def foo + | puts "a" + |end + |""".stripMargin + + val cpg = code(fileContent, "Test0.rb") + + cpg.file.name("Test0.rb").content.head shouldBe fileContent + } + + "Content of method" in { + + val fooFunc = + """def foo + | puts "a" + |end""".stripMargin + + val cpg = code(s"""$fooFunc""".stripMargin) + + val method = cpg.method.name("foo").head + + method.content.head shouldBe fooFunc + } + + "Content of Class" in { + val cls = + """class Animal + |end""".stripMargin + + val cpg = code(s"""$cls""".stripMargin) + val animal = cpg.typeDecl.name("Animal").head + + animal.content.head shouldBe cls + } + + "Content of Module" in { + val mod = """module Foo + |end""".stripMargin + + val cpg = code(mod) + val module = cpg.typeDecl.name("Foo").head + + module.content.head shouldBe mod + } + + "Method and Class content" in { + val cls = + """class Animal + |end""".stripMargin + + val fooFunc = + """def foo + | puts "a" + |end""".stripMargin + + val cpg = code(s"""$cls + |$fooFunc""".stripMargin) + + val method = cpg.method.name("foo").head + val animal = cpg.typeDecl.name("Animal").head + + method.content.head shouldBe fooFunc + animal.content.head shouldBe cls + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala index a24197513520..9fd240e3a3fa 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala @@ -1,9 +1,10 @@ package io.joern.rubysrc2cpg.querying +import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.GlobalTypes.kernelPrefix import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal} import io.shiftleft.semanticcpg.language.* class ControlStructureTests extends RubyCode2CpgFixture { @@ -93,6 +94,9 @@ class ControlStructureTests extends RubyCode2CpgFixture { val List(breakNode) = cpg.break.l breakNode.code shouldBe "break" breakNode.lineNumber shouldBe Some(8) + + // `loop` is lowered as a do-while loop with a true condition + cpg.controlStructure.condition("true").size shouldBe 1 } "`if-end` statement is represented by an `IF` CONTROL_STRUCTURE node" in { @@ -293,7 +297,7 @@ class ControlStructureTests extends RubyCode2CpgFixture { whileCond.code shouldBe "true" whileCond.lineNumber shouldBe Some(2) - putsHi.methodFullName shouldBe s"$kernelPrefix:puts" + putsHi.methodFullName shouldBe s"$kernelPrefix.puts" putsHi.code shouldBe "puts 'hi'" putsHi.lineNumber shouldBe Some(2) } @@ -334,27 +338,29 @@ class ControlStructureTests extends RubyCode2CpgFixture { |end |""".stripMargin) - val List(rescueNode) = cpg.method("test1").tryBlock.l - rescueNode.controlStructureType shouldBe ControlStructureTypes.TRY - val List(body, rescueBody1, rescueBody2, rescueBody3, elseBody, ensureBody) = rescueNode.astChildren.l - body.ast.isLiteral.code.l shouldBe List("1") - body.order shouldBe 1 + inside(cpg.method("test1").controlStructure.l) { + case tryStruct :: rescue1Struct :: rescue2Struct :: rescue3Struct :: elseStruct :: ensureStruct :: Nil => + tryStruct.controlStructureType shouldBe ControlStructureTypes.TRY + val body = tryStruct.astChildren.head + body.ast.isLiteral.code.l shouldBe List("1") - rescueBody1.ast.isLiteral.code.l shouldBe List("2") - rescueBody1.order shouldBe 2 + rescue1Struct.controlStructureType shouldBe ControlStructureTypes.CATCH + rescue1Struct.ast.isLocal.code.l shouldBe List("e") + rescue1Struct.ast.isLiteral.code.l shouldBe List("2") - rescueBody2.ast.isLiteral.code.l shouldBe List("3") - rescueBody2.order shouldBe 2 + rescue2Struct.controlStructureType shouldBe ControlStructureTypes.CATCH + rescue2Struct.ast.isLiteral.code.l shouldBe List("3") - rescueBody3.ast.isLiteral.code.l shouldBe List("4") - rescueBody3.order shouldBe 2 + rescue3Struct.controlStructureType shouldBe ControlStructureTypes.CATCH + rescue3Struct.ast.isLiteral.code.l shouldBe List("4") - elseBody.ast.isLiteral.code.l shouldBe List("5") - elseBody.order shouldBe 2 - - ensureBody.ast.isLiteral.code.l shouldBe List("6") - ensureBody.order shouldBe 3 + elseStruct.controlStructureType shouldBe ControlStructureTypes.ELSE + elseStruct.ast.isLiteral.code.l shouldBe List("5") + ensureStruct.controlStructureType shouldBe ControlStructureTypes.FINALLY + ensureStruct.ast.isLiteral.code.l shouldBe List("6") + case xs => fail(s"Expected 6 structures, got ${xs.code.mkString(",")}") + } } "`begin ... ensure ... end is represented by a `TRY` CONTROL_STRUCTURE node" in { @@ -367,18 +373,21 @@ class ControlStructureTests extends RubyCode2CpgFixture { | end |end |""".stripMargin) - val List(rescueNode) = cpg.method("test2").tryBlock.l - rescueNode.controlStructureType shouldBe ControlStructureTypes.TRY - val List(body, defaultElseBody, ensureBody) = rescueNode.astChildren.l - body.ast.isLiteral.code.l shouldBe List("1") - body.order shouldBe 1 + inside(cpg.method("test2").controlStructure.l) { + case tryStruct :: defaultElseStruct :: ensureStruct :: Nil => + tryStruct.controlStructureType shouldBe ControlStructureTypes.TRY + val body = tryStruct.astChildren.head + body.ast.isLiteral.code.l shouldBe List("1") + + defaultElseStruct.controlStructureType shouldBe ControlStructureTypes.ELSE + defaultElseStruct.ast.isLiteral.code.l shouldBe List("nil") - defaultElseBody.ast.isLiteral.code.l shouldBe List("nil") - ensureBody.order shouldBe 3 + ensureStruct.controlStructureType shouldBe ControlStructureTypes.FINALLY + ensureStruct.ast.isLiteral.code.l shouldBe List("2") - ensureBody.ast.isLiteral.code.l shouldBe List("2") - ensureBody.order shouldBe 3 + case xs => fail(s"Expected two structures, got ${xs.code.mkString(",")}") + } } "`for .. in` control structure" should { @@ -404,12 +413,25 @@ class ControlStructureTests extends RubyCode2CpgFixture { forEachNode.controlStructureType shouldBe ControlStructureTypes.FOR inside(forEachNode.astChildren.l) { - case (iteratorNode: Identifier) :: (iterableNode: Identifier) :: (doBody: Block) :: Nil => - iteratorNode.code shouldBe "i" - iterableNode.code shouldBe "x" - // We use .ast as there will be an implicit return node here - doBody.ast.isCall.code.headOption shouldBe Option("puts x - i") - case _ => fail("No node for iterable found in `for-in` statement") + case (idxLocal: Local) :: (iVarLocal: Local) :: (initAssign: Call) :: (cond: Call) :: (update: Call) :: (forBlock: Block) :: Nil => + idxLocal.name shouldBe "_idx_" + idxLocal.typeFullName shouldBe Defines.getBuiltInType(Defines.Integer) + + iVarLocal.name shouldBe "i" + + initAssign.code shouldBe "_idx_ = 0" + initAssign.name shouldBe Operators.assignment + initAssign.methodFullName shouldBe Operators.assignment + + cond.code shouldBe "_idx_ < x.length" + cond.name shouldBe Operators.lessThan + cond.methodFullName shouldBe Operators.lessThan + + update.code shouldBe "i = x[_idx_++]" + update.name shouldBe Operators.assignment + update.methodFullName shouldBe Operators.assignment + + case xs => fail(s"Expected 6 children for `forEachNode`, got [${xs.code.mkString(",")}]") } inside(forEachNode.astChildren.isBlock.l) { @@ -429,13 +451,25 @@ class ControlStructureTests extends RubyCode2CpgFixture { forEachNode.controlStructureType shouldBe ControlStructureTypes.FOR inside(forEachNode.astChildren.l) { - case (iteratorNode: Identifier) :: (iterableNode: Call) :: (doBody: Block) :: Nil => - iteratorNode.code shouldBe "i" - iterableNode.code shouldBe "1..x" - iterableNode.name shouldBe Operators.range - // We use .ast as there will be an implicit return node here - doBody.ast.isCall.code.headOption shouldBe Option("puts x + i") - case _ => fail("Invalid `for-in` children nodes") + case (idxLocal: Local) :: (iVarLocal: Local) :: (initAssign: Call) :: (cond: Call) :: (update: Call) :: (forBlock: Block) :: Nil => + idxLocal.name shouldBe "_idx_" + idxLocal.typeFullName shouldBe Defines.getBuiltInType(Defines.Integer) + + iVarLocal.name shouldBe "i" + + initAssign.code shouldBe "_idx_ = 0" + initAssign.name shouldBe Operators.assignment + initAssign.methodFullName shouldBe Operators.assignment + + cond.code shouldBe "_idx_ < 1..x.length" + cond.name shouldBe Operators.lessThan + cond.methodFullName shouldBe Operators.lessThan + + update.code shouldBe "i = 1..x[_idx_++]" + update.name shouldBe Operators.assignment + update.methodFullName shouldBe Operators.assignment + + case xs => fail(s"Expected 6 children for `forEachNode`, got [${xs.code.mkString(",")}]") } case _ => fail("No control structure node found for `for-in`.") @@ -518,4 +552,167 @@ class ControlStructureTests extends RubyCode2CpgFixture { } } } + + "Generate continue node for next" in { + val cpg = code(""" + |for i in arr do + | next if i % 2 == 0 + |end + |""".stripMargin) + + inside(cpg.controlStructure.controlStructureType(ControlStructureTypes.CONTINUE).l) { + case nextControl :: Nil => + nextControl.code shouldBe "next" + case xs => fail(s"Expected next to be continue, got [${xs.code.mkString(",")}]") + } + } + + "A `raise` call with a string argument should generate a `throw` control structure with explicit `StandardError.new` call" in { + val cpg = code("raise 'Hello, world!'") + inside(cpg.controlStructure.l) { + case (ctrlStruct: ControlStructure) :: Nil => + ctrlStruct.code shouldBe "raise 'Hello, world!'" + ctrlStruct.controlStructureType shouldBe ControlStructureTypes.THROW + + val constructorBlock = ctrlStruct.astChildren.head.asInstanceOf[Block] + constructorBlock.ast.isCall.where(_.name(Operators.alloc)).nonEmpty shouldBe true + + val initialize = constructorBlock.ast.isCall.name(Defines.Initialize).head + initialize.code shouldBe "StandardError.new('Hello, world!')" + val helloWorld = initialize.argument(1).asInstanceOf[Literal] + helloWorld.code shouldBe "'Hello, world!'" + case xs => fail(s"Expected single `throw` call, got [${xs.code.mkString(",")}]") + } + } + + "A `raise` call with an explicit error argument should generate a `throw` control structure" in { + val cpg = code("raise ZeroDivisionError.new 'b should not be 0'") + inside(cpg.controlStructure.l) { + case (ctrlStruct: ControlStructure) :: Nil => + ctrlStruct.code shouldBe "raise ZeroDivisionError.new 'b should not be 0'" + ctrlStruct.controlStructureType shouldBe ControlStructureTypes.THROW + + val constructorBlock = ctrlStruct.astChildren.head.asInstanceOf[Block] + constructorBlock.ast.isCall.where(_.name(Operators.alloc)).nonEmpty shouldBe true + + val initialize = constructorBlock.ast.isCall.name(Defines.Initialize).head + initialize.code shouldBe "ZeroDivisionError.new 'b should not be 0'" + val errMsg = initialize.argument(1).asInstanceOf[Literal] + errMsg.code shouldBe "'b should not be 0'" + case xs => fail(s"Expected single `throw` call, got [${xs.code.mkString(",")}]") + } + } + + "Ternary if" in { + val cpg = code(""" + |class Api::V1::UsersController < ApplicationController + | def index + | respond_with @user.admin ? User.all : @user + | end + |end + |""".stripMargin) + + inside(cpg.method.name("index").l) { + case indexMethod :: Nil => + inside(indexMethod.call.name(Operators.conditional).l) { + case ternary :: Nil => + ternary.code shouldBe "@user.admin ? User.all : @user" + + inside(ternary.argument.l) { + case condition :: (leftOpt: Block) :: (rightOpt: Block) :: Nil => + condition.code shouldBe "( = @user).admin" + condition.ast.isFieldIdentifier.code.l shouldBe List("@user", "admin") + + leftOpt.ast.fieldAccess.code.head shouldBe "User.all" + leftOpt.ast.isFieldIdentifier.code.l shouldBe List("User", "all") + + rightOpt.ast.fieldAccess.code.head shouldBe "self.@user" + rightOpt.ast.isFieldIdentifier.code.head shouldBe "@user" + + case xs => fail(s"Expected two arguments, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected one call for ternary, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected one method, got ${xs.name.mkString(",")}") + } + } + + "RETURN keyword in logicalAndExpression" in { + val cpg = code(""" + |def foo + | if (a == 1 && return) + | puts a + | end + |end + |""".stripMargin) + + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + + val List(_: Call, returnCall: Return) = ifStruct.condition.isCall.argument.l: @unchecked + returnCall.code shouldBe "return" + + case xs => fail(s"Expected one control strucuture, got [${xs.code.mkString(",")}]") + } + } + + "RETURN keyword in logicalOrExpression" in { + val cpg = code(""" + |def foo + | if (a == 10 || return) + | puts a + | end + |end + |""".stripMargin) + + inside(cpg.method.name("foo").controlStructure.l) { + case orIfStruct :: Nil => + orIfStruct.controlStructureType shouldBe ControlStructureTypes.IF + + val List(_: Call, returnCall: Return) = orIfStruct.condition.isCall.argument.l: @unchecked + returnCall.code shouldBe "return" + case xs => fail(s"Expected one IF structure, got [${xs.code.mkString(",")}]") + } + } + + "ForEach loops" in { + val cpg = code(""" + |fibNumbers = [0, 1, 1, 2, 3, 5, 8, 13] + |for num in fibNumbers + | puts num + |end + |""".stripMargin) + + inside(cpg.method.isModule.controlStructure.l) { + case forEachNode :: Nil => + forEachNode.controlStructureType shouldBe ControlStructureTypes.FOR + + inside(forEachNode.astChildren.l) { + case (idxLocal: Local) :: (numLocal: Local) :: (initAssign: Call) :: (cond: Call) :: (update: Call) :: (forBlock: Block) :: Nil => + idxLocal.name shouldBe "_idx_" + idxLocal.typeFullName shouldBe Defines.getBuiltInType(Defines.Integer) + + numLocal.name shouldBe "num" + + initAssign.code shouldBe "_idx_ = 0" + initAssign.name shouldBe Operators.assignment + initAssign.methodFullName shouldBe Operators.assignment + + cond.code shouldBe "_idx_ < fibNumbers.length" + cond.name shouldBe Operators.lessThan + cond.methodFullName shouldBe Operators.lessThan + + update.code shouldBe "num = fibNumbers[_idx_++]" + update.name shouldBe Operators.assignment + update.methodFullName shouldBe Operators.assignment + + val List(putsCall) = cpg.call.nameExact("puts").l + putsCall.astParent shouldBe forBlock + + case xs => fail(s"Expected 6 children for `forEachNode`, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one node for `forEach` loop, got [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DependencyTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DependencyTests.scala index 106efaf63b19..453bb6e03d1b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DependencyTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DependencyTests.scala @@ -2,9 +2,10 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines -import io.joern.rubysrc2cpg.passes.Defines as RubyDefines +import io.joern.rubysrc2cpg.passes.{DependencyPass, Defines as RubyDefines} import io.shiftleft.codepropertygraph.generated.nodes.{Block, Identifier} import io.shiftleft.semanticcpg.language.* +import io.joern.rubysrc2cpg.passes.Defines.Main class DependencyTests extends RubyCode2CpgFixture { @@ -13,7 +14,7 @@ class DependencyTests extends RubyCode2CpgFixture { val cpg = code(DependencyTests.GEMFILELOCK, "Gemfile.lock") "result in dependency nodes of the set packages" in { - inside(cpg.dependency.nameNot(RubyDefines.Resolver).l) { + inside(cpg.dependency.nameNot(RubyDefines.Resolver).versionNot(DependencyPass.CORE_GEM_VERSION).l) { case aruba :: bcrypt :: betterErrors :: Nil => aruba.name shouldBe "aruba" aruba.version shouldBe "0.14.12" @@ -35,7 +36,7 @@ class DependencyTests extends RubyCode2CpgFixture { val cpg = code(DependencyTests.GEMFILE, "Gemfile") "result in dependency nodes of the set packages" in { - inside(cpg.dependency.nameNot(RubyDefines.Resolver).l) { + inside(cpg.dependency.nameNot(RubyDefines.Resolver).versionNot(DependencyPass.CORE_GEM_VERSION).l) { case aruba :: bcrypt :: coffeeRails :: Nil => aruba.name shouldBe "aruba" aruba.version shouldBe "2.5.1" @@ -58,7 +59,10 @@ class DependencyTests extends RubyCode2CpgFixture { "be preferred over a normal Gemfile" in { // Our Gemfile.lock specifies exact versions whereas the Gemfile does not - cpg.dependency.nameNot(RubyDefines.Resolver).forall(d => !d.version.isBlank) shouldBe true + cpg.dependency + .nameNot(RubyDefines.Resolver) + .versionNot(DependencyPass.CORE_GEM_VERSION) + .forall(d => !d.version.isBlank) shouldBe true } } @@ -94,9 +98,9 @@ class DownloadDependencyTest extends RubyCode2CpgFixture(downloadDependencies = case (v: Identifier) :: (block: Block) :: Nil => v.dynamicTypeHintFullName should contain("dummy_logger.Main_module.Main_outer_class") - inside(block.astChildren.isCall.nameExact("new").headOption) { + inside(block.astChildren.isCall.nameExact(RubyDefines.Initialize).headOption) { case Some(constructorCall) => - constructorCall.methodFullName shouldBe s"dummy_logger.Main_module.Main_outer_class:${RubyDefines.Initialize}" + constructorCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName case None => fail(s"Expected constructor call, did not find one") } case xs => fail(s"Expected two arguments under the constructor assignment, got [${xs.code.mkString(", ")}]") @@ -108,9 +112,9 @@ class DownloadDependencyTest extends RubyCode2CpgFixture(downloadDependencies = case (g: Identifier) :: (block: Block) :: Nil => g.dynamicTypeHintFullName should contain("dummy_logger.Help") - inside(block.astChildren.isCall.name("new").headOption) { + inside(block.astChildren.isCall.name(RubyDefines.Initialize).headOption) { case Some(constructorCall) => - constructorCall.methodFullName shouldBe s"dummy_logger.Help:${RubyDefines.Initialize}" + constructorCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName case None => fail(s"Expected constructor call, did not find one") } case xs => fail(s"Expected two arguments under the constructor assignment, got [${xs.code.mkString(", ")}]") @@ -120,12 +124,12 @@ class DownloadDependencyTest extends RubyCode2CpgFixture(downloadDependencies = // TODO: This requires type propagation "recognise methodFullName for `first_fun`" ignore { cpg.call.name("first_fun").head.methodFullName should equal( - "dummy_logger::program:Main_module:Main_outer_class:first_fun" + s"dummy_logger.$Main.Main_module.Main_outer_class.first_fun" ) cpg.call .name("help_print") .head - .methodFullName shouldBe "dummy_logger::program:Help:help_print" + .methodFullName shouldBe s"dummy_logger.$Main:Help:help_print" } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DestructuredAssignmentsTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DestructuredAssignmentsTests.scala index 6e71c4a963f8..32c8da7e121d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DestructuredAssignmentsTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DestructuredAssignmentsTests.scala @@ -281,4 +281,126 @@ class DestructuredAssignmentsTests extends RubyCode2CpgFixture { } + "Destructered Assignment with splat in the middle" in { + val cpg = code(""" + |a, *b, c = 1, 2, 3, 4, 5, 6 + |""".stripMargin) + + inside(cpg.call.name(Operators.assignment).l) { + case aAssignment :: bAssignment :: cAssignment :: Nil => + aAssignment.code shouldBe "a, *b, c = 1, 2, 3, 4, 5, 6" + bAssignment.code shouldBe "a, *b, c = 1, 2, 3, 4, 5, 6" + cAssignment.code shouldBe "a, *b, c = 1, 2, 3, 4, 5, 6" + + val List(a: Identifier, lit: Literal) = aAssignment.argumentOut.toList: @unchecked + a.name shouldBe "a" + lit.code shouldBe "1" + + val List(splat: Identifier, arr: Call) = bAssignment.argumentOut.toList: @unchecked + splat.name shouldBe "b" + arr.name shouldBe Operators.arrayInitializer + + inside(arr.argumentOut.l) { + case (two: Literal) :: (three: Literal) :: (four: Literal) :: (five: Literal) :: Nil => + two.code shouldBe "2" + three.code shouldBe "3" + four.code shouldBe "4" + five.code shouldBe "5" + case _ => fail("Unexpected number of array elements in `*`'s assignment") + } + + val List(c: Identifier, cLiteral: Literal) = cAssignment.argumentOut.toList: @unchecked + c.name shouldBe "c" + cLiteral.code shouldBe "6" + case xs => fail(s"Expected three assignments, got ${xs.code.mkString(",")}") + } + } + + "Destructured assignment with naked splat" in { + val cpg = code(""" + |*, a = 1, 2, 3 + |""".stripMargin) + + inside(cpg.assignment.l) { + case splatAssignment :: aAssignment :: Nil => + aAssignment.code shouldBe "*, a = 1, 2, 3" + splatAssignment.code shouldBe "*, a = 1, 2, 3" + + val List(a: Identifier, lit: Literal) = aAssignment.argumentOut.toList: @unchecked + a.name shouldBe "a" + lit.code shouldBe "3" + + val List(splat: Identifier, arr: Call) = splatAssignment.argumentOut.toList: @unchecked + splat.name shouldBe "_" + arr.name shouldBe Operators.arrayInitializer + inside(arr.argumentOut.l) { + case (one: Literal) :: (two: Literal) :: Nil => + one.code shouldBe "1" + two.code shouldBe "2" + case _ => fail("Unexpected number of array elements in `*`'s assignment") + } + case _ => fail("Unexpected number of assignments found") + } + } + + "Destructered Assignment RHS" in { + val cpg = code(""" + |a, *b, c = 1, 2, *d, *f, 4 + |""".stripMargin) + + inside(cpg.call.name(Operators.assignment).l) { + case aAssignment :: bAssignment :: cAssignment :: Nil => + aAssignment.code shouldBe "a, *b, c = 1, 2, *d, *f, 4" + bAssignment.code shouldBe "a, *b, c = 1, 2, *d, *f, 4" + cAssignment.code shouldBe "a, *b, c = 1, 2, *d, *f, 4" + + val List(a: Identifier, aLiteral: Literal) = aAssignment.argumentOut.toList: @unchecked + a.name shouldBe "a" + aLiteral.code shouldBe "1" + + val List(splat: Identifier, arr: Call) = bAssignment.argumentOut.toList: @unchecked + splat.name shouldBe "b" + arr.name shouldBe Operators.arrayInitializer + + inside(arr.argumentOut.l) { + case (two: Literal) :: (d: Call) :: (f: Call) :: Nil => + two.code shouldBe "2" + + d.code shouldBe "*d" + d.methodFullName shouldBe RubyOperators.splat + + f.code shouldBe "*f" + f.methodFullName shouldBe RubyOperators.splat + + case xs => fail(s"Unexpected number of array elements in `*`'s assignment, got ${xs.code.mkString(",")}") + } + + val List(c: Identifier, cLiteral: Literal) = cAssignment.argumentOut.toList: @unchecked + c.name shouldBe "c" + cLiteral.code shouldBe "4" + + case xs => fail(s"Expected 3 assignments, got ${xs.code.mkString(",")}") + } + } + + "multi-assignments as a return value" should { + + val cpg = code(""" + |def f + | a, b = 1, 2 # => return [1, 2] + |end + |""".stripMargin) + + "create an explicit return of the LHS values as an array" in { + val arrayLiteral = cpg.method.name("f").methodReturn.toReturn.astChildren.isCall.head + + arrayLiteral.name shouldBe Operators.arrayInitializer + arrayLiteral.methodFullName shouldBe Operators.arrayInitializer + arrayLiteral.code shouldBe "a, b = 1, 2" + + arrayLiteral.astChildren.isIdentifier.code.l shouldBe List("a", "b") + } + + } + } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala index 4b89a14b3c87..0d678849a43e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/DoBlockTests.scala @@ -1,8 +1,10 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.GlobalTypes.builtinPrefix +import io.joern.rubysrc2cpg.passes.Defines.{Initialize, Main} import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -21,7 +23,7 @@ class DoBlockTests extends RubyCode2CpgFixture { | |""".stripMargin) - "create an anonymous method with associated type declaration" in { + "create an anonymous method with associated type declaration and wrapper type" in { inside(cpg.method.isModule.l) { case program :: Nil => inside(program.astChildren.collectAll[Method].l) { @@ -29,18 +31,22 @@ class DoBlockTests extends RubyCode2CpgFixture { foo.name shouldBe "foo" closureMethod.name shouldBe "0" - closureMethod.fullName shouldBe "Test0.rb:::program:0" + closureMethod.fullName shouldBe s"Test0.rb:$Main.0" case xs => fail(s"Expected a two method nodes, instead got [${xs.code.mkString(", ")}]") } inside(program.astChildren.collectAll[TypeDecl].isLambda.l) { case closureType :: Nil => closureType.name shouldBe "0" - closureType.fullName shouldBe "Test0.rb:::program:0" + closureType.fullName shouldBe s"Test0.rb:$Main.0" + case xs => fail(s"Expected a one closure type node, instead got [${xs.code.mkString(", ")}]") + } + inside(program.astChildren.collectAll[TypeDecl].name(".*Proc").l) { + case closureType :: Nil => val callMember = closureType.member.nameExact("call").head callMember.typeFullName shouldBe Defines.Any - callMember.dynamicTypeHintFullName shouldBe Seq("Test0.rb:::program:0") + callMember.dynamicTypeHintFullName shouldBe Seq(s"Test0.rb:$Main.0") case xs => fail(s"Expected a one closure type node, instead got [${xs.code.mkString(", ")}]") } case xs => fail(s"Expected a single program module, instead got [${xs.code.mkString(", ")}]") @@ -48,9 +54,9 @@ class DoBlockTests extends RubyCode2CpgFixture { } "create a method ref argument with populated type full name, which corresponds to the method type" in { - val methodRefArg = cpg.call("foo").argument(1).head.asInstanceOf[MethodRef] + val typeRefArg = cpg.call("foo").argument(1).head.asInstanceOf[TypeRef] val lambdaTypeDecl = cpg.typeDecl("0").head - methodRefArg.typeFullName shouldBe lambdaTypeDecl.fullName + typeRefArg.typeFullName shouldBe s"${lambdaTypeDecl.fullName}&Proc" } "have no parameters in the closure declaration" in { @@ -79,19 +85,19 @@ class DoBlockTests extends RubyCode2CpgFixture { |""".stripMargin) "create an anonymous method with associated type declaration" in { - inside(cpg.method.nameExact(":program").l) { + inside(cpg.method.isModule.l) { case program :: Nil => inside(program.astChildren.collectAll[Method].l) { case closureMethod :: Nil => closureMethod.name shouldBe "0" - closureMethod.fullName shouldBe "Test0.rb:::program:0" + closureMethod.fullName shouldBe s"Test0.rb:$Main.0" case xs => fail(s"Expected a one method nodes, instead got [${xs.code.mkString(", ")}]") } - inside(program.astChildren.collectAll[TypeDecl].l) { + inside(program.astChildren.collectAll[TypeDecl].isLambda.l) { case closureType :: Nil => closureType.name shouldBe "0" - closureType.fullName shouldBe "Test0.rb:::program:0" + closureType.fullName shouldBe s"Test0.rb:$Main.0" case xs => fail(s"Expected a one closure type node, instead got [${xs.code.mkString(", ")}]") } case xs => fail(s"Expected a single program module, instead got [${xs.code.mkString(", ")}]") @@ -108,13 +114,13 @@ class DoBlockTests extends RubyCode2CpgFixture { "specify the closure reference as an argument to the member call with block" in { inside(cpg.call("each").argument.l) { - case (myArray: Identifier) :: (lambdaRef: MethodRef) :: Nil => + case (myArray: Identifier) :: (lambdaRef: TypeRef) :: Nil => myArray.argumentIndex shouldBe 0 myArray.name shouldBe "my_array" myArray.code shouldBe "my_array" lambdaRef.argumentIndex shouldBe 1 - lambdaRef.methodFullName shouldBe "Test0.rb:::program:0" + lambdaRef.typeFullName shouldBe s"Test0.rb:$Main.0&Proc" case xs => fail(s"Expected `each` call to have call and method ref arguments, instead got [${xs.code.mkString(", ")}]") } @@ -141,20 +147,20 @@ class DoBlockTests extends RubyCode2CpgFixture { |""".stripMargin) "create an anonymous method with associated type declaration" in { - inside(cpg.method.nameExact(":program").l) { + inside(cpg.method.isModule.l) { case program :: Nil => inside(program.astChildren.collectAll[Method].l) { case closureMethod :: Nil => closureMethod.name shouldBe "0" - closureMethod.fullName shouldBe "Test0.rb:::program:0" + closureMethod.fullName shouldBe s"Test0.rb:$Main.0" closureMethod.isLambda.nonEmpty shouldBe true case xs => fail(s"Expected a one method nodes, instead got [${xs.code.mkString(", ")}]") } - inside(program.astChildren.collectAll[TypeDecl].l) { + inside(program.astChildren.collectAll[TypeDecl].isLambda.l) { case closureType :: Nil => closureType.name shouldBe "0" - closureType.fullName shouldBe "Test0.rb:::program:0" + closureType.fullName shouldBe s"Test0.rb:$Main.0" closureType.isLambda.nonEmpty shouldBe true case xs => fail(s"Expected a one closure type node, instead got [${xs.code.mkString(", ")}]") } @@ -173,13 +179,13 @@ class DoBlockTests extends RubyCode2CpgFixture { "specify the closure reference as an argument to the member call with block" in { inside(cpg.call("each").argument.l) { - case (hash: Identifier) :: (lambdaRef: MethodRef) :: Nil => + case (hash: Identifier) :: (lambdaRef: TypeRef) :: Nil => hash.argumentIndex shouldBe 0 hash.name shouldBe "hash" hash.code shouldBe "hash" lambdaRef.argumentIndex shouldBe 1 - lambdaRef.methodFullName shouldBe "Test0.rb:::program:0" + lambdaRef.typeFullName shouldBe s"Test0.rb:$Main.0&Proc" case xs => fail(s"Expected `each` call to have call and method ref arguments, instead got [${xs.code.mkString(", ")}]") } @@ -207,14 +213,16 @@ class DoBlockTests extends RubyCode2CpgFixture { |""".stripMargin) // Basic assertions for expected behaviour - "create the declarations for the closure" in { - inside(cpg.method(".*").l) { + "create the declarations for the closure with captured local" in { + inside(cpg.method.isLambda.l) { case m :: Nil => m.name should startWith("") + val myValue = m.local.nameExact("myValue").head + myValue.closureBindingId shouldBe Option(s"Test0.rb:$Main.myValue") case xs => fail(s"Expected exactly one closure method decl, instead got [${xs.code.mkString(",")}]") } - inside(cpg.typeDecl(".*").l) { + inside(cpg.typeDecl.isLambda.l) { case m :: Nil => m.name should startWith("") case xs => fail(s"Expected exactly one closure type decl, instead got [${xs.code.mkString(",")}]") @@ -224,17 +232,17 @@ class DoBlockTests extends RubyCode2CpgFixture { "annotate the nodes via CAPTURE bindings" in { cpg.all.collectAll[ClosureBinding].l match { case myValue :: Nil => - myValue.closureOriginalName.head shouldBe "myValue" + myValue.closureOriginalName shouldBe Option("myValue") inside(myValue._localViaRefOut) { case Some(local) => local.name shouldBe "myValue" - local.method.fullName.headOption shouldBe Option("Test0.rb:::program") + local.method.fullName.headOption shouldBe Option(s"Test0.rb:$Main") case None => fail("Expected closure binding refer to the captured local") } inside(myValue._captureIn.l) { - case (x: MethodRef) :: Nil => x.methodFullName shouldBe "Test0.rb:::program:0" - case xs => fail(s"Expected single method ref binding but got [${xs.mkString(",")}]") + case (x: TypeRef) :: Nil => x.typeFullName shouldBe s"Test0.rb:$Main.0&Proc" + case xs => fail(s"Expected single method ref binding but got [${xs.mkString(",")}]") } case xs => @@ -260,15 +268,16 @@ class DoBlockTests extends RubyCode2CpgFixture { inside(constrBlock.astChildren.l) { case (tmpLocal: Local) :: (tmpAssign: Call) :: (newCall: Call) :: (_: Identifier) :: Nil => tmpLocal.name shouldBe "" - tmpAssign.code shouldBe " = Array.new(x) { |i| i += 1 }" + tmpAssign.code shouldBe s" = Array.$Initialize" - newCall.name shouldBe "new" - newCall.methodFullName shouldBe s"$builtinPrefix.Array:initialize" + newCall.name shouldBe Initialize + newCall.methodFullName shouldBe Defines.DynamicCallUnknownFullName + newCall.dynamicTypeHintFullName should contain(s"$builtinPrefix.Array.$Initialize") inside(newCall.argument.l) { - case (_: Identifier) :: (x: Identifier) :: (closure: MethodRef) :: Nil => + case (_: Identifier) :: (x: Identifier) :: (closure: TypeRef) :: Nil => x.name shouldBe "x" - closure.methodFullName should endWith("0") + closure.typeFullName should endWith("0&Proc") case xs => fail(s"Expected a base, `x`, and closure ref, instead got [${xs.code.mkString(",")}]") } case xs => @@ -308,13 +317,167 @@ class DoBlockTests extends RubyCode2CpgFixture { "create a call `test_name` with a test name and lambda argument" in { inside(cpg.call.nameExact("test_name").argument.l) { - case (_: Identifier) :: (testName: Literal) :: (testMethod: MethodRef) :: Nil => + case (_: Identifier) :: (testName: Literal) :: (testMethod: TypeRef) :: Nil => testName.code shouldBe "'Foo'" - testMethod.referencedMethod.call.nameExact("puts").nonEmpty shouldBe true + cpg.method + .fullNameExact(testMethod.typ.referencedTypeDecl.member.name("call").dynamicTypeHintFullName.toSeq*) + .call + .nameExact("puts") + .nonEmpty shouldBe true case xs => fail(s"Expected a literal and method ref argument, instead got $xs") } } } + "A lambda with arrow syntax" should { + + val cpg = code(""" + |arrow_lambda = ->(y) { y } + |""".stripMargin) + + "create a lambda method with a `y` parameter" in { + inside(cpg.method.isLambda.headOption) { + case Some(lambda) => + lambda.code shouldBe "{ y }" + lambda.parameter.name.l shouldBe List("self", "y") + case xs => fail(s"Expected a lambda method") + } + } + + "create a method ref assigned to `arrow_lambda`" in { + inside(cpg.method.isModule.assignment.code("arrow_lambda.*").headOption) { + case Some(lambdaAssign) => + lambdaAssign.target.asInstanceOf[Identifier].name shouldBe "arrow_lambda" + lambdaAssign.source.asInstanceOf[TypeRef].typeFullName shouldBe s"Test0.rb:$Main.0&Proc" + case xs => fail(s"Expected an assignment to a lambda") + } + } + + } + + "A lambda with lambda keyword syntax" should { + + val cpg = code(""" + |a_lambda = lambda { |y| y } + |""".stripMargin) + + "create a lambda method with a `y` parameter" in { + inside(cpg.method.isLambda.headOption) { + case Some(lambda) => + lambda.code shouldBe "{ |y| y }" + lambda.parameter.name.l shouldBe List("self", "y") + case xs => fail(s"Expected a lambda method") + } + } + + "create a method ref assigned to `arrow_lambda`" in { + inside(cpg.method.isModule.assignment.code("a_lambda.*").headOption) { + case Some(lambdaAssign) => + lambdaAssign.target.asInstanceOf[Identifier].name shouldBe "a_lambda" + lambdaAssign.source.asInstanceOf[TypeRef].typeFullName shouldBe s"Test0.rb:$Main.0&Proc" + case xs => fail(s"Expected an assignment to a lambda") + } + } + + } + + "One local node for variable in lambda only" in { + val cpg = code(""" + | def get_pto_schedule + | begin + | jfs = [] + | schedules.each do |s| + | hash = Hash.new + | hash[:id] = s[:id] + | hash[:title] = s[:event_name] + | hash[:start] = s[:date_begin] + | hash[:end] = s[:date_end] + | jfs << hash + | end + | rescue + | end + | end + |""".stripMargin) + + inside(cpg.local.l) { + case jfsOutsideLocal :: hashInsideLocal :: tmp0 :: jfsCapturedLocal :: tmp1 :: Nil => + jfsOutsideLocal.closureBindingId shouldBe None + hashInsideLocal.closureBindingId shouldBe None + jfsCapturedLocal.closureBindingId shouldBe Some("Test0.rb:
.get_pto_schedule.jfs") + + tmp0.name shouldBe "" + tmp1.name shouldBe "" + case xs => fail(s"Expected 5 locals, got ${xs.code.mkString(",")}") + } + + inside(cpg.method.isLambda.local.l) { + case hashLocal :: _ :: jfsLocal :: Nil => + hashLocal.closureBindingId shouldBe None + jfsLocal.closureBindingId shouldBe Some("Test0.rb:
.get_pto_schedule.jfs") + case xs => fail(s"Expected 3 locals in lambda, got ${xs.code.mkString(",")}") + } + } + + "Various do-block parameters" should { + val cpg = code(""" + |f { |a, (b, c), *d, e, (f, *g), **h, &i| } + |""".stripMargin) + + "Generate correct parameters" in { + inside(cpg.method.isLambda.parameter.l) { + case _ :: aParam :: tmp0Param :: dParam :: eParam :: tmp1Param :: hParam :: iParam :: Nil => + aParam.name shouldBe "a" + aParam.code shouldBe "a" + + tmp0Param.name shouldBe "" + tmp0Param.code shouldBe "" + + dParam.name shouldBe "d" + dParam.code shouldBe "*d" + + eParam.name shouldBe "e" + eParam.code shouldBe "e" + + tmp1Param.name shouldBe "" + tmp1Param.code shouldBe "" + + hParam.name shouldBe "h" + hParam.code shouldBe "**h" + + iParam.name shouldBe "i" + iParam.code shouldBe "&i" + case xs => fail(s"Expected 8 parameters, got [${xs.name.mkString(", ")}]") + } + } + + "Generate required locals" in { + inside(cpg.method.isLambda.body.local.l) { + case bLocal :: cLocal :: fLocal :: gSplatLocal :: Nil => + bLocal.code shouldBe "b" + cLocal.code shouldBe "c" + + fLocal.code shouldBe "f" + gSplatLocal.code shouldBe "g" + case xs => fail(s"Expected 4 locals, got [${xs.name.mkString(", ")}]") + } + } + + "Generate required `assignment` calls" in { + inside(cpg.method.isLambda.call(Operators.assignment).l) { + case bAssign :: cAssign :: fAssign :: gAssign :: Nil => + bAssign.code shouldBe "b = *" + cAssign.code shouldBe "c = *" + + fAssign.code shouldBe "f = *" + gAssign.code shouldBe "*g = *" + case xs => fail(s"Expected 4 assignments, got [${xs.code.mkString(", ")}]") + } + } + + "Return nil and not the desugaring" in { + val nilLiteral = cpg.method.isLambda.methodReturn.toReturn.astChildren.isLiteral.head + nilLiteral.code shouldBe "return nil" + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FieldAccessTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FieldAccessTests.scala index 8f6608977daa..3212b9310e70 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FieldAccessTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FieldAccessTests.scala @@ -4,44 +4,29 @@ import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Identifier, TypeRef} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.shiftleft.semanticcpg.language.* +import io.joern.rubysrc2cpg.passes.Defines.Main +import io.joern.rubysrc2cpg.passes.Defines class FieldAccessTests extends RubyCode2CpgFixture { - "`x.y` is represented by an `x.y` CALL without arguments" in { + "`x.y` is represented by a `x.y` field access" in { val cpg = code(""" - |x.y - |""".stripMargin) + |x = Foo.new + |x.y + |""".stripMargin) - inside(cpg.call("y").headOption) { + inside(cpg.fieldAccess.code("x.y").headOption) { case Some(xyCall) => - xyCall.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH - xyCall.lineNumber shouldBe Some(2) + xyCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + xyCall.name shouldBe Operators.fieldAccess + xyCall.methodFullName shouldBe Operators.fieldAccess + xyCall.lineNumber shouldBe Some(3) xyCall.code shouldBe "x.y" - - inside(xyCall.argumentOption(0)) { - case Some(receiver: Call) => - receiver.name shouldBe Operators.fieldAccess - receiver.code shouldBe "self.x" - case _ => fail("Expected an field access receiver") - } - - inside(xyCall.receiver.headOption) { - case Some(xyBase: Call) => - xyBase.name shouldBe Operators.fieldAccess - xyBase.code shouldBe "x.y" - - val selfX = xyBase.argument(1).asInstanceOf[Call] - selfX.code shouldBe "self.x" - - val yIdentifier = xyBase.argument(2).asInstanceOf[FieldIdentifier] - yIdentifier.code shouldBe "y" - case _ => fail("Expected an field access receiver") - } - case None => fail("Expected a call with the name `y`") + case None => fail("Expected a field access with the code `x.y`") } } - "`self.x` should correctly create a `this` node field base" in { + "`self.x` should correctly create a `self` node field base" in { // Example from railsgoat val cpg = code(""" @@ -55,17 +40,21 @@ class FieldAccessTests extends RubyCode2CpgFixture { |end |""".stripMargin) - inside(cpg.call.name("sick_days_earned").l) { + inside(cpg.fieldAccess.code("self.sick_days_earned").l) { case sickDays :: _ => sickDays.code shouldBe "self.sick_days_earned" - sickDays.name shouldBe "sick_days_earned" - sickDays.dispatchType shouldBe DispatchTypes.DYNAMIC_DISPATCH + sickDays.name shouldBe Operators.fieldAccess + sickDays.methodFullName shouldBe Operators.fieldAccess + sickDays.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH inside(sickDays.argument.l) { - case (self: Identifier) :: Nil => + case (self: Identifier) :: (sickDaysId: FieldIdentifier) :: Nil => self.name shouldBe "self" self.code shouldBe "self" self.typeFullName should endWith("PaidTimeOff") + + sickDaysId.canonicalName shouldBe "@sick_days_earned" + sickDaysId.code shouldBe "sick_days_earned" case xs => fail(s"Expected exactly two field access arguments, instead got [${xs.code.mkString(", ")}]") } case Nil => fail("Expected at least one call with `self` base, but got none.") @@ -83,8 +72,8 @@ class FieldAccessTests extends RubyCode2CpgFixture { | end |end | - |Base64::decode64 # self.Base64.decode64() - |Baz::func1 # self.Baz.func1() + |Base64::decode64() # self.Base64.decode64() + |Baz::func1() # self.Baz.func1() | |# self.Foo = TYPE_REF Foo |class Foo @@ -105,7 +94,7 @@ class FieldAccessTests extends RubyCode2CpgFixture { bazAssign.code shouldBe "self.Baz" val bazTypeRef = baz.argument(2).asInstanceOf[TypeRef] - bazTypeRef.typeFullName shouldBe "Test0.rb:::program.Baz" + bazTypeRef.typeFullName shouldBe s"Test0.rb:$Main.Baz" bazTypeRef.code shouldBe "module Baz (...)" val fooAssign = foo.argument(1).asInstanceOf[Call] @@ -113,27 +102,26 @@ class FieldAccessTests extends RubyCode2CpgFixture { fooAssign.code shouldBe "self.Foo" val fooTypeRef = foo.argument(2).asInstanceOf[TypeRef] - fooTypeRef.typeFullName shouldBe "Test0.rb:::program.Foo" + fooTypeRef.typeFullName shouldBe s"Test0.rb:$Main.Foo" fooTypeRef.code shouldBe "class Foo (...)" case _ => fail(s"Expected two type ref assignments on the module level") } } "give external type accesses on script-level the `self.` base" in { - val call = cpg.method.isModule.call.codeExact("Base64::decode64").head + val call = cpg.method.isModule.call.nameExact("decode64").head call.name shouldBe "decode64" - val base = call.argument(0).asInstanceOf[Call] - base.name shouldBe Operators.fieldAccess - base.code shouldBe "self.Base64" + val base = call.argument(0).asInstanceOf[Identifier] + base.code shouldBe "" val receiver = call.receiver.isCall.head receiver.name shouldBe Operators.fieldAccess - receiver.code shouldBe "Base64.decode64" + receiver.code shouldBe "( = Base64).decode64" val selfArg1 = receiver.argument(1).asInstanceOf[Call] - selfArg1.name shouldBe Operators.fieldAccess - selfArg1.code shouldBe "self.Base64" + selfArg1.name shouldBe Operators.assignment + selfArg1.code shouldBe " = Base64" val selfArg2 = receiver.argument(2).asInstanceOf[FieldIdentifier] selfArg2.canonicalName shouldBe "decode64" @@ -141,20 +129,19 @@ class FieldAccessTests extends RubyCode2CpgFixture { } "give internal type accesses on script-level the `self.` base" in { - val call = cpg.method.isModule.call.codeExact("Baz::func1").head + val call = cpg.method.isModule.call.nameExact("func1").head call.name shouldBe "func1" - val base = call.argument(0).asInstanceOf[Call] - base.name shouldBe Operators.fieldAccess - base.code shouldBe "self.Baz" + val base = call.argument(0).asInstanceOf[Identifier] + base.code shouldBe "" val receiver = call.receiver.isCall.head receiver.name shouldBe Operators.fieldAccess - receiver.code shouldBe "Baz.func1" + receiver.code shouldBe "( = Baz).func1" val selfArg1 = receiver.argument(1).asInstanceOf[Call] - selfArg1.name shouldBe Operators.fieldAccess - selfArg1.code shouldBe "self.Baz" + selfArg1.name shouldBe Operators.assignment + selfArg1.code shouldBe " = Baz" val selfArg2 = receiver.argument(2).asInstanceOf[FieldIdentifier] selfArg2.canonicalName shouldBe "func1" @@ -186,17 +173,16 @@ class FieldAccessTests extends RubyCode2CpgFixture { val call = cpg.method.nameExact("func").call.nameExact("func1").head call.name shouldBe "func1" - val base = call.argument(0).asInstanceOf[Call] - base.name shouldBe Operators.fieldAccess - base.code shouldBe "self.Baz" + val base = call.argument(0).asInstanceOf[Identifier] + base.code shouldBe "" val receiver = call.receiver.isCall.head receiver.name shouldBe Operators.fieldAccess - receiver.code shouldBe "Baz.func1" + receiver.code shouldBe "( = Baz).func1" val selfArg1 = receiver.argument(1).asInstanceOf[Call] - selfArg1.name shouldBe Operators.fieldAccess - selfArg1.code shouldBe "self.Baz" + selfArg1.name shouldBe Operators.assignment + selfArg1.code shouldBe " = Baz" val selfArg2 = receiver.argument(2).asInstanceOf[FieldIdentifier] selfArg2.canonicalName shouldBe "func1" @@ -213,7 +199,7 @@ class FieldAccessTests extends RubyCode2CpgFixture { | end | module C | # TYPE_REF A B func - | A::B::func + | A::B::func() | end | end |end @@ -222,23 +208,24 @@ class FieldAccessTests extends RubyCode2CpgFixture { "create `TYPE_REF` targets for the field accesses" in { val call = cpg.call.nameExact("func").head - val base = call.argument(0).asInstanceOf[Call] - base.name shouldBe Operators.fieldAccess - base.code shouldBe "A::B" - - base.argument(1).asInstanceOf[TypeRef].typeFullName shouldBe "Test0.rb:::program.A" - base.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "B" + val base = call.argument(0).asInstanceOf[Identifier] + base.code shouldBe "" val receiver = call.receiver.isCall.head receiver.name shouldBe Operators.fieldAccess - receiver.code shouldBe "A::B.func" + receiver.code shouldBe "( = A::B).func" val selfArg1 = receiver.argument(1).asInstanceOf[Call] - selfArg1.name shouldBe Operators.fieldAccess - selfArg1.code shouldBe "A::B" + selfArg1.name shouldBe Operators.assignment + selfArg1.code shouldBe " = A::B" + + selfArg1.argument(1).asInstanceOf[Identifier].code shouldBe s"" + + val abRhs = selfArg1.argument(2).asInstanceOf[Call] + abRhs.code shouldBe "A::B" - selfArg1.argument(1).asInstanceOf[TypeRef].typeFullName shouldBe "Test0.rb:::program.A" - selfArg1.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "B" + abRhs.argument(1).asInstanceOf[TypeRef].typeFullName shouldBe s"Test0.rb:$Main.A" + abRhs.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "B" val selfArg2 = receiver.argument(2).asInstanceOf[FieldIdentifier] selfArg2.canonicalName shouldBe "func" diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/HashTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/HashTests.scala index 2cd5e4caa398..ad0bcdaf98b3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/HashTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/HashTests.scala @@ -30,7 +30,7 @@ class HashTests extends RubyCode2CpgFixture { val List(assocCall) = hashCall.inCall.astSiblings.assignment.l val List(x, one) = assocCall.argument.l - x.code shouldBe "[x]" + x.code shouldBe "[:x]" one.code shouldBe "1" } @@ -209,4 +209,62 @@ class HashTests extends RubyCode2CpgFixture { } } + "Splatting argument in hash" in { + val cpg = code(""" + |a = {**x, **y} + |""".stripMargin) + + inside(cpg.call.name(RubyOperators.hashInitializer).l) { + case hashCall :: Nil => + val List(xSplatCall, ySplatCall) = hashCall.inCall.astSiblings.isCall.l + xSplatCall.code shouldBe "**x" + xSplatCall.methodFullName shouldBe RubyOperators.splat + + ySplatCall.code shouldBe "**y" + ySplatCall.methodFullName shouldBe RubyOperators.splat + case xs => fail(s"Expected call to hashInitializer, [${xs.code.mkString(",")}]") + } + } + + "Function call in hash" in { + val cpg = code(""" + |a = {**foo(bar)} + |""".stripMargin) + + inside(cpg.call.name(RubyOperators.hashInitializer).l) { + case hashInitializer :: Nil => + val List(splatCall) = hashInitializer.inCall.astSiblings.isCall.l + splatCall.code shouldBe "**foo(bar)" + splatCall.name shouldBe RubyOperators.splat + + val List(splatCallArg: Call) = splatCall.argument.l: @unchecked + + splatCallArg.code shouldBe "foo(bar)" + + val List(selfCallArg, barCallArg) = splatCallArg.argument.l + barCallArg.code shouldBe "self.bar" + case xs => fail(s"Expected one call for init, got [${xs.code.mkString(",")}]") + } + } + + "Function call without parentheses" in { + val cpg = code(""" + |a = {**(foo 13)} + |""".stripMargin) + + inside(cpg.call.name(RubyOperators.hashInitializer).l) { + case hashInitializer :: Nil => + val List(splatCall) = hashInitializer.inCall.astSiblings.isCall.l + splatCall.code shouldBe "**(foo 13)" + splatCall.name shouldBe RubyOperators.splat + + val List(splatCallArg: Call) = splatCall.argument.l: @unchecked + + splatCallArg.code shouldBe "foo 13" + + val List(selfCallArg, literalCallArg) = splatCallArg.argument.l + literalCallArg.code shouldBe "13" + case xs => fail(s"Expected one call for init, got [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ImportTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ImportTests.scala index 5551d82ec6b1..c39025784a43 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ImportTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ImportTests.scala @@ -1,10 +1,13 @@ package io.joern.rubysrc2cpg.querying +import io.joern.rubysrc2cpg.passes.Defines +import io.joern.rubysrc2cpg.passes.Defines.{Initialize, Main} +import io.joern.rubysrc2cpg.passes.GlobalTypes.{builtinPrefix, kernelPrefix} import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.{ImplicitRequirePass, ImportsPass, TypeImportInfo} +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.nodes.Literal import io.shiftleft.semanticcpg.language.* -import io.joern.rubysrc2cpg.RubySrc2Cpg -import io.joern.rubysrc2cpg.Config -import scala.util.{Success, Failure} import org.scalatest.Inspectors class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with Inspectors { @@ -21,6 +24,42 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In call.argument.where(_.argumentIndexGt(0)).code.l shouldBe List("'test'") } + "`require_relative 'test'` is a CALL node with an IMPORT node pointing to it" in { + val cpg = code(""" + |require_relative 'test' + |""".stripMargin) + val List(importNode) = cpg.imports.l + importNode.importedEntity shouldBe Some("test") + importNode.importedAs shouldBe Some("test") + val List(call) = importNode.call.l + call.callee.name.l shouldBe List("require_relative") + call.argument.where(_.argumentIndexGt(0)).code.l shouldBe List("'test'") + } + + "`load 'test'` is a CALL node with an IMPORT node pointing to it" in { + val cpg = code(""" + |load 'test' + |""".stripMargin) + val List(importNode) = cpg.imports.l + importNode.importedEntity shouldBe Some("test") + importNode.importedAs shouldBe Some("test") + val List(call) = importNode.call.l + call.callee.name.l shouldBe List("load") + call.argument.where(_.argumentIndexGt(0)).code.l shouldBe List("'test'") + } + + "`require_all 'test'` is a CALL node with an IMPORT node pointing to it" in { + val cpg = code(""" + |require_all 'test' + |""".stripMargin) + val List(importNode) = cpg.imports.l + importNode.importedEntity shouldBe Some("test") + importNode.importedAs shouldBe Some("test") + val List(call) = importNode.call.l + call.callee.name.l shouldBe List("require_all") + call.argument.where(_.argumentIndexGt(0)).code.l shouldBe List("'test'") + } + "`begin require 'test' rescue LoadError end` has a CALL node with an IMPORT node pointing to it" in { val cpg = code(""" |begin @@ -59,8 +98,15 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In ) val List(newCall) = - cpg.method.name(":program").filename("t1.rb").ast.isCall.methodFullName(".*:initialize").methodFullName.l - newCall should startWith(s"${path}.rb:") + cpg.method.isModule + .filename("t1.rb") + .ast + .isCall + .dynamicTypeHintFullName + .filter(x => x.startsWith(path) && x.endsWith(Initialize)) + .l + + newCall should startWith(s"$path.rb:") } } @@ -86,11 +132,183 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In |""".stripMargin) val List(methodName) = - cpg.method.name("bar").ast.isCall.methodFullName(".*::program\\.(A|B):foo").methodFullName.l + cpg.method.name("bar").ast.isCall.methodFullName(s".*\\.$Main\\.(A|B).foo").methodFullName.l methodName should endWith(s"${moduleName}:foo") } } + "implicitly imported types in base class" should { + val cpg = code( + """ + |class MyController < ApplicationController + |end + |""".stripMargin, + "app/controllers/my_controller.rb" + ) + .moreCode( + """ + |class ApplicationController + |end + |""".stripMargin, + "app/controllers/application_controller.rb" + ) + .moreCode( + """ + |GEM + | remote: https://rubygems.org/ + | specs: + | zeitwerk (2.2.1) + |""".stripMargin, + "Gemfile.lock" + ) + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*my_controller.rb")).toList) { case List(i) => + i.importedAs shouldBe Some("app/controllers/application_controller") + i.importedEntity shouldBe Some("app/controllers/application_controller") + } + } + } + + "implicitly imported types in base class that are qualified names" should { + val cpg = code( + """ + |class MyController < Controllers::ApplicationController + |end + |""".stripMargin, + "app/controllers/my_controller.rb" + ) + .moreCode( + """ + |module Controllers + | class ApplicationController + | end + |end + |""".stripMargin, + "app/controllers/controllers.rb" + ) + .moreCode( + """ + |GEM + | remote: https://rubygems.org/ + | specs: + | zeitwerk (2.2.1) + |""".stripMargin, + "Gemfile.lock" + ) + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*my_controller.rb")).toList) { case List(i) => + i.importedAs shouldBe Some("app/controllers/controllers") + i.importedEntity shouldBe Some("app/controllers/controllers") + } + } + } + + "implicitly imported types that are qualified names in an include statement" should { + val cpg = code( + """ + |module MyController + | include Controllers::ApplicationController + |end + |""".stripMargin, + "app/controllers/my_controller.rb" + ) + .moreCode( + """ + |module Controllers + | class ApplicationController + | end + |end + |""".stripMargin, + "app/controllers/controllers.rb" + ) + .moreCode( + """ + |GEM + | remote: https://rubygems.org/ + | specs: + | zeitwerk (2.2.1) + |""".stripMargin, + "Gemfile.lock" + ) + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*my_controller.rb")).toList) { case List(i) => + i.importedAs shouldBe Some("app/controllers/controllers") + i.importedEntity shouldBe Some("app/controllers/controllers") + } + } + } + + "implicitly imported types in include statement" should { + val cpg = code( + """ + |class MyController + | include ApplicationController + |end + |""".stripMargin, + "app/controllers/my_controller.rb" + ) + .moreCode( + """ + |class ApplicationController + |end + |""".stripMargin, + "app/controllers/application_controller.rb" + ) + .moreCode( + """ + |GEM + | remote: https://rubygems.org/ + | specs: + | zeitwerk (2.2.1) + |""".stripMargin, + "Gemfile.lock" + ) + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*my_controller.rb")).toList) { case List(i) => + i.importedAs shouldBe Some("app/controllers/application_controller") + i.importedEntity shouldBe Some("app/controllers/application_controller") + } + } + } + + "implicitly imported types in extend statement" should { + val cpg = code( + """ + |class MyController + | extend ApplicationController + |end + |""".stripMargin, + "app/controllers/my_controller.rb" + ) + .moreCode( + """ + |class ApplicationController + |end + |""".stripMargin, + "app/controllers/application_controller.rb" + ) + .moreCode( + """ + |GEM + | remote: https://rubygems.org/ + | specs: + | zeitwerk (2.2.1) + |""".stripMargin, + "Gemfile.lock" + ) + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*my_controller.rb")).toList) { case List(i) => + i.importedAs shouldBe Some("app/controllers/application_controller") + i.importedEntity shouldBe Some("app/controllers/application_controller") + } + } + } + "implicitly imported types (common in frameworks like Ruby on Rails)" should { val cpg = code( @@ -107,7 +325,7 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In | end |end | - |B::bar + |B::bar() |""".stripMargin, "bar/B.rb" ) @@ -119,7 +337,9 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In ) .moreCode( """ - |B.bar + |def func() + | B.bar() + |end |""".stripMargin, "Bar.rb" ) @@ -157,6 +377,15 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In cpg.imports.where(_.call.file.name(".*B.rb")).size shouldBe 0 } + "create a `require` call following the simplified format" in { + val require = cpg.call("require").head + require.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + require.methodFullName shouldBe s"$kernelPrefix.require" + + val strLit = require.argument(1).asInstanceOf[Literal] + strLit.typeFullName shouldBe s"$kernelPrefix.String" + } + } "Builtin Types type-map" should { @@ -170,20 +399,23 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In "resolve calls to builtin functions" in { inside(cpg.call.methodFullName("(pp|csv).*").l) { - case csvParseCall :: csvTableInitCall :: ppCall :: Nil => - csvParseCall.methodFullName shouldBe "csv.CSV:parse" - ppCall.methodFullName shouldBe "pp.PP:pp" - csvTableInitCall.methodFullName shouldBe "csv.CSV.Table:initialize" - case xs => fail(s"Expected three calls, got [${xs.code.mkString(",")}] instead") + case csvParseCall :: csvTableCall :: ppCall :: Nil => + csvParseCall.methodFullName shouldBe "csv.CSV.parse" + csvTableCall.methodFullName shouldBe "csv.CSV.Table.initialize" + ppCall.methodFullName shouldBe "pp.PP.pp" + case xs => fail(s"Expected calls, got [${xs.code.mkString(",")}] instead") } + + // TODO: fixme - set is empty +// cpg.call(Initialize).dynamicTypeHintFullName.toSet should contain("csv.CSV.Table.initialize") } } "`require_all` on a directory" should { val cpg = code(""" |require_all './dir' - |Module1.foo - |Module2.foo + |Module1.foo() + |Module2.foo() |""".stripMargin) .moreCode( """ @@ -206,8 +438,8 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In "allow the resolution for all modules in that directory" in { cpg.call("foo").methodFullName.l shouldBe List( - "dir/module1.rb:::program.Module1:foo", - "dir/module2.rb:::program.Module2:foo" + s"dir/module1.rb:$Main.Module1.foo", + s"dir/module2.rb:$Main.Module2.foo" ) } } @@ -245,9 +477,9 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In |require 'file2' |require 'file3' | - |File1::foo # lib/file1.rb::program:foo - |File2::foo # lib/file2.rb::program:foo - |File3::foo # src/file3.rb::program:foo + |File1::foo # lib/file1.rb.
.foo + |File2::foo # lib/file2.rb.
.foo + |File3::foo # src/file3.rb.
.foo |""".stripMargin, "main.rb" ).moreCode( @@ -279,11 +511,40 @@ class ImportTests extends RubyCode2CpgFixture(withPostProcessing = true) with In "resolve the calls directly" in { inside(cpg.call.name("foo.*").l) { case foo1 :: foo2 :: foo3 :: Nil => - foo1.methodFullName shouldBe "lib/file1.rb:::program.File1:foo" - foo2.methodFullName shouldBe "lib/file2.rb:::program.File2:foo" - foo3.methodFullName shouldBe "src/file3.rb:::program.File3:foo" + foo1.methodFullName shouldBe s"lib/file1.rb:$Main.File1.foo" + foo2.methodFullName shouldBe s"lib/file2.rb:$Main.File2.foo" + foo3.methodFullName shouldBe s"src/file3.rb:$Main.File3.foo" case xs => fail(s"Expected 3 calls, got [${xs.code.mkString(",")}] instead") } } } } + +class ImportWithAutoloadedExternalGemsTests extends RubyCode2CpgFixture(withPostProcessing = false) { + + "use of a type specified as external" should { + + val cpg = code( + """ + |x = Base64.encode("Hello, world!") + |Bar::Foo.new + |""".stripMargin, + "encoder.rb" + ) + + ImplicitRequirePass(cpg, TypeImportInfo("Base64", "base64") :: TypeImportInfo("Bar", "foobar") :: Nil) + .createAndApply() + ImportsPass(cpg).createAndApply() + + "result in require statement of the file containing the symbol" in { + inside(cpg.imports.where(_.call.file.name(".*encoder.rb")).toList) { case List(i1, i2) => + i1.importedAs shouldBe Some("base64") + i1.importedEntity shouldBe Some("base64") + + i2.importedAs shouldBe Some("foobar") + i2.importedEntity shouldBe Some("foobar") + } + } + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IndexAccessTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IndexAccessTests.scala index 3a420aa3265e..8ea753448535 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IndexAccessTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IndexAccessTests.scala @@ -47,4 +47,27 @@ class IndexAccessTests extends RubyCode2CpgFixture { two.code shouldBe "2" } + "Index Access with `.[](index)`" in { + val cpg = code(""" + |class Foo + | def extract_url + | @params.dig(:event, :links)&.first&.[](:url) + | end + |end + |""".stripMargin) + + inside(cpg.call.name(Operators.indexAccess).l) { + case indexCall :: Nil => + indexCall.code shouldBe "@params.dig(:event, :links)&.first&.[](:url)" + + inside(indexCall.argument.l) { + case target :: index :: Nil => + target.code shouldBe "( = @params.dig(:event, :links))&.first" + index.code shouldBe ":url" + case xs => fail(s"Expected target and index, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one index access, got [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/LiteralTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/LiteralTests.scala index 9b579319aaa4..7a3d86516a9b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/LiteralTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/LiteralTests.scala @@ -248,4 +248,22 @@ class LiteralTests extends RubyCode2CpgFixture { formatValueCall.methodFullName shouldBe Operators.formatString } + "-> Lambda literal" in { + val cpg = code(""" + |-> (a, *b, &c) {} + |""".stripMargin) + + inside(cpg.method.isLambda.l) { + case lambdaLiteral :: Nil => + inside(lambdaLiteral.parameter.l) { + case _ :: aParam :: bParam :: cParam :: Nil => + aParam.code shouldBe "a" + bParam.code shouldBe "*b" + cParam.code shouldBe "&c" + case xs => fail(s"Expected four parameters, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one lambda, got [${xs.name.mkString(",")}]") + } + } + } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodReturnTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodReturnTests.scala index d2cc7fd8775a..17687062f503 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodReturnTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodReturnTests.scala @@ -1,10 +1,10 @@ package io.joern.rubysrc2cpg.querying -import io.joern.rubysrc2cpg.passes.Defines.RubyOperators -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.Operators +import io.joern.rubysrc2cpg.passes.Defines.{Main, RubyOperators} import io.joern.rubysrc2cpg.passes.GlobalTypes.kernelPrefix -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Literal, Method, MethodRef, Return} +import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { @@ -72,7 +72,7 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { r.lineNumber shouldBe Some(3) val List(c: Call) = r.astChildren.isCall.l - c.methodFullName shouldBe s"$kernelPrefix:puts" + c.methodFullName shouldBe s"$kernelPrefix.puts" c.lineNumber shouldBe Some(3) c.code shouldBe "puts x" } @@ -339,7 +339,7 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { } } - "implicit RETURN node for ASSOCIATION" in { + "implicit RETURN node for super call" in { val cpg = code(""" |def j | super(only: ["a"]) @@ -350,11 +350,11 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { case jMethod :: Nil => inside(jMethod.methodReturn.toReturn.l) { case retAssoc :: Nil => - retAssoc.code shouldBe "only: [\"a\"]" + retAssoc.code shouldBe "super(only: [\"a\"])" val List(call: Call) = retAssoc.astChildren.l: @unchecked - call.name shouldBe RubyOperators.association - call.code shouldBe "only: [\"a\"]" + call.name shouldBe "super" + call.code shouldBe "super(only: [\"a\"])" case xs => fail(s"Expected exactly one return nodes, instead got [${xs.code.mkString(",")}]") } case _ => fail("Only one method expected") @@ -380,7 +380,7 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { inside(bar.astChildren.collectAll[Method].l) { case closureMethod :: Nil => closureMethod.name shouldBe "0" - closureMethod.fullName shouldBe "Test0.rb:::program:bar:0" + closureMethod.fullName shouldBe s"Test0.rb:$Main.bar.0" case xs => fail(s"Expected closure method, but found ${xs.code.mkString(", ")} instead") } @@ -392,8 +392,8 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { returnCall.name shouldBe "foo" - val List(_, arg: MethodRef) = returnCall.argument.l: @unchecked - arg.methodFullName shouldBe "Test0.rb:::program:bar:0" + val List(_, arg: TypeRef) = returnCall.argument.l: @unchecked + arg.typeFullName shouldBe s"Test0.rb:$Main.bar.0&Proc" case xs => fail(s"Expected one call for return, but found ${xs.code.mkString(", ")} instead") } @@ -439,4 +439,78 @@ class MethodReturnTests extends RubyCode2CpgFixture(withDataFlow = true) { } } + "a return in an expression position without arguments should generate a return node with no children" in { + val cpg = code(""" + |def foo + | return unless baz() + | bar() + |end + |""".stripMargin) + + inside(cpg.method.nameExact("foo").ast.isReturn.headOption) { + case Some(ret) => + ret.code shouldBe "return" + ret.astChildren.size shouldBe 0 + ret.astParent.astParent.code shouldBe "return unless baz()" + case None => fail(s"Expected at least one return node") + } + } + + "a return with multiple values" in { + val cpg = code(""" + |def foo + | return 1, :z => 1 + |end + |""".stripMargin) + + inside(cpg.method.nameExact("foo").ast.isReturn.headOption) { + case Some(ret) => + val List(oneLiteral: Literal, zAssoc: Call) = ret.astChildren.l: @unchecked + oneLiteral.code shouldBe "1" + zAssoc.code shouldBe ":z => 1" + zAssoc.methodFullName shouldBe RubyOperators.association + + inside(zAssoc.argument.l) { + case (key: Literal) :: (value: Literal) :: Nil => + key.code shouldBe ":z" + value.code shouldBe "1" + case xs => fail(s"Expected two args, got ${xs.code.mkString(",")}") + } + case None => fail(s"Expected at least one retrun node") + } + } + + "Return with methodInvocationWithoutParentheses" in { + val cpg = code(""" + |def foo() + | return render json: {}, status: :internal_server_error unless success + |end + |""".stripMargin) + + inside(cpg.method.name("foo").body.astChildren.isControlStructure.l) { + case ifNode :: Nil => + ifNode.controlStructureType shouldBe ControlStructureTypes.IF + + val List(notCall: Call) = ifNode.condition.l: @unchecked + notCall.methodFullName shouldBe Operators.logicalNot + + val List(ifReturnTrue: Return) = ifNode.whenTrue.isBlock.astChildren.isReturn.l + ifReturnTrue.code shouldBe "return render json: {}, status: :internal_server_error" + + val List(_, jsonArg: Block, statusArg: Literal) = ifReturnTrue.astChildren.isCall.argument.l: @unchecked + jsonArg.argumentName shouldBe Some("json") + jsonArg.code shouldBe "" + + val List(_: Identifier, hashInitCall: Call) = jsonArg.astChildren.isCall.argument.l: @unchecked + hashInitCall.methodFullName shouldBe RubyOperators.hashInitializer + + statusArg.argumentName shouldBe Some("status") + statusArg.code shouldBe ":internal_server_error" + + val List(ifReturnFalse: Return) = ifNode.whenFalse.isBlock.astChildren.isReturn.l + ifReturnFalse.code shouldBe "return nil" + + case xs => fail(s"Expected two method returns, got [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala index 4e6b8bc2daa7..ee887bcce687 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala @@ -1,23 +1,26 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.Defines as RDefines +import io.joern.rubysrc2cpg.passes.Defines.Main import io.joern.rubysrc2cpg.passes.GlobalTypes.kernelPrefix import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, NodeTypes, Operators} import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.{Assignment, FieldAccess} class MethodTests extends RubyCode2CpgFixture { "`def f(x) = 1`" should { val cpg = code(""" |def f(x) = 1 + |f(1) |""".stripMargin) "be represented by a METHOD node" in { val List(f) = cpg.method.name("f").l - f.fullName shouldBe "Test0.rb:::program:f" + f.fullName shouldBe s"Test0.rb:$Main.f" f.isExternal shouldBe false f.lineNumber shouldBe Some(2) f.numberOfLines shouldBe 1 @@ -26,21 +29,33 @@ class MethodTests extends RubyCode2CpgFixture { x.index shouldBe 1 x.isVariadic shouldBe false x.lineNumber shouldBe Some(2) + + val List(fSelf) = f.parameter.name(RDefines.Self).l + fSelf.index shouldBe 0 + fSelf.isVariadic shouldBe false + fSelf.lineNumber shouldBe Some(2) + fSelf.referencingIdentifiers.size shouldBe 0 + + val List(mSelf) = cpg.method.isModule.parameter.name(RDefines.Self).l + mSelf.index shouldBe 0 + mSelf.isVariadic shouldBe false + mSelf.lineNumber shouldBe Some(1) + mSelf.referencingIdentifiers.size shouldBe 3 } "have a corresponding bound type" in { val List(fType) = cpg.typeDecl("f").l - fType.fullName shouldBe "Test0.rb:::program:f" + fType.fullName shouldBe s"Test0.rb:$Main.f" fType.code shouldBe "def f(x) = 1" - fType.astParentFullName shouldBe "Test0.rb:::program" + fType.astParentFullName shouldBe s"Test0.rb:$Main" fType.astParentType shouldBe NodeTypes.METHOD val List(fMethod) = fType.iterator.boundMethod.l - fType.fullName shouldBe "Test0.rb:::program:f" + fType.fullName shouldBe s"Test0.rb:$Main.f" } "create a 'fake' method for the file" in { - val List(m) = cpg.method.nameExact(RDefines.Program).l - m.fullName shouldBe "Test0.rb:::program" + val List(m) = cpg.method.nameExact(RDefines.Main).l + m.fullName shouldBe s"Test0.rb:$Main" m.isModule.nonEmpty shouldBe true } } @@ -54,7 +69,7 @@ class MethodTests extends RubyCode2CpgFixture { val List(f) = cpg.method.name("f").l - f.fullName shouldBe "Test0.rb:::program:f" + f.fullName shouldBe s"Test0.rb:$Main.f" f.isExternal shouldBe false f.lineNumber shouldBe Some(2) f.numberOfLines shouldBe 3 @@ -68,7 +83,7 @@ class MethodTests extends RubyCode2CpgFixture { val List(f) = cpg.method.name("f").l - f.fullName shouldBe "Test0.rb:::program:f" + f.fullName shouldBe s"Test0.rb:$Main.f" f.isExternal shouldBe false f.lineNumber shouldBe Some(2) f.numberOfLines shouldBe 1 @@ -86,7 +101,7 @@ class MethodTests extends RubyCode2CpgFixture { val List(f) = cpg.method.name("f").l - f.fullName shouldBe "Test0.rb:::program:f" + f.fullName shouldBe s"Test0.rb:$Main.f" f.isExternal shouldBe false f.lineNumber shouldBe Some(2) f.numberOfLines shouldBe 1 @@ -193,7 +208,7 @@ class MethodTests extends RubyCode2CpgFixture { inside(funcF.parameter.l) { case thisParam :: xParam :: Nil => thisParam.code shouldBe RDefines.Self - thisParam.typeFullName shouldBe "Test0.rb:::program.C" + thisParam.typeFullName shouldBe s"Test0.rb:$Main.C" thisParam.index shouldBe 0 thisParam.isVariadic shouldBe false @@ -227,7 +242,7 @@ class MethodTests extends RubyCode2CpgFixture { inside(funcF.parameter.l) { case thisParam :: xParam :: Nil => thisParam.code shouldBe RDefines.Self - thisParam.typeFullName shouldBe "Test0.rb:::program.C" + thisParam.typeFullName shouldBe s"Test0.rb:$Main.C" thisParam.index shouldBe 0 thisParam.isVariadic shouldBe false @@ -351,7 +366,7 @@ class MethodTests extends RubyCode2CpgFixture { case thisParam :: xParam :: Nil => thisParam.name shouldBe RDefines.Self thisParam.code shouldBe "F" - thisParam.typeFullName shouldBe "Test0.rb:::program.F" + thisParam.typeFullName shouldBe s"Test0.rb:$Main.F" xParam.name shouldBe "x" case xs => fail(s"Expected two parameters, got ${xs.name.mkString(", ")}") @@ -361,7 +376,7 @@ class MethodTests extends RubyCode2CpgFixture { case thisParam :: xParam :: Nil => thisParam.name shouldBe RDefines.Self thisParam.code shouldBe "F" - thisParam.typeFullName shouldBe "Test0.rb:::program.F" + thisParam.typeFullName shouldBe s"Test0.rb:$Main.F" xParam.name shouldBe "x" xParam.code shouldBe "x" @@ -374,17 +389,17 @@ class MethodTests extends RubyCode2CpgFixture { // TODO: we cannot bind baz as this is a dynamic assignment to `F` which is trickier to determine // Also, double check bindings "have bindings to the singleton module TYPE_DECL" ignore { - cpg.typeDecl.name("F").methodBinding.methodFullName.l shouldBe List("Test0.rb:::program.F:bar") + cpg.typeDecl.name("F").methodBinding.methodFullName.l shouldBe List(s"Test0.rb:$Main.F.bar") } - "baz should not exist in the :program block" in { - inside(cpg.method.name(":program").l) { + "baz should not exist in the
block" in { + inside(cpg.method.isModule.l) { case prog :: Nil => inside(prog.block.astChildren.isMethod.name("baz").l) { case Nil => // passing case case _ => fail("Baz should not exist under program method block") } - case _ => fail("Expected one Method for :program") + case _ => fail("Expected one Method for ") } } } @@ -399,7 +414,7 @@ class MethodTests extends RubyCode2CpgFixture { "be represented by a METHOD node" in { inside(cpg.method.name("exists\\?").l) { case existsMethod :: Nil => - existsMethod.fullName shouldBe "Test0.rb:::program:exists?" + existsMethod.fullName shouldBe s"Test0.rb:$Main.exists?" existsMethod.isExternal shouldBe false inside(existsMethod.methodReturn.cfgIn.l) { @@ -509,20 +524,16 @@ class MethodTests extends RubyCode2CpgFixture { |""".stripMargin) "Should be represented as a TRY structure" in { - inside(cpg.method.name("foo").tryBlock.l) { - case tryBlock :: Nil => - tryBlock.controlStructureType shouldBe ControlStructureTypes.TRY - - inside(tryBlock.astChildren.l) { - case body :: ensureBody :: Nil => - body.ast.isLiteral.code.l shouldBe List("1") - body.order shouldBe 1 - - ensureBody.ast.isLiteral.code.l shouldBe List("2") - ensureBody.order shouldBe 3 - case xs => fail(s"Expected body and ensureBody, got ${xs.code.mkString(", ")} instead") - } - case xs => fail(s"Expected one method, found ${xs.method.name.mkString(", ")} instead") + inside(cpg.method.name("foo").controlStructure.l) { + case tryStruct :: ensureStruct :: Nil => + tryStruct.controlStructureType shouldBe ControlStructureTypes.TRY + val body = tryStruct.astChildren.head + body.ast.isLiteral.code.l shouldBe List("1") + + ensureStruct.controlStructureType shouldBe ControlStructureTypes.FINALLY + ensureStruct.ast.isLiteral.code.l shouldBe List("2") + + case xs => fail(s"Expected two structures, got ${xs.code.mkString(",")}") } } } @@ -548,23 +559,23 @@ class MethodTests extends RubyCode2CpgFixture { leftArg.name shouldBe "a" rightArg.name shouldBe "hexdigest" - rightArg.code shouldBe "Digest::MD5.hexdigest(password)" + rightArg.code shouldBe "( = Digest::MD5).hexdigest(password)" - inside(rightArg.argument.l) { - case (md5: Call) :: (passwordArg: Identifier) :: Nil => - md5.name shouldBe Operators.fieldAccess - md5.code shouldBe "Digest::MD5" + val hexDigestFa = rightArg.receiver.head.asInstanceOf[FieldAccess] + hexDigestFa.code shouldBe "( = Digest::MD5).hexdigest" - val md5Base = md5.argument(1).asInstanceOf[Call] - md5.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "MD5" + val tmp1Assign = hexDigestFa.argument(1).asInstanceOf[Assignment] + tmp1Assign.code shouldBe " = Digest::MD5" - md5Base.name shouldBe Operators.fieldAccess - md5Base.code shouldBe "self.Digest" + val md5Fa = tmp1Assign.source.asInstanceOf[FieldAccess] + md5Fa.code shouldBe "( = Digest)::MD5" - md5Base.argument(1).asInstanceOf[Identifier].name shouldBe RDefines.Self - md5Base.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "Digest" - case xs => fail(s"Expected identifier and call, got ${xs.code.mkString(", ")} instead") - } + val tmp0Assign = md5Fa.argument(1).asInstanceOf[Assignment] + tmp0Assign.code shouldBe " = Digest" + + val digestFa = tmp0Assign.source.asInstanceOf[FieldAccess] + digestFa.argument(1).asInstanceOf[Identifier].name shouldBe RDefines.Self + digestFa.argument(2).asInstanceOf[FieldIdentifier].canonicalName shouldBe "Digest" case xs => fail(s"Expected 2 arguments, got ${xs.code.mkString(", ")} instead") } case None => fail("Expected if-condition") @@ -601,7 +612,7 @@ class MethodTests extends RubyCode2CpgFixture { ) "be directly under :program" in { - inside(cpg.method.name(RDefines.Program).filename("t1.rb").assignment.l) { + inside(cpg.method.name(RDefines.Main).filename("t1.rb").assignment.l) { case moduleAssignment :: classAssignment :: methodAssignment :: Nil => moduleAssignment.code shouldBe "self.A = module A (...)" classAssignment.code shouldBe "self.B = class B (...)" @@ -611,7 +622,7 @@ class MethodTests extends RubyCode2CpgFixture { case (lhs: Call) :: (rhs: TypeRef) :: Nil => lhs.code shouldBe "self.A" lhs.name shouldBe Operators.fieldAccess - rhs.typeFullName shouldBe "t1.rb:::program.A" + rhs.typeFullName shouldBe s"t1.rb:$Main.A" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } @@ -619,7 +630,7 @@ class MethodTests extends RubyCode2CpgFixture { case (lhs: Call) :: (rhs: TypeRef) :: Nil => lhs.code shouldBe "self.B" lhs.name shouldBe Operators.fieldAccess - rhs.typeFullName shouldBe "t1.rb:::program.B" + rhs.typeFullName shouldBe s"t1.rb:$Main.B" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } @@ -627,8 +638,8 @@ class MethodTests extends RubyCode2CpgFixture { case (lhs: Call) :: (rhs: MethodRef) :: Nil => lhs.code shouldBe "self.c" lhs.name shouldBe Operators.fieldAccess - rhs.methodFullName shouldBe "t1.rb:::program:c" - rhs.typeFullName shouldBe "t1.rb:::program:c" + rhs.methodFullName shouldBe s"t1.rb:$Main.c" + rhs.typeFullName shouldBe s"t1.rb:$Main.c" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } @@ -637,7 +648,7 @@ class MethodTests extends RubyCode2CpgFixture { } "not be present in other files" in { - inside(cpg.method.name(RDefines.Program).filename("t2.rb").assignment.l) { + inside(cpg.method.name(RDefines.Main).filename("t2.rb").assignment.l) { case classAssignment :: methodAssignment :: Nil => classAssignment.code shouldBe "self.D = class D (...)" methodAssignment.code shouldBe "self.e = def e (...)" @@ -646,7 +657,7 @@ class MethodTests extends RubyCode2CpgFixture { case (lhs: Call) :: (rhs: TypeRef) :: Nil => lhs.code shouldBe "self.D" lhs.name shouldBe Operators.fieldAccess - rhs.typeFullName shouldBe "t2.rb:::program.D" + rhs.typeFullName shouldBe s"t2.rb:$Main.D" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } @@ -654,8 +665,8 @@ class MethodTests extends RubyCode2CpgFixture { case (lhs: Call) :: (rhs: MethodRef) :: Nil => lhs.code shouldBe "self.e" lhs.name shouldBe Operators.fieldAccess - rhs.methodFullName shouldBe "t2.rb:::program:e" - rhs.typeFullName shouldBe "t2.rb:::program:e" + rhs.methodFullName shouldBe s"t2.rb:$Main.e" + rhs.typeFullName shouldBe s"t2.rb:$Main.e" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } @@ -664,15 +675,340 @@ class MethodTests extends RubyCode2CpgFixture { } "be placed in order of definition" in { - inside(cpg.method.name(RDefines.Program).filename("t1.rb").block.astChildren.l) { + inside(cpg.method.name(RDefines.Main).filename("t1.rb").block.astChildren.isCall.l) { case (a1: Call) :: (a2: Call) :: (a3: Call) :: (a4: Call) :: (a5: Call) :: Nil => a1.code shouldBe "self.A = module A (...)" - a2.code shouldBe "self::A::" + a2.code shouldBe "( = self::A)::()" a3.code shouldBe "self.B = class B (...)" - a4.code shouldBe "self::B::" + a4.code shouldBe "( = self::B)::()" a5.code shouldBe "self.c = def c (...)" case xs => fail(s"Expected assignments to appear before definitions, instead got [${xs.mkString("\n")}]") } } } + + "Splatting and normal argument" in { + val cpg = code(""" + |def foo(*x, y) + |end + |""".stripMargin) + + inside(cpg.method.name("foo").l) { + case fooMethod :: Nil => + inside(fooMethod.method.parameter.l) { + case selfArg :: splatArg :: normalArg :: Nil => + splatArg.code shouldBe "*x" + splatArg.index shouldBe 1 + + normalArg.code shouldBe "y" + normalArg.index shouldBe 2 + case xs => fail(s"Expected two parameters, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one method, got [${xs.code.mkString(",")}]") + } + } + + "Splatting argument in call" in { + val cpg = code(""" + |def foo(a, b) + |end + | + |x = 1,2 + |foo(*x, y) + |""".stripMargin) + + inside(cpg.call.name("foo").l) { + case fooCall :: Nil => + inside(fooCall.argument.l) { + case selfArg :: xArg :: yArg :: Nil => + xArg.code shouldBe "*x" + yArg.code shouldBe "self.y" + case xs => fail(s"Expected two args, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one call to foo, got [${xs.code.mkString(",")}]") + } + } + + "a nested method declaration inside of a do-block should connect the member node to the bound type decl" in { + val cpg = code(""" + |foo do + | def bar + | end + |end + |""".stripMargin) + + val parentType = cpg.member("bar").typeDecl.head + parentType.isLambda should not be empty + parentType.methodBinding.methodFullName.head should endWith("0") + } + + "a method that is redefined should have a counter suffixed to ensure uniqueness" in { + val cpg = code(""" + |def foo;end + |def bar;end + |def foo;end + |def foo;end + |""".stripMargin) + + cpg.method.name("(foo|bar).*").name.l shouldBe List("foo", "bar", "foo", "foo") + cpg.method.name("(foo|bar).*").fullName.l shouldBe List( + s"Test0.rb:$Main.foo", + s"Test0.rb:$Main.bar", + s"Test0.rb:$Main.foo0", + s"Test0.rb:$Main.foo1" + ) + } + + "MemberCall with a function name the same as a reserved keyword" in { + val cpg = code(""" + |batch.retry!() + |""".stripMargin) + + inside(cpg.call.name(".*retry!").l) { + case batchCall :: Nil => + batchCall.name shouldBe "retry!" + batchCall.code shouldBe "( = batch).retry!()" + + inside(batchCall.receiver.l) { + case (receiverCall: Call) :: Nil => + receiverCall.name shouldBe Operators.fieldAccess + receiverCall.code shouldBe "( = batch).retry!" + + val selfBatch = receiverCall.argument(1).asInstanceOf[Call] + selfBatch.code shouldBe " = batch" + + val retry = receiverCall.argument(2).asInstanceOf[FieldIdentifier] + retry.code shouldBe "retry!" + + case xs => fail(s"Expected one receiver for call, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one method for batch.retry, got [${xs.code.mkString(",")}]") + } + } + + "Call with :: syntax and reserved keyword" in { + val cpg = code(""" + |batch::retry!() + |""".stripMargin) + + inside(cpg.call.name(".*retry!").l) { + case batchCall :: Nil => + batchCall.name shouldBe "retry!" + batchCall.code shouldBe "( = batch)::retry!()" + + inside(batchCall.receiver.l) { + case (receiverCall: Call) :: Nil => + receiverCall.name shouldBe Operators.fieldAccess + receiverCall.code shouldBe "( = batch).retry!" + + val selfBatch = receiverCall.argument(1).asInstanceOf[Call] + selfBatch.code shouldBe " = batch" + + val retry = receiverCall.argument(2).asInstanceOf[FieldIdentifier] + retry.code shouldBe "retry!" + + case xs => fail(s"Expected one receiver for call, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one method for batch.retry, got [${xs.code.mkString(",")}]") + } + } + + "Call with reserved keyword as base and call name using . notation" in { + val cpg = code(""" + |retry.retry!() + |""".stripMargin) + + inside(cpg.call.name(".*retry!").l) { + case batchCall :: Nil => + batchCall.name shouldBe "retry!" + batchCall.code shouldBe "( = retry).retry!()" + + inside(batchCall.receiver.l) { + case (receiverCall: Call) :: Nil => + receiverCall.name shouldBe Operators.fieldAccess + receiverCall.code shouldBe "( = retry).retry!" + + val selfBatch = receiverCall.argument(1).asInstanceOf[Call] + selfBatch.code shouldBe " = retry" + + val retry = receiverCall.argument(2).asInstanceOf[FieldIdentifier] + retry.code shouldBe "retry!" + + case xs => fail(s"Expected one receiver for call, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one method for batch.retry, got [${xs.code.mkString(",")}]") + } + } + + "Call with reserved keyword as base and call name" in { + val cpg = code(""" + |retry::retry!() + |""".stripMargin) + + inside(cpg.call.name(".*retry!").l) { + case batchCall :: Nil => + batchCall.name shouldBe "retry!" + batchCall.code shouldBe "( = retry)::retry!()" + + inside(batchCall.receiver.l) { + case (receiverCall: Call) :: Nil => + receiverCall.name shouldBe Operators.fieldAccess + receiverCall.code shouldBe "( = retry).retry!" + + val selfBatch = receiverCall.argument(1).asInstanceOf[Call] + selfBatch.code shouldBe " = retry" + + val retry = receiverCall.argument(2).asInstanceOf[FieldIdentifier] + retry.code shouldBe "retry!" + + case xs => fail(s"Expected one receiver for call, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one method for batch.retry, got [${xs.code.mkString(",")}]") + } + } + + "%x should be represented as a call to EXEC" in { + val cpg = code(""" + |%x(ls -l) + |""".stripMargin) + + inside(cpg.call.name("exec").l) { + case execCall :: Nil => + execCall.name shouldBe "exec" + inside(execCall.argument.l) { + case selfArg :: lsArg :: Nil => + selfArg.code shouldBe "self" + lsArg.code shouldBe "ls -l" + case xs => fail(s"expected 2 arguments, got [${xs.code.mkString(",")}]") + } + case xs => fail(s"Expected one call to exec, got [${xs.code.mkString(",")}]") + } + } + + "MemberAccessCommand with two parameters" in { + val cpg = code("foo&.bar 1,2") + + inside(cpg.call.name("bar").l) { + case barCall :: Nil => + inside(barCall.argument.l) { + case _ :: (arg1: Literal) :: (arg2: Literal) :: Nil => + arg1.code shouldBe "1" + arg1.typeFullName shouldBe RDefines.getBuiltInType(RDefines.Integer) + + arg2.code shouldBe "2" + arg2.typeFullName shouldBe RDefines.getBuiltInType(RDefines.Integer) + case xs => fail(s"Expected three args, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one call, got [${xs.code.mkString(",")}]") + } + } + + "Method def in class defined in a namespace" in { + val cpg = code(""" + |class Api::V1::MobileController + | def show + | end + |end + |""".stripMargin) + + inside(cpg.method.name("show").l) { + case showMethod :: Nil => + showMethod.astParentFullName shouldBe "Test0.rb:
.Api.V1.MobileController" + showMethod.astParentType shouldBe NodeTypes.TYPE_DECL + case xs => fail(s"Expected one methood, got ${xs.name.mkString(",")}") + } + } + + "Method def with mandatory arg after splat arg" in { + val cpg = code(""" + |def foo(a=1, *b, c) + |end + |""".stripMargin) + + inside(cpg.method.name("foo").parameter.l) { + case _ :: aParam :: bParam :: cParam :: Nil => + aParam.code shouldBe "a=1" + bParam.code shouldBe "*b" + cParam.code shouldBe "c" + case xs => fail(s"Expected 4 params, got ${xs.code.mkString(",")}") + } + } + + "Unnamed proc parameters" should { + val cpg = code(""" + |def outer_method(&) + | puts "In outer_method" + | inner_method(&) + |end + | + |def inner_method(&) + | puts "In inner_method" + | yield if block_given? + |end + | + |outer_method do + | puts "Hello from the block!" + |end + |""".stripMargin) + + "generate and reference proc param" in { + inside(cpg.method.name("outer_method").l) { + case outerMethod :: Nil => + val List(_, procParam) = outerMethod.parameter.l + procParam.name shouldBe "" + + inside(outerMethod.call.name("inner_method").argument.l) { + case _ :: procParamArg :: Nil => + procParamArg.code shouldBe "" + case xs => fail(s"Expected two arguments, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected one method def, got [${xs.name.mkString(",")}]") + } + } + + "call correct proc param in `yield`" in { + inside(cpg.method.name("inner_method").l) { + case innerMethod :: Nil => + val List(_, procParam) = innerMethod.parameter.l + procParam.name shouldBe "" + + innerMethod.call.nameExact("call").argument.isIdentifier.name.l shouldBe List("") + + case xs => fail(s"Expected one method def, got [${xs.name.mkString(",")}]") + } + } + } + + "lambdas as arguments to a long chained call" should { + val cpg = code(""" + |def foo(xs, total_ys, hex_values) + | xs.map.with_index { |f, i| [f / total_ys, hex_values[i]] } # 1 + | .sort_by { |r| -r[0] } # 2 + | .reject { |r| r[1].size == 8 && r[1].end_with?('00') } # 3 + | .map { |r| Foo::Bar::Baz.new(*r[1][0..5].scan(/../).map { |c| c.to_i(16) }) } # 4 & 5 + | .slice(0, quantity) + | end + |""".stripMargin) + + "not write lambda nodes that are already assigned to some temp variable" in { + cpg.typeRef.typeFullName(".*Proc").size shouldBe 5 + cpg.typeRef.whereNot(_.astParent).size shouldBe 0 + } + + "resolve cached lambdas correctly" in { + def getLineNumberOfLambdaForCall(callName: String) = + cpg.call.nameExact(callName).argument.isTypeRef.typ.referencedTypeDecl.lineNumber.head + + getLineNumberOfLambdaForCall("with_index") shouldBe 3 + getLineNumberOfLambdaForCall("sort_by") shouldBe 4 + getLineNumberOfLambdaForCall("reject") shouldBe 5 + getLineNumberOfLambdaForCall("map") shouldBe 6 + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ModuleTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ModuleTests.scala index 818e191e8b8d..236be162c313 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ModuleTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ModuleTests.scala @@ -1,7 +1,10 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.passes.Defines +import io.joern.rubysrc2cpg.passes.Defines.Main import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.NodeTypes +import io.shiftleft.codepropertygraph.generated.nodes.{File, NamespaceBlock} import io.shiftleft.semanticcpg.language.* class ModuleTests extends RubyCode2CpgFixture { @@ -14,7 +17,7 @@ class ModuleTests extends RubyCode2CpgFixture { val List(m) = cpg.typeDecl.name("M").l - m.fullName shouldBe "Test0.rb:::program.M" + m.fullName shouldBe s"Test0.rb:$Main.M" m.lineNumber shouldBe Some(2) m.baseType.l shouldBe List() m.member.name.l shouldBe List(Defines.TypeDeclBody) @@ -31,11 +34,47 @@ class ModuleTests extends RubyCode2CpgFixture { val List(m) = cpg.typeDecl.name("M1").l - m.fullName shouldBe "Test0.rb:::program.M1" + m.fullName shouldBe s"Test0.rb:$Main.M1" m.lineNumber shouldBe Some(2) m.baseType.l shouldBe List() m.member.name.l shouldBe List(Defines.TypeDeclBody) m.method.name.l shouldBe List(Defines.TypeDeclBody) } + "Module defined in Namespace" in { + val cpg = code(""" + |module Api::V1::MobileController + |end + |""".stripMargin) + + inside(cpg.namespaceBlock.fullNameExact("Api.V1").typeDecl.l) { + case mobileNamespace :: mobileClassNamespace :: Nil => + mobileNamespace.name shouldBe "MobileController" + mobileNamespace.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + + mobileClassNamespace.name shouldBe "MobileController" + mobileClassNamespace.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + case xs => fail(s"Expected two namespace blocks, got ${xs.code.mkString(",")}") + } + + inside(cpg.typeDecl.name("MobileController").l) { + case mobileTypeDecl :: Nil => + mobileTypeDecl.name shouldBe "MobileController" + mobileTypeDecl.fullName shouldBe "Test0.rb:
.Api.V1.MobileController" + mobileTypeDecl.astParentFullName shouldBe "Api.V1" + mobileTypeDecl.astParentType shouldBe NodeTypes.NAMESPACE_BLOCK + + mobileTypeDecl.astParent.isNamespaceBlock shouldBe true + + val namespaceDecl = mobileTypeDecl.astParent.asInstanceOf[NamespaceBlock] + namespaceDecl.name shouldBe "Api.V1" + namespaceDecl.filename shouldBe "Test0.rb" + + namespaceDecl.astParent.isFile shouldBe true + val parentFileDecl = namespaceDecl.astParent.asInstanceOf[File] + parentFileDecl.name shouldBe "Test0.rb" + + case xs => fail(s"Expected one class decl, got [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala index 67abaca49a42..566e84cf8756 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ProcParameterAndYieldTests.scala @@ -1,84 +1,120 @@ package io.joern.rubysrc2cpg.querying +import io.joern.rubysrc2cpg.passes.Defines.RubyOperators import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.Operators -import org.scalatest.Inspectors -import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* +import org.scalatest.Inspectors class ProcParameterAndYieldTests extends RubyCode2CpgFixture with Inspectors { - "Methods" should { - "with a yield expression" should { - "with a proc parameter" should { - val cpg1 = code("def foo(&b) yield end") - val cpg2 = code("def self.foo(&b) yield end") - val cpgs = List(cpg1, cpg2) - - "have a single block argument" in { - forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l shouldBe List("b")) - } - "represent the yield as a conditional with a call and return node as children" in { - forAll(cpgs) { cpg => - inside(cpg.method("foo").call.nameExact(Operators.conditional).code("yield").astChildren.l) { - case List(cond: Expression, call: Call, ret: Return) => { - cond.code shouldBe "" - call.name shouldBe "b" - call.code shouldBe "b" - ret.code shouldBe "yield" - } - } - } - } - } - - "without a proc parameter" should { - val cpg = code(""" - |def foo() yield end - |def self.bar() yield end - |""".stripMargin) - - "have a call to a block parameter" in { - cpg.method("foo").call.code("yield").astChildren.isCall.code("").name.l shouldBe List( - "" - ) - cpg.method("bar").call.code("yield").astChildren.isCall.code("").name.l shouldBe List( - "" - ) - } + "a method with an explicit proc parameter should create an invocation of it's `call` member" in { + val cpg = code("def foo(&b) yield end") - "add a block argument" in { - val List(param1) = cpg.method("foo").parameter.code("&.*").l - param1.name shouldBe "" - param1.index shouldBe 1 + val foo = cpg.method("foo").head - val List(param2) = cpg.method("bar").parameter.code("&.*").l - param2.name shouldBe "" - param2.index shouldBe 1 - } - } - - "with yield arguments" should { - val cpg = code("def foo(x) yield(x) end") - "replace the yield with a call to the block parameter with arguments" in { - val List(call) = cpg.call.codeExact("yield(x)").astChildren.isCall.codeExact("").l - call.name shouldBe "" - call.argument.code.l shouldBe List("self", "x") - } + val bParam = foo.parameter.last + bParam.name shouldBe "b" + bParam.code shouldBe "&b" + bParam.index shouldBe 1 - } + inside(foo.call.nameExact("call").argument.l) { case selfBase :: Nil => + selfBase.code shouldBe "b" } + } + + "a singleton method with an explicit proc parameter should create an invocation of it's `call` member" in { + val cpg = code("def self.foo(&b) yield end") - "that don't have a yield nor a proc parameter" should { - val cpg1 = code("def foo() end") - val cpg2 = code("def self.foo() end") - val cpgs = List(cpg1, cpg2) + val foo = cpg.method("foo").head - "not add a block argument" in { - forAll(cpgs)(_.method("foo").parameter.code("&.*").name.l should be(empty)) - } + val bParam = foo.parameter.last + bParam.name shouldBe "b" + bParam.code shouldBe "&b" + bParam.index shouldBe 1 + + inside(foo.call.nameExact("call").argument.l) { case selfBase :: Nil => + selfBase.code shouldBe "b" } + } + + "a method with an implicit proc parameter should create an invocation using a unique parameter name" in { + val cpg = code(""" + |def foo() yield end + |def self.bar() yield end + |""".stripMargin) + + val foo = cpg.method("foo").head + val bar = cpg.method("bar").head + + val fooParam = foo.parameter.last + fooParam.name shouldBe "" + fooParam.code shouldBe "&" + fooParam.index shouldBe 1 + + val barParam = bar.parameter.last + barParam.name shouldBe "" + barParam.code shouldBe "&" + barParam.index shouldBe 1 + foo.call.nameExact("call").argument.isIdentifier.name.l shouldBe List("") + bar.call.nameExact("call").argument.isIdentifier.name.l shouldBe List("") } + "a method with an implicit proc parameter should create an invocation of it's `call` member with given arguments" in { + val cpg = code("def foo(x) yield(x) end") + + val foo = cpg.method("foo").head + + val List(xParam, procParam) = foo.parameter.l.takeRight(2) + + xParam.name shouldBe "x" + xParam.index shouldBe 1 + + procParam.name shouldBe "" + procParam.code shouldBe "&" + procParam.index shouldBe 2 + + inside(foo.call.nameExact("call").argument.l) { case selfBase :: x :: Nil => + selfBase.code shouldBe "" + selfBase.argumentIndex shouldBe 0 + x.code shouldBe "x" + x.argumentIndex shouldBe 1 + } + } + + "a method without a yield nor proc parameter should not have either modelled" in { + val cpg1 = code("def foo() end") + val cpg2 = code("def self.foo() end") + val cpgs = List(cpg1, cpg2) + + forAll(cpgs)(cpg => { + cpg.method("foo").parameter.code("&.*").name.l should be(empty) + cpg.method("foo").call.nameExact("call").name.l should be(empty) + }) + } + + "A Yield statement with multiple arguments" in { + val cpg = code(""" + |def foo + | yield 1, :z => 2 + |end + |""".stripMargin) + + inside(cpg.method.name("foo").call.nameExact("call").l) { + case yieldCall :: Nil => + inside(yieldCall.argument.l) { + case (base: Identifier) :: (oneLiteral: Literal) :: (twoLiteral: Literal) :: Nil => + base.name shouldBe "" + base.code shouldBe "" + + oneLiteral.code shouldBe "1" + oneLiteral.argumentIndex shouldBe 1 + twoLiteral.code shouldBe "2" + twoLiteral.argumentName shouldBe Some("z") + case xs => fail(s"Expected two arguments for yieldCall, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected one call for yield, got ${xs.code.mkString(",")}") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RangeTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RangeTests.scala index 72db05379666..28633186b7cf 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RangeTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RangeTests.scala @@ -23,4 +23,31 @@ class RangeTests extends RubyCode2CpgFixture { upperBound.code shouldBe "1" } + "`0..` is represented by a `range` operator call with infinity upperbound" in { + val cpg = code("0..") + val List(range) = cpg.call(Operators.range).l + + range.methodFullName shouldBe Operators.range + range.code shouldBe "0.." + range.lineNumber shouldBe Some(1) + + val List(lowerBound, upperBound) = range.argument.l + + lowerBound.code shouldBe "0" + upperBound.code shouldBe "Float::INFINITY" + } + + "`..0` is represented by a `range` operator call with infinity lowerbound" in { + val cpg = code("..0") + val List(range) = cpg.call(Operators.range).l + + range.methodFullName shouldBe Operators.range + range.code shouldBe "..0" + range.lineNumber shouldBe Some(1) + + val List(lowerBound, upperBound) = range.argument.l + + lowerBound.code shouldBe "-Float::INFINITY" + upperBound.code shouldBe "0" + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RegexTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RegexTests.scala index 674a39f6f40d..d5d892cb1e8d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RegexTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/RegexTests.scala @@ -12,7 +12,7 @@ class RegexTests extends RubyCode2CpgFixture(withPostProcessing = true) { |0 |""".stripMargin) cpg.call(RubyOperators.regexpMatch).methodFullName.l shouldBe List( - s"$kernelPrefix.String:${RubyOperators.regexpMatch}" + s"$kernelPrefix.String.${RubyOperators.regexpMatch}" ) } "`/x/ =~ 'y'` is a member call `/x/.=~ 'y'" in { @@ -20,7 +20,7 @@ class RegexTests extends RubyCode2CpgFixture(withPostProcessing = true) { |0 |""".stripMargin) cpg.call(RubyOperators.regexpMatch).methodFullName.l shouldBe List( - s"$kernelPrefix.Regexp:${RubyOperators.regexpMatch}" + s"$kernelPrefix.Regexp.${RubyOperators.regexpMatch}" ) } @@ -33,7 +33,7 @@ class RegexTests extends RubyCode2CpgFixture(withPostProcessing = true) { inside(cpg.controlStructure.isIf.l) { case regexIf :: Nil => - regexIf.condition.isCall.methodFullName.l shouldBe List(s"$kernelPrefix.Regexp:${RubyOperators.regexpMatch}") + regexIf.condition.isCall.methodFullName.l shouldBe List(s"$kernelPrefix.Regexp.${RubyOperators.regexpMatch}") inside(regexIf.condition.isCall.argument.l) { case (lhs: Literal) :: (rhs: Literal) :: Nil => diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SetterTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SetterTests.scala deleted file mode 100644 index 7381411226bb..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SetterTests.scala +++ /dev/null @@ -1,29 +0,0 @@ -package io.joern.rubysrc2cpg.querying - -import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.semanticcpg.language.* - -class SetterTests extends RubyCode2CpgFixture { - - "`x.y=1` is represented by a `x.y=` CALL with argument `1`" in { - val cpg = code("""x = Foo.new - |x.y = 1 - |""".stripMargin) - - val List(setter) = cpg.call("y=").l - val List(fieldAccess) = cpg.fieldAccess.l - - setter.code shouldBe "x.y = 1" - setter.lineNumber shouldBe Some(2) - setter.receiver.l shouldBe List(fieldAccess) - - fieldAccess.code shouldBe "x.y=" - fieldAccess.lineNumber shouldBe Some(2) - fieldAccess.fieldIdentifier.code.l shouldBe List("y=") - - val List(_, one) = setter.argument.l - one.code shouldBe "1" - one.lineNumber shouldBe Some(2) - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala index ff2dd238f5b2..7310bf6b4365 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala @@ -2,8 +2,9 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal} -import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} import io.shiftleft.semanticcpg.language.* +import io.joern.rubysrc2cpg.passes.Defines as RubyDefines class SingleAssignmentTests extends RubyCode2CpgFixture { @@ -42,34 +43,54 @@ class SingleAssignmentTests extends RubyCode2CpgFixture { rhs.code shouldBe "1" } - "`||=` is represented by an `assignmentOr` operator call" in { + "`||=` is represented by a lowered if call to .nil?" in { val cpg = code(""" - |x ||= false + |def foo + | x ||= false + |end |""".stripMargin) - val List(assignment) = cpg.call(Operators.assignmentOr).l - assignment.code shouldBe "x ||= false" - assignment.lineNumber shouldBe Some(2) - assignment.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("x.nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "x = false" + val List(lhs, rhs) = assignmentCall.argument.l + lhs.code shouldBe "x" + rhs.code shouldBe "false" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } - val List(lhs, rhs) = assignment.argument.l - lhs.code shouldBe "x" - rhs.code shouldBe "false" + case xs => fail(s"Expected one control structure, got ${xs.code.mkString(",")}") + } } - "`&&=` is represented by an `assignmentAnd` operator call" in { + "`&&=` is represented by lowered if call to .nil?" in { val cpg = code(""" - |x &&= true + |def foo + | x &&= true + |end |""".stripMargin) - val List(assignment) = cpg.call(Operators.assignmentAnd).l - assignment.code shouldBe "x &&= true" - assignment.lineNumber shouldBe Some(2) - assignment.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("!x.nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "x = true" + val List(lhs, rhs) = assignmentCall.argument.l + lhs.code shouldBe "x" + rhs.code shouldBe "true" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } - val List(lhs, rhs) = assignment.argument.l - lhs.code shouldBe "x" - rhs.code shouldBe "true" + case xs => fail(s"Expected one control structure, got ${xs.code.mkString(",")}") + } } "`/=` is represented by an `assignmentDivision` operator call" in { @@ -233,4 +254,255 @@ class SingleAssignmentTests extends RubyCode2CpgFixture { } } + "Bracket Assignments" in { + val cpg = code(""" + | def get_pto_schedule + | begin + | schedules = current_user.paid_time_off.schedule + | jfs = [] + | schedules.each do |s| + | hash = Hash.new + | hash[:id] = s[:id] + | hash[:title] = s[:event_name] + | hash[:start] = s[:date_begin] + | hash[:end] = s[:date_end] + | jfs << hash + | end + | rescue + | end + | respond_to do |format| + | format.json { render json: jfs.to_json } + | end + | end + |""".stripMargin) + + inside(cpg.method.isLambda.l) { + case scheduleLambda :: _ :: _ :: Nil => + inside(scheduleLambda.call.name(Operators.assignment).l) { + case _ :: id :: title :: start :: end :: _ :: Nil => + id.code shouldBe "hash[:id] = s[:id]" + + inside(id.argument.l) { + case (lhs: Call) :: (rhs: Call) :: Nil => + lhs.methodFullName shouldBe Operators.indexAccess + lhs.code shouldBe "hash[:id]" + + rhs.methodFullName shouldBe Operators.indexAccess + rhs.code shouldBe "s[:id]" + + inside(lhs.argument.l) { + case base :: (index: Literal) :: Nil => + index.typeFullName shouldBe RubyDefines.getBuiltInType(RubyDefines.Symbol) + case xs => fail(s"Expected base and index, got [${xs.code.mkString(",")}]") + } + + inside(rhs.argument.l) { + case base :: (index: Literal) :: Nil => + index.typeFullName shouldBe RubyDefines.getBuiltInType(RubyDefines.Symbol) + case xs => fail(s"Expected base and index, got [${xs.code.mkString(",")}]") + } + + case xs => fail(s"Expected lhs and rhs, got ${xs.code.mkString(";")}]") + } + case xs => fail(s"Expected six assignemnts, got [${xs.code.mkString(";")}]") + } + case xs => fail(s"Expected three lambdas, got ${xs.size} lambdas instead") + } + } + + "Bracketed ||= is represented by a lowered if call to .nil?" in { + val cpg = code(""" + |def foo + | hash[:id] ||= s[:id] + |end + |""".stripMargin) + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("( = hash[:id]).nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "hash[:id] = s[:id]" + val List(lhs, rhs) = assignmentCall.argument.l + lhs.code shouldBe "hash[:id]" + rhs.code shouldBe "s[:id]" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } + + case xs => fail(s"Expected one control structure, got ${xs.code.mkString(",")}") + } + } + + "Bracketed +=" in { + val cpg = code(""" + |hash[:id] += s[:id] + |""".stripMargin) + + inside(cpg.call.name(Operators.assignmentPlus).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "hash[:id] += s[:id]" + assignmentCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + + inside(assignmentCall.argument.l) { + case lhs :: rhs :: Nil => + lhs.code shouldBe "hash[:id]" + rhs.code shouldBe "s[:id]" + case xs => fail(s"Expected lhs and rhs arguments, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected on assignmentOr call, got ${xs.code.mkString(",")}") + } + } + + "Bracketed &&= is represented by a lowere if call to .nil?" in { + val cpg = code(""" + |def foo + | hash[:id] &&= s[:id] + |end + |""".stripMargin) + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("!hash[:id].nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "hash[:id] = s[:id]" + val List(lhs, rhs) = assignmentCall.argument.l + lhs.code shouldBe "hash[:id]" + rhs.code shouldBe "s[:id]" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } + + case xs => fail(s"Expected one control structure, got ${xs.code.mkString(",")}") + } + } + + "Bracketed /=" in { + val cpg = code(""" + |hash[:id] /= s[:id] + |""".stripMargin) + + inside(cpg.call.name(Operators.assignmentDivision).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "hash[:id] /= s[:id]" + assignmentCall.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH + + inside(assignmentCall.argument.l) { + case lhs :: rhs :: Nil => + lhs.code shouldBe "hash[:id]" + rhs.code shouldBe "s[:id]" + case xs => fail(s"Expected lhs and rhs arguments, got ${xs.code.mkString(",")}") + } + case xs => fail(s"Expected on assignmentOr call, got ${xs.code.mkString(",")}") + } + } + + "Single ||= Assignment" in { + val cpg = code(""" + |def foo + | A.B ||= c 1 + |end + |""".stripMargin) + + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("( = A.B).nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "A.B = c 1" + val List(lhs, rhs: Call) = assignmentCall.argument.l: @unchecked + lhs.code shouldBe "A.B" + + rhs.code shouldBe "c 1" + val List(_, litArg) = rhs.argument.l + litArg.code shouldBe "1" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } + + case xs => fail(s"Expected one if statement, got ${xs.code.mkString(",")}") + } + } + + "Single &&= Assignment" in { + val cpg = code(""" + |def foo + | A.B &&= c 1 + |end + |""".stripMargin) + + inside(cpg.method.name("foo").controlStructure.l) { + case ifStruct :: Nil => + ifStruct.controlStructureType shouldBe ControlStructureTypes.IF + ifStruct.condition.code.l shouldBe List("!A.B.nil?") + + inside(ifStruct.whenTrue.ast.isCall.name(Operators.assignment).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "A.B = c 1" + val List(lhs: Call, rhs: Call) = assignmentCall.argument.l: @unchecked + lhs.code shouldBe "A.B" + lhs.methodFullName shouldBe Operators.fieldAccess + + rhs.code shouldBe "c 1" + val List(_, litArg) = rhs.argument.l + litArg.code shouldBe "1" + case xs => fail(s"Expected assignment call in true branch, got ${xs.code.mkString}") + } + + case xs => fail(s"Expected one if statement, got ${xs.code.mkString(",")}") + } + } + + "+= assignment operator" in { + val cpg = code(""" + |A::b += 1 + |""".stripMargin) + + inside(cpg.call.name(Operators.assignmentPlus).l) { + case assignmentCall :: Nil => + val List(lhs: Call, rhs) = assignmentCall.argument.l: @unchecked + + lhs.code shouldBe "A.b" + lhs.methodFullName shouldBe Operators.fieldAccess + + rhs.code shouldBe "1" + case xs => fail(s"Expected one call for assignment, got ${xs.code.mkString(",")}") + } + } + + "*= assignment operator" in { + val cpg = code(""" + |A::b *= 1 + |""".stripMargin) + + inside(cpg.call.name(Operators.assignmentMultiplication).l) { + case assignmentCall :: Nil => + assignmentCall.code shouldBe "A::b *= 1" + val List(lhs: Call, rhs) = assignmentCall.argument.l: @unchecked + + lhs.code shouldBe "A.b" + lhs.methodFullName shouldBe Operators.fieldAccess + + rhs.code shouldBe "1" + case xs => fail(s"Expected one call for assignment, got ${xs.code.mkString(",")}") + } + } + + "MethodInvocationWithoutParentheses multiple call args" in { + val cpg = code(""" + |def gl_badge_tag(*args, &block) + | render :some_symbol, &block + |end + |""".stripMargin) + + inside(cpg.call.name("render").argument.l) { + case _ :: (symbolArg: Literal) :: (blockArg: Identifier) :: Nil => + symbolArg.code shouldBe ":some_symbol" + blockArg.code shouldBe "block" + + case xs => fail(s"Expected two args, found [${xs.code.mkString(",")}]") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala index c8ec8791bd1d..5de1fe45df79 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala @@ -1,42 +1,71 @@ package io.joern.rubysrc2cpg.testfixtures +import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} -import io.joern.rubysrc2cpg.deprecated.utils.PackageTable import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} +import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.testfixtures.* -import io.joern.x2cpg.{ValidationMode, X2Cpg} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import org.scalatest.Tag - -import java.io.File import org.scalatest.Inside -trait RubyFrontend(useDeprecatedFrontend: Boolean, withDownloadDependencies: Boolean) extends LanguageFrontend { +import java.io.File +import java.nio.file.Files +import scala.jdk.CollectionConverters.* + +trait RubyFrontend( + withDownloadDependencies: Boolean, + disableFileContent: Boolean, + antlrDebugging: Boolean, + antlrProfiling: Boolean +) extends LanguageFrontend { override val fileSuffix: String = ".rb" implicit val config: Config = getConfig() .map(_.asInstanceOf[Config]) .getOrElse(Config().withSchemaValidation(ValidationMode.Enabled)) - .withUseDeprecatedFrontend(useDeprecatedFrontend) .withDownloadDependencies(withDownloadDependencies) + .withDisableFileContent(disableFileContent) + .withAntlrDebugging(antlrDebugging) + .withAntlrProfiling(antlrProfiling) override def execute(sourceCodeFile: File): Cpg = { - new RubySrc2Cpg().createCpg(sourceCodeFile.getAbsolutePath).get + val cpg = new RubySrc2Cpg().createCpg(sourceCodeFile.getAbsolutePath).get + if (antlrProfiling) { + if (sourceCodeFile.isDirectory) { + Files + .walk(sourceCodeFile.toPath) + .iterator() + .asScala + .filter(_.getFileName.toString.endsWith(".log")) + .map(_.toFile) + .foreach(printAntlrProfilingInfo) + } else { + printAntlrProfilingInfo(sourceCodeFile) + } + } + cpg + } + + private def printAntlrProfilingInfo(logfile: File): Unit = { + if (logfile.exists()) { + println(Files.readString(logfile.toPath)) + logfile.delete() // cleanup + } } } class DefaultTestCpgWithRuby( - packageTable: Option[PackageTable], - useDeprecatedFrontend: Boolean, - downloadDependencies: Boolean = false + downloadDependencies: Boolean = false, + disableFileContent: Boolean = true, + antlrDebugging: Boolean = false, + antlrProfiling: Boolean ) extends DefaultTestCpg - with RubyFrontend(useDeprecatedFrontend, downloadDependencies) + with RubyFrontend(downloadDependencies, disableFileContent, antlrDebugging, antlrProfiling) with SemanticTestCpg { override protected def applyPasses(): Unit = { @@ -45,31 +74,26 @@ class DefaultTestCpgWithRuby( } override protected def applyPostProcessingPasses(): Unit = { - packageTable match { - case Some(table) => - RubySrc2Cpg.packageTableInfo.set(table) - case None => - } RubySrc2Cpg.postProcessingPasses(this, config).foreach(_.createAndApply()) } - } class RubyCode2CpgFixture( withPostProcessing: Boolean = false, withDataFlow: Boolean = false, downloadDependencies: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, - packageTable: Option[PackageTable] = None, - useDeprecatedFrontend: Boolean = false + disableFileContent: Boolean = true, + semantics: Semantics = DefaultSemantics(), + antlrDebugging: Boolean = false, + antlrProfiling: Boolean = false ) extends Code2CpgFixture(() => - new DefaultTestCpgWithRuby(packageTable, useDeprecatedFrontend, downloadDependencies) + new DefaultTestCpgWithRuby(downloadDependencies, disableFileContent, antlrDebugging, antlrProfiling) .withOssDataflow(withDataFlow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) with Inside - with SemanticCpgTestFixture(extraFlows) { + with SemanticCpgTestFixture(semantics) { implicit val resolver: ICallResolver = NoResolve @@ -79,17 +103,13 @@ class RubyCode2CpgFixture( } } -class RubyCfgTestCpg(useDeprecatedFrontend: Boolean = true, downloadDependencies: Boolean = false) - extends CfgTestCpg - with RubyFrontend(useDeprecatedFrontend, downloadDependencies) { +class RubyCfgTestCpg( + downloadDependencies: Boolean = false, + disableFileContent: Boolean = true, + antlrDebugging: Boolean = false, + antlrProfiling: Boolean = false +) extends CfgTestCpg + with RubyFrontend(downloadDependencies, disableFileContent, antlrDebugging, antlrProfiling) { override val fileSuffix: String = ".rb" } - -/** Denotes a test which has been similarly ported to the new frontend. - */ -object SameInNewFrontend extends Tag("SameInNewFrontend") - -/** Denotes a test which has been ported to the new frontend, but has different expectations. - */ -object DifferentInNewFrontend extends Tag("DifferentInNewFrontend") diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala new file mode 100644 index 000000000000..c1ef223a76a3 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala @@ -0,0 +1,101 @@ +package io.joern.rubysrc2cpg.testfixtures + +import better.files.File as BFile +import io.joern.rubysrc2cpg.Config +import io.joern.rubysrc2cpg.parser.{AstPrinter, ResourceManagedParser, RubyParser} +import io.joern.x2cpg.SourceFiles +import io.joern.x2cpg.utils.{ConcurrentTaskUtil, TestCodeWriter} +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpecLike +import org.slf4j.LoggerFactory + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.util.{Failure, Success, Using} + +class RubyParserFixture + extends RubyFrontend( + withDownloadDependencies = false, + disableFileContent = true, + antlrDebugging = false, + antlrProfiling = false + ) + with TestCodeWriter + with AnyWordSpecLike + with Matchers { + private val RubySourceFileExtensions: Set[String] = Set(".rb") + private val logger = LoggerFactory.getLogger(this.getClass) + private var fileNameCounter = 0 + + def generateParserTasks( + resourceManagedParser: ResourceManagedParser, + config: Config, + inputPath: String + ): Iterator[() => RubyParser.ProgramContext] = { + SourceFiles + .determine( + inputPath, + RubySourceFileExtensions, + ignoredDefaultRegex = Option(config.defaultIgnoredFilesRegex), + ignoredFilesRegex = Option(config.ignoredFilesRegex), + ignoredFilesPath = Option(config.ignoredFiles) + ) + .map(fileName => + () => + resourceManagedParser.parse(BFile(config.inputPath), fileName) match { + case Failure(exception) => throw exception + case Success(ctx) => ctx + } + ) + .iterator + } + + def writeCode(code: String, extension: String): Path = { + val tmpDir = BFile.newTemporaryDirectory("x2cpgTestTmpDir").deleteOnExit() + val tmpPath = tmpDir.path + val codeFiles = { + val fileName = { + val filename = s"Test$fileNameCounter$extension" + fileNameCounter += 1 + filename + } + + val filePath = Path.of(fileName) + if (filePath.getParent != null) { + Files.createDirectories(tmpPath.resolve(filePath.getParent)) + } + val codeAsBytes = code.getBytes(StandardCharsets.UTF_8) + val codeFile = tmpPath.resolve(filePath) + Files.write(codeFile, codeAsBytes) + codeFilePreProcessing(codeFile) + codeFile + } + + tmpPath + } + + def parseCode(code: String): List[RubyParser.ProgramContext] = { + val tempPath = writeCode(code, ".rb") + + Using.resource(new ResourceManagedParser(config.antlrCacheMemLimit)) { parser => + ConcurrentTaskUtil.runUsingThreadPool(generateParserTasks(parser, config, tempPath.toString)).flatMap { + case Failure(exception) => logger.warn(s"Could not parse file, skipping - ", exception); None + case Success(ctx) => Option(ctx) + } + } + } + + def test(code: String, expected: String = null): Unit = { + val astPrinter = parseCode(code).headOption match { + case Some(head) => Option(AstPrinter().visit(head)) + case None => None + } + + astPrinter match { + case Some(ast) => + val compareTo = if (expected != null) expected else code + ast shouldBe compareTo + case None => fail("AST Printer failed") + } + } +} diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala index b9d071df2273..2d2d37dd6257 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala @@ -4,6 +4,7 @@ import io.joern.swiftsrc2cpg.Frontend.* import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} import io.joern.x2cpg.utils.Environment import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -34,14 +35,19 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new SwiftSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new SwiftSrc2Cpg()) with FrontendHTTPServer[Config, SwiftSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() def run(config: Config, swiftsrc2cpg: SwiftSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - if (Environment.pathExists(absPath)) { - swiftsrc2cpg.run(config.withInputPath(absPath)) - } else { - System.exit(1) + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + if (Environment.pathExists(absPath)) { + swiftsrc2cpg.run(config.withInputPath(absPath)) + } else { + System.exit(1) + } } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala index 43d16968e363..af843e9d2568 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala @@ -21,7 +21,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewTypeRef import io.shiftleft.codepropertygraph.generated.ModifierTypes import io.shiftleft.codepropertygraph.generated.nodes.File.PropertyDefaults import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import scala.collection.mutable @@ -121,10 +121,10 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse case null => notHandledYet(node) } - override protected def line(node: SwiftNode): Option[Int] = node.startLine.map(Integer.valueOf) - override protected def column(node: SwiftNode): Option[Int] = node.startColumn.map(Integer.valueOf) - override protected def lineEnd(node: SwiftNode): Option[Int] = node.endLine.map(Integer.valueOf) - override protected def columnEnd(node: SwiftNode): Option[Int] = node.endColumn.map(Integer.valueOf) + override protected def line(node: SwiftNode): Option[Int] = node.startLine + override protected def column(node: SwiftNode): Option[Int] = node.startColumn + override protected def lineEnd(node: SwiftNode): Option[Int] = node.endLine + override protected def columnEnd(node: SwiftNode): Option[Int] = node.endColumn private val lineOffsetTable = OffsetUtils.getLineOffsetTable(Option(parserResult.fileContent)) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala index 20b47f96f7bb..7a60e48fe4d2 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala @@ -10,6 +10,7 @@ import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.GuardStmtSyntax import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.InitializerDeclSyntax import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.SwiftNode import io.joern.x2cpg.frontendspecific.swiftsrc2cpg.Defines +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.{Ast, ValidationMode} import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode} import io.shiftleft.codepropertygraph.generated.nodes.NewNode @@ -18,7 +19,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import io.shiftleft.codepropertygraph.generated.ControlStructureTypes import io.shiftleft.codepropertygraph.generated.PropertyNames -import io.shiftleft.passes.IntervalKeyPool import scala.collection.mutable diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ImportsPass.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ImportsPass.scala index 963e5e4e66c4..d89d9b8f5dbe 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ImportsPass.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/ImportsPass.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.passes.frontend.XImportsPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment /** This pass creates `IMPORT` nodes by looking for calls to `require`. `IMPORT` nodes are linked to existing dependency diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala index 75a5d0348985..1e78013d2b25 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/SwiftTypeNodePass.scala @@ -3,15 +3,14 @@ package io.joern.swiftsrc2cpg.passes import io.shiftleft.codepropertygraph.generated.Cpg import io.joern.x2cpg.passes.frontend.TypeNodePass import io.shiftleft.semanticcpg.language.* -import io.shiftleft.passes.KeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import scala.collection.mutable object SwiftTypeNodePass { - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) { + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) { override def fullToShortName(typeName: String): String = { typeName match { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala index 1b2ef98e9a94..88b07c32917d 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala @@ -65,7 +65,7 @@ object AstGenRunner { val astGenCommand = path.getOrElse("SwiftAstGen") val localPath = path.flatMap(File(_).parentOption.map(_.pathAsString)).getOrElse(".") val debugMsgPath = path.getOrElse("PATH") - ExternalCommand.run(s"$astGenCommand -h", localPath).toOption match { + ExternalCommand.run(Seq(astGenCommand, "-h"), localPath).toOption match { case Some(_) => logger.debug(s"Using SwiftAstGen from $debugMsgPath") true @@ -140,7 +140,7 @@ class AstGenRunner(config: Config) { } private def runAstGenNative(in: File, out: File): Try[Seq[String]] = - ExternalCommand.run(s"$astGenCommand -o $out", in.toString()) + ExternalCommand.run(Seq(astGenCommand, "-o", out.toString), in.toString()) private def checkParsedFiles(files: List[String], in: File): List[String] = { val numOfParsedFiles = files.size diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala index b2ba122f8a5d..84ff1422e78c 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala @@ -4,19 +4,20 @@ import scala.util.Failure import scala.util.Success import scala.util.Try -object ExternalCommand extends io.joern.x2cpg.utils.ExternalCommand { +object ExternalCommand { - override def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - result match { - case Success(0) => + import io.joern.x2cpg.utils.ExternalCommand.ExternalCommandResult + + def run(command: Seq[String], cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { + io.joern.x2cpg.utils.ExternalCommand.run(command, cwd, mergeStdErrInStdOut = true, extraEnv) match { + case ExternalCommandResult(0, stdOut, _) => Success(stdOut) - case Success(_) if stdErr.isEmpty && stdOut.nonEmpty => + case ExternalCommandResult(_, stdOut, stdErr) if stdErr.isEmpty && stdOut.nonEmpty => // SwiftAstGen exits with exit code != 0 on Windows. // To catch with we specifically handle the empty stdErr here. Success(stdOut) - case _ => - val allOutput = stdOut ++ stdErr - Failure(new RuntimeException(allOutput.mkString(System.lineSeparator()))) + case ExternalCommandResult(_, stdOut, _) => + Failure(new RuntimeException(stdOut.mkString(System.lineSeparator()))) } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala index e7de93beec41..15ac7441b461 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/dataflow/DataFlowTests.scala @@ -8,7 +8,6 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.Identifier import io.shiftleft.codepropertygraph.generated.nodes.Literal import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.toNodeTraversal class DataFlowTests extends DataFlowCodeToCpgSuite { @@ -871,7 +870,7 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { cpg .call("bar") .outE(EdgeTypes.REACHING_DEF) - .count(_.inNode() == cpg.ret.head) shouldBe 1 + .count(_.dst == cpg.ret.head) shouldBe 1 } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromFileTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromFileTests.scala index 401379610cca..74070859371c 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromFileTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromFileTests.scala @@ -3,7 +3,7 @@ package io.joern.swiftsrc2cpg.io import better.files.File import io.joern.swiftsrc2cpg.testfixtures.SwiftSrc2CpgSuite import io.shiftleft.semanticcpg.codedumper.CodeDumper -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..fccf57f5b5e8 --- /dev/null +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala @@ -0,0 +1,82 @@ +package io.joern.swiftsrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class SwiftSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("swiftsrc2cpgTestsHttpTest") + val file = dir / "main.swift" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |func main() { + | println($indexStr) + |}""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.swiftsrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.swiftsrc2cpg.Main.stop() + } + + "Using swiftsrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("swiftsrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("swiftsrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ActorTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ActorTests.scala index 9f35b700b653..ac7a7b581e3a 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ActorTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ActorTests.scala @@ -3,9 +3,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class ActorTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AvailabilityQueryTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AvailabilityQueryTests.scala index bc5471c202f6..f49f1b07708d 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AvailabilityQueryTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AvailabilityQueryTests.scala @@ -3,9 +3,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class AvailabilityQueryTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BorrowExprTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BorrowExprTests.scala index 346292310e0b..41d5bfb74245 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BorrowExprTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BorrowExprTests.scala @@ -3,7 +3,7 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class BorrowExprTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BuiltinWordTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BuiltinWordTests.scala index 1619e9046cf2..01e2535d6108 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BuiltinWordTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/BuiltinWordTests.scala @@ -3,7 +3,7 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class BuiltinWordTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ConflictMarkersTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ConflictMarkersTests.scala index 83c8113a0304..1c102a80e460 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ConflictMarkersTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ConflictMarkersTests.scala @@ -3,7 +3,7 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class ConflictMarkersTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/CopyExprTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/CopyExprTests.scala index a37615b6b7c0..74cdd5dd2ad3 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/CopyExprTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/CopyExprTests.scala @@ -3,7 +3,7 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CopyExprTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/DeclarationTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/DeclarationTests.scala index d296aa4a8c6e..07ba4e56a134 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/DeclarationTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/DeclarationTests.scala @@ -2,9 +2,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class DeclarationTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/EnumTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/EnumTests.scala index 4c06817fe338..eb1f77e8ac99 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/EnumTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/EnumTests.scala @@ -4,9 +4,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class EnumTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala index a34be3b027d8..fe36cd54b82f 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala @@ -2,9 +2,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class ExpressionTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ForeachTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ForeachTests.scala index 034f2cd05a6b..82baf8f5365f 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ForeachTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ForeachTests.scala @@ -2,9 +2,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class ForeachTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/StatementTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/StatementTests.scala index 071f028346f7..99102b6f6b62 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/StatementTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/StatementTests.scala @@ -2,9 +2,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class StatementTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SuperTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SuperTests.scala index 3991c1d143f5..c3e538e433a3 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SuperTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SuperTests.scala @@ -4,7 +4,7 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class SuperTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SwitchTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SwitchTests.scala index a49ca2a5a8b5..1532d02ad0da 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SwitchTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/SwitchTests.scala @@ -4,9 +4,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class SwitchTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ToplevelLibraryTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ToplevelLibraryTests.scala index eda0969c3d9c..a70e3645c9ba 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ToplevelLibraryTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ToplevelLibraryTests.scala @@ -4,9 +4,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class ToplevelLibraryTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TryTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TryTests.scala index cc04f6a56439..92f9e092680e 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TryTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TryTests.scala @@ -4,9 +4,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class TryTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TypealiasTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TypealiasTests.scala index 97ecb3006bdc..9a974501a878 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TypealiasTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/TypealiasTests.scala @@ -4,9 +4,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class TypealiasTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/WhileTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/WhileTests.scala index df30834877dd..6138b719d8e1 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/WhileTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/WhileTests.scala @@ -2,9 +2,9 @@ package io.joern.swiftsrc2cpg.passes.ast import io.joern.swiftsrc2cpg.testfixtures.AstSwiftSrc2CpgSuite -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* class WhileTests extends AstSwiftSrc2CpgSuite { diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala index 451f2dfefeb0..af0875c95404 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/DataFlowCodeToCpgSuite.scala @@ -27,7 +27,7 @@ class DataFlowCodeToCpgSuite extends Code2CpgFixture(() => new DataFlowTestCpg() protected implicit val context: EngineContext = EngineContext() protected def flowToResultPairs(path: Path): List[(String, Integer)] = - path.resultPairs().collect { case (firstElement: String, secondElement: Option[Integer]) => + path.resultPairs().collect { case (firstElement: String, secondElement) => (firstElement, secondElement.getOrElse(-1)) } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/SwiftSrc2CpgSuite.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/SwiftSrc2CpgSuite.scala index 1eeebaffd7d5..2aa8cc29b730 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/SwiftSrc2CpgSuite.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/testfixtures/SwiftSrc2CpgSuite.scala @@ -1,16 +1,17 @@ package io.joern.swiftsrc2cpg.testfixtures -import io.joern.dataflowengineoss.semanticsloader.FlowSemantic +import io.joern.dataflowengineoss.DefaultSemantics +import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} import io.joern.x2cpg.testfixtures.Code2CpgFixture class SwiftSrc2CpgSuite( fileSuffix: String = ".swift", withOssDataflow: Boolean = false, - extraFlows: List[FlowSemantic] = List.empty, + semantics: Semantics = DefaultSemantics(), withPostProcessing: Boolean = false ) extends Code2CpgFixture(() => new SwiftDefaultTestCpg(fileSuffix) .withOssDataflow(withOssDataflow) - .withExtraFlows(extraFlows) + .withSemantics(semantics) .withPostProcessingPasses(withPostProcessing) ) diff --git a/joern-cli/frontends/x2cpg/build.sbt b/joern-cli/frontends/x2cpg/build.sbt index dcf0bde3b5cc..4aa7fbfb1376 100644 --- a/joern-cli/frontends/x2cpg/build.sbt +++ b/joern-cli/frontends/x2cpg/build.sbt @@ -4,10 +4,12 @@ dependsOn(Projects.semanticcpg) libraryDependencies ++= Seq( /* Start: AST Gen Dependencies */ - "com.lihaoyi" %% "upickle" % Versions.upickle, - "com.typesafe" % "config" % Versions.typeSafeConfig, - "com.michaelpollmeier" % "versionsort" % Versions.versionSort, + "com.lihaoyi" %% "upickle" % Versions.upickle, + "com.typesafe" % "config" % Versions.typeSafeConfig, + "com.michaelpollmeier" % "versionsort" % Versions.versionSort, + "org.apache.commons" % "commons-exec" % Versions.commonsExec, /* End: AST Gen Dependencies */ + "net.freeutils" % "jlhttp" % Versions.jlhttp, "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling % Optional, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala index ea5f9f8bd5f4..b9f8bed7e67c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Ast.scala @@ -1,11 +1,11 @@ package io.joern.x2cpg -import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.{DiffGraphBuilder, EdgeTypes} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.nodes.AstNode.PropertyDefaults import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.DiffGraphBuilder -import overflowdb.SchemaViolationException +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder +import flatgraph.SchemaViolationException case class AstEdge(src: NewNode, dst: NewNode) @@ -49,6 +49,10 @@ object Ast { ast.bindsEdges.foreach { edge => diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.BINDS) } + + ast.captureEdges.foreach { edge => + diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.CAPTURE) + } } def neighbourValidation(src: NewNode, dst: NewNode, edge: String)(implicit @@ -58,7 +62,7 @@ object Ast { !(src.isValidOutNeighbor(edge, dst) && dst.isValidInNeighbor(edge, src)) ) { throw new SchemaViolationException( - s"Malformed AST detected: (${src.label()}) -[$edge]-> (${dst.label()}) violates the schema." + s"Malformed AST detected: (${src.label}) -[$edge]-> (${dst.label}) violates the schema." ) } @@ -92,7 +96,8 @@ case class Ast( refEdges: collection.Seq[AstEdge] = Vector.empty, bindsEdges: collection.Seq[AstEdge] = Vector.empty, receiverEdges: collection.Seq[AstEdge] = Vector.empty, - argEdges: collection.Seq[AstEdge] = Vector.empty + argEdges: collection.Seq[AstEdge] = Vector.empty, + captureEdges: collection.Seq[AstEdge] = Vector.empty )(implicit withSchemaValidation: ValidationMode = ValidationMode.Disabled) { def root: Option[NewNode] = nodes.headOption @@ -114,7 +119,8 @@ case class Ast( argEdges = argEdges ++ other.argEdges, receiverEdges = receiverEdges ++ other.receiverEdges, refEdges = refEdges ++ other.refEdges, - bindsEdges = bindsEdges ++ other.bindsEdges + bindsEdges = bindsEdges ++ other.bindsEdges, + captureEdges = captureEdges ++ other.captureEdges ) } @@ -126,7 +132,8 @@ case class Ast( argEdges = argEdges ++ other.argEdges, receiverEdges = receiverEdges ++ other.receiverEdges, refEdges = refEdges ++ other.refEdges, - bindsEdges = bindsEdges ++ other.bindsEdges + bindsEdges = bindsEdges ++ other.bindsEdges, + captureEdges = captureEdges ++ other.captureEdges ) } @@ -217,14 +224,25 @@ case class Ast( this.copy(receiverEdges = receiverEdges ++ dsts.map(AstEdge(src, _))) } + def withCaptureEdge(src: NewNode, dst: NewNode): Ast = { + Ast.neighbourValidation(src, dst, EdgeTypes.CAPTURE) + this.copy(captureEdges = captureEdges ++ List(AstEdge(src, dst))) + } + + def withCaptureEdges(src: NewNode, dsts: Seq[NewNode]): Ast = { + dsts.foreach(dst => Ast.neighbourValidation(src, dst, EdgeTypes.CAPTURE)) + this.copy(captureEdges = captureEdges ++ dsts.map(AstEdge(src, _))) + } + /** Returns a deep copy of the sub tree rooted in `node`. If `order` is set, then the `order` and `argumentIndex` * fields of the new root node are set to `order`. If `replacementNode` is set, then this replaces `node` in the new * copy. */ def subTreeCopy(node: AstNodeNew, argIndex: Int = -1, replacementNode: Option[AstNodeNew] = None): Ast = { - val newNode = replacementNode match + val newNode = replacementNode match { case Some(n) => n case None => node.copy + } if (argIndex != -1) { // newNode.order = argIndex newNode match { @@ -249,6 +267,7 @@ case class Ast( val newRefEdges = refEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst))) val newBindsEdges = bindsEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst))) val newReceiverEdges = receiverEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst))) + val newCaptureEdges = captureEdges.filter(_.src == node).map(x => AstEdge(newNode, newIfExists(x.dst))) Ast(newNode) .copy( @@ -256,7 +275,8 @@ case class Ast( conditionEdges = newConditionEdges, refEdges = newRefEdges, bindsEdges = newBindsEdges, - receiverEdges = newReceiverEdges + receiverEdges = newReceiverEdges, + captureEdges = newCaptureEdges ) .withChildren(newChildren) } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala index 8df7a930c78c..9a271db76fff 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstCreatorBase.scala @@ -1,13 +1,13 @@ package io.joern.x2cpg import io.joern.x2cpg.passes.frontend.MetaDataPass +import io.joern.x2cpg.utils.IntervalKeyPool import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, ModifierTypes} -import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: ValidationMode) { val diffGraph: DiffGraphBuilder = Cpg.newDiffGraphBuilder @@ -88,7 +88,7 @@ abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: V ): Ast = Ast(method) .withChildren(parameters) - .withChild(Ast(NewBlock())) + .withChild(Ast(NewBlock().typeFullName(Defines.Any))) .withChildren(modifiers.map(Ast(_))) .withChild(Ast(methodReturn)) @@ -113,7 +113,7 @@ abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: V methodNode.filename(fileName.get) } val staticModifier = NewModifier().modifierType(ModifierTypes.STATIC) - val body = blockAst(NewBlock(), initAsts) + val body = blockAst(NewBlock().typeFullName(Defines.Any), initAsts) val methodReturn = newMethodReturnNode(returnType, None, None, None) methodAst(methodNode, Nil, body, methodReturn, List(staticModifier)) } @@ -150,9 +150,9 @@ abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: V def wrapMultipleInBlock(asts: Seq[Ast], lineNumber: Option[Int]): Ast = { asts.toList match { - case Nil => blockAst(NewBlock().lineNumber(lineNumber)) + case Nil => blockAst(NewBlock().typeFullName(Defines.Any).lineNumber(lineNumber)) case ast :: Nil => ast - case astList => blockAst(NewBlock().lineNumber(lineNumber), astList) + case astList => blockAst(NewBlock().typeFullName(Defines.Any).lineNumber(lineNumber), astList) } } @@ -310,8 +310,8 @@ abstract class AstCreatorBase(filename: String)(implicit withSchemaValidation: V .withReceiverEdges(callNode, receiverRoot) } - def setArgumentIndices(arguments: Seq[Ast]): Unit = { - var currIndex = 1 + def setArgumentIndices(arguments: Seq[Ast], start: Int = 1): Unit = { + var currIndex = start arguments.foreach { a => a.root match { case Some(x: ExpressionNew) => diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala index 2b2832960813..217814f6d4d0 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/AstNodeBuilder.scala @@ -234,7 +234,7 @@ trait AstNodeBuilder[Node, NodeProcessor] { this: NodeProcessor => } protected def blockNode(node: Node): NewBlock = { - blockNode(node, BlockDefaults.Code, BlockDefaults.TypeFullName) + blockNode(node, BlockDefaults.Code, Defines.Any) } protected def blockNode(node: Node, code: String, typeFullName: String): NewBlock = { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Imports.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Imports.scala index a46e41fdabaf..7d523c6aba44 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Imports.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/Imports.scala @@ -2,7 +2,7 @@ package io.joern.x2cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{CallBase, NewImport} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder object Imports { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/SourceFiles.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/SourceFiles.scala index 24b224a15fab..f8ac012141fc 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/SourceFiles.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/SourceFiles.scala @@ -5,17 +5,59 @@ import better.files.* import org.slf4j.LoggerFactory import java.io.FileNotFoundException +import java.nio.file.FileVisitor +import java.nio.file.FileVisitResult +import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.Files import scala.util.matching.Regex +import scala.jdk.CollectionConverters.SetHasAsJava + object SourceFiles { private val logger = LoggerFactory.getLogger(getClass) + /** Hack to have a FileVisitor in place that will continue iterating files even if an IOException happened during + * traversal. + */ + private final class FailsafeFileVisitor extends FileVisitor[Path] { + + private val seenFiles = scala.collection.mutable.Set.empty[Path] + + def files(): Set[File] = seenFiles.map(File(_)).toSet + + override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { + FileVisitResult.CONTINUE + } + + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + seenFiles.addOne(file) + FileVisitResult.CONTINUE + } + + override def visitFileFailed(file: Path, exc: java.io.IOException): FileVisitResult = { + exc match { + case e: java.nio.file.FileSystemLoopException => logger.warn(s"Ignoring '$file' (cyclic symlink)") + case other => logger.warn(s"Ignoring '$file'", other) + } + FileVisitResult.CONTINUE + } + + override def postVisitDirectory(dir: Path, exc: java.io.IOException): FileVisitResult = FileVisitResult.CONTINUE + } + private def isIgnoredByFileList(filePath: String, ignoredFiles: Seq[String]): Boolean = { - val isInIgnoredFiles = ignoredFiles.exists { - case ignorePath if File(ignorePath).isDirectory => filePath.startsWith(ignorePath) - case ignorePath => filePath == ignorePath + val filePathFile = File(filePath) + if (!filePathFile.exists || !filePathFile.isReadable) { + logger.debug(s"'$filePath' ignored (not readable or broken symlink)") + return true + } + val isInIgnoredFiles = ignoredFiles.exists { ignorePath => + val ignorePathFile = File(ignorePath) + ignorePathFile.exists && + (ignorePathFile.contains(filePathFile, strict = false) || ignorePathFile.isSameFileAs(filePathFile)) } if (isInIgnoredFiles) { logger.debug(s"'$filePath' ignored (--exclude)") @@ -64,7 +106,7 @@ object SourceFiles { && !ignoredFilesRegex.exists(isIgnoredByRegex(file, inputPath, _)) && !ignoredFilesPath.exists(isIgnoredByFileList(file, _)) - private def filterFiles( + def filterFiles( files: List[String], inputPath: String, ignoredDefaultRegex: Option[Seq[Regex]] = None, @@ -106,7 +148,11 @@ object SourceFiles { val matchingFiles = files.filter(hasSourceFileExtension).map(_.toString) val matchingFilesFromDirs = dirs - .flatMap(_.listRecursively) + .flatMap { dir => + val visitor = new FailsafeFileVisitor + Files.walkFileTree(dir.path, visitOptions.toSet.asJava, Int.MaxValue, visitor) + visitor.files() + } .filter(hasSourceFileExtension) .map(_.pathAsString) @@ -117,21 +163,17 @@ object SourceFiles { * unexpected and hard-to-debug issues in the results. */ private def assertAllExist(files: Set[File]): Unit = { - val (existant, nonExistant) = files.partition(_.isReadable) - val nonReadable = existant.filterNot(_.isReadable) - - if (nonExistant.nonEmpty || nonReadable.nonEmpty) { - logErrorWithPaths("Source input paths do not exist", nonExistant.map(_.canonicalPath)) - + val (existent, nonExistent) = files.partition(_.exists) + val nonReadable = existent.filterNot(_.isReadable) + if (nonExistent.nonEmpty || nonReadable.nonEmpty) { + logErrorWithPaths("Source input paths do not exist", nonExistent.map(_.canonicalPath)) logErrorWithPaths("Source input paths exist, but are not readable", nonReadable.map(_.canonicalPath)) - throw FileNotFoundException("Invalid source paths provided") } } private def logErrorWithPaths(message: String, paths: Iterable[String]): Unit = { val pathsArray = paths.toArray.sorted - pathsArray.lengthCompare(1) match { case cmp if cmp < 0 => // pathsArray is empty, so don't log anything case cmp if cmp == 0 => logger.error(s"$message: ${paths.head}") diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala index ca5b223ec8b1..d73248d3df72 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/X2Cpg.scala @@ -2,11 +2,11 @@ package io.joern.x2cpg import better.files.File import io.joern.x2cpg.X2Cpg.{applyDefaultOverlays, withErrorsToConsole} +import io.joern.x2cpg.frontendspecific.FrontendArgsDelimitor import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} import org.slf4j.LoggerFactory -import overflowdb.Config import scopt.OParser import java.io.PrintWriter @@ -15,12 +15,14 @@ import scala.util.matching.Regex import scala.util.{Failure, Success, Try} object X2CpgConfig { + def defaultInputPath: String = "" def defaultOutputPath: String = "cpg.bin" } trait X2CpgConfig[R <: X2CpgConfig[R]] { - var inputPath: String = "" - var outputPath: String = X2CpgConfig.defaultOutputPath + var inputPath: String = X2CpgConfig.defaultInputPath + var outputPath: String = X2CpgConfig.defaultOutputPath + var serverMode: Boolean = false def withInputPath(inputPath: String): R = { this.inputPath = Paths.get(inputPath).toAbsolutePath.normalize().toString @@ -32,6 +34,11 @@ trait X2CpgConfig[R <: X2CpgConfig[R]] { this.asInstanceOf[R] } + def withServerMode(x: Boolean): R = { + this.serverMode = x + this.asInstanceOf[R] + } + var defaultIgnoredFilesRegex: Seq[Regex] = Seq.empty var ignoredFilesRegex: Regex = "".r var ignoredFiles: Seq[String] = Seq.empty @@ -74,6 +81,7 @@ trait X2CpgConfig[R <: X2CpgConfig[R]] { def withInheritedFields(config: R): R = { this.inputPath = config.inputPath this.outputPath = config.outputPath + this.serverMode = config.serverMode this.defaultIgnoredFilesRegex = config.defaultIgnoredFilesRegex this.ignoredFilesRegex = config.ignoredFilesRegex this.ignoredFiles = config.ignoredFiles @@ -113,9 +121,10 @@ object DependencyDownloadConfig { * @param frontend * the frontend to use for CPG creation */ -abstract class X2CpgMain[T <: X2CpgConfig[T], X <: X2CpgFrontend[?]](val cmdLineParser: OParser[Unit, T], frontend: X)( - implicit defaultConfig: T -) { +abstract class X2CpgMain[T <: X2CpgConfig[T], X <: X2CpgFrontend[T]]( + val cmdLineParser: OParser[Unit, T], + val frontend: X +)(implicit defaultConfig: T) { private val logger = LoggerFactory.getLogger(classOf[X2CpgMain[T, X]]) @@ -149,7 +158,6 @@ abstract class X2CpgMain[T <: X2CpgConfig[T], X <: X2CpgFrontend[?]](val cmdLine run(config, frontend) } catch { case ex: Throwable => - println(ex.getMessage) ex.printStackTrace() System.exit(1) } @@ -163,7 +171,7 @@ abstract class X2CpgMain[T <: X2CpgConfig[T], X <: X2CpgFrontend[?]](val cmdLine /** Trait that represents a CPG generator, where T is the frontend configuration class. */ -trait X2CpgFrontend[T <: X2CpgConfig[?]] { +trait X2CpgFrontend[T <: X2CpgConfig[T]] { /** Create a CPG according to given configuration. Returns CPG wrapped in a `Try`, making it possible to detect and * inspect exceptions in CPG generation. To be provided by the frontend. @@ -173,15 +181,24 @@ trait X2CpgFrontend[T <: X2CpgConfig[?]] { /** Create CPG according to given configuration, printing errors to the console if they occur. The CPG is closed and * not returned. */ + @throws[Throwable]("if createCpg throws any Throwable") def run(config: T): Unit = { withErrorsToConsole(config) { _ => createCpg(config) match { case Success(cpg) => - cpg.close() + cpg.close() // persists to disk Success(cpg) case Failure(exception) => Failure(exception) } + } match { + case Failure(exception) => + // We explicitly rethrow the exception so that every frontend will + // terminate with exit code 1 if there was an exception during createCpg. + // Frontend maintainer may want to catch that RuntimeException on their end + // to add custom error handling. + throw exception + case Success(_) => // this is fine } } @@ -209,12 +226,12 @@ trait X2CpgFrontend[T <: X2CpgConfig[?]] { * exists, it is the file name of the resulting CPG. Otherwise, the CPG is held in memory. */ def createCpg(inputName: String, outputName: Option[String])(implicit defaultConfig: T): Try[Cpg] = { - val defaultWithInputPath = defaultConfig.withInputPath(inputName).asInstanceOf[T] + val defaultWithInputPath = defaultConfig.withInputPath(inputName) val config = if (!outputName.contains(X2CpgConfig.defaultOutputPath)) { if (outputName.isEmpty) { - defaultWithInputPath.withOutputPath("").asInstanceOf[T] + defaultWithInputPath.withOutputPath("") } else { - defaultWithInputPath.withOutputPath(outputName.get).asInstanceOf[T] + defaultWithInputPath.withOutputPath(outputName.get) } } else { defaultWithInputPath @@ -281,6 +298,10 @@ object X2Cpg { .text( "add the raw source code to the content field of FILE nodes to allow for method source retrieval via offset fields (disabled by default)" ), + opt[Unit]("server") + .action((_, c) => c.withServerMode(true)) + .hidden() + .text("runs this frontend in server mode (disabled by default)"), opt[Unit]("disable-file-content") .action((_, c) => c.withDisableFileContent(true)) .hidden() @@ -293,19 +314,16 @@ object X2Cpg { /** Create an empty CPG, backed by the file at `optionalOutputPath` or in-memory if `optionalOutputPath` is empty. */ def newEmptyCpg(optionalOutputPath: Option[String] = None): Cpg = { - val odbConfig = optionalOutputPath - .map { outputPath => - val outFile = File(outputPath) + optionalOutputPath match { + case Some(outputPath) => + lazy val outFile = File(outputPath) if (outputPath != "" && outFile.exists) { logger.info("Output file exists, removing: " + outputPath) outFile.delete() } - Config.withDefaults.withStorageLocation(outputPath) - } - .getOrElse { - Config.withDefaults() - } - Cpg.withConfig(odbConfig) + Cpg.withStorage(outFile.path) + case None => Cpg.empty + } } /** Apply function `applyPasses` to a newly created CPG. The CPG is wrapped in a `Try` and returned. On failure, the @@ -368,7 +386,7 @@ object X2Cpg { } /** Strips surrounding quotation characters from a string. - * @param s + * @param str * the target string. * @return * the stripped string. diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala index c9a9a514efab..6852d6c611ce 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala @@ -46,24 +46,14 @@ object AstGenRunner { packagePath: URL ) - def executableDir(implicit metaData: AstGenProgramMetaData): String = { - val dir = metaData.packagePath.toString - val indexOfLib = dir.lastIndexOf("lib") - val fixedDir = if (indexOfLib != -1) { - new java.io.File(dir.substring("file:".length, indexOfLib)).toString - } else { - val indexOfTarget = dir.lastIndexOf("target") - if (indexOfTarget != -1) { - new java.io.File(dir.substring("file:".length, indexOfTarget)).toString - } else { - "." - } - } - Paths.get(fixedDir, "/bin/astgen").toAbsolutePath.toString - } + def executableDir(implicit metaData: AstGenProgramMetaData): String = + ExternalCommand + .executableDir(Paths.get(metaData.packagePath.toURI)) + .resolve("astgen") + .toString def hasCompatibleAstGenVersion(compatibleVersion: String)(implicit metaData: AstGenProgramMetaData): Boolean = { - ExternalCommand.run(s"$metaData.name -version", ".").toOption.map(_.mkString.strip()) match { + ExternalCommand.run(Seq(metaData.name, "-version"), ".").successOption.map(_.mkString.strip()) match { case Some(installedVersion) if installedVersion != "unknown" && Try(VersionHelper.compare(installedVersion, compatibleVersion)).toOption.getOrElse(-1) >= 0 => @@ -74,7 +64,8 @@ object AstGenRunner { s"Found local ${metaData.name} v$installedVersion in systems PATH but ${metaData.name} requires at least v$compatibleVersion" ) false - case _ => false + case _ => + false } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/javasrc2cpg/JavaTypeRecoveryPassGenerator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/javasrc2cpg/JavaTypeRecoveryPassGenerator.scala index 607bc70fa3dd..936595fc045b 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/javasrc2cpg/JavaTypeRecoveryPassGenerator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/javasrc2cpg/JavaTypeRecoveryPassGenerator.scala @@ -5,7 +5,7 @@ import io.joern.x2cpg.passes.frontend.* import io.shiftleft.codepropertygraph.generated.{Cpg, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class JavaTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) extends XTypeRecoveryPassGenerator[Method](cpg, config) { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ConstClosurePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ConstClosurePass.scala index fedb8e0f77d7..f9b4b0f339c9 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ConstClosurePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ConstClosurePass.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Method, MethodRef} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** A pass that identifies assignments of closures to constants and updates `METHOD` nodes accordingly. */ diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala index 3264b4e73f05..2d9c8072f1f2 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/JavaScriptTypeRecovery.scala @@ -5,10 +5,10 @@ import io.joern.x2cpg.Defines.ConstructorMethodName import io.joern.x2cpg.passes.frontend.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class JavaScriptTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) extends XTypeRecoveryPassGenerator[File](cpg, config) { @@ -48,9 +48,9 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui override protected def prepopulateSymbolTableEntry(x: AstNode): Unit = x match { case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) != Defines.Any => - val typeFullName = x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) - val typeHints = symbolTable.get(LocalVar(x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any))) - typeFullName + if x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) != Defines.Any => + val typeFullName = x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) + val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName lazy val cpgTypeFullName = cpg.typeDecl.nameExact(typeFullName).fullName.toSet val resolvedTypeHints = if (typeHints.nonEmpty) symbolTable.put(x, typeHints) @@ -59,9 +59,8 @@ private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBui if (!resolvedTypeHints.contains(typeFullName) && resolvedTypeHints.sizeIs == 1) builder.setNodeProperty(x, PropertyNames.TYPE_FULL_NAME, resolvedTypeHints.head) - case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty[String]).nonEmpty => - val possibleTypes = x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty[String]) + case x @ (_: Identifier | _: Local | _: MethodParameterIn) if x.property(Properties.PossibleTypes).nonEmpty => + val possibleTypes = x.property(Properties.PossibleTypes) if (possibleTypes.sizeIs == 1 && !possibleTypes.contains("ANY")) { val typeFullName = possibleTypes.head val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ObjectPropertyCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ObjectPropertyCallLinker.scala index cb09e7df9d4d..4b77b193ed03 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ObjectPropertyCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/jssrc2cpg/ObjectPropertyCallLinker.scala @@ -3,7 +3,6 @@ package io.joern.x2cpg.frontendspecific.jssrc2cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, MethodRef} import io.shiftleft.codepropertygraph.generated.{Cpg, PropertyNames} import io.shiftleft.passes.CpgPass -import overflowdb.BatchedUpdate import io.shiftleft.semanticcpg.language.* /** Perform a simple analysis to find a common pattern in JavaScript where objects are dynamically assigned function @@ -13,7 +12,7 @@ import io.shiftleft.semanticcpg.language.* */ class ObjectPropertyCallLinker(cpg: Cpg) extends CpgPass(cpg) { - override def run(builder: BatchedUpdate.DiffGraphBuilder): Unit = { + override def run(builder: DiffGraphBuilder): Unit = { def propertyCallRegexPattern(withMatchingGroup: Boolean): String = "^(?:\\{.*\\}|.*):\\(" + (if withMatchingGroup then "(.*)" else ".*") + "\\):.*$" diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/package.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/package.scala index 7434394eb0b4..e6eebfa8eba4 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/package.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/package.scala @@ -10,4 +10,7 @@ package io.joern.x2cpg * Otherwise we'll end in jar hell with various incompatible versions of many different dependencies, and complex * issues with things like OSGI and JPMS. */ -package object frontendspecific +package object frontendspecific { + // Special string used to separate joern-parse opts from frontend-specific opts + val FrontendArgsDelimitor = "--frontend-args" +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala index 9c7857a95ccd..6399506585c6 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeRecovery.scala @@ -8,7 +8,7 @@ import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, Operators, import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.{Assignment, FieldAccess} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import scala.collection.mutable @@ -150,7 +150,7 @@ private class RecoverForPhpFile(cpg: Cpg, cu: NamespaceBlock, builder: DiffGraph symbolTable.append(head, callees) case _ => Set.empty } - val returnTypes = extractTypes(ret.argumentOut.l) + val returnTypes = extractTypes(ret.argumentOut.cast[CfgNode].l) existingTypes.addAll(returnTypes) /* Check whether method return is already known, and if so, remove dummy value */ @@ -221,7 +221,7 @@ private class RecoverForPhpFile(cpg: Cpg, cu: NamespaceBlock, builder: DiffGraph .getOrElse(XTypeRecovery.DummyIndexAccess) else x.name - val collectionVar = Option(c.argumentOut.l match { + val collectionVar = Option(c.argumentOut.cast[CfgNode].l match { case List(i: Identifier, idx: Literal) => CollectionVar(i.name, idx.code) case List(i: Identifier, idx: Identifier) => CollectionVar(i.name, idx.code) case List(c: Call, idx: Call) => CollectionVar(callName(c), callName(idx)) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeStubsParser.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeStubsParser.scala index e8f2cb4cec7c..c7d43c33f4e9 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeStubsParser.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/php2cpg/PhpTypeStubsParser.scala @@ -9,7 +9,6 @@ import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate import scopt.OParser import java.io.File as JFile @@ -47,7 +46,7 @@ class PhpTypeStubsParserPass(cpg: Cpg, config: XTypeStubsParserConfig = XTypeStu arr } - override def runOnPart(builder: overflowdb.BatchedUpdate.DiffGraphBuilder, part: KnownFunction): Unit = { + override def runOnPart(builder: DiffGraphBuilder, part: KnownFunction): Unit = { /* calculate the result of this part - this is done as a concurrent task */ val builtinMethod = cpg.method.fullNameExact(part.name).l builtinMethod.foreach(mNode => { @@ -73,7 +72,7 @@ class PhpTypeStubsParserPass(cpg: Cpg, config: XTypeStubsParserConfig = XTypeStu def scanParamTypes(pTypesRawArr: List[String]): Seq[Seq[String]] = pTypesRawArr.map(paramTypeRaw => paramTypeRaw.split(",").map(_.strip).toSeq).toSeq - protected def setTypes(builder: overflowdb.BatchedUpdate.DiffGraphBuilder, n: StoredNode, types: Seq[String]): Unit = + protected def setTypes(builder: DiffGraphBuilder, n: StoredNode, types: Seq[String]): Unit = if (types.size == 1) builder.setNodeProperty(n, PropertyNames.TYPE_FULL_NAME, types.head) else builder.setNodeProperty(n, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, types) } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala index 5675afe23e63..2e70aea7e38f 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/DynamicTypeHintFullNamePass.scala @@ -5,7 +5,6 @@ import io.shiftleft.codepropertygraph.generated.{Cpg, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, MethodParameterIn, MethodReturn, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate import java.io.File import java.util.regex.{Matcher, Pattern} @@ -74,7 +73,7 @@ class DynamicTypeHintFullNamePass(cpg: Cpg) extends ForkJoinParallelCpgPass[CfgN fullName.replaceFirst("\\.py:", "").replaceAll(Pattern.quote(File.separator), ".") private def setTypeHints( - diffGraph: BatchedUpdate.DiffGraphBuilder, + diffGraph: DiffGraphBuilder, node: StoredNode, typeHint: String, alias: String, diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeHintCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeHintCallLinker.scala index cfaa52cc2d9d..3a7edff71a72 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeHintCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeHintCallLinker.scala @@ -8,7 +8,7 @@ import io.shiftleft.semanticcpg.language.* class PythonTypeHintCallLinker(cpg: Cpg) extends XTypeHintCallLinker(cpg) { - override def calls: Iterator[Call] = super.calls.nameNot("^(import).*") + override def calls: Iterator[Call] = super.calls.whereNot(_.isImport) override def calleeNames(c: Call): Seq[String] = super.calleeNames(c).map { // Python call from a type diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeRecovery.scala index 6449cfc8e013..ccdbf86c4aec 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/pysrc2cpg/PythonTypeRecovery.scala @@ -13,7 +13,7 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.importresolver.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder private class PythonTypeRecovery(cpg: Cpg, state: XTypeRecoveryState, iteration: Int) extends XTypeRecovery[File](cpg, state, iteration) { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/Constants.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/Constants.scala new file mode 100644 index 000000000000..821a95293b68 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/Constants.scala @@ -0,0 +1,93 @@ +package io.joern.x2cpg.frontendspecific.rubysrc2cpg + +object Constants { + + val builtinPrefix = "__core" + val kernelPrefix = s"$builtinPrefix.Kernel" + val Initialize = "initialize" + val Main = "
" + + /* Source: https://ruby-doc.org/3.2.2/Kernel.html + * + * We comment-out methods that require an explicit "receiver" (target of member access.) + */ + val kernelFunctions: Set[String] = Set( + "Array", + "Complex", + "Float", + "Hash", + "Integer", + "Rational", + "String", + "__callee__", + "__dir__", + "__method__", + "abort", + "at_exit", + "autoload", + "autoload?", + "binding", + "block_given?", + "callcc", + "caller", + "caller_locations", + "catch", + "chomp", + "chomp!", + "chop", + "chop!", + // "class", + // "clone", + "eval", + "exec", + "exit", + "exit!", + "fail", + "fork", + "format", + // "frozen?", + "gets", + "global_variables", + "gsub", + "gsub!", + "iterator?", + "lambda", + "load", + "local_variables", + "loop", + "open", + "p", + "print", + "printf", + "proc", + "putc", + "puts", + "raise", + "rand", + "readline", + "readlines", + "require", + "require_all", + "require_relative", + "select", + "set_trace_func", + "sleep", + "spawn", + "sprintf", + "srand", + "sub", + "sub!", + "syscall", + "system", + "tap", + "test", + // "then", + "throw", + "trace_var", + // "trap", + "untrace_var", + "warn" + // "yield_self", + ) + +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImplicitRequirePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImplicitRequirePass.scala new file mode 100644 index 000000000000..54bb109f515a --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImplicitRequirePass.scala @@ -0,0 +1,196 @@ +package io.joern.x2cpg.frontendspecific.rubysrc2cpg + +import io.joern.x2cpg.Defines +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes, Operators} +import io.shiftleft.passes.ForkJoinParallelCpgPass +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.{Assignment, FieldAccess} +import org.apache.commons.text.CaseUtils + +import java.util.regex.Pattern +import scala.annotation.tailrec +import scala.collection.mutable + +/** A tuple holding the (name, importPath) for types in the analysis. + */ +case class TypeImportInfo(name: String, importPath: String) + +/** In some Ruby frameworks, it is common to have an autoloader library that implicitly loads requirements onto the + * stack. This pass makes these imports explicit. The most popular one is Zeitwerk which we check in `Gemsfile.lock` to enable this pass. + * + * @param externalTypes + * a list of additional types to consider that may be importable but are not in the CPG. + */ +class ImplicitRequirePass(cpg: Cpg, externalTypes: Seq[TypeImportInfo] = Nil) + extends ForkJoinParallelCpgPass[Method](cpg) { + + /** A tuple holding information about the type import info, additionally with a boolean indicating if it is external + * or not. + */ + private case class TypeImportInfoWithProvidence(info: TypeImportInfo, isExternal: Boolean) + private val typeNameToImportInfo = mutable.Map.empty[String, Seq[TypeImportInfoWithProvidence]] + + private val Require: String = "require" + private val Self: String = "self" + private val Initialize: String = "initialize" + private val Clazz: String = "" + + override def init(): Unit = { + val importableTypeInfo = cpg.typeDecl + .isExternal(false) + .filter { typeDecl => + // zeitwerk will match types that share the name of the path. + // This match is insensitive to camel case, i.e, foo_bar will match type FooBar. + val fileName = typeDecl.filename.split(Array('/', '\\')).last + val typeName = typeDecl.name + ImplicitRequirePass.isAutoloadable(typeName, fileName) + } + .map { typeDecl => + val typeImportInfo = TypeImportInfo(typeDecl.name, ImplicitRequirePass.normalizePath(typeDecl.filename)) + TypeImportInfoWithProvidence(typeImportInfo, typeDecl.isExternal) + } + .l + // Group types by symbol and add to map for quicker retrieval later + typeNameToImportInfo.addAll(importableTypeInfo.groupBy { case TypeImportInfoWithProvidence(typeImportInfo, _) => + typeImportInfo.name + }) + typeNameToImportInfo.addAll(externalTypes.map(TypeImportInfoWithProvidence(_, true)).groupBy { + case TypeImportInfoWithProvidence(typeImportInfo, _) => typeImportInfo.name + }) + } + + private def getFieldBaseFromString(fieldAccessString: String): String = { + val normalizedFieldAccessString = fieldAccessString.replace("::", ".") + normalizedFieldAccessString.split('.').headOption.getOrElse(normalizedFieldAccessString) + } + + override def generateParts(): Array[Method] = + cpg.method.isModule.whereNot(_.astChildren.isCall.nameExact(Require)).toArray + + /** Collects methods within a module. + */ + private def findMethodsViaAstChildren(module: Method): Iterator[Method] = { + // TODO For now we have to go via the full name regex because the AST is not yet linked + // at the execution time of this pass. + // Iterator(module) ++ module.astChildren.flatMap { + // case x: TypeDecl => x.method.flatMap(findMethodsViaAstChildren) + // case x: Method => Iterator(x) ++ x.astChildren.collectAll[Method].flatMap(findMethodsViaAstChildren) + // case _ => Iterator.empty + // } + cpg.method.fullName(Pattern.quote(module.fullName) + ".*") + } + + override def runOnPart(builder: DiffGraphBuilder, moduleMethod: Method): Unit = { + val possiblyImportedSymbols = mutable.ArrayBuffer.empty[String] + val currPath = ImplicitRequirePass.normalizePath(moduleMethod.filename) + + val typeDecl = cpg.typeDecl.fullName(Pattern.quote(moduleMethod.fullName) + ".*").l + typeDecl.inheritsFromTypeFullName + .filterNot(_.endsWith(Clazz)) + .map(getFieldBaseFromString) + .foreach(possiblyImportedSymbols.append) + + val methodsOfModule = findMethodsViaAstChildren(moduleMethod).toList + val callsOfModule = methodsOfModule.ast.isCall.toList + + val symbolsGatheredFromCalls = callsOfModule + .flatMap { + case x if x.name == Initialize => + x.receiver.headOption.flatMap { + case x: TypeRef => Option(getFieldBaseFromString(x.code)) + case x: Identifier => Option(x.name) + case x: Call if x.name == Operators.fieldAccess => + Option(fieldAccessBase(x.asInstanceOf[FieldAccess])) + case _ => None + }.iterator + case x if x.methodFullName == Operators.fieldAccess => + fieldAccessBase(x.asInstanceOf[FieldAccess]) :: Nil + case _ => + Iterator.empty + } + .filterNot(_.isBlank) + + possiblyImportedSymbols.appendAll(symbolsGatheredFromCalls) + + var currOrder = moduleMethod.block.astChildren.size + possiblyImportedSymbols.distinct + .flatMap { identifierName => + typeNameToImportInfo + .getOrElse(identifierName, Seq.empty) + .sortBy { case TypeImportInfoWithProvidence(_, isExternal) => + isExternal // sorting booleans puts false (internal) first + } + .collectFirst { + // ignore an import to a file that defines the type + case TypeImportInfoWithProvidence(TypeImportInfo(_, importPath), _) if importPath != currPath => + importPath -> createRequireCall(builder, importPath) + } + } + .distinctBy { case (importPath, _) => importPath } + .foreach { case (_, requireCall) => + requireCall.order(currOrder) + builder.addEdge(moduleMethod.block, requireCall, EdgeTypes.AST) + currOrder += 1 + } + } + + private def createRequireCall(builder: DiffGraphBuilder, path: String): NewCall = { + val requireCallNode = NewCall() + .name(Require) + .code(s"$Require '$path'") + .methodFullName(s"$kernelPrefix.$Require") + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .typeFullName(Defines.Any) + builder.addNode(requireCallNode) + // Create literal argument + val pathLiteralNode = + NewLiteral().code(s"'$path'").typeFullName(s"$kernelPrefix.String").argumentIndex(1).order(2) + builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.AST) + builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.ARGUMENT) + requireCallNode + } + + private def fieldAccessBase(fa: FieldAccess): String = fieldAccessParts(fa).headOption.getOrElse(fa.argument(1).code) + + @tailrec + private def fieldAccessParts(fa: FieldAccess): Seq[String] = { + fa.argument(1) match { + case subFa: Call if subFa.name == Operators.fieldAccess => fieldAccessParts(subFa.asInstanceOf[FieldAccess]) + case self: Identifier if self.name == Self => fa.fieldIdentifier.map(_.canonicalName).toSeq + case assignCall: Call if assignCall.name == Operators.assignment => + val assign = assignCall.asInstanceOf[Assignment] + // Handle the tmp var assign of qualified names + (assign.target, assign.source) match { + case (lhs: Identifier, rhs: Call) if lhs.name.startsWith(" + fieldAccessParts(rhs.asInstanceOf[FieldAccess]) + case _ => Seq.empty + } + case _ => Seq.empty + } + } + +} + +object ImplicitRequirePass { + + /** Determines if the given type name and its corresponding parent file name allow for the type to be autoloaded by + * zeitwerk. + * @return + * true if the type is autoloadable from the given filename. + */ + def isAutoloadable(typeName: String, fileName: String): Boolean = { + // We use lowercase as something like `openssl` and `OpenSSL` don't give obvious clues where capitalisation occurs + val strippedFileName = normalizePath(fileName).toLowerCase + val lowerCaseTypeName = typeName.toLowerCase + lowerCaseTypeName == strippedFileName.toLowerCase || lowerCaseTypeName == CaseUtils + .toCamelCase(strippedFileName, true, '_', '-') + .toLowerCase + } + + private def normalizePath(path: String): String = path.replace("\\", "/").stripSuffix(".rb") + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImportsPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImportsPass.scala similarity index 71% rename from joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImportsPass.scala rename to joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImportsPass.scala index 8467d44ee88f..ea8a427e426d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImportsPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/ImportsPass.scala @@ -1,4 +1,4 @@ -package io.joern.rubysrc2cpg.passes +package io.joern.x2cpg.frontendspecific.rubysrc2cpg import io.joern.x2cpg.Imports.createImportNodeAndLink import io.joern.x2cpg.X2Cpg.stripQuotes @@ -9,9 +9,9 @@ import io.shiftleft.semanticcpg.language.* class ImportsPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { - private val importCallName: Seq[String] = Seq("require", "load", "require_relative", "require_all") - - override def generateParts(): Array[Call] = cpg.call.nameExact(importCallName*).toArray + override def generateParts(): Array[Call] = { + cpg.call.nameExact(ImportsPass.ImportCallNames.toSeq*).isStatic.toArray + } override def runOnPart(diffGraph: DiffGraphBuilder, call: Call): Unit = { val importedEntity = stripQuotes(call.argument.isLiteral.code.l match { @@ -22,3 +22,7 @@ class ImportsPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) { if (call.name == "require_all") importNode.isWildcard(true) } } + +object ImportsPass { + val ImportCallNames: Set[String] = Set("require", "load", "require_relative", "require_all") +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyImportResolverPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyImportResolverPass.scala similarity index 90% rename from joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyImportResolverPass.scala rename to joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyImportResolverPass.scala index f52733204573..298d1e94b976 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyImportResolverPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyImportResolverPass.scala @@ -1,14 +1,12 @@ -package io.joern.rubysrc2cpg.passes +package io.joern.x2cpg.frontendspecific.rubysrc2cpg import better.files.File -import io.joern.rubysrc2cpg.deprecated.utils.PackageTable -import io.joern.x2cpg.Defines as XDefines -import io.shiftleft.semanticcpg.language.importresolver.* +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants.* import io.joern.x2cpg.passes.frontend.XImportResolverPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.semanticcpg.language.* -import io.joern.rubysrc2cpg.passes.Defines as RDefines +import io.shiftleft.semanticcpg.language.importresolver.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import java.io.File as JFile @@ -47,7 +45,7 @@ class RubyImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) { .flatMap(fullName => Seq( ResolvedTypeDecl(fullName), - ResolvedMethod(s"$fullName.${Defines.Initialize}", "new", fullName.split("[.]").lastOption) + ResolvedMethod(s"$fullName.${Initialize}", "new", fullName.split("[.]").lastOption) ) ) .toSet @@ -61,7 +59,7 @@ class RubyImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) { // Expose methods which are directly present in a file, without any module, TypeDecl val resolvedMethods = cpg.method .where(_.file.name(filePattern)) - .where(_.nameExact(RDefines.Program)) + .where(_.nameExact(Main)) .astChildren .astChildren .isMethod diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeHintCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeHintCallLinker.scala similarity index 84% rename from joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeHintCallLinker.scala rename to joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeHintCallLinker.scala index 333245e40dda..5244b021fb4a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeHintCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeHintCallLinker.scala @@ -1,6 +1,7 @@ -package io.joern.rubysrc2cpg.passes +package io.joern.x2cpg.frontendspecific.rubysrc2cpg import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, NewMethod} import io.shiftleft.semanticcpg.language.* @@ -26,8 +27,8 @@ class RubyTypeHintCallLinker(cpg: Cpg) extends XTypeHintCallLinker(cpg) { } val name = if (methodName.contains(pathSep) && methodName.length > methodName.lastIndexOf(pathSep) + 1) - val strippedMethod = methodName.stripPrefix(s"${GlobalTypes.kernelPrefix}:") - if GlobalTypes.kernelFunctions.contains(strippedMethod) then strippedMethod + val strippedMethod = methodName.stripPrefix(s"$kernelPrefix.") + if kernelFunctions.contains(strippedMethod) then strippedMethod else methodName.substring(methodName.lastIndexOf(pathSep) + pathSep.length) else methodName createMethodStub(name, methodName, call.argumentOut.size, isExternal, builder) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeRecoveryPassGenerator.scala similarity index 83% rename from joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala rename to joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeRecoveryPassGenerator.scala index a9afaf7b9d67..1171a97e1e9a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPassGenerator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/rubysrc2cpg/RubyTypeRecoveryPassGenerator.scala @@ -1,13 +1,13 @@ -package io.joern.rubysrc2cpg.passes +package io.joern.x2cpg.frontendspecific.rubysrc2cpg import io.joern.x2cpg.Defines as XDefines +import io.joern.x2cpg.frontendspecific.rubysrc2cpg.Constants.* import io.joern.x2cpg.passes.frontend.* import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt -import io.shiftleft.codepropertygraph.generated.{Cpg, Operators, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder, Operators, PropertyNames} +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import io.shiftleft.semanticcpg.language.{types, *} -import overflowdb.BatchedUpdate.DiffGraphBuilder class RubyTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) extends XTypeRecoveryPassGenerator[File](cpg, config) { @@ -20,6 +20,8 @@ private class RubyTypeRecovery(cpg: Cpg, state: XTypeRecoveryState, iteration: I override def compilationUnits: Iterator[File] = cpg.file.iterator + override def isParallel: Boolean = false + override def generateRecoveryForCompilationUnitTask( unit: File, builder: DiffGraphBuilder @@ -40,7 +42,7 @@ private class RecoverForRubyFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, /** A heuristic method to determine if a call name is a constructor or not. */ override protected def isConstructor(name: String): Boolean = - !name.isBlank && (name == "new" || name == Defines.Initialize) + !name.isBlank && (name == "new" || name == Initialize) override protected def hasTypes(node: AstNode): Boolean = node match { case x: Call if !x.methodFullName.startsWith("") => @@ -55,9 +57,14 @@ private class RecoverForRubyFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, case x @ (_: Identifier | _: Local | _: MethodParameterIn) => symbolTable.append(x, x.getKnownTypes) case call: Call => val tnfs = - if call.methodFullName == XDefines.DynamicCallUnknownFullName || call.methodFullName.startsWith("") - then (call.dynamicTypeHintFullName ++ call.possibleTypes).distinct - else (call.methodFullName +: (call.dynamicTypeHintFullName ++ call.possibleTypes)).distinct + if ( + call.name != "initialize" && (call.methodFullName == XDefines.DynamicCallUnknownFullName || call.methodFullName + .startsWith("")) + ) { + (call.dynamicTypeHintFullName ++ call.possibleTypes).distinct + } else { + (call.methodFullName +: (call.dynamicTypeHintFullName ++ call.possibleTypes)).distinct + } symbolTable.append(call, tnfs.toSet) case _ => @@ -96,7 +103,7 @@ private class RecoverForRubyFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, else fieldAccessParents .filter(_.endsWith(fieldAccessName.stripSuffix(s".${c.name}"))) - .map(x => s"$x:${c.name}") + .map(x => s"$x.${c.name}") } else { types } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala index a40fd813531c..50f1f2bc8fcc 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/frontendspecific/swiftsrc2cpg/SwiftTypeRecovery.scala @@ -5,10 +5,10 @@ import io.joern.x2cpg.Defines.ConstructorMethodName import io.joern.x2cpg.passes.frontend.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class SwiftTypeRecoveryPassGenerator(cpg: Cpg, config: XTypeRecoveryConfig = XTypeRecoveryConfig()) extends XTypeRecoveryPassGenerator[File](cpg, config) { @@ -47,9 +47,9 @@ private class RecoverForSwiftFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, override protected def prepopulateSymbolTableEntry(x: AstNode): Unit = x match { case x @ (_: Identifier | _: Local | _: MethodParameterIn) - if x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) != Defines.Any => - val typeFullName = x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any) - val typeHints = symbolTable.get(LocalVar(x.property(PropertyNames.TYPE_FULL_NAME, Defines.Any))) - typeFullName + if x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) != Defines.Any => + val typeFullName = x.propertyOption(Properties.TypeFullName).getOrElse(Defines.Any) + val typeHints = symbolTable.get(LocalVar(typeFullName)) - typeFullName lazy val cpgTypeFullName = cpg.typeDecl.nameExact(typeFullName).fullName.toSet val resolvedTypeHints = if (typeHints.nonEmpty) symbolTable.put(x, typeHints) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala index a14daa5f8ad5..a23a71fb4f5b 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/Base.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.passes.CpgPassBase import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} -import io.joern.x2cpg.passes.base._ +import io.joern.x2cpg.passes.base.* object Base { val overlayName: String = "base" @@ -31,11 +31,7 @@ class Base extends LayerCreator { override val description: String = Base.description override def create(context: LayerCreatorContext): Unit = { - val cpg = context.cpg - cpg.graph.indexManager.createNodePropertyIndex(PropertyNames.FULL_NAME) - Base.passes(cpg).zipWithIndex.foreach { case (pass, index) => - runPass(pass, context, index) - } + Base.passes(context.cpg).foreach(_.createAndApply()) } // LayerCreators need one-arg constructor, because they're called by reflection from io.joern.console.Run diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/CallGraph.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/CallGraph.scala index 45ec40470400..b821a148f623 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/CallGraph.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/CallGraph.scala @@ -22,10 +22,7 @@ class CallGraph extends LayerCreator { override val dependsOn: List[String] = List(TypeRelations.overlayName) override def create(context: LayerCreatorContext): Unit = { - val cpg = context.cpg - CallGraph.passes(cpg).zipWithIndex.foreach { case (pass, index) => - runPass(pass, context, index) - } + CallGraph.passes(context.cpg).foreach(_.createAndApply()) } // LayerCreators need one-arg constructor, because they're called by reflection from io.joern.console.Run diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/ControlFlow.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/ControlFlow.scala index d0cdb3b1de5c..c19dcc9bb704 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/ControlFlow.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/ControlFlow.scala @@ -3,7 +3,7 @@ package io.joern.x2cpg.layers import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} import io.joern.x2cpg.passes.controlflow.CfgCreationPass import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass @@ -31,10 +31,7 @@ class ControlFlow extends LayerCreator { override val dependsOn: List[String] = List(Base.overlayName) override def create(context: LayerCreatorContext): Unit = { - val cpg = context.cpg - ControlFlow.passes(cpg).zipWithIndex.foreach { case (pass, index) => - runPass(pass, context, index) - } + ControlFlow.passes(context.cpg).foreach(_.createAndApply()) } // LayerCreators need one-arg constructor, because they're called by reflection from io.joern.console.Run diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpAst.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpAst.scala index cbbe9ceaf31d..cf06abf67ed7 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpAst.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpAst.scala @@ -1,7 +1,7 @@ package io.joern.x2cpg.layers import better.files.File -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class AstDumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCdg.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCdg.scala index 2528d7107758..6e92efc27ac8 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCdg.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCdg.scala @@ -1,7 +1,7 @@ package io.joern.x2cpg.layers import better.files.File -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class CdgDumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCfg.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCfg.scala index 1f78125b4177..0b3000c31fc8 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCfg.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/DumpCfg.scala @@ -1,7 +1,7 @@ package io.joern.x2cpg.layers import better.files.File -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} case class CfgDumpOptions(var outDir: String) extends LayerCreatorOptions {} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/TypeRelations.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/TypeRelations.scala index 82fc0d70dadd..e143138c88b5 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/TypeRelations.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/layers/TypeRelations.scala @@ -20,10 +20,7 @@ class TypeRelations extends LayerCreator { override val dependsOn: List[String] = List(Base.overlayName) override def create(context: LayerCreatorContext): Unit = { - val cpg = context.cpg - TypeRelations.passes(cpg).zipWithIndex.foreach { case (pass, index) => - runPass(pass, context, index) - } + TypeRelations.passes(context.cpg).foreach(_.createAndApply()) } // Layers need one-arg constructor, because they're called by reflection from io.joern.console.Run diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala index 703607803eee..53d16ff44297 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala @@ -1,12 +1,13 @@ package io.joern.x2cpg.passes.base import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.passes.ForkJoinParallelCpgPass +import io.shiftleft.semanticcpg.language.* import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide * method stubs and type decl stubs. @@ -15,7 +16,7 @@ class ContainsEdgePass(cpg: Cpg) extends ForkJoinParallelCpgPass[AstNode](cpg) { import ContainsEdgePass._ override def generateParts(): Array[AstNode] = - cpg.graph.nodes(sourceTypes*).asScala.map(_.asInstanceOf[AstNode]).toArray + cpg.graph.nodes(sourceTypes*).cast[AstNode].toArray override def runOnPart(dstGraph: DiffGraphBuilder, source: AstNode): Unit = { // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala index a045ef5971e3..b5bef4d0c258 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala @@ -1,9 +1,8 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{NewFile, StoredNode} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{File, NewFile, StoredNode} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal @@ -25,7 +24,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { } def createFileIfDoesNotExist(srcNode: StoredNode, destFullName: String): Unit = { - if (destFullName != srcNode.propertyDefaultValue(PropertyNames.FILENAME)) { + if (destFullName != File.PropertyDefaults.Name) { val dstFullName = if (destFullName == "") { FileTraversal.UNKNOWN } else { destFullName } val newFile = newFileNameToNode.getOrElseUpdate( @@ -42,7 +41,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { // Create SOURCE_FILE edges from nodes of various types to FILE linkToSingle( cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcNodes = cpg.graph.nodes(srcLabels*).cast[StoredNode].toList, srcLabels = srcLabels, dstNodeLabel = NodeTypes.FILE, edgeType = EdgeTypes.SOURCE_FILE, @@ -50,6 +49,7 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { originalFileNameToNode.get(x) }, dstFullNameKey = PropertyNames.FILENAME, + dstDefaultPropertyValue = File.PropertyDefaults.Name, dstGraph, Some(createFileIfDoesNotExist) ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala index 99b19f36d7a4..19f2bf802a87 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala @@ -3,7 +3,7 @@ package io.joern.x2cpg.passes.base import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} /** Adds a METHOD_PARAMETER_OUT for each METHOD_PARAMETER_IN to the graph and connects those with a PARAMETER_LINK edge. diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala index 7fa2f42016e5..794127e77402 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala @@ -3,12 +3,11 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.Defines import io.joern.x2cpg.passes.base.MethodStubCreator.createMethodStub import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, EvaluationStrategies, NodeTypes} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder import scala.collection.mutable import scala.util.Try @@ -24,7 +23,7 @@ class MethodStubCreator(cpg: Cpg) extends CpgPass(cpg) { private val methodFullNameToNode = mutable.LinkedHashMap[String, Method]() private val methodToParameterCount = mutable.LinkedHashMap[CallSummary, Int]() - override def run(dstGraph: BatchedUpdate.DiffGraphBuilder): Unit = { + override def run(dstGraph: DiffGraphBuilder): Unit = { for (method <- cpg.method) { methodFullNameToNode.put(method.fullName, method) } @@ -121,7 +120,7 @@ object MethodStubCreator { val blockNode = NewBlock() .order(1) .argumentIndex(1) - .typeFullName("ANY") + .typeFullName(Defines.Any) dstGraph.addNode(blockNode) dstGraph.addEdge(methodNode, blockNode, EdgeTypes.AST) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala index 9d6724e7a607..417c8b24afbf 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewNamespace import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** Creates NAMESPACE nodes and connects NAMESPACE_BLOCKs to corresponding NAMESPACE nodes. * diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala index a5a3dc2ae508..9323da89cf01 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn.PropertyDefaults import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* /** Old CPGs use the `order` field to indicate the parameter index while newer CPGs use the `parameterIndex` field. This * pass checks whether `parameterIndex` is not set, in which case the value of `order` is copied over. diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala index 778c9cd42a9e..7d658d7efb2f 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewTypeDecl, TypeDeclBase} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.{FileTraversal, NamespaceTraversal} /** This pass has no other pass as prerequisite. For each `TYPE` node that does not have a corresponding `TYPE_DECL` diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala index 00547bf2142d..b51a934c0e98 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala @@ -1,14 +1,12 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{Local, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import overflowdb.Node -import overflowdb.traversal.* - -class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { +import io.shiftleft.semanticcpg.language.* +class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[StoredNode]](cpg) with LinkingUtil { private val srcLabels = List( NodeTypes.METHOD_PARAMETER_IN, NodeTypes.METHOD_PARAMETER_OUT, @@ -24,11 +22,11 @@ class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wi NodeTypes.UNKNOWN ) - def generateParts(): Array[List[Node]] = { - cpg.graph.nodes(srcLabels*).toList.grouped(MAX_BATCH_SIZE).toArray + def generateParts(): Array[List[StoredNode]] = { + cpg.graph.nodes(srcLabels*).cast[StoredNode].toList.grouped(MAX_BATCH_SIZE).toArray } - def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { + def runOnPart(builder: DiffGraphBuilder, part: List[StoredNode]): Unit = { linkToSingle( cpg = cpg, srcNodes = part, @@ -37,6 +35,7 @@ class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wi edgeType = EdgeTypes.EVAL_TYPE, dstNodeMap = typeFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.TYPE_FULL_NAME, + dstDefaultPropertyValue = Local.PropertyDefaults.TypeFullName, dstGraph = builder, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala index 65dbae189c28..d851b16e201d 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala @@ -1,21 +1,19 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.{Type, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import overflowdb.Node -import overflowdb.traversal.* - -class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { +import io.shiftleft.semanticcpg.language.* +class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[StoredNode]](cpg) with LinkingUtil { private val srcLabels = List(NodeTypes.TYPE) - def generateParts(): Array[List[Node]] = { - cpg.graph.nodes(srcLabels*).toList.grouped(MAX_BATCH_SIZE).toArray + def generateParts(): Array[List[StoredNode]] = { + cpg.graph.nodes(srcLabels*).cast[StoredNode].toList.grouped(MAX_BATCH_SIZE).toArray } - def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { + def runOnPart(builder: DiffGraphBuilder, part: List[StoredNode]): Unit = { linkToSingle( cpg = cpg, srcNodes = part, @@ -24,6 +22,7 @@ class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wit edgeType = EdgeTypes.REF, dstNodeMap = typeDeclFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.TYPE_DECL_FULL_NAME, + dstDefaultPropertyValue = Type.PropertyDefaults.TypeDeclFullName, dstGraph = builder, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala index e12539269822..ed3496e7e94c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala @@ -2,15 +2,14 @@ package io.joern.x2cpg.passes.callgraph import io.joern.x2cpg.Defines.DynamicCallUnknownFullName import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, StoredNode, Type, TypeDecl} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.{NodeDb, NodeRef} import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* /** We compute the set of possible call-targets for each dynamic call, and add them as CALL edges to the graph, based on * call.methodFullName, method.name and method.signature, the inheritance hierarchy and the AST of typedecls and @@ -56,10 +55,10 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { initMaps() // ValidM maps class C and method name N to the set of // func ptrs implementing N for C and its subclasses - for ( - typeDecl <- cpg.typeDecl; + for { + typeDecl <- cpg.typeDecl method <- typeDecl._methodViaAstOut - ) { + } { val methodName = method.fullName val candidates = allSubclasses(typeDecl.fullName).flatMap { staticLookup(_, method) } validM.put(methodName, candidates) @@ -114,8 +113,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { if (visitedNodes.contains(cur)) return visitedNodes visitedNodes.addOne(cur) - (if (inSuperDirection) cpg.typeDecl.fullNameExact(cur.fullName).flatMap(_.inheritsFromOut.referencedTypeDecl) - else cpg.typ.fullNameExact(cur.fullName).flatMap(_.inheritsFromIn)) + (if (inSuperDirection) cpg.typeDecl.fullNameExact(cur.fullName)._typeViaInheritsFromOut.referencedTypeDecl + else cpg.typ.fullNameExact(cur.fullName).inheritsFromIn) .collectAll[TypeDecl] .to(mutable.LinkedHashSet) match { case classesToEval if classesToEval.isEmpty => visitedNodes @@ -174,16 +173,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { validM.get(call.methodFullName) match { case Some(tgts) => - val callsOut = call.callOut.fullName.toSetImmutable - val tgtMs = tgts - .flatMap(destMethod => - if (cpg.graph.indexManager.isIndexed(PropertyNames.FULL_NAME)) { - methodFullNameToNode(destMethod) - } else { - cpg.method.fullNameExact(destMethod).headOption - } - ) - .toSet + val callsOut = call._callOut.cast[Method].fullName.toSetImmutable + val tgtMs = tgts.flatMap(destMethod => methodFullNameToNode(destMethod)).toSet // Non-overridden methods linked as external stubs should be excluded if they are detected val (externalMs, internalMs) = tgtMs.partition(_.isExternal) (if (externalMs.nonEmpty && internalMs.nonEmpty) internalMs else tgtMs) @@ -209,8 +200,8 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { } } - private def nodesWithFullName(x: String): Iterable[NodeRef[? <: NodeDb]] = - cpg.graph.indexManager.lookup(PropertyNames.FULL_NAME, x).asScala + private def nodesWithFullName(x: String): Iterator[StoredNode] = + cpg.graph.nodesWithProperty(PropertyNames.FULL_NAME, x).cast[StoredNode] private def methodFullNameToNode(x: String): Option[Method] = nodesWithFullName(x).collectFirst { case x: Method => x } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala index 86174f9a872d..e0411f8dead4 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala @@ -1,28 +1,27 @@ package io.joern.x2cpg.passes.callgraph import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.CpgPass -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide * method stubs and type decl stubs. */ class MethodRefLinker(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - private val srcLabels = List(NodeTypes.METHOD_REF) - override def run(dstGraph: DiffGraphBuilder): Unit = { // Create REF edges from METHOD_REFs to METHOD linkToSingle( cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcNodes = cpg.methodRef.l, srcLabels = List(NodeTypes.METHOD_REF), dstNodeLabel = NodeTypes.METHOD, edgeType = EdgeTypes.REF, dstNodeMap = methodFullNameToNode(cpg, _), dstFullNameKey = PropertyNames.METHOD_FULL_NAME, + dstDefaultPropertyValue = Method.PropertyDefaults.FullName, dstGraph, None ) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala index e9156f217b49..f9f1332af728 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/NaiveCallLinker.scala @@ -3,8 +3,7 @@ package io.joern.x2cpg.passes.callgraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.jIteratortoTraversal +import io.shiftleft.semanticcpg.language.* /** Link remaining unlinked calls to methods only by their name (not full name) * @param cpg diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala index f1dab9252fb7..cf2ce37a0f43 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala @@ -3,7 +3,7 @@ package io.joern.x2cpg.passes.controlflow import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.controlflow.cfgcreation.CfgCreator /** A pass that creates control flow graphs from abstract syntax trees. diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgcreation/CfgCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgcreation/CfgCreator.scala index ac157243ac15..f59b5776ffd8 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgcreation/CfgCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgcreation/CfgCreator.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.CfgEdgeType import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, EdgeTypes, Operators} import io.shiftleft.semanticcpg.language.* -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder /** Translation of abstract syntax trees into control flow graphs * @@ -174,10 +174,18 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) { cfgForChildren(node) case ControlStructureTypes.MATCH => cfgForMatchExpression(node) + case ControlStructureTypes.THROW => + cfgForThrowStatement(node) case _ => Cfg.empty } + protected def cfgForThrowStatement(node: ControlStructure): Cfg = { + val throwExprCfg = node.astChildren.find(_.order == 1).map(cfgFor).getOrElse(Cfg.empty) + val concatedNatedCfg = throwExprCfg ++ Cfg(entryNode = Option(node)) + concatedNatedCfg.copy(edges = concatedNatedCfg.edges ++ singleEdge(node, exitNode)) + } + /** The CFG for a break/continue statements contains only the break/continue statement as a single entry node. The * fringe is empty, that is, appending another CFG to the break statement will not result in the creation of an edge * from the break statement to the entry point of the other CFG. Labeled breaks are treated like gotos and are added @@ -368,16 +376,17 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) { val loopExprCfg = children.find(_.order == nLocals + 3).map(cfgFor).getOrElse(Cfg.empty) val bodyCfg = children.find(_.order == nLocals + 4).map(cfgFor).getOrElse(Cfg.empty) - val innerCfg = conditionCfg ++ bodyCfg ++ loopExprCfg - val entryNode = (initExprCfg ++ innerCfg).entryNode + val innerCfg = bodyCfg ++ loopExprCfg + val loopEntryNode = conditionCfg.entryNode.orElse(innerCfg.entryNode) + val entryNode = initExprCfg.entryNode.orElse(loopEntryNode) - val newEdges = edgesFromFringeTo(initExprCfg, innerCfg.entryNode) ++ - edgesFromFringeTo(innerCfg, innerCfg.entryNode) ++ - edgesFromFringeTo(conditionCfg, bodyCfg.entryNode, TrueEdge) ++ { + val newEdges = edgesFromFringeTo(initExprCfg, loopEntryNode) ++ + edgesFromFringeTo(innerCfg, loopEntryNode) ++ + edgesFromFringeTo(conditionCfg, innerCfg.entryNode.orElse(conditionCfg.entryNode), TrueEdge) ++ { if (loopExprCfg.entryNode.isDefined) { edges(takeCurrentLevel(bodyCfg.continues), loopExprCfg.entryNode) } else { - edges(takeCurrentLevel(bodyCfg.continues), innerCfg.entryNode) + edges(takeCurrentLevel(bodyCfg.continues), loopEntryNode) } } @@ -385,7 +394,7 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) { .from(initExprCfg, conditionCfg, loopExprCfg, bodyCfg) .copy( entryNode = entryNode, - edges = newEdges ++ initExprCfg.edges ++ innerCfg.edges, + edges = newEdges ++ initExprCfg.edges ++ conditionCfg.edges ++ innerCfg.edges, fringe = conditionCfg.fringe.withEdgeType(FalseEdge) ++ takeCurrentLevel(bodyCfg.breaks).map((_, AlwaysEdge)), breaks = reduceAndFilterLevel(bodyCfg.breaks), continues = reduceAndFilterLevel(bodyCfg.continues) @@ -461,18 +470,32 @@ class CfgCreator(entryNode: Method, diffGraph: DiffGraphBuilder) { val diffGraphs = edgesFromFringeTo(conditionCfg, trueCfg.entryNode) ++ edgesFromFringeTo(conditionCfg, falseCfg.entryNode) - Cfg - .from(conditionCfg, trueCfg, falseCfg) - .copy( - entryNode = conditionCfg.entryNode, - edges = diffGraphs ++ conditionCfg.edges ++ trueCfg.edges ++ falseCfg.edges, - fringe = trueCfg.fringe ++ { + val ifStatementFringe = + if (trueCfg.entryNode.isEmpty && falseCfg.entryNode.isEmpty) { + conditionCfg.fringe.withEdgeType(AlwaysEdge) + } else { + val trueFringe = if (trueCfg.entryNode.isDefined) { + trueCfg.fringe + } else { + conditionCfg.fringe.withEdgeType(TrueEdge) + } + + val falseFringe = if (falseCfg.entryNode.isDefined) { falseCfg.fringe } else { conditionCfg.fringe.withEdgeType(FalseEdge) } - } + + trueFringe ++ falseFringe + } + + Cfg + .from(conditionCfg, trueCfg, falseCfg) + .copy( + entryNode = conditionCfg.entryNode, + edges = diffGraphs ++ conditionCfg.edges ++ trueCfg.edges ++ falseCfg.edges, + fringe = ifStatementFringe ) } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala index e0823e835ee7..b05e62aee1be 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorFrontier.scala @@ -17,7 +17,7 @@ class CfgDominatorFrontier[NodeType](cfgAdapter: CfgAdapter[NodeType], domTreeAd private def withIDom(x: NodeType, preds: Seq[NodeType]) = doms(x).map(i => (x, preds, i)) - def calculate(cfgNodes: Seq[NodeType]): mutable.Map[NodeType, mutable.Set[NodeType]] = { + def calculate(cfgNodes: Iterator[NodeType]): mutable.Map[NodeType, mutable.Set[NodeType]] = { val domFrontier = mutable.Map.empty[NodeType, mutable.Set[NodeType]] for { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala index 933ba72fdf2b..69e33e8773a0 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{Method, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import scala.collection.mutable diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala index 5bca67fd685f..4fc9f82d346f 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala @@ -13,7 +13,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.{ Unknown } import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.controlflow.cfgdominator.{CfgDominatorFrontier, ReverseCpgCfgAdapter} import org.slf4j.{Logger, LoggerFactory} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/Dereference.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/Dereference.scala deleted file mode 100644 index 2ee1c7ae355a..000000000000 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/Dereference.scala +++ /dev/null @@ -1,35 +0,0 @@ -package io.joern.x2cpg.passes.frontend - -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ - -object Dereference { - - def apply(cpg: Cpg): Dereference = cpg.metaData.language.headOption match { - case Some(Languages.NEWC) => CDereference() - case _ => DefaultDereference() - } - -} - -sealed trait Dereference { - - def dereferenceTypeFullName(fullName: String): String - -} - -case class CDereference() extends Dereference { - - /** Types from C/C++ can be annotated with * to indicate being a reference. As our CPG schema currently lacks a - * separate field for that information the * is part of the type full name and needs to be removed when linking. - */ - override def dereferenceTypeFullName(fullName: String): String = fullName.replace("*", "") - -} - -case class DefaultDereference() extends Dereference { - - override def dereferenceTypeFullName(fullName: String): String = fullName - -} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala index 179c31d4b6d1..5aef81da1a82 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/TypeNodePass.scala @@ -1,26 +1,22 @@ package io.joern.x2cpg.passes.frontend import io.joern.x2cpg.passes.frontend.TypeNodePass.fullToShortName -import io.shiftleft.codepropertygraph.generated.Cpg +import io.joern.x2cpg.Defines +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties} import io.shiftleft.codepropertygraph.generated.nodes.NewType -import io.shiftleft.passes.{KeyPool, CpgPass} -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.codepropertygraph.generated.PropertyNames +import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import scala.collection.mutable -import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal /** Creates a `TYPE` node for each type in `usedTypes` as well as all inheritsFrom type names in the CPG * * Alternatively, set `getTypesFromCpg = true`. If this is set, the `registeredTypes` argument will be ignored. * Instead, type nodes will be created for every unique `TYPE_FULL_NAME` value in the CPG. */ -class TypeNodePass protected ( - registeredTypes: List[String], - cpg: Cpg, - keyPool: Option[KeyPool], - getTypesFromCpg: Boolean -) extends CpgPass(cpg, "types", keyPool) { +class TypeNodePass protected (registeredTypes: List[String], cpg: Cpg, getTypesFromCpg: Boolean) + extends CpgPass(cpg, "types") { protected def typeDeclTypes: mutable.Set[String] = { val typeDeclTypes = mutable.Set[String]() @@ -33,9 +29,8 @@ class TypeNodePass protected ( protected def typeFullNamesFromCpg: Set[String] = { cpg.all - .map(_.property(PropertyNames.TYPE_FULL_NAME)) + .map(_.property(Properties.TypeFullName)) .filter(_ != null) - .map(_.toString) .toSet } @@ -51,7 +46,9 @@ class TypeNodePass protected ( val usedTypesSet = typeDeclTypes ++ typeFullNameValues usedTypesSet.remove("") val usedTypes = - (usedTypesSet.filterInPlace(!_.endsWith(NamespaceTraversal.globalNamespaceName)).toArray :+ "ANY").toSet.sorted + (usedTypesSet + .filterInPlace(!_.endsWith(NamespaceTraversal.globalNamespaceName)) + .toArray :+ Defines.Any).toSet.sorted usedTypes.foreach { typeName => val shortName = fullToShortName(typeName) @@ -65,15 +62,20 @@ class TypeNodePass protected ( } object TypeNodePass { - def withTypesFromCpg(cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(Nil, cpg, keyPool, getTypesFromCpg = true) + def withTypesFromCpg(cpg: Cpg): TypeNodePass = { + new TypeNodePass(Nil, cpg, getTypesFromCpg = true) } - def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg, keyPool: Option[KeyPool] = None): TypeNodePass = { - new TypeNodePass(registeredTypes, cpg, keyPool, getTypesFromCpg = false) + def withRegisteredTypes(registeredTypes: List[String], cpg: Cpg): TypeNodePass = { + new TypeNodePass(registeredTypes, cpg, getTypesFromCpg = false) } def fullToShortName(typeName: String): String = { - typeName.takeWhile(_ != ':').split('.').lastOption.getOrElse(typeName) + if (typeName.endsWith(">")) { + // special case for typeFullName with generics as suffix + typeName.takeWhile(c => c != ':' && c != '<').split('.').lastOption.getOrElse(typeName) + } else { + typeName.takeWhile(_ != ':').split('.').lastOption.getOrElse(typeName) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala index 4a496b1b2855..252bc2d28e01 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala @@ -4,7 +4,7 @@ import better.files.File import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.utils.IOUtils import org.slf4j.LoggerFactory diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XImportsPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XImportsPass.scala index bb81e30b77ff..3d6ab6bbba53 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XImportsPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XImportsPass.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.Imports.createImportNodeAndLink import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment abstract class XImportsPass(cpg: Cpg) extends ForkJoinParallelCpgPass[(Call, Assignment)](cpg) { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala index 6f42f087cb0a..7a3adaf38617 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XInheritanceFullNamePass.scala @@ -2,7 +2,7 @@ package io.joern.x2cpg.passes.frontend import io.joern.x2cpg.passes.base.TypeDeclStubCreator import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames} import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language.* @@ -41,7 +41,7 @@ abstract class XInheritanceFullNamePass(cpg: Cpg) extends ForkJoinParallelCpgPas inheritedTypes == Seq("ANY") || inheritedTypes == Seq("object") || inheritedTypes.isEmpty private def extractTypeDeclFromNode(node: AstNode): Option[String] = node match { - case x: Call if x.isCallForImportOut.nonEmpty => + case x: Call if x._isCallForImportOut.nonEmpty => x.isCallForImportOut.importedEntity.map { case imp if relativePathPattern.matcher(imp).matches() => imp.split(pathSep).toList match { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala index 7447a091f588..92733810ca93 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeHintCallLinker.scala @@ -4,9 +4,8 @@ import io.joern.x2cpg.passes.base.MethodStubCreator import io.joern.x2cpg.passes.frontend.XTypeRecovery.isDummyType import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.proto.cpg.Cpg.DispatchTypes import io.shiftleft.semanticcpg.language.* import java.util.regex.Pattern @@ -151,7 +150,7 @@ abstract class XTypeHintCallLinker(cpg: Cpg) extends CpgPass(cpg) { name, fullName, "", - DispatchTypes.DYNAMIC_DISPATCH.name(), + DispatchTypes.DYNAMIC_DISPATCH, argSize, builder, isExternal, diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala index b82ed185384c..894671b56646 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala @@ -1,23 +1,21 @@ package io.joern.x2cpg.passes.frontend import io.joern.x2cpg.{Defines, X2CpgConfig} -import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes, NodeTypes, Operators, PropertyNames} import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.passes.{CpgPass, CpgPassBase, ForkJoinParallelCpgPass} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.importresolver.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.{Assignment, FieldAccess} import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder -import scopt.OParser +import scopt.{DefaultOParserSetup, OParser} import java.util.regex.Pattern import scala.annotation.tailrec import scala.collection.mutable -import scala.util.{Failure, Success, Try} import scala.util.matching.Regex +import scala.util.{Failure, Success, Try} /** @param iterations * the number of iterations to run. @@ -31,7 +29,14 @@ object XTypeRecoveryConfig { def parse(cmdLineArgs: Seq[String]): XTypeRecoveryConfig = { OParser - .parse(parserOptions, cmdLineArgs, XTypeRecoveryConfig()) + .parse( + parserOptions, + cmdLineArgs, + XTypeRecoveryConfig(), + new DefaultOParserSetup { + override def errorOnUnknownArgument = false + } + ) .getOrElse( throw new RuntimeException( s"unable to parse XTypeRecoveryConfig from commandline arguments ${cmdLineArgs.mkString(" ")}" @@ -96,8 +101,7 @@ class XTypeRecoveryState(val config: XTypeRecoveryConfig = XTypeRecoveryConfig() object XTypeRecoveryPassGenerator { private def linkMembersToTheirRefs(cpg: Cpg, builder: DiffGraphBuilder): Unit = { - import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromIteratorExt - import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt + import io.joern.x2cpg.passes.frontend.XTypeRecovery.{AllNodeTypesFromIteratorExt, AllNodeTypesFromNodeExt} def getFieldBaseTypes(fieldAccess: FieldAccess): Iterator[TypeDecl] = { fieldAccess @@ -151,7 +155,7 @@ abstract class XTypeRecoveryPassGenerator[CompilationUnitType <: AstNode]( if (postTypeRecoveryAndPropagation) res.append( new CpgPass(cpg): - override def run(builder: BatchedUpdate.DiffGraphBuilder): Unit = { + override def run(builder: DiffGraphBuilder): Unit = { XTypeRecoveryPassGenerator.linkMembersToTheirRefs(cpg, builder) } ) @@ -257,17 +261,13 @@ object XTypeRecovery { */ def isDummyType(typ: String): Boolean = DummyTokens.exists(typ.contains) - @deprecated("please use XTypeRecoveryConfig.parserOptionsForParserConfig", since = "2.0.415") - def parserOptions[R <: X2CpgConfig[R] & TypeRecoveryParserConfig[R]]: OParser[?, R] = - XTypeRecoveryConfig.parserOptionsForParserConfig - // The below are convenience calls for accessing type properties, one day when this pass uses `Tag` nodes instead of // the symbol table then perhaps this would work out better implicit class AllNodeTypesFromNodeExt(x: StoredNode) { def allTypes: Iterator[String] = - (x.property(PropertyNames.TYPE_FULL_NAME, "ANY") +: - (x.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq.empty) - ++ x.property(PropertyNames.POSSIBLE_TYPES, Seq.empty))).iterator + (x.propertyOption(Properties.TypeFullName).getOrElse("ANY") +: + (x.property(Properties.DynamicTypeHintFullName) ++ + x.property(Properties.PossibleTypes))).iterator def getKnownTypes: Set[String] = { x.allTypes.toSet.filterNot(XTypeRecovery.unknownTypePattern.matches) @@ -301,8 +301,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( state: XTypeRecoveryState ) extends Runnable { - import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt - import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromIteratorExt + import io.joern.x2cpg.passes.frontend.XTypeRecovery.{AllNodeTypesFromIteratorExt, AllNodeTypesFromNodeExt} protected val logger: Logger = LoggerFactory.getLogger(getClass) @@ -436,7 +435,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( * @param a * assignment call pointer. */ - protected def visitAssignments(a: Assignment): Set[String] = visitAssignmentArguments(a.argumentOut.l) + protected def visitAssignments(a: Assignment): Set[String] = + visitAssignmentArguments(a.argumentOut.cast[CfgNode].l) protected def visitAssignmentArguments(args: List[AstNode]): Set[String] = args match { case List(i: Identifier, b: Block) => visitIdentifierAssignedToBlock(i, b) @@ -555,7 +555,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( isFieldCache.getOrElseUpdate(i, isFieldUncached(i)) protected def isFieldUncached(i: Identifier): Boolean = - i.method.typeDecl.member.nameExact(i.name).nonEmpty + Try(i.method.typeDecl.member.nameExact(i.name).nonEmpty).getOrElse(false) /** Associates the types with the identifier. This may sometimes be an identifier that should be considered a field * which this method uses [[isField]] to determine. @@ -569,7 +569,9 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( val fieldName = getFieldName(fa).split(Pattern.quote(pathSep)).last Try(cpg.member.nameExact(fieldName).typeDecl.fullName.filterNot(_.contains("ANY")).toSet) match case Failure(exception) => - logger.warn("Unable to obtain name of member's parent type declaration", exception) + logger.warn( + s"Unable to obtain name of member's parent type declaration: ${cpg.member.nameExact(fieldName).propertiesMap.mkString(",")}" + ) Set.empty case Success(typeDeclNames) => typeDeclNames } @@ -810,7 +812,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( case ::(_: TypeRef, ::(f: FieldIdentifier, _)) => f.canonicalName case xs => - logger.warn(s"Unhandled field structure ${xs.map(x => (x.label, x.code)).mkString(",")} @ ${debugLocation(fa)}") + val debugInfo = xs.collect { case x: CfgNode => (x.label(), x.code) }.mkString(",") + logger.warn(s"Unhandled field structure $debugInfo @ ${debugLocation(fa)}") wrapName("") } } @@ -1231,7 +1234,8 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( lazy val existingTypes = storedNode.getKnownTypes val hasUnknownTypeFullName = storedNode - .property(PropertyNames.TYPE_FULL_NAME, Defines.Any) + .propertyOption(Properties.TypeFullName) + .getOrElse(Defines.Any) .matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) if (types.nonEmpty && (hasUnknownTypeFullName || types.toSet != existingTypes)) { @@ -1270,10 +1274,12 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode]( */ protected def storeDefaultTypeInfo(n: StoredNode, types: Seq[String]): Unit = val hasUnknownType = - n.property(PropertyNames.TYPE_FULL_NAME, Defines.Any).matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) + n.propertyOption(Properties.TypeFullName) + .getOrElse(Defines.Any) + .matches(XTypeRecovery.unknownTypePattern.pattern.pattern()) if (types.toSet != n.getKnownTypes || (hasUnknownType && types.nonEmpty)) { - setTypes(n, (n.property(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq.empty) ++ types).distinct) + setTypes(n, (n.propertyOption(PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME).getOrElse(Seq.empty) ++ types).distinct) } /** If there is only 1 type hint then this is set to the `typeFullName` property and `dynamicTypeHintFullName` is diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala index 3714a9c3b6c1..6d9509df1d56 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.utils.LinkingUtil class AliasLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala index e60479b1be34..1d5bc0bc552d 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala @@ -1,13 +1,14 @@ package io.joern.x2cpg.passes.typerelations -import io.joern.x2cpg.passes.frontend.Dereference import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Member, StoredNode} import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.Call +import io.shiftleft.codepropertygraph.generated.nodes.Member +import io.shiftleft.codepropertygraph.generated.nodes.StoredNode import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.language.operatorextension.{OpNodes, allFieldAccessTypes} -import io.shiftleft.semanticcpg.utils.MemberAccess +import io.shiftleft.semanticcpg.language.operatorextension.OpNodes +import io.shiftleft.semanticcpg.language.operatorextension.allFieldAccessTypes import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* @@ -67,18 +68,16 @@ class FieldAccessLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { dstFullNameKey: String, dstGraph: DiffGraphBuilder ): Unit = { - val dereference = Dereference(cpg) - cpg.graph.nodes(srcLabels*).asScala.cast[SRC_NODE_TYPE].filterNot(_.outE(edgeType).hasNext).foreach { srcNode => + cpg.graph.nodes(srcLabels*).cast[SRC_NODE_TYPE].filterNot(_.outE(edgeType).hasNext).foreach { srcNode => if (!srcNode.outE(edgeType).hasNext) { getDstFullNames(srcNode).foreach { dstFullName => - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - dstNodeMap(dereferenceDstFullName) match { + dstNodeMap(dstFullName) match { case Some(dstNode) => dstGraph.addEdge(srcNode, dstNode, edgeType) case None if dstNodeMap(dstFullName).isDefined => dstGraph.addEdge(srcNode, dstNodeMap(dstFullName).get, edgeType) case None => - logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dstFullName) } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala index 5f96ba2e76dc..d1a3af47136d 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala @@ -1,10 +1,11 @@ package io.joern.x2cpg.passes.typerelations import io.shiftleft.codepropertygraph.generated.Cpg +import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.joern.x2cpg.utils.LinkingUtil +import io.shiftleft.semanticcpg.language.* /** Create INHERITS_FROM edges from `TYPE_DECL` nodes to `TYPE` nodes. */ diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala index 0e85d34636df..30b0d5e569a5 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala @@ -1,39 +1,108 @@ package io.joern.x2cpg.utils -import java.util.concurrent.ConcurrentLinkedQueue -import scala.sys.process.{Process, ProcessLogger} -import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters.* +import org.slf4j.LoggerFactory -trait ExternalCommand { +import java.io.BufferedReader +import java.io.File +import java.io.InputStreamReader +import java.nio.file.Path +import java.nio.file.Paths +import scala.jdk.CollectionConverters.* +import scala.util.control.NonFatal +import scala.util.Failure +import scala.util.Success +import scala.util.Try - protected val IsWin: Boolean = scala.util.Properties.isWin +object ExternalCommand { - // do not prepend any shell layer by default - // individual frontends may override this - protected val shellPrefix: Seq[String] = Nil + private val logger = LoggerFactory.getLogger(ExternalCommand.getClass) - protected def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - result match { - case Success(0) => - Success(stdOut) - case _ => + case class ExternalCommandResult(exitCode: Int, stdOut: Seq[String], stdErr: Seq[String]) { + def successOption: Option[Seq[String]] = exitCode match { + case 0 => Some(stdOut) + case _ => None + } + def toTry: Try[Seq[String]] = exitCode match { + case 0 => Success(stdOut) + case nonZeroExitCode => val allOutput = stdOut ++ stdErr - Failure(new RuntimeException(allOutput.mkString(System.lineSeparator()))) + val message = s"""Process exited with code $nonZeroExitCode. Output: + |${allOutput.mkString(System.lineSeparator())} + |""".stripMargin + Failure(new RuntimeException(message)) } } - def run(command: String, cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { - val stdOutOutput = new ConcurrentLinkedQueue[String] - val stdErrOutput = new ConcurrentLinkedQueue[String] - val processLogger = ProcessLogger(stdOutOutput.add, stdErrOutput.add) - val process = shellPrefix match { - case Nil => Process(command, new java.io.File(cwd), extraEnv.toList*) - case _ => Process(shellPrefix :+ command, new java.io.File(cwd), extraEnv.toList*) + def run( + command: Seq[String], + cwd: String, + mergeStdErrInStdOut: Boolean = false, + extraEnv: Map[String, String] = Map.empty + ): ExternalCommandResult = { + val builder = new ProcessBuilder() + .command(command.toArray*) + .directory(new File(cwd)) + .redirectErrorStream(mergeStdErrInStdOut) + builder.environment().putAll(extraEnv.asJava) + + val stdOut = scala.collection.mutable.ArrayBuffer.empty[String] + val stdErr = scala.collection.mutable.ArrayBuffer.empty[String] + + try { + val process = builder.start() + + val outputReaderThread = new Thread(() => { + val outputReader = new BufferedReader(new InputStreamReader(process.getInputStream)) + outputReader.lines.iterator.forEachRemaining(stdOut.addOne) + }) + + val errorReaderThread = new Thread(() => { + val errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream)) + errorReader.lines.iterator.forEachRemaining(stdErr.addOne) + }) + + outputReaderThread.start() + errorReaderThread.start() + + val returnValue = process.waitFor() + outputReaderThread.join() + errorReaderThread.join() + + process.getInputStream.close() + process.getOutputStream.close() + process.getErrorStream.close() + process.destroy() + + if (stdErr.nonEmpty) logger.warn(s"subprocess stderr: ${stdErr.mkString(System.lineSeparator())}") + ExternalCommandResult(returnValue, stdOut.toSeq, stdErr.toSeq) + } catch { + case NonFatal(exception) => + ExternalCommandResult(1, Seq.empty, stdErr = Seq(exception.getMessage)) } - handleRunResult(Try(process.!(processLogger)), stdOutOutput.asScala.toSeq, stdErrOutput.asScala.toSeq) } -} + /** Finds the absolute path to the executable directory (e.g. `/path/to/javasrc2cpg/bin`). Based on the package path + * of a loaded classfile based on some (potentially flakey?) filename heuristics. Context: we want to be able to + * invoke the x2cpg frontends from any directory, not just their install directory, and then invoke other + * executables, like astgen, php-parser et al. + */ + def executableDir(packagePath: Path): Path = { + val packagePathAbsolute = packagePath.toAbsolutePath + val fixedDir = + if (packagePathAbsolute.toString.contains("lib")) { + var dir = packagePathAbsolute + while (dir.toString.contains("lib")) + dir = dir.getParent + dir + } else if (packagePathAbsolute.toString.contains("target")) { + var dir = packagePathAbsolute + while (dir.toString.contains("target")) + dir = dir.getParent + dir + } else { + Paths.get(".") + } -object ExternalCommand extends ExternalCommand + fixedDir.resolve("bin/").toAbsolutePath + } +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala new file mode 100644 index 000000000000..0faa1f1fa216 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/KeyPool.scala @@ -0,0 +1,80 @@ +package io.joern.x2cpg.utils + +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} + +/** A pool of long integers. Using the method `next`, the pool provides the next id in a thread-safe manner. */ +trait KeyPool { + def next: Long +} + +/** A key pool that returns the integers of the interval [first, last] in a thread-safe manner. + */ +class IntervalKeyPool(val first: Long, val last: Long) extends KeyPool { + + /** Get next number in interval or raise if number is larger than `last` + */ + def next: Long = { + if (!valid) { + throw new IllegalStateException("Call to `next` on invalidated IntervalKeyPool.") + } + val n = cur.incrementAndGet() + if (n > last) { + throw new RuntimeException("Pool exhausted") + } else { + n + } + } + + /** Split key pool into `numberOfPartitions` partitions of mostly equal size. Invalidates the current pool to ensure + * that the user does not continue to use both the original pool and pools derived from it via `split`. + */ + def split(numberOfPartitions: Int): Iterator[IntervalKeyPool] = { + valid = false + if (numberOfPartitions == 0) { + Iterator() + } else { + val curFirst = cur.get() + val k = (last - curFirst) / numberOfPartitions + (1 to numberOfPartitions).map { i => + val poolFirst = curFirst + (i - 1) * k + new IntervalKeyPool(poolFirst, poolFirst + k - 1) + }.iterator + } + } + + private val cur: AtomicLong = new AtomicLong(first - 1) + private var valid: Boolean = true +} + +/** A key pool that returns elements of `seq` in order in a thread-safe manner. + */ +class SequenceKeyPool(seq: Seq[Long]) extends KeyPool { + + val seqLen: Int = seq.size + var cur = new AtomicInteger(-1) + + override def next: Long = { + val i = cur.incrementAndGet() + if (i >= seqLen) { + throw new RuntimeException("Pool exhausted") + } else { + seq(i) + } + } +} + +object KeyPoolCreator { + + /** Divide the keyspace into n intervals and return a list of corresponding key pools. + */ + def obtain(n: Long, minValue: Long = 0, maxValue: Long = Long.MaxValue): List[IntervalKeyPool] = { + val nIntervals = Math.max(n, 1) + val intervalLen: Long = (maxValue - minValue) / nIntervals + List.range(0L, nIntervals).map { i => + val first = i * intervalLen + minValue + val last = first + intervalLen - 1 + new IntervalKeyPool(first, last) + } + } + +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala index 9fd6290b4d8b..0c87eb423e48 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala @@ -1,20 +1,17 @@ package io.joern.x2cpg.utils -import io.joern.x2cpg.passes.frontend.Dereference -import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Properties, PropertyNames} +import io.shiftleft.codepropertygraph.generated.{Cpg, Properties, PropertyNames} +import io.shiftleft.codepropertygraph.generated.nodes.NamespaceBlock +import io.shiftleft.codepropertygraph.generated.nodes.Type +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.traversal.* -import overflowdb.traversal.ChainedImplicitsTemp.* -import overflowdb.{Node, NodeDb, NodeRef, PropertyKey} -import scala.collection.mutable import scala.jdk.CollectionConverters.* trait LinkingUtil { - import overflowdb.BatchedUpdate.DiffGraphBuilder + import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder val MAX_BATCH_SIZE: Int = 100 @@ -32,61 +29,49 @@ trait LinkingUtil { def namespaceBlockFullNameToNode(cpg: Cpg, x: String): Option[NamespaceBlock] = nodesWithFullName(cpg, x).collectFirst { case x: NamespaceBlock => x } - def nodesWithFullName(cpg: Cpg, x: String): mutable.Seq[NodeRef[? <: NodeDb]] = - cpg.graph.indexManager.lookup(PropertyNames.FULL_NAME, x).asScala + def nodesWithFullName(cpg: Cpg, x: String): Iterator[StoredNode] = + cpg.graph.nodesWithProperty(propertyName = PropertyNames.FULL_NAME, value = x).cast[StoredNode] /** For all nodes `n` with a label in `srcLabels`, determine the value of `n.\$dstFullNameKey`, use that to lookup the * destination node in `dstNodeMap`, and create an edge of type `edgeType` between `n` and the destination node. */ - protected def linkToSingle( cpg: Cpg, - srcNodes: List[Node], + srcNodes: List[StoredNode], srcLabels: List[String], dstNodeLabel: String, edgeType: String, dstNodeMap: String => Option[StoredNode], dstFullNameKey: String, + dstDefaultPropertyValue: Any, dstGraph: DiffGraphBuilder, dstNotExistsHandler: Option[(StoredNode, String) => Unit] ): Unit = { - val dereference = Dereference(cpg) var loggedDeprecationWarning = false srcNodes.foreach { srcNode => // If the source node does not have any outgoing edges of this type // This check is just required for backward compatibility if (srcNode.outE(edgeType).isEmpty) { - val key = new PropertyKey[String](dstFullNameKey) srcNode - .propertyOption(key) - .filter { dstFullName => - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - srcNode.propertyDefaultValue(dstFullNameKey) != dereferenceDstFullName - } - .ifPresent { dstFullName => + .propertyOption[String](dstFullNameKey) + .filter { dstFullName => dstDefaultPropertyValue != dstFullName } + .map { dstFullName => // for `UNKNOWN` this is not always set, so we're using an Option here - val srcStoredNode = srcNode.asInstanceOf[StoredNode] - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - dstNodeMap(dereferenceDstFullName) match { + dstNodeMap(dstFullName) match { case Some(dstNode) => - dstGraph.addEdge(srcStoredNode, dstNode, edgeType) + dstGraph.addEdge(srcNode, dstNode, edgeType) case None if dstNodeMap(dstFullName).isDefined => - dstGraph.addEdge(srcStoredNode, dstNodeMap(dstFullName).get, edgeType) + dstGraph.addEdge(srcNode, dstNodeMap(dstFullName).get, edgeType) case None if dstNotExistsHandler.isDefined => - dstNotExistsHandler.get(srcStoredNode, dereferenceDstFullName) + dstNotExistsHandler.get(srcNode, dstFullName) case _ => - logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dstFullName) } } } else { srcNode.out(edgeType).property(Properties.FullName).nextOption() match { - case Some(dstFullName) => - dstGraph.setNodeProperty( - srcNode.asInstanceOf[StoredNode], - dstFullNameKey, - dereference.dereferenceTypeFullName(dstFullName) - ) - case None => logger.info(s"Missing outgoing edge of type $edgeType from node $srcNode") + case Some(dstFullName) => dstGraph.setNodeProperty(srcNode, dstFullNameKey, dstFullName) + case None => logger.info(s"Missing outgoing edge of type $edgeType from node $srcNode") } if (!loggedDeprecationWarning) { logger.info( @@ -110,23 +95,21 @@ trait LinkingUtil { dstGraph: DiffGraphBuilder ): Unit = { var loggedDeprecationWarning = false - val dereference = Dereference(cpg) - cpg.graph.nodes(srcLabels*).asScala.cast[SRC_NODE_TYPE].foreach { srcNode => + cpg.graph.nodes(srcLabels*).cast[SRC_NODE_TYPE].foreach { srcNode => if (!srcNode.outE(edgeType).hasNext) { getDstFullNames(srcNode).foreach { dstFullName => - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - dstNodeMap(dereferenceDstFullName) match { + dstNodeMap(dstFullName) match { case Some(dstNode) => dstGraph.addEdge(srcNode, dstNode, edgeType) case None if dstNodeMap(dstFullName).isDefined => dstGraph.addEdge(srcNode, dstNodeMap(dstFullName).get, edgeType) case None => - logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dstFullName) } } } else { val dstFullNames = srcNode.out(edgeType).property(Properties.FullName).l - dstGraph.setNodeProperty(srcNode, dstFullNameKey, dstFullNames.map(dereference.dereferenceTypeFullName)) + dstGraph.setNodeProperty(srcNode, dstFullNameKey, dstFullNames) if (!loggedDeprecationWarning) { logger.info( s"Using deprecated CPG format with already existing $edgeType edge between" + diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala index 4fd0d93df7b5..5cfdb1f50769 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala @@ -18,10 +18,8 @@ case class DependencyResolverParams( ) object DependencyResolver { - private val logger = LoggerFactory.getLogger(getClass) - private val defaultGradleProjectName = "app" - private val defaultGradleConfigurationName = "compileClasspath" - private val MaxSearchDepth: Int = 4 + private val logger = LoggerFactory.getLogger(getClass) + private val MaxSearchDepth: Int = 4 def getCoordinates( projectDir: Path, @@ -31,9 +29,10 @@ object DependencyResolver { if (isMavenBuildFile(buildFile)) // TODO: implement None - else if (isGradleBuildFile(buildFile)) - getCoordinatesForGradleProject(buildFile.getParent, defaultGradleConfigurationName) - else { + else if (isGradleBuildFile(buildFile)) { + // TODO: Don't limit this to the default configuration name + getCoordinatesForGradleProject(buildFile.getParent, "compileClasspath") + } else { logger.warn(s"Found unsupported build file $buildFile") Nil } @@ -46,7 +45,9 @@ object DependencyResolver { projectDir: Path, configuration: String ): Option[collection.Seq[String]] = { - val lines = ExternalCommand.run(s"gradle dependencies --configuration $configuration", projectDir.toString) match { + val lines = ExternalCommand + .run(Seq("gradle", "dependencies", "--configuration,", configuration), projectDir.toString) + .toTry match { case Success(lines) => lines case Failure(exception) => logger.warn( @@ -84,12 +85,14 @@ object DependencyResolver { projectDir: Path ): Option[collection.Seq[String]] = { logger.info("resolving Gradle dependencies at {}", projectDir) - val gradleProjectName = params.forGradle.getOrElse(GradleConfigKeys.ProjectName, defaultGradleProjectName) - val gradleConfiguration = - params.forGradle.getOrElse(GradleConfigKeys.ConfigurationName, defaultGradleConfigurationName) - GradleDependencies.get(projectDir, gradleProjectName, gradleConfiguration) match { - case Some(deps) => Some(deps) - case None => + val maybeProjectNameOverride = params.forGradle.get(GradleConfigKeys.ProjectName) + val maybeConfigurationOverride = params.forGradle.get(GradleConfigKeys.ConfigurationName) + + GradleDependencies.get(projectDir, maybeProjectNameOverride, maybeConfigurationOverride) match { + case dependenciesMap if dependenciesMap.values.exists(_.nonEmpty) => + Option(dependenciesMap.values.flatten.toSet.toSeq) + + case _ => logger.warn(s"Could not download Gradle dependencies for project at path `$projectDir`") None } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/GradleDependencies.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/GradleDependencies.scala index bc8c13918df7..f867cb86391b 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/GradleDependencies.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/GradleDependencies.scala @@ -1,19 +1,40 @@ package io.joern.x2cpg.utils.dependency -import better.files._ +import better.files.* import org.gradle.tooling.{GradleConnector, ProjectConnection} -import org.gradle.tooling.model.GradleProject +import org.gradle.tooling.model.{GradleProject, ProjectIdentifier, Task} import org.gradle.tooling.model.build.BuildEnvironment import org.slf4j.LoggerFactory import java.io.ByteArrayOutputStream import java.nio.file.{Files, Path} -import java.io.{File => JFile} +import java.io.File as JFile import java.util.stream.Collectors -import scala.jdk.CollectionConverters._ +import scala.collection.mutable +import scala.jdk.CollectionConverters.* import scala.util.{Failure, Random, Success, Try, Using} -case class GradleProjectInfo(gradleVersion: String, tasks: Seq[String], hasAndroidSubproject: Boolean = false) { +case class ProjectNameInfo(projectName: String, isSubproject: Boolean) { + override def toString: String = { + if (isSubproject) + s":$projectName" + else + projectName + } + + def makeGradleTaskName(taskName: String): String = { + if (isSubproject) + s"$projectName:$taskName" + else + taskName + } +} + +case class GradleProjectInfo( + subprojects: Map[ProjectNameInfo, List[String]], + gradleVersion: String, + hasAndroidSubproject: Boolean +) { def gradleVersionMajorMinor(): (Int, Int) = { def isValidPart(part: String) = part.forall(Character.isDigit) val parts = gradleVersion.split('.') @@ -27,45 +48,73 @@ case class GradleProjectInfo(gradleVersion: String, tasks: Seq[String], hasAndro } } -object Constants { - val aarFileExtension = "aar" - val gradleAndroidPropertyPrefix = "android." - val gradlePropertiesTaskName = "properties" - val jarInsideAarFileName = "classes.jar" -} - case class GradleDepsInitScript(contents: String, taskName: String, destinationDir: Path) object GradleDependencies { - private val logger = LoggerFactory.getLogger(getClass) - private val initScriptPrefix = "x2cpg.init.gradle" - private val taskNamePrefix = "x2cpgCopyDeps" - private val tempDirPrefix = "x2cpgDependencies" + private val aarFileExtension = "aar" + private val gradleAndroidPropertyPrefix = "android" + private val gradlePropertiesTaskName = "properties" + private val jarInsideAarFileName = "classes.jar" + private val defaultConfigurationName = "releaseRuntimeClasspath" + private val initScriptPrefix = "x2cpg.init.gradle" + private val taskNamePrefix = "x2cpgCopyDeps" + private val tempDirPrefix = "x2cpgDependencies" + private val defaultGradleAppName = "app" + + private val logger = LoggerFactory.getLogger(getClass) // works with Gradle 5.1+ because the script makes use of `task.register`: // https://docs.gradle.org/current/userguide/task_configuration_avoidance.html - private def gradle5OrLaterAndroidInitScript( - taskName: String, - destination: String, - gradleProjectName: String, - gradleConfigurationName: String - ): String = { + private def getInitScriptContent(taskName: String, destination: String, projectInfo: GradleProjectInfo): String = { + val projectConfigurationString = projectInfo.subprojects + .map { case (projectNameInfo, configurationNames) => + val quotedConfigurationNames = configurationNames.map(name => s"\"$name\"").mkString(", ") + s"\"${projectNameInfo.projectName}\": [$quotedConfigurationNames]" + } + .mkString(", ") + + val taskCreationFunction = projectInfo.gradleVersionMajorMinor() match { + case (major, minor) if major >= 5 && minor >= 1 => "tasks.register" + case _ => "tasks.create" + } + + val androidTaskDefinition = Option.when(projectInfo.hasAndroidSubproject)(s""" + |def androidDepsCopyTaskName = taskName + "_androidDeps" + | $taskCreationFunction(androidDepsCopyTaskName, Copy) { + | duplicatesStrategy = 'include' + | into destinationDir + | from project.configurations.find { it.name.equals("androidApis") } + | } + |""".stripMargin) + + val dependsOnAndroidTask = Option.when(projectInfo.hasAndroidSubproject)("dependsOn androidDepsCopyTaskName") + s""" |allprojects { | afterEvaluate { project -> | def taskName = "$taskName" | def destinationDir = "${destination.replaceAll("\\\\", "/")}" - | def gradleProjectName = "$gradleProjectName" - | def gradleConfigurationName = "$gradleConfigurationName" + | def gradleProjectConfigurations = [$projectConfigurationString] | - | if (project.name.equals(gradleProjectName)) { + | if (gradleProjectConfigurations.containsKey(project.name)) { + | def gradleConfigurationNames = gradleProjectConfigurations.get(project.name) + | | def compileDepsCopyTaskName = taskName + "_compileDeps" - | tasks.register(compileDepsCopyTaskName, Copy) { - | def selectedConfig = project.configurations.find { it.name.equals(gradleConfigurationName) } + | $taskCreationFunction(compileDepsCopyTaskName, Copy) { + | + | def selectedConfigs = project.configurations.findAll { + | configuration -> gradleConfigurationNames.contains(configuration.getName()) + | } + | | def componentIds = [] - | if (selectedConfig != null) { - | componentIds = selectedConfig.incoming.resolutionResult.allDependencies.collect { it.selected.id } + | if (!selectedConfigs.isEmpty()) { + | for (selectedConfig in selectedConfigs) { + | componentIds = selectedConfig.incoming.resolutionResult.allDependencies.findAll { + | dep -> dep instanceof org.gradle.api.internal.artifacts.result.DefaultResolvedDependencyResult + | } .collect { it.selected.id } + | } | } + | | def result = dependencies.createArtifactResolutionQuery() | .forComponents(componentIds) | .withArtifacts(JvmLibrary, SourcesArtifact) @@ -74,14 +123,9 @@ object GradleDependencies { | into destinationDir | from result.resolvedComponents.collect { it.getArtifacts(SourcesArtifact).collect { it.file } } | } - | def androidDepsCopyTaskName = taskName + "_androidDeps" - | tasks.register(androidDepsCopyTaskName, Copy) { - | duplicatesStrategy = 'include' - | into destinationDir - | from project.configurations.find { it.name.equals("androidApis") } - | } - | tasks.register(taskName, Copy) { - | dependsOn androidDepsCopyTaskName + | ${androidTaskDefinition.getOrElse("")} + | $taskCreationFunction(taskName, Copy) { + | ${dependsOnAndroidTask.getOrElse("")} | dependsOn compileDepsCopyTaskName | } | } @@ -90,40 +134,9 @@ object GradleDependencies { |""".stripMargin } - // this init script _should_ work with Gradle >=4, but has not been tested thoroughly - // TODO: add test cases for older Gradle versions - private def gradle5OrLaterInitScript( - taskName: String, - destination: String, - gradleConfigurationName: String - ): String = { - val into = destination.replaceAll("\\\\", "/") - val fromConfigurations = - Set(s"from configurations.$gradleConfigurationName", "from configurations.runtimeClasspath").mkString("\n") - s""" - |allprojects { - | apply plugin: 'java' - | task $taskName(type: Copy) { - | $fromConfigurations - | into "$into" - | } - |} - |""".stripMargin - } - - private def makeInitScript( - destinationDir: Path, - forAndroid: Boolean, - gradleProjectName: String, - gradleConfigurationName: String - ): GradleDepsInitScript = { + private def makeInitScript(destinationDir: Path, projectInfo: GradleProjectInfo): GradleDepsInitScript = { val taskName = taskNamePrefix + "_" + (Random.alphanumeric take 8).toList.mkString - val content = - if (forAndroid) { - gradle5OrLaterAndroidInitScript(taskName, destinationDir.toString, gradleProjectName, gradleConfigurationName) - } else { - gradle5OrLaterInitScript(taskName, destinationDir.toString, gradleConfigurationName) - } + val content = getInitScriptContent(taskName, destinationDir.toString, projectInfo) GradleDepsInitScript(content, taskName, destinationDir) } @@ -131,39 +144,170 @@ object GradleDependencies { GradleConnector.newConnector().forProjectDirectory(projectDir).connect() } - private def getGradleProjectInfo(projectDir: Path, projectName: String): Option[GradleProjectInfo] = { + private def getConfigurationsWithDependencies(dependenciesOutput: String): List[String] = { + // TODO: this is a heuristic for matching configuration names based on a sample of open source projects. + // either add more options to this or revise the approach completely if this turns out to miss too much. + val configurationNameRegex = raw"(\S*([rR]elease|[rR]untime)\S*) -.+$$".r + val lines = dependenciesOutput.lines.iterator().asScala + val results = mutable.Set[String]() + + while (lines.hasNext) { + val line = lines.next() + line match { + case configurationNameRegex(configurationName, _) if lines.hasNext => + val next = lines.next() + if (next != "No dependencies") { + results.addOne(configurationName) + } + lines.takeWhile(_.nonEmpty) + + case _ => + lines.takeWhile(_.nonEmpty) + } + } + + results.filterNot(_.toLowerCase.contains("test")).toList + } + + private def getGradleProjectInfo( + projectDir: Path, + projectNameOverride: Option[String], + configurationNameOverride: Option[String] + ): Option[GradleProjectInfo] = { Try(makeConnection(projectDir.toFile)) match { case Success(gradleConnection) => Using.resource(gradleConnection) { connection => try { val buildEnv = connection.getModel[BuildEnvironment](classOf[BuildEnvironment]) val project = connection.getModel[GradleProject](classOf[GradleProject]) - val hasAndroidPrefixGradleProperty = - runGradleTask(connection, Constants.gradlePropertiesTaskName) match { + + val availableProjectNames = ProjectNameInfo(project.getName, false) :: project.getChildren.asScala + .map(child => ProjectNameInfo(child.getName, true)) + .toList + + val availableProjectNamesString = availableProjectNames.mkString(" ") + + logger.debug(s"Found gradle project names ${availableProjectNames.mkString(" ")}") + + val selectedProjectNames = if (projectNameOverride.isDefined) { + val overrideName = projectNameOverride.get + availableProjectNames.find(_.projectName == overrideName) match { + case Some(projectInfo) => + logger.debug(s"Only fetching dependencies for overridden project name $overrideName") + projectInfo :: Nil + + case None => + logger.warn( + s"Project name override was specified for dependency fetching ($overrideName), but no such project found." + ) + logger.warn( + s"Falling back to fetching dependencies for all available project names: $availableProjectNamesString" + ) + availableProjectNames + } + } else { + availableProjectNames.find(_.projectName == defaultGradleAppName) match { + case Some(defaultProjectInfo) => + // TODO: This is a temporary check to avoid issues that could arise from subprojects using conflicting + // versions of dependencies. Ideally dependencies for all of these projects will be fetched with + // any conflicts handled in the consumer. + logger.debug(s"Found project with default name ($defaultGradleAppName)") + logger.debug(s"Fetching dependencies only for default project ($defaultGradleAppName)") + defaultProjectInfo :: Nil + + case None => + logger.debug(s"No project name override or project with default name ($defaultGradleAppName) found.") + logger.debug(s"Fetching dependencies for all available projects: $availableProjectNamesString") + availableProjectNames + } + } + + val selectedConfigurations = selectedProjectNames.flatMap { projectNameInfo => + val dependenciesTaskName = projectNameInfo.makeGradleTaskName("dependencies") + + val availableConfigurations = runGradleTask(connection, dependenciesTaskName) match { case Some(out) => - out.split('\n').exists(_.startsWith(Constants.gradleAndroidPropertyPrefix)) - case None => false + getConfigurationsWithDependencies(out) match { + case Nil => + logger.debug(s"No configurations with dependencies found for project $projectNameInfo") + Nil + case deps => + logger.debug( + s"Found the following configurations with dependencies for project $projectNameInfo: ${deps.mkString(", ")}" + ) + deps + } + case None => + logger.warn(s"Failure executing dependencies task $dependenciesTaskName") + Nil } - val info = GradleProjectInfo( - buildEnv.getGradle.getGradleVersion, - project.getTasks.asScala.map(_.getName).toSeq, - hasAndroidPrefixGradleProperty - ) - if (hasAndroidPrefixGradleProperty) { - val validProjectNames = List(project.getName) ++ project.getChildren.getAll.asScala.map(_.getName) - logger.debug(s"Found Gradle projects: ${validProjectNames.mkString(",")}") - if (!validProjectNames.contains(projectName)) { - val validProjectNamesStr = validProjectNames.mkString(",") - logger.warn( - s"The provided Gradle project name `$projectName` is is not part of the valid project names: `$validProjectNamesStr`" - ) - None + + val availableConfigurationsString = availableConfigurations.mkString(", ") + + val selectedConfigurations = if (availableConfigurations.isEmpty) { + // Skip logging below, since no available configurations already logged + Nil + } else if (configurationNameOverride.isDefined) { + val overrideName = configurationNameOverride.get + availableConfigurations.find(_ == overrideName) match { + case Some(configurationName) => + logger.debug(s"Only fetching dependencies for overridden configuration $overrideName") + configurationName :: Nil + + case None => + logger.warn( + s"Configuration name override was specified for dependency fetching ($overrideName), but no such configuration found for project $projectNameInfo." + ) + logger.warn( + s"Falling back to fetching dependencies for all available configurations: $availableConfigurationsString" + ) + availableConfigurations + } } else { - Some(info) + availableConfigurations.find(_ == defaultConfigurationName) match { + case Some(defaultConfigurationName) => + // TODO: This is a temporary check to avoid issues that could arise from subprojects using conflicting + // versions of dependencies. Ideally dependencies for all of these configurations will be fetched with + // any conflicts handled in the consumer. + logger.debug( + s"Found default configuration name ($defaultConfigurationName) for project $projectNameInfo" + ) + logger.debug( + s"Fetching dependencies only for default configuration ($defaultConfigurationName) for project $projectNameInfo" + ) + defaultConfigurationName :: Nil + + case None => + logger.debug( + s"No configuration override or configuration with default name ($defaultConfigurationName) found for project $projectNameInfo." + ) + logger.debug( + s"Fetching dependencies for all available configurations for project $projectNameInfo: $availableConfigurationsString" + ) + availableConfigurations + } + } + + Option.when(selectedConfigurations.nonEmpty) { + projectNameInfo -> selectedConfigurations + } + }.toMap + + val includesAndroidProject = selectedProjectNames.exists { projectNameInfo => + val propertiesTaskName = projectNameInfo.makeGradleTaskName(gradlePropertiesTaskName) + + runGradleTask(connection, propertiesTaskName) match { + case Some(out) => + out.lines().iterator().asScala.exists(_.startsWith(gradleAndroidPropertyPrefix)) + case None => false } - } else { - Some(info) } + + val gradleVersion = buildEnv.getGradle.getGradleVersion + + val gradleProjectInfo = GradleProjectInfo(selectedConfigurations, gradleVersion, includesAndroidProject) + + Option(gradleProjectInfo) } catch { case t: Throwable => logger.warn(s"Caught exception while trying use Gradle connection: ${t.getMessage}") @@ -198,15 +342,17 @@ object GradleDependencies { private def runGradleTask( connection: ProjectConnection, - initScript: GradleDepsInitScript, + taskName: String, + destinationDir: Path, initScriptPath: String ): Option[collection.Seq[String]] = { Using.resources(new ByteArrayOutputStream, new ByteArrayOutputStream) { case (stdoutStream, stderrStream) => - logger.info(s"Executing gradle task '${initScript.taskName}'...") + logger.debug(s"Executing gradle task '${taskName}'...") + Try( connection .newBuild() - .forTasks(initScript.taskName) + .forTasks(taskName) .withArguments("--init-script", initScriptPath) .setStandardOutput(stdoutStream) .setStandardError(stderrStream) @@ -215,14 +361,26 @@ object GradleDependencies { case Success(_) => val result = Files - .list(initScript.destinationDir) + .list(destinationDir) .collect(Collectors.toList[Path]) .asScala .map(_.toAbsolutePath.toString) - logger.info(s"Resolved `${result.size}` dependency files.") + logger.info(s"Task $taskName resolved `${result.size}` dependency files.") Some(result) case Failure(ex) => logger.warn(s"Caught exception while executing Gradle task: ${ex.getMessage}") + val androidSdkError = "Define a valid SDK location with an ANDROID_HOME environment variable" + if (stderrStream.toString.contains(androidSdkError)) { + logger.warn( + "A missing Android SDK configuration caused gradle dependency fetching failures. Please define a valid SDK location with an ANDROID_HOME environment variable or by setting the sdk.dir path in your project's local properties file" + ) + } + if (stderrStream.toString.contains("Could not compile initialization script")) { + val scriptContents = File(initScriptPath).contentAsString + logger.debug( + s"########## INITIALIZATION_SCRIPT ##########\n$scriptContents\n###########################################" + ) + } logger.debug(s"Gradle task execution stdout: \n$stdoutStream") logger.debug(s"Gradle task execution stderr: \n$stderrStream") None @@ -231,14 +389,14 @@ object GradleDependencies { } private def extractClassesJarFromAar(aar: File): Option[Path] = { - val newPath = aar.path.toString.replaceFirst(Constants.aarFileExtension + "$", "jar") + val newPath = aar.path.toString.replaceFirst(aarFileExtension + "$", "jar") val aarUnzipDirSuffix = ".unzipped" val outDir = File(aar.path.toString + aarUnzipDirSuffix) - aar.unzipTo(outDir, _.getName == Constants.jarInsideAarFileName) + aar.unzipTo(outDir, _.getName == jarInsideAarFileName) val outFile = File(newPath) val classesJarEntries = outDir.listRecursively - .filter(_.path.getFileName.toString == Constants.jarInsideAarFileName) + .filter(_.path.getFileName.toString == jarInsideAarFileName) .toList if (classesJarEntries.size != 1) { logger.warn(s"Found aar file without `classes.jar` inside at path ${aar.path}") @@ -258,60 +416,61 @@ object GradleDependencies { // a destination directory. private[dependency] def get( projectDir: Path, - projectName: String, - configurationName: String - ): Option[collection.Seq[String]] = { - logger.info(s"Fetching Gradle project information at path `$projectDir` with project name `$projectName`.") - getGradleProjectInfo(projectDir, projectName) match { + projectNameOverride: Option[String], + configurationNameOverride: Option[String] + ): Map[String, collection.Seq[String]] = { + logger.info(s"Fetching Gradle project information at path `$projectDir`.") + getGradleProjectInfo(projectDir, projectNameOverride, configurationNameOverride) match { case Some(projectInfo) if projectInfo.gradleVersionMajorMinor()._1 < 5 => logger.warn(s"Unsupported Gradle version `${projectInfo.gradleVersion}`") - None + Map.empty + case Some(projectInfo) => Try(File.newTemporaryDirectory(tempDirPrefix).deleteOnExit()) match { case Success(destinationDir) => Try(File.newTemporaryFile(initScriptPrefix).deleteOnExit()) match { case Success(initScriptFile) => - val initScript = - makeInitScript(destinationDir.path, projectInfo.hasAndroidSubproject, projectName, configurationName) + val initScript = makeInitScript(destinationDir.path, projectInfo) initScriptFile.write(initScript.contents) - logger.info( - s"Downloading dependencies for configuration `$configurationName` of project `$projectName` at `$projectDir` into `$destinationDir`..." - ) Try(makeConnection(projectDir.toFile)) match { case Success(connection) => Using.resource(connection) { c => - runGradleTask(c, initScript, initScriptFile.pathAsString) match { - case Some(deps) => - Some(deps.map { d => - if (!d.endsWith(Constants.aarFileExtension)) d + projectInfo.subprojects.keys.flatMap { projectNameInfo => + val taskName = projectNameInfo.makeGradleTaskName(initScript.taskName) + + runGradleTask(c, taskName, initScript.destinationDir, initScriptFile.pathAsString) map { deps => + val depsOutput = deps.map { d => + if (!d.endsWith(aarFileExtension)) d else extractClassesJarFromAar(File(d)) match { case Some(path) => path.toString case None => d } - }) - case None => None - } + } + + projectNameInfo.projectName -> depsOutput + } + }.toMap } case Failure(ex) => logger.warn(s"Caught exception while trying to establish a Gradle connection: ${ex.getMessage}") logger.debug(s"Full exception: ", ex) - None + Map.empty } case Failure(ex) => logger.warn(s"Could not create temporary file for Gradle init script: ${ex.getMessage}") logger.debug(s"Full exception: ", ex) - None + Map.empty } case Failure(ex) => logger.warn(s"Could not create temporary directory for saving dependency files: ${ex.getMessage}") logger.debug("Full exception: ", ex) - None + Map.empty } case None => logger.warn("Could not fetch Gradle project information") - None + Map.empty } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala index 4594359e9a31..c4cc36ab168b 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala @@ -14,17 +14,25 @@ object MavenDependencies { // also separate this from fetchCommandWithOpts to log a version that clearly separates options we provide from // options specified by the user via the MAVEN_CLI_OPTS environment variable, while also making it clear that this // environment variable is being considered. - private val fetchCommand = - s"mvn $$$MavenCliOpts --fail-never -B dependency:build-classpath -DincludeScope=compile -Dorg.slf4j.simpleLogger.defaultLogLevel=info -Dorg.slf4j.simpleLogger.logFile=System.out" + private val fetchArgs = + Vector( + "--fail-never", + "-B", + "dependency:build-classpath", + "-DincludeScope=compile", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=info", + "-Dorg.slf4j.simpleLogger.logFile=System.out" + ) - private val fetchCommandWithOpts = { + private val fetchCommandWithOpts: Seq[String] = { // These options suppress output, so if they're provided we won't get any results. // "-q" and "--quiet" are the only ones that would realistically be used. val optionsToStrip = Set("-h", "--help", "-q", "--quiet", "-v", "--version") - val mavenOpts = Option(System.getenv(MavenCliOpts)).getOrElse("") - val mavenOptsStripped = mavenOpts.split(raw"\s").filterNot(optionsToStrip.contains).mkString(" ") - fetchCommand.replace(s"$$$MavenCliOpts", mavenOptsStripped) + val cli = org.apache.commons.exec.CommandLine("mvn") + cli.addArguments(System.getenv(MavenCliOpts), false) // a null from getenv() does not add any argument + + cli.toStrings.toIndexedSeq.filterNot(optionsToStrip.contains) ++ fetchArgs } private def logErrors(output: String): Unit = { @@ -34,13 +42,13 @@ object MavenDependencies { "The compile class path may be missing or partial.\n" + "Results will suffer from poor type information.\n" + "To fix this issue, please ensure that the below command can be executed successfully from the project root directory:\n" + - fetchCommand + "\n\n", + s"mvn $MavenCliOpts " + fetchArgs.mkString(" ") + "\n\n", output ) } private[dependency] def get(projectDir: Path): Option[collection.Seq[String]] = { - val lines = ExternalCommand.run(fetchCommandWithOpts, projectDir.toString) match { + val lines = ExternalCommand.run(fetchCommandWithOpts, projectDir.toString).toTry match { case Success(lines) => if (lines.contains("[INFO] Build failures were ignored.")) { logErrors(lines.mkString(System.lineSeparator())) diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPClient.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPClient.scala new file mode 100644 index 000000000000..f32c2ad88ddb --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPClient.scala @@ -0,0 +1,64 @@ +package io.joern.x2cpg.utils.server + +import java.io.IOException +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.URI +import java.net.http.HttpRequest.BodyPublishers +import java.net.http.HttpResponse.BodyHandlers +import scala.util.Failure +import scala.util.Success +import scala.util.Try + +/** Represents an HTTP client for interacting with a frontend server. + * + * This class provides functionality to create and send HTTP requests to a specified frontend server. The server's host + * needs to be configured. + * + * @param port + * The port of the frontend server. + */ +case class FrontendHTTPClient(port: Int) { + + /** The underlying HTTP client used to send requests. */ + private val underlyingClient: HttpClient = HttpClient.newBuilder().build() + + /** Builds an HTTP POST request with the given arguments. + * + * The request is sent to the configured host and port, with a URI path defined by `FrontendHTTPDefaults.route`. The + * request body is constructed from the `args` array, which is concatenated into a single string separated by "&" and + * sent as `application/x-www-form-urlencoded`. + * + * @param args + * An array of arguments to be included in the POST request body. + * @return + * The constructed `HttpRequest` object. + */ + def buildRequest(args: Array[String]): HttpRequest = { + HttpRequest + .newBuilder() + .uri(URI.create(s"http://localhost:$port/run")) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(BodyPublishers.ofString(args.mkString("&"))) + .build() + } + + /** Sends the given HTTP request and returns the response body if successful. + * + * This method sends the provided `HttpRequest` built with `buildRequest` using the underlying HTTP client. If the + * response status code is 200, the response body is returned as a `Success`. If the status code indicates a failure, + * a `Failure` containing an `IOException` with the error details is returned. + * + * @param req + * The `HttpRequest` to be sent. + * @return + * A `Try[String]` containing the response body in case of success, or an `IOException` in case of failure. + */ + def sendRequest(req: HttpRequest): Try[String] = { + val resp = underlyingClient.send(req, BodyHandlers.ofString()) + resp match { + case r if r.statusCode() == 200 => Success(resp.body()) + case r => Failure(new IOException(s"Sending request failed with code ${r.statusCode()}: ${r.body()}")) + } + } +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala new file mode 100644 index 000000000000..14ec8c04bc88 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala @@ -0,0 +1,184 @@ +package io.joern.x2cpg.utils.server + +import io.joern.x2cpg.X2Cpg +import io.joern.x2cpg.X2CpgConfig +import io.joern.x2cpg.X2CpgFrontend +import io.joern.x2cpg.X2CpgMain +import net.freeutils.httpserver.HTTPServer +import net.freeutils.httpserver.HTTPServer.Context +import org.slf4j.LoggerFactory + +import java.util.concurrent.Executors +import java.util.concurrent.ExecutorService +import scala.annotation.tailrec +import scala.jdk.CollectionConverters.ListHasAsScala +import scala.util.Failure +import scala.util.Random +import scala.util.Success +import scala.util.Try + +/** Companion object for `FrontendHTTPServer` providing default executor configurations. */ +object FrontendHTTPServer { + + /** ExecutorService for single-threaded execution. */ + def singleThreadExecutor(): ExecutorService = Executors.newSingleThreadExecutor() + + /** ExecutorService for cached thread pool execution. */ + def cachedThreadPoolExecutor(): ExecutorService = Executors.newCachedThreadPool() + + /** Default ExecutorService used by `FrontendHTTPServer`. */ + def defaultExecutor(): ExecutorService = cachedThreadPoolExecutor() + +} + +/** A trait representing a frontend HTTP server for handling operations any subclass of `X2CpgMain` may offer via its + * main function. This trait provides methods and configurations for setting up an HTTP server that processes requests + * related to `X2CpgMain`. It includes handling request execution either in a single-threaded or multi-threaded manner, + * depending on the executor configuration. + * + * @tparam T + * The type parameter representing the X2Cpg configuration. + * @tparam X + * The type parameter representing the X2Cpg frontend. + */ +trait FrontendHTTPServer[T <: X2CpgConfig[T], X <: X2CpgFrontend[T]] { this: X2CpgMain[T, X] => + + /** Logger instance for logging server-related information. */ + private val logger = LoggerFactory.getLogger(this.getClass) + + /** Optionally holds the underlying HTTP server instance. */ + private var underlyingServer: Option[HTTPServer] = None + + /** Creates a new default configuration for the inheriting `X2CpgFrontend`. + * + * This method should be overridden by implementations to provide the default configuration object needed for the + * `X2CpgFrontend` operation. + * + * @return + * A new instance of the configuration `T`. + */ + protected def newDefaultConfig(): T + + /** ExecutorService used to execute HTTP requests. + * + * This can be overridden to switch between single-threaded and multi-threaded execution. By default, it uses the + * cached thread pool executor from `FrontendHTTPServer`. + */ + protected val executor: ExecutorService = FrontendHTTPServer.defaultExecutor() + + /** Handler for HTTP requests, providing functionality to handle specific routes. + * + * @param server + * The underlying HTTP server instance. + */ + protected class FrontendHTTPHandler(val server: HTTPServer) { + + /** Handles POST requests to the "/run" endpoint. + * + * This method is annotated to handle POST requests directed to the `/run` path. The request `req` is expected to + * include `input`, `output`, and (optionally) frontend arguments (unbounded). The request is expected to be sent + * `application/x-www-form-urlencoded`. The provided `X2CpgFrontend` is run with these input/output/arguments and + * the resulting CPG output path is returned in the response `resp` and status code 200. In case of a failure, + * status code 400 is sent together with a response containing the reason. + * + * @param req + * The HTTP request received by the server. + * @param resp + * The HTTP response to be sent by the server. + * @return + * The HTTP status code for the response. + */ + @Context(value = "/run", methods = Array("POST")) + def run(req: server.Request, resp: server.Response): Int = { + resp.getHeaders.add("Content-Type", "text/plain") + resp.getHeaders.add("Connection", "close") + + val params = req.getParamsList.asScala + val outputDir = params + .collectFirst { case Array(arg, value) if arg == "output" => value } + .getOrElse(X2CpgConfig.defaultOutputPath) + val arguments = params.collect { + case Array(arg, value) if arg == "input" => Array(value) + case Array(arg, value) if value.strip().isEmpty => Array(s"--$arg") + case Array(arg, value) => Array(s"--$arg", value) + }.flatten + logger.debug("Got POST with arguments: " + arguments.mkString(" ")) + + val config = X2Cpg + .parseCommandLine(arguments.toArray, cmdLineParser, newDefaultConfig()) + .getOrElse(newDefaultConfig()) + Try(frontend.run(config)) match { + case Failure(exception) => + resp.send(400, exception.getMessage) + case Success(_) => + resp.send(200, outputDir) + } + 0 + } + } + + /** Stops the underlying HTTP server if it is running. + * + * This method checks if the `underlyingServer` is defined and, if so, stops the server. It also logs a debug message + * indicating that the server has been stopped. If the server is not running, this method does nothing. + */ + def stop(): Unit = { + underlyingServer.foreach { server => + executor.shutdown() + server.stop() + logger.debug("Server stopped.") + } + } + + private def randomPort(): Int = { + val random = new Random() + 10000 + random.nextInt(65000) + } + + private def internalServerStart(): Try[Int] = { + val port = randomPort() + try { + val server = new HTTPServer(port) + val host = server.getVirtualHost(null) + host.addContexts(new FrontendHTTPHandler(server)) + server.setExecutor(executor) + server.start() + underlyingServer = Some(server) + Success(port) + } catch { + case exception: Throwable => Failure(exception) + } finally { + Runtime.getRuntime.addShutdownHook(new Thread(() => { + stop() + })) + } + } + + private def retryUntilSuccess[F](f: () => Try[F], maxAttempts: Int): F = { + @tailrec + def attempt(remainingAttempts: Int): F = { + f() match { + case Success(port) => port + case Failure(_) if remainingAttempts > 1 => attempt(remainingAttempts - 1) + case Failure(exception) => throw exception + } + } + attempt(maxAttempts) + } + + /** Starts the HTTP server. + * + * This method initializes the `underlyingServer`, sets the executor, and adds the appropriate contexts using the + * `FrontendHTTPHandler`. It then starts the server and prints the server's port to stdout. Additionally, a shutdown + * hook is added to ensure that the server is properly stopped when the application is terminated. + * + * @return + * The port this server is bound to which is chosen randomly until success (default number of attempts: 10) + */ + def startup(): Int = { + val port = retryUntilSuccess(internalServerStart, maxAttempts = 10) + println(s"FrontendHTTPServer started on port $port") + port + } + +} diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala index 86fbb6c1860c..15dbf7497572 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/AstTests.scala @@ -1,9 +1,9 @@ package io.joern.x2cpg -import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, NewCall, NewClosureBinding, NewIdentifier} +import flatgraph.SchemaViolationException +import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, Call, NewCall, NewClosureBinding, NewIdentifier} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.SchemaViolationException class AstTests extends AnyWordSpec with Matchers { @@ -35,7 +35,7 @@ class AstTests extends AnyWordSpec with Matchers { copied.root match { case Some(root: NewCall) => root should not be Some(moo) - root.properties("NAME") shouldBe "moo" + root.properties(Call.PropertyNames.Name) shouldBe "moo" root.argumentIndex shouldBe 123 case _ => fail() } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/SourceFilesTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/SourceFilesTests.scala index 2ebc2ffab906..cddfdd11d2e6 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/SourceFilesTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/SourceFilesTests.scala @@ -1,6 +1,6 @@ package io.joern.x2cpg -import better.files._ +import better.files.* import io.joern.x2cpg.utils.IgnoreInWindows import io.shiftleft.utils.ProjectRoot import org.scalatest.matchers.should.Matchers @@ -8,7 +8,7 @@ import org.scalatest.wordspec.AnyWordSpec import org.scalatest.Inside import java.nio.file.attribute.PosixFilePermissions -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.util.Try import java.io.FileNotFoundException @@ -53,6 +53,24 @@ class SourceFilesTests extends AnyWordSpec with Matchers with Inside { } + "do not throw an exception" when { + "one of the input files is a broken symlink" in { + File.usingTemporaryDirectory() { tmpDir => + (tmpDir / "a.c").touch() + val symlink = (tmpDir / "broken.c").symbolicLinkTo(File("does/not/exist.c")) + symlink.exists shouldBe false + symlink.isReadable shouldBe false + val ignored = (tmpDir / "ignored.c").touch() + val result = Try( + SourceFiles + .determine(tmpDir.canonicalPath, cSourceFileExtensions, ignoredFilesPath = Some(Seq(ignored.pathAsString))) + ) + result.isFailure shouldBe false + result.getOrElse(List.empty).size shouldBe 1 + } + } + } + "throw an exception" when { "the input file does not exist" in { diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala index 18ea217c43ef..7c9acf560a51 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/X2CpgTests.scala @@ -12,8 +12,7 @@ class X2CpgTests extends AnyWordSpec with Matchers { "create an empty in-memory CPG when no output path is given" in { val cpg = X2Cpg.newEmptyCpg(None) - cpg.graph.V.hasNext shouldBe false - cpg.graph.E.hasNext shouldBe false + cpg.graph.allNodes.hasNext shouldBe false cpg.close() } @@ -22,9 +21,9 @@ class X2CpgTests extends AnyWordSpec with Matchers { file.delete() file.exists shouldBe false val cpg = X2Cpg.newEmptyCpg(Some(file.path.toString)) + cpg.close() file.exists shouldBe true Files.size(file.path) should not be 0 - cpg.close() } "overwrite existing file to create empty CPG" in { @@ -32,11 +31,10 @@ class X2CpgTests extends AnyWordSpec with Matchers { file.exists shouldBe true Files.size(file.path) shouldBe 0 val cpg = X2Cpg.newEmptyCpg(Some(file.path.toString)) - cpg.graph.V.hasNext shouldBe false - cpg.graph.E.hasNext shouldBe false + cpg.graph.allNodes.hasNext shouldBe false + cpg.close() file.exists shouldBe true Files.size(file.path) should not be 0 - cpg.close() } } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala index 9b4d4e59c4d5..eadfe3f51682 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorFrontierTests.scala @@ -1,48 +1,62 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.controlflow.cfgdominator.{CfgAdapter, CfgDominator, CfgDominatorFrontier, DomTreeAdapter} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.{NewUnknown, StoredNode} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ - -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { - private class TestCfgAdapter extends CfgAdapter[Node] { - override def successors(node: Node): IterableOnce[Node] = - node.out("CFG").asScala + private class TestCfgAdapter extends CfgAdapter[StoredNode] { + override def successors(node: StoredNode): Iterator[StoredNode] = + node.out("CFG").cast[StoredNode] - override def predecessors(node: Node): IterableOnce[Node] = - node.in("CFG").asScala + override def predecessors(node: StoredNode): Iterator[StoredNode] = + node.in("CFG").cast[StoredNode] } - private class TestDomTreeAdapter(immediateDominators: scala.collection.Map[Node, Node]) extends DomTreeAdapter[Node] { - override def immediateDominator(cfgNode: Node): Option[Node] = { + private class TestDomTreeAdapter(immediateDominators: scala.collection.Map[StoredNode, StoredNode]) + extends DomTreeAdapter[StoredNode] { + override def immediateDominator(cfgNode: StoredNode): Option[StoredNode] = { immediateDominators.get(cfgNode) } } "Cfg dominance frontier test" in { - val graph = OverflowDbTestInstance.create - - val v0 = graph + "UNKNOWN" - val v1 = graph + "UNKNOWN" - val v2 = graph + "UNKNOWN" - val v3 = graph + "UNKNOWN" - val v4 = graph + "UNKNOWN" - val v5 = graph + "UNKNOWN" - val v6 = graph + "UNKNOWN" - - v0 --- "CFG" --> v1 - v1 --- "CFG" --> v2 - v2 --- "CFG" --> v3 - v2 --- "CFG" --> v5 - v3 --- "CFG" --> v4 - v4 --- "CFG" --> v2 - v4 --- "CFG" --> v5 - v5 --- "CFG" --> v6 + val cpg = Cpg.empty + val graph = cpg.graph + + val v0 = graph.addNode(NewUnknown()) + val v1 = graph.addNode(NewUnknown()) + val v2 = graph.addNode(NewUnknown()) + val v3 = graph.addNode(NewUnknown()) + val v4 = graph.addNode(NewUnknown()) + val v5 = graph.addNode(NewUnknown()) + val v6 = graph.addNode(NewUnknown()) + + // TODO MP get arrow syntax back +// v0 --- "CFG" --> v1 +// v1 --- "CFG" --> v2 +// v2 --- "CFG" --> v3 +// v2 --- "CFG" --> v5 +// v3 --- "CFG" --> v4 +// v4 --- "CFG" --> v2 +// v4 --- "CFG" --> v5 +// v5 --- "CFG" --> v6 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v1, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v3, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v3, v4, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v5, v6, EdgeTypes.CFG) + } val cfgAdapter = new TestCfgAdapter val cfgDominatorCalculator = new CfgDominator(cfgAdapter) @@ -50,7 +64,7 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { val domTreeAdapter = new TestDomTreeAdapter(immediateDominators) val cfgDominatorFrontier = new CfgDominatorFrontier(cfgAdapter, domTreeAdapter) - val dominanceFrontier = cfgDominatorFrontier.calculate(graph.nodes.asScala.toList) + val dominanceFrontier = cfgDominatorFrontier.calculate(cpg.all) dominanceFrontier.get(v0) shouldBe None dominanceFrontier.get(v1) shouldBe None @@ -62,14 +76,20 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { } "Cfg domiance frontier with dead code test" in { - val graph = OverflowDbTestInstance.create - - val v0 = graph + "UNKNOWN" - val v1 = graph + "UNKNOWN" // This node simulates dead code as it is not reachable from the entry v0. - val v2 = graph + "UNKNOWN" - - v0 --- "CFG" --> v2 - v1 --- "CFG" --> v2 + val cpg = Cpg.empty + val graph = cpg.graph + + val v0 = graph.addNode(NewUnknown()) + val v1 = graph.addNode(NewUnknown()) // This node simulates dead code as it is not reachable from the entry v0. + val v2 = graph.addNode(NewUnknown()) + + // TODO MP get arrow syntax back +// v0 --- "CFG" --> v2 +// v1 --- "CFG" --> v2 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + } val cfgAdapter = new TestCfgAdapter val cfgDominatorCalculator = new CfgDominator(cfgAdapter) @@ -77,7 +97,7 @@ class CfgDominatorFrontierTests extends AnyWordSpec with Matchers { val domTreeAdapter = new TestDomTreeAdapter(immediateDominators) val cfgDominatorFrontier = new CfgDominatorFrontier(cfgAdapter, domTreeAdapter) - val dominanceFrontier = cfgDominatorFrontier.calculate(graph.nodes.asScala.toList) + val dominanceFrontier = cfgDominatorFrontier.calculate(cpg.all) dominanceFrontier.get(v0) shouldBe None dominanceFrontier.apply(v1) shouldBe Set(v2) diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala index c8955335485a..1b5b615676ea 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/CfgDominatorPassTests.scala @@ -1,80 +1,93 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodReturn, NewUnknown} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class CfgDominatorPassTests extends AnyWordSpec with Matchers { "Have correct DOMINATE/POST_DOMINATE edges after CfgDominatorPass run." in { - val graph = OverflowDbTestInstance.create - val cpg = new Cpg(graph) + val cpg = Cpg.empty + val graph = cpg.graph - val v0 = graph + NodeTypes.METHOD - val v1 = graph + NodeTypes.UNKNOWN - val v2 = graph + NodeTypes.UNKNOWN - val v3 = graph + NodeTypes.UNKNOWN - val v4 = graph + NodeTypes.UNKNOWN - val v5 = graph + NodeTypes.UNKNOWN - val v6 = graph + NodeTypes.METHOD_RETURN + val v0 = graph.addNode(NewMethod()) + val v1 = graph.addNode(NewUnknown()) + val v2 = graph.addNode(NewUnknown()) + val v3 = graph.addNode(NewUnknown()) + val v4 = graph.addNode(NewUnknown()) + val v5 = graph.addNode(NewUnknown()) + val v6 = graph.addNode(NewMethodReturn()) - v0 --- EdgeTypes.AST --> v6 + // TODO MP get arrow syntax back +// v0 --- EdgeTypes.AST --> v6 +// +// v0 --- EdgeTypes.CFG --> v1 +// v1 --- EdgeTypes.CFG --> v2 +// v2 --- EdgeTypes.CFG --> v3 +// v2 --- EdgeTypes.CFG --> v5 +// v3 --- EdgeTypes.CFG --> v4 +// v4 --- EdgeTypes.CFG --> v2 +// v4 --- EdgeTypes.CFG --> v5 +// v5 --- EdgeTypes.CFG --> v6 + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(v0, v6, EdgeTypes.AST) - v0 --- EdgeTypes.CFG --> v1 - v1 --- EdgeTypes.CFG --> v2 - v2 --- EdgeTypes.CFG --> v3 - v2 --- EdgeTypes.CFG --> v5 - v3 --- EdgeTypes.CFG --> v4 - v4 --- EdgeTypes.CFG --> v2 - v4 --- EdgeTypes.CFG --> v5 - v5 --- EdgeTypes.CFG --> v6 + diffGraphBuilder.addEdge(v0, v1, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v1, v2, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v3, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v2, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v3, v4, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v4, v5, EdgeTypes.CFG) + diffGraphBuilder.addEdge(v5, v6, EdgeTypes.CFG) + } val dominatorTreePass = new CfgDominatorPass(cpg) dominatorTreePass.createAndApply() - val v0Dominates = v0.out(EdgeTypes.DOMINATE).asScala.toList + val v0Dominates = v0.out(EdgeTypes.DOMINATE).l v0Dominates.size shouldBe 1 v0Dominates.toSet shouldBe Set(v1) - val v1Dominates = v1.out(EdgeTypes.DOMINATE).asScala.toList + val v1Dominates = v1.out(EdgeTypes.DOMINATE).l v1Dominates.size shouldBe 1 v1Dominates.toSet shouldBe Set(v2) - val v2Dominates = v2.out(EdgeTypes.DOMINATE).asScala.toList + val v2Dominates = v2.out(EdgeTypes.DOMINATE).l v2Dominates.size shouldBe 2 v2Dominates.toSet shouldBe Set(v3, v5) - val v3Dominates = v3.out(EdgeTypes.DOMINATE).asScala.toList + val v3Dominates = v3.out(EdgeTypes.DOMINATE).l v3Dominates.size shouldBe 1 v3Dominates.toSet shouldBe Set(v4) - val v4Dominates = v4.out(EdgeTypes.DOMINATE).asScala.toList + val v4Dominates = v4.out(EdgeTypes.DOMINATE).l v4Dominates.size shouldBe 0 - val v5Dominates = v5.out(EdgeTypes.DOMINATE).asScala.toList + val v5Dominates = v5.out(EdgeTypes.DOMINATE).l v5Dominates.size shouldBe 1 v5Dominates.toSet shouldBe Set(v6) - val v6Dominates = v6.out(EdgeTypes.DOMINATE).asScala.toList + val v6Dominates = v6.out(EdgeTypes.DOMINATE).l v6Dominates.size shouldBe 0 - val v6PostDominates = v6.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v6PostDominates = v6.out(EdgeTypes.POST_DOMINATE).l v6PostDominates.size shouldBe 1 v6PostDominates.toSet shouldBe Set(v5) - val v5PostDominates = v5.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v5PostDominates = v5.out(EdgeTypes.POST_DOMINATE).l v5PostDominates.size shouldBe 2 v5PostDominates.toSet shouldBe Set(v2, v4) - val v4PostDominates = v4.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v4PostDominates = v4.out(EdgeTypes.POST_DOMINATE).l v4PostDominates.size shouldBe 1 v4PostDominates.toSet shouldBe Set(v3) - val v3PostDominates = v3.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v3PostDominates = v3.out(EdgeTypes.POST_DOMINATE).l v3PostDominates.size shouldBe 0 - val v2PostDominates = v2.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v2PostDominates = v2.out(EdgeTypes.POST_DOMINATE).l v2PostDominates.size shouldBe 1 v2PostDominates.toSet shouldBe Set(v1) - val v1PostDominates = v1.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v1PostDominates = v1.out(EdgeTypes.POST_DOMINATE).l v1PostDominates.size shouldBe 1 v1PostDominates.toSet shouldBe Set(v0) - val v0PostDominates = v0.out(EdgeTypes.POST_DOMINATE).asScala.toList + val v0PostDominates = v0.out(EdgeTypes.POST_DOMINATE).l v0PostDominates.size shouldBe 0 } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala index 329ccb09d32a..ba049a318622 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/ContainsEdgePassTest.scala @@ -1,14 +1,13 @@ package io.joern.x2cpg.passes -import io.shiftleft.OverflowDbTestInstance -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes, NodeTypes} +import flatgraph.misc.TestUtils.* import io.joern.x2cpg.passes.base.ContainsEdgePass +import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewFile, NewMethod, NewTypeDecl} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ - -import scala.jdk.CollectionConverters._ class ContainsEdgePassTest extends AnyWordSpec with Matchers { @@ -16,26 +15,26 @@ class ContainsEdgePassTest extends AnyWordSpec with Matchers { "Files " can { "contain Methods" in Fixture { fixture => - fixture.methodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.fileVertex) + fixture.methodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.fileVertex) } "contain Classes" in Fixture { fixture => - fixture.typeDeclVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.fileVertex) + fixture.typeDeclVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.fileVertex) } } "Classes " can { "contain Methods" in Fixture { fixture => - fixture.typeMethodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.typeDeclVertex) + fixture.typeMethodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.typeDeclVertex) } } "Methods " can { "contain Methods" in Fixture { fixture => - fixture.innerMethodVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.methodVertex) + fixture.innerMethodVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.methodVertex) } "contain expressions" in Fixture { fixture => - fixture.expressionVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.methodVertex) - fixture.innerExpressionVertex.in(EdgeTypes.CONTAINS).asScala.toList shouldBe List(fixture.innerMethodVertex) + fixture.expressionVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.methodVertex) + fixture.innerExpressionVertex.in(EdgeTypes.CONTAINS).l shouldBe List(fixture.innerMethodVertex) } } @@ -43,23 +42,34 @@ class ContainsEdgePassTest extends AnyWordSpec with Matchers { object ContainsEdgePassTest { private class Fixture { - private val graph = OverflowDbTestInstance.create + private val cpg = Cpg.empty + private val graph = cpg.graph - val fileVertex = graph + NodeTypes.FILE - val typeDeclVertex = graph + NodeTypes.TYPE_DECL - val typeMethodVertex = graph + NodeTypes.METHOD - val methodVertex = graph + NodeTypes.METHOD - val innerMethodVertex = graph + NodeTypes.METHOD - val expressionVertex = graph + NodeTypes.CALL - val innerExpressionVertex = graph + NodeTypes.CALL + val fileVertex = graph.addNode(NewFile()) + val typeDeclVertex = graph.addNode(NewTypeDecl()) + val typeMethodVertex = graph.addNode(NewMethod()) + val methodVertex = graph.addNode(NewMethod()) + val innerMethodVertex = graph.addNode(NewMethod()) + val expressionVertex = graph.addNode(NewCall()) + val innerExpressionVertex = graph.addNode(NewCall()) - fileVertex --- EdgeTypes.AST --> typeDeclVertex - typeDeclVertex --- EdgeTypes.AST --> typeMethodVertex + // TODO MP get arrow syntax back +// fileVertex --- EdgeTypes.AST --> typeDeclVertex +// typeDeclVertex --- EdgeTypes.AST --> typeMethodVertex +// +// fileVertex --- EdgeTypes.AST --> methodVertex +// methodVertex --- EdgeTypes.AST --> innerMethodVertex +// methodVertex --- EdgeTypes.AST --> expressionVertex +// innerMethodVertex --- EdgeTypes.AST --> innerExpressionVertex + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(fileVertex, typeDeclVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(typeDeclVertex, typeMethodVertex, EdgeTypes.AST) - fileVertex --- EdgeTypes.AST --> methodVertex - methodVertex --- EdgeTypes.AST --> innerMethodVertex - methodVertex --- EdgeTypes.AST --> expressionVertex - innerMethodVertex --- EdgeTypes.AST --> innerExpressionVertex + diffGraphBuilder.addEdge(fileVertex, methodVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(methodVertex, innerMethodVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(methodVertex, expressionVertex, EdgeTypes.AST) + diffGraphBuilder.addEdge(innerMethodVertex, innerExpressionVertex, EdgeTypes.AST) + } val containsEdgeCalculator = new ContainsEdgePass(new Cpg(graph)) containsEdgeCalculator.createAndApply() diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MemberAccessLinkerTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MemberAccessLinkerTests.scala index cf9a848532ef..208e89d2850c 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MemberAccessLinkerTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MemberAccessLinkerTests.scala @@ -1,8 +1,8 @@ package io.joern.x2cpg.passes -import io.shiftleft.codepropertygraph.generated._ +import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewMember} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala index ac281490f0ee..6977bd1456cb 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/MethodDecoratorPassTests.scala @@ -1,30 +1,32 @@ package io.joern.x2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn +import flatgraph.misc.TestUtils.* +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.base.MethodDecoratorPass import io.joern.x2cpg.testfixtures.EmptyGraphFixture import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ class MethodDecoratorPassTests extends AnyWordSpec with Matchers { "MethodDecoratorTest" in EmptyGraphFixture { graph => - val method = graph + NodeTypes.METHOD - val parameterIn = graph - .+( - NodeTypes.METHOD_PARAMETER_IN, - Properties.Code -> "p1", - Properties.Order -> 1, - Properties.Name -> "p1", - Properties.EvaluationStrategy -> EvaluationStrategies.BY_REFERENCE, - Properties.TypeFullName -> "some.Type", - Properties.LineNumber -> 10 - ) - .asInstanceOf[MethodParameterIn] + val method = graph.addNode(NewMethod()) + val parameterIn = graph.addNode( + NewMethodParameterIn() + .code("p1") + .order(1) + .name("p1") + .evaluationStrategy(EvaluationStrategies.BY_REFERENCE) + .typeFullName("some.Type") + .lineNumber(10) + ) - method --- EdgeTypes.AST --> parameterIn + // TODO MP get arrow syntax back +// method --- EdgeTypes.AST --> parameterIn + graph.applyDiff { diffGraphBuilder => + diffGraphBuilder.addEdge(method, parameterIn, EdgeTypes.AST) + } val methodDecorator = new MethodDecoratorPass(new Cpg(graph)) methodDecorator.createAndApply() diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala index 63e0d7ec1e86..2f22f3c62b2e 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/passes/NamespaceCreatorTests.scala @@ -1,22 +1,22 @@ package io.joern.x2cpg.passes -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} -import io.shiftleft.semanticcpg.language._ +import flatgraph.misc.TestUtils.addNode +import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes} +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.base.NamespaceCreator import io.joern.x2cpg.testfixtures.EmptyGraphFixture +import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb._ class NamespaceCreatorTests extends AnyWordSpec with Matchers { "NamespaceCreateor test " in EmptyGraphFixture { graph => val cpg = new Cpg(graph) - val block1 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace1") - val block2 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace1") - val block3 = graph + (NodeTypes.NAMESPACE_BLOCK, Properties.Name -> "namespace2") + val block1 = graph.addNode(NewNamespaceBlock().name("namespace1")) + val block2 = graph.addNode(NewNamespaceBlock().name("namespace1")) + val block3 = graph.addNode(NewNamespaceBlock().name("namespace2")) - val namespaceCreator = new NamespaceCreator(new Cpg(graph)) + val namespaceCreator = new NamespaceCreator(cpg) namespaceCreator.createAndApply() val namespaces = cpg.namespace.l diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/CfgTestFixture.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/CfgTestFixture.scala index 14c57a6fb97f..b8292db40bb4 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/CfgTestFixture.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/CfgTestFixture.scala @@ -4,7 +4,7 @@ import io.joern.x2cpg.passes.controlflow.CfgCreationPass import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.CfgEdgeType import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Method} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* abstract class CfgTestCpg extends TestCpg { override protected def applyPasses(): Unit = { @@ -30,7 +30,7 @@ class CfgTestFixture[T <: CfgTestCpg](testCpgFactory: () => T) extends Code2CpgF ExpectationInfo(pair._1, pair._2, pair._3) } - def expected(pairs: ExpectationInfo*)(implicit cpg: Cpg): Set[String] = { + def expected(pairs: ExpectationInfo*)(implicit cpg: Cpg): List[String] = { pairs.map { case ExpectationInfo(code, index, _) => cpg.method.ast.isCfgNode.toVector .collect { @@ -38,11 +38,11 @@ class CfgTestFixture[T <: CfgTestCpg](testCpgFactory: () => T) extends Code2CpgF } .lift(index) .getOrElse(fail(s"No node found for code = '$code' and index '$index'!")) - }.toSet + }.toList } // index is zero based and describes which node to take if multiple node match the code string. - def succOf(code: String, index: Int = 0)(implicit cpg: Cpg): Set[String] = { + def succOf(code: String, index: Int = 0)(implicit cpg: Cpg): List[String] = { cpg.method.ast.isCfgNode.toVector .collect { case node if matchCode(node, code) => node @@ -52,10 +52,10 @@ class CfgTestFixture[T <: CfgTestCpg](testCpgFactory: () => T) extends Code2CpgF ._cfgOut .cast[CfgNode] .code - .toSetImmutable + .toList } - def succOf(code: String, nodeType: String)(implicit cpg: Cpg): Set[String] = { + def succOf(code: String, nodeType: String)(implicit cpg: Cpg): List[String] = { cpg.method.ast.isCfgNode .label(nodeType) .toVector @@ -66,6 +66,6 @@ class CfgTestFixture[T <: CfgTestCpg](testCpgFactory: () => T) extends Code2CpgF ._cfgOut .cast[CfgNode] .code - .toSetImmutable + .toList } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala index 4a378c095580..36c6dcb2c481 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/EmptyGraphFixture.scala @@ -1,12 +1,11 @@ package io.joern.x2cpg.testfixtures -import io.shiftleft.OverflowDbTestInstance -import overflowdb.Graph +import flatgraph.Graph +import io.shiftleft.codepropertygraph.generated.Cpg + +import scala.util.Using object EmptyGraphFixture { - def apply[T](fun: Graph => T): T = { - val graph = OverflowDbTestInstance.create - try fun(graph) - finally { graph.close() } - } + def apply[T](fun: Graph => T): T = + Using.resource(Cpg.empty.graph)(fun) } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala index b70cddfb4c56..7dac8f7b7bee 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/testfixtures/TestCpg.scala @@ -1,9 +1,9 @@ package io.joern.x2cpg.testfixtures +import flatgraph.Graph import io.joern.x2cpg.X2CpgConfig import io.joern.x2cpg.utils.TestCodeWriter import io.shiftleft.codepropertygraph.generated.Cpg -import overflowdb.Graph import java.nio.file.{Files, Path} import java.util.Comparator @@ -11,7 +11,7 @@ import java.util.Comparator // Lazily populated test CPG which is created upon first access to the underlying graph. // The trait LanguageFrontend is mixed in and not property/field of this class in order // to allow the configuration of language frontend specific properties on the CPG object. -abstract class TestCpg extends Cpg() with LanguageFrontend with TestCodeWriter { +abstract class TestCpg extends Cpg(Cpg.empty.graph) with LanguageFrontend with TestCodeWriter { private var _graph = Option.empty[Graph] protected var _withPostProcessing = false diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala index 006af0bfff99..ef89b57fd2b3 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala @@ -4,15 +4,41 @@ import better.files.File import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import scala.util.Properties.isWin import scala.util.{Failure, Success} class ExternalCommandTest extends AnyWordSpec with Matchers { + private def cwd = File.currentWorkingDirectory.pathAsString + "ExternalCommand.run" should { "be able to run `ls` successfully" in { File.usingTemporaryDirectory("sample") { sourceDir => - val cmd = "ls " + sourceDir.pathAsString - ExternalCommand.run(cmd, sourceDir.pathAsString) should be a Symbol("success") + val cmd = Seq("ls", sourceDir.pathAsString) + ExternalCommand.run(cmd, sourceDir.pathAsString).toTry should be a Symbol("success") + } + } + + "report exit code and stdout/stderr for nonzero exit code" in { + ExternalCommand.run(Seq("ls", "/does/not/exist"), cwd).toTry match { + case result: Success[_] => + fail(s"expected failure, but got $result") + case Failure(exception) => + exception.getMessage should include("Process exited with code") // exit code `2` on linux, `1` on mac... + exception.getMessage should include("No such file or directory") // again, different errors on mac and linux + } + } + + "report error for io exception (e.g. for nonexisting command)" in { + ExternalCommand.run(Seq("/command/does/not/exist"), cwd).toTry match { + case result: Success[_] => + fail(s"expected failure, but got $result") + case Failure(exception) => + exception.getMessage should include("""Cannot run program "/command/does/not/exist"""") + if (isWin) + exception.getMessage should include("The system cannot find the file") + else + exception.getMessage should include("No such file or directory") } } } diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala new file mode 100644 index 000000000000..4815c4827332 --- /dev/null +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/KeyPoolTests.scala @@ -0,0 +1,74 @@ +package io.joern.x2cpg.utils + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class KeyPoolTests extends AnyWordSpec with Matchers { + + "IntervalKeyPool" should { + "return [first, ..., last] and then raise" in { + val keyPool = new IntervalKeyPool(10, 19) + List.range(0, 10).map(_ => keyPool.next) shouldBe List.range(10, 20) + assertThrows[RuntimeException] { keyPool.next } + assertThrows[RuntimeException] { keyPool.next } + } + + "allow splitting into multiple pools" in { + val keyPool = new IntervalKeyPool(1, 1000) + val pools = keyPool.split(11).toList + assertThrows[IllegalStateException] { keyPool.next } + pools.size shouldBe 11 + // Pools should all have the same size + pools + .map { x => + (x.last - x.first) + } + .distinct + .size shouldBe 1 + // Pools should be pairwise disjoint + val keySets = pools.map { x => + (x.first to x.last).toSet + } + keySets.combinations(2).foreach { + case List(x: Set[Long], y: Set[Long]) => + x.intersect(y).isEmpty shouldBe true + case _ => + fail() + } + } + + "return empty iterator when asked to create 0 partitions" in { + val keyPool = new IntervalKeyPool(1, 1000) + keyPool.split(0).hasNext shouldBe false + } + + } + + "SequenceKeyPool" should { + "return elements of sequence one by one and then raise" in { + val seq = List[Long](1, 2, 3) + val keyPool = new SequenceKeyPool(seq) + List.range(0, 3).map(_ => keyPool.next) shouldBe seq + assertThrows[RuntimeException] { keyPool.next } + assertThrows[RuntimeException] { keyPool.next } + } + } + + "KeyPoolCreator" should { + "split into n pools and honor minimum value" in { + val minValue = 10 + val pools = KeyPoolCreator.obtain(3, minValue) + pools.size shouldBe 3 + pools match { + case List(pool1, pool2, pool3) => + pool1.first shouldBe minValue + pool1.last should be < pool2.first + pool2.last should be < pool3.first + pool3.last shouldBe Long.MaxValue - 1 + case _ => fail() + } + } + + } + +} diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala index 58bde517e1bd..0a8c79b14727 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala @@ -31,7 +31,7 @@ class DependencyResolverTests extends AnyWordSpec with Matchers { "test maven dependency resolution" ignore { // check that `mvn` is available - otherwise test will fail with only some logged warnings... withClue("`mvn` must be installed in order for this test to work...") { - ExternalCommand.run("mvn --version", ".").get.exists(_.contains("Apache Maven")) shouldBe true + ExternalCommand.run(Seq("mvn", "--version"), ".").successOption.exists(_.contains("Apache Maven")) shouldBe true } @nowarn // otherwise scalac warns that this might be an interpolated expression diff --git a/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala b/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala index d40ae3550443..274920f8a7d4 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/CpgBasedTool.scala @@ -4,22 +4,23 @@ import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.cpgloading.CpgLoaderConfig import io.shiftleft.semanticcpg.layers.LayerCreatorContext -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader object CpgBasedTool { + def loadFromFile(filename: String): Cpg = + CpgLoader.load(filename) + /** Load code property graph from overflowDB * * @param filename * name of the file that stores the CPG */ - def loadFromOdb(filename: String): Cpg = { - val odbConfig = overflowdb.Config.withDefaults().withStorageLocation(filename) - val config = CpgLoaderConfig().withOverflowConfig(odbConfig).doNotCreateIndexesOnLoad - io.shiftleft.codepropertygraph.cpgloading.CpgLoader.loadFromOverflowDb(config) - } + @deprecated("use `loadFromFile` instead", "joern v3") + def loadFromOdb(filename: String): Cpg = + loadFromFile(filename) /** Add the data flow layer to the CPG if it does not exist yet. */ diff --git a/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala b/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala index 4cc406f104d5..3adc182a4eb3 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/DefaultOverlays.scala @@ -3,7 +3,7 @@ package io.joern.joerncli import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.layers._ +import io.shiftleft.semanticcpg.layers.* object DefaultOverlays { @@ -16,7 +16,7 @@ object DefaultOverlays { * the filename of the cpg */ def create(storeFilename: String, maxNumberOfDefinitions: Int = defaultMaxNumberOfDefinitions): Cpg = { - val cpg = CpgBasedTool.loadFromOdb(storeFilename) + val cpg = CpgBasedTool.loadFromFile(storeFilename) applyDefaultOverlays(cpg) val context = new LayerCreatorContext(cpg) val options = new OssDataFlowOptions(maxNumberOfDefinitions) diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala index 6b0b78612602..7bea6d43f6aa 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernExport.scala @@ -2,21 +2,21 @@ package io.joern.joerncli import better.files.Dsl.* import better.files.File +import flatgraph.{Accessors, Edge, GNode} +import flatgraph.formats.ExportResult +import flatgraph.formats.dot.DotExporter +import flatgraph.formats.graphml.GraphMLExporter +import flatgraph.formats.graphson.GraphSONExporter +import flatgraph.formats.neo4jcsv.Neo4jCsvExporter import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.layers.dataflows.* -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.{NoSemantics, Semantics} import io.joern.joerncli.CpgBasedTool.exitIfInvalid import io.joern.x2cpg.layers.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.semanticcpg.language.{toAstNodeMethods, toNodeTypeStarters} +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.* -import overflowdb.formats.ExportResult -import overflowdb.formats.dot.DotExporter -import overflowdb.formats.graphml.GraphMLExporter -import overflowdb.formats.graphson.GraphSONExporter -import overflowdb.formats.neo4jcsv.Neo4jCsvExporter -import overflowdb.{Edge, Node} import java.nio.file.{Path, Paths} import scala.collection.mutable @@ -64,7 +64,7 @@ object JoernExport { exitIfInvalid(outDir, config.cpgFileName) mkdir(File(outDir)) - Using.resource(CpgBasedTool.loadFromOdb(config.cpgFileName)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(config.cpgFileName)) { cpg => exportCpg(cpg, config.repr, config.format, Paths.get(outDir).toAbsolutePath) } } @@ -96,7 +96,7 @@ object JoernExport { def exportCpg(cpg: Cpg, representation: Representation.Value, format: Format.Value, outDir: Path): Unit = { implicit val semantics: Semantics = DefaultSemantics() - if (semantics.elements.isEmpty) { + if (semantics == NoSemantics) { System.err.println("Warning: semantics are empty.") } @@ -105,15 +105,15 @@ object JoernExport { format match { case Format.Dot if representation == Representation.All || representation == Representation.Cpg => - exportWithOdbFormat(cpg, representation, outDir, DotExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, DotExporter) case Format.Dot => exportDot(representation, outDir, context) case Format.Neo4jCsv => - exportWithOdbFormat(cpg, representation, outDir, Neo4jCsvExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, Neo4jCsvExporter) case Format.Graphml => - exportWithOdbFormat(cpg, representation, outDir, GraphMLExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, GraphMLExporter) case Format.Graphson => - exportWithOdbFormat(cpg, representation, outDir, GraphSONExporter) + exportWithFlatgraphFormat(cpg, representation, outDir, GraphSONExporter) case other => throw new NotImplementedError(s"repr=$representation not yet supported for format=$format") } @@ -133,11 +133,11 @@ object JoernExport { } } - private def exportWithOdbFormat( + private def exportWithFlatgraphFormat( cpg: Cpg, repr: Representation.Value, outDir: Path, - exporter: overflowdb.formats.Exporter + exporter: flatgraph.formats.Exporter ): Unit = { val ExportResult(nodeCount, edgeCount, _, additionalInfo) = repr match { case Representation.All => @@ -154,7 +154,7 @@ object JoernExport { windowsFilenameDeduplicationHelper ) val outFileName = outDir.resolve(relativeFilename) - exporter.runExport(nodes, subGraph.edges, outFileName) + exporter.runExport(cpg.graph.schema, nodes, subGraph.edges, outFileName) } .reduce(plus) } else { @@ -220,12 +220,12 @@ object JoernExport { private def emptyExportResult = ExportResult(0, 0, Seq.empty, Option("Empty CPG")) - case class MethodSubGraph(methodName: String, methodFilename: String, nodes: Set[Node]) { + case class MethodSubGraph(methodName: String, methodFilename: String, nodes: Set[GNode]) { def edges: Set[Edge] = { for { node <- nodes - edge <- node.bothE.asScala - if nodes.contains(edge.inNode) && nodes.contains(edge.outNode) + edge <- Accessors.getEdgesOut(node) + if nodes.contains(edge.dst) } yield edge } } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala index 07cf60a4ef28..211bcd1ec0b4 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernFlow.scala @@ -28,7 +28,7 @@ object JoernFlow { } debugOut("Loading graph... ") - val cpg = CpgBasedTool.loadFromOdb(config.cpgFileName) + val cpg = CpgBasedTool.loadFromFile(config.cpgFileName) debugOut("[DONE]\n") implicit val resolver: ICallResolver = NoResolve diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernParse.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernParse.scala index 34e2ff7a2889..1bfed24fc7f6 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernParse.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernParse.scala @@ -4,15 +4,14 @@ import better.files.File import io.joern.console.cpgcreation.{CpgGenerator, cpgGeneratorForLanguage, guessLanguage} import io.joern.console.{FrontendConfig, InstallConfig} import io.joern.joerncli.CpgBasedTool.newCpgCreatedString +import io.joern.x2cpg.frontendspecific.FrontendArgsDelimitor import io.shiftleft.codepropertygraph.generated.Languages import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} object JoernParse { - // Special string used to separate joern-parse opts from frontend-specific opts - val ArgsDelimitor = "--frontend-args" val DefaultCpgOutFile = "cpg.bin" var generator: CpgGenerator = scala.compiletime.uninitialized @@ -64,7 +63,7 @@ object JoernParse { note("Misc") help("help").text("display this help message") - note(s"Args specified after the $ArgsDelimitor separator will be passed to the front-end verbatim") + note(s"Args specified after the $FrontendArgsDelimitor separator will be passed to the front-end verbatim") } private def run(args: Array[String]): Try[String] = { diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala index c5728589c15a..77a52b8d3ef9 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala @@ -4,18 +4,20 @@ import better.files.* import io.joern.console.scan.{ScanPass, outputFindings} import io.joern.console.{BridgeBase, DefaultArgumentProvider, Query, QueryDatabase} import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.{Semantics, NoSemantics} import io.joern.joerncli.JoernScan.getQueriesFromQueryDb import io.joern.joerncli.Scan.{allTag, defaultTag} import io.joern.joerncli.console.ReplBridge import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.semanticcpg.language.{DefaultNodeExtensionFinder, NodeExtensionFinder} import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} + import java.io.PrintStream import org.json4s.native.Serialization import org.json4s.{Formats, NoTypeHints} + import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* object JoernScanConfig { val defaultDbVersion: String = "latest" @@ -128,7 +130,7 @@ object JoernScan extends BridgeBase { } private def dumpQueriesAsJson(outFileName: String): Unit = { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val formats: AnyRef & Formats = Serialization.formats(NoTypeHints) val queryDb = new QueryDatabase(new JoernDefaultArgumentProvider(0)) better.files @@ -179,7 +181,7 @@ object JoernScan extends BridgeBase { } private def queryNames(): List[String] = { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) getQueriesFromQueryDb(new JoernDefaultArgumentProvider(0)).map(_.name) } @@ -274,7 +276,7 @@ class Scan(options: ScanOptions)(implicit engineContext: EngineContext) extends println("No queries matched current filter selection (total number of queries: `" + allQueries.length + "`)") return } - runPass(new ScanPass(context.cpg, queriesAfterFilter), context) + ScanPass(context.cpg, queriesAfterFilter).createAndApply() outputFindings(context.cpg) } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala index c16f09eb428a..453d370eed5b 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernSlice.scala @@ -122,7 +122,7 @@ object JoernSlice { } else { config.inputPath.pathAsString } - Using.resource(CpgBasedTool.loadFromOdb(inputCpgPath)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(inputCpgPath)) { cpg => checkAndApplyOverlays(cpg) // Slice the CPG (config match { diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala index 57ba6673bcc0..d32bceb28f33 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala @@ -15,11 +15,10 @@ import scala.util.hashing.MurmurHash3 class BagOfPropertiesForNodes extends EmbeddingGenerator[AstNode, (String, String)] { override def structureToString(pair: (String, String)): String = pair._1 + ":" + pair._2 - override def extractObjects(cpg: Cpg): Iterator[AstNode] = cpg.graph.V.collect { case x: AstNode => x } + override def extractObjects(cpg: Cpg): Iterator[AstNode] = cpg.astNode override def enumerateSubStructures(obj: AstNode): List[(String, String)] = { val relevantFieldTypes = Set(PropertyNames.NAME, PropertyNames.FULL_NAME, PropertyNames.CODE) - val relevantFields = obj - .propertiesMap() + val relevantFields = obj.propertiesMap .entrySet() .asScala .toList @@ -136,7 +135,7 @@ object JoernVectors { def main(args: Array[String]) = { parseConfig(args).foreach { config => exitIfInvalid(config.outDir, config.cpgFileName) - Using.resource(CpgBasedTool.loadFromOdb(config.cpgFileName)) { cpg => + Using.resource(CpgBasedTool.loadFromFile(config.cpgFileName)) { cpg => val generator = new BagOfPropertiesForNodes() val embedding = generator.embed(cpg) println("{") @@ -150,8 +149,8 @@ object JoernVectors { traversalToJson(embedding.vectors, generator.vectorToString) println(",\"edges\":") traversalToJson( - cpg.graph.edges().map { x => - Map("src" -> x.outNode().id(), "dst" -> x.inNode().id(), "label" -> x.label()) + cpg.graph.allEdges.map { edge => + Map("src" -> edge.src.id, "dst" -> edge.dst.id, "label" -> edge.label) }, generator.defaultToString ) diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala index 93b7ce1a7bc5..607e9a3074aa 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala @@ -1,6 +1,6 @@ package io.joern.joerncli.console -import better.files._ +import better.files.* import io.joern.console.defaultAvailableWidthProvider import io.joern.console.workspacehandling.{ProjectFile, WorkspaceLoader} import io.joern.console.{Console, ConsoleConfig, InstallConfig} diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernProject.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernProject.scala index df1bd9fb86e9..ecfaa2f7dd96 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernProject.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernProject.scala @@ -2,7 +2,7 @@ package io.joern.joerncli.console import io.joern.console.workspacehandling.{Project, ProjectFile} import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import io.shiftleft.codepropertygraph.generated.Cpg import java.nio.file.Path @@ -11,5 +11,5 @@ class JoernProject( projectFile: ProjectFile, path: Path, cpg: Option[Cpg] = None, - var context: EngineContext = EngineContext(Semantics.empty) + var context: EngineContext = EngineContext(NoSemantics) ) extends Project(projectFile, path, cpg) {} diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala index f3107c8ff907..d5c8925c5ee1 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala @@ -6,26 +6,21 @@ object Predefined { val shared: Seq[String] = Seq( - "import _root_.io.joern.console._", - "import _root_.io.joern.joerncli.console.JoernConsole._", - "import _root_.io.shiftleft.codepropertygraph.Cpg.docSearchPackages", - "import _root_.io.shiftleft.codepropertygraph.generated.Cpg", - "import _root_.io.shiftleft.codepropertygraph.cpgloading._", - "import _root_.io.shiftleft.codepropertygraph.generated._", - "import _root_.io.shiftleft.codepropertygraph.generated.nodes._", - "import _root_.io.shiftleft.codepropertygraph.generated.edges._", - "import _root_.io.joern.dataflowengineoss.language._", - "import _root_.io.shiftleft.semanticcpg.language._", - "import overflowdb._", - "import overflowdb.traversal.{`package` => _, help => _, _}", - "import scala.jdk.CollectionConverters._", + "import _root_.io.joern.console.*", + "import _root_.io.joern.joerncli.console.JoernConsole.*", + "import _root_.io.shiftleft.codepropertygraph.cpgloading.*", + "import _root_.io.shiftleft.codepropertygraph.generated.{help => _, _}", + "import _root_.io.shiftleft.codepropertygraph.generated.nodes.*", + "import _root_.io.joern.dataflowengineoss.language.*", + "import _root_.io.shiftleft.semanticcpg.language.*", + "import scala.jdk.CollectionConverters.*", "implicit val resolver: ICallResolver = NoResolve", "implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder" ) val forInteractiveShell: Seq[String] = { shared ++ - Seq("import _root_.io.joern.joerncli.console.Joern._") ++ + Seq("import _root_.io.joern.joerncli.console.Joern.*") ++ Run.codeForRunCommand().linesIterator ++ Help.codeForHelpCommand(classOf[io.joern.joerncli.console.JoernConsole]).linesIterator ++ Seq("ossDataFlowOptions = opts.ossdataflow") diff --git a/joern-cli/src/test/scala/io/joern/joerncli/GenerationTests.scala b/joern-cli/src/test/scala/io/joern/joerncli/GenerationTests.scala index 6cf0a8e14e88..3f6ae67c4ba8 100644 --- a/joern-cli/src/test/scala/io/joern/joerncli/GenerationTests.scala +++ b/joern-cli/src/test/scala/io/joern/joerncli/GenerationTests.scala @@ -1,7 +1,7 @@ package io.joern.joerncli import better.files.File -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/joern-cli/src/universal/schema-extender/build.sbt b/joern-cli/src/universal/schema-extender/build.sbt index 7632c73b0f74..17b2c2cc81fd 100644 --- a/joern-cli/src/universal/schema-extender/build.sbt +++ b/joern-cli/src/universal/schema-extender/build.sbt @@ -1,10 +1,10 @@ name := "schema-extender" -ThisBuild / scalaVersion := "3.4.1" +ThisBuild / scalaVersion := "3.4.2" val cpgVersion = IO.read(file("cpg-version")) -val generateDomainClasses = taskKey[Seq[File]]("generate overflowdb domain classes for our schema") +val generateDomainClasses = taskKey[Seq[File]]("generate domain classes for our schema") val joernInstallPath = settingKey[String]("path to joern installation, e.g. `/home/username/bin/joern/joern-cli` or `../../joern/joern-cli`") @@ -33,9 +33,9 @@ ThisBuild / libraryDependencies ++= Seq( lazy val schema = project .in(file("schema")) .settings(generateDomainClasses := { - val outputRoot = target.value / "odb-codegen" + val outputRoot = target.value / "fg-codegen" FileUtils.deleteRecursively(outputRoot) - val invoked = (Compile / runMain).toTask(s" CpgExtCodegen schema/target/odb-codegen").value + val invoked = (Compile / runMain).toTask(s" CpgExtCodegen schema/target/fg-codegen").value FileUtils.listFilesRecursively(outputRoot) }) diff --git a/joern-cli/src/universal/schema-extender/project/FileUtils.scala b/joern-cli/src/universal/schema-extender/project/FileUtils.scala index dc61c44afee2..4bd8aaae7980 100644 --- a/joern-cli/src/universal/schema-extender/project/FileUtils.scala +++ b/joern-cli/src/universal/schema-extender/project/FileUtils.scala @@ -1,6 +1,6 @@ import java.io.File import java.nio.file.Files -import scala.collection.JavaConverters._ +import scala.collection.JavaConverters.* object FileUtils { diff --git a/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala b/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala index a94cfedd5857..5bdd11cdac44 100644 --- a/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala +++ b/joern-cli/src/universal/schema-extender/schema/src/main/scala/CpgExtCodegen.scala @@ -1,14 +1,13 @@ -import io.shiftleft.codepropertygraph.schema._ -import overflowdb.codegen.CodeGen -import overflowdb.schema.SchemaBuilder -import overflowdb.schema.Property.ValueType - -import java.io.File +import io.shiftleft.codepropertygraph.schema.* +import flatgraph.codegen.DomainClassesGenerator +import flatgraph.schema.SchemaBuilder +import flatgraph.schema.Property.ValueType +import java.nio.file.Paths object CpgExtCodegen { def main(args: Array[String]): Unit = { val outputDir = args.headOption - .map(new File(_)) + .map(Paths.get(_)) .getOrElse(throw new AssertionError("please pass outputDir as first parameter")) val builder = new SchemaBuilder(domainShortName = "Cpg", basePackage = "io.shiftleft.codepropertygraph.generated") @@ -24,6 +23,6 @@ object CpgExtCodegen { cpgSchema.fs.file.addProperties(exampleProperty) // END extensions for this build - new CodeGen(builder.build).run(outputDir) + new DomainClassesGenerator(builder.build).run(outputDir) } } diff --git a/joern-cli/src/universal/schema-extender/test.sh b/joern-cli/src/universal/schema-extender/test.sh index 309ab80b710a..4c3fba8ac3b6 100755 --- a/joern-cli/src/universal/schema-extender/test.sh +++ b/joern-cli/src/universal/schema-extender/test.sh @@ -9,6 +9,6 @@ set -x #verbose on # we should now be able to use our new `EXAMPLE_NODE` node mkdir -p scripts echo 'assert(nodes.ExampleNode.Label == "EXAMPLE_NODE") -assert(nodes.ExampleNode.PropertyNames.all.contains("EXAMPLE_PROPERTY"))' > scripts/SchemaExtenderTest.sc +assert(nodes.ExampleNode.PropertyNames.ExampleProperty == "EXAMPLE_PROPERTY")' > scripts/SchemaExtenderTest.sc ./joern --script scripts/SchemaExtenderTest.sc diff --git a/joern-install.sh b/joern-install.sh index 487177c9cd0d..e3734fec69a9 100755 --- a/joern-install.sh +++ b/joern-install.sh @@ -190,7 +190,6 @@ else sudo ln -sf "$JOERN_INSTALL_DIR"/joern-cli/joern-export "$JOERN_LINK_DIR" || true sudo ln -sf "$JOERN_INSTALL_DIR"/joern-cli/joern-flow "$JOERN_LINK_DIR" || true sudo ln -sf "$JOERN_INSTALL_DIR"/joern-cli/joern-scan "$JOERN_LINK_DIR" || true - sudo ln -sf "$JOERN_INSTALL_DIR"/joern-cli/joern-stats "$JOERN_LINK_DIR" || true sudo ln -sf "$JOERN_INSTALL_DIR"/joern-cli/joern-slice "$JOERN_LINK_DIR" || true fi fi diff --git a/macros/build.sbt b/macros/build.sbt index d54e400c5e88..51f63dbd8026 100644 --- a/macros/build.sbt +++ b/macros/build.sbt @@ -3,8 +3,9 @@ name := "macros" dependsOn(Projects.semanticcpg % Test) libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "net.oneandone.reflections8" % "reflections8" % "0.11.7", + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) enablePlugins(JavaAppPackaging) diff --git a/macros/src/main/scala/io/joern/console/QueryDatabase.scala b/macros/src/main/scala/io/joern/console/QueryDatabase.scala index 31ca139b5c13..54933ef15478 100644 --- a/macros/src/main/scala/io/joern/console/QueryDatabase.scala +++ b/macros/src/main/scala/io/joern/console/QueryDatabase.scala @@ -5,7 +5,7 @@ import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder} import java.lang.reflect.{Method, Parameter} import scala.annotation.unused -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* trait QueryBundle diff --git a/macros/src/test/scala/io/joern/console/QueryDatabaseTests.scala b/macros/src/test/scala/io/joern/console/QueryDatabaseTests.scala index 6f026090c225..3dd84bca6ed0 100644 --- a/macros/src/test/scala/io/joern/console/QueryDatabaseTests.scala +++ b/macros/src/test/scala/io/joern/console/QueryDatabaseTests.scala @@ -1,7 +1,7 @@ package io.joern.console import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should import org.scalatest.wordspec.AnyWordSpec diff --git a/macros/src/test/scala/io/joern/macros/QueryMacroTests.scala b/macros/src/test/scala/io/joern/macros/QueryMacroTests.scala index 5476f9eaa48b..254a7cd3ae9d 100644 --- a/macros/src/test/scala/io/joern/macros/QueryMacroTests.scala +++ b/macros/src/test/scala/io/joern/macros/QueryMacroTests.scala @@ -4,8 +4,8 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import io.joern.macros.QueryMacros.withStrRep -import io.joern.console._ -import io.shiftleft.semanticcpg.language._ +import io.joern.console.* +import io.shiftleft.semanticcpg.language.* class QueryMacroTests extends AnyWordSpec with Matchers { "Query macros" should { diff --git a/project/Versions.scala b/project/Versions.scala index 9e75b78f4c4f..8d7bd2fb009c 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -9,16 +9,18 @@ object Versions { val catsEffect = "3.5.4" val cfr = "0.152" val commonsCompress = "1.26.2" + val commonsExec = "1.4.0" val commonsIo = "2.16.0" val commonsLang = "3.14.0" val commonsText = "1.12.0" val eclipseCdt = "8.4.0.202401242025" val eclipseCore = "3.20.100" val eclipseText = "3.14.0" - val ghidra = "11.0_PUBLIC_20231222-2" + val ghidra = "11.1.2_PUBLIC_20240709-1" val gradleTooling = "8.3" val jacksonDatabind = "2.17.0" val javaParser = "3.25.9" + val jlhttp = "3.1" val json4s = "4.0.7" val lombok = "1.18.32" val mavenArcheologist = "0.0.10" diff --git a/project/build.properties b/project/build.properties index 081fdbbc7625..0b699c3052d7 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.0 +sbt.version=1.10.2 diff --git a/querydb/src/main/scala/io/joern/dumpq/Main.scala b/querydb/src/main/scala/io/joern/dumpq/Main.scala index b2b012e5da07..32f6c8cf3098 100644 --- a/querydb/src/main/scala/io/joern/dumpq/Main.scala +++ b/querydb/src/main/scala/io/joern/dumpq/Main.scala @@ -2,7 +2,7 @@ package io.joern.dumpq import io.joern.console.{DefaultArgumentProvider, QueryDatabase} import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import org.json4s.{Formats, NoTypeHints} import org.json4s.native.Serialization @@ -13,7 +13,7 @@ object Main { } def dumpQueries(): Unit = { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val formats: AnyRef & Formats = Serialization.formats(NoTypeHints) val queryDb = new QueryDatabase(new JoernDefaultArgumentProvider(0)) diff --git a/querydb/src/main/scala/io/joern/scanners/android/ArbitraryFileWrites.scala b/querydb/src/main/scala/io/joern/scanners/android/ArbitraryFileWrites.scala index 97e78f177457..a897527bd642 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/ArbitraryFileWrites.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/ArbitraryFileWrites.scala @@ -1,15 +1,15 @@ package io.joern.scanners.android -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* object ArbitraryFileWrites extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve // todo: improve accuracy, might lead to high number of false positives diff --git a/querydb/src/main/scala/io/joern/scanners/android/ExternalStorage.scala b/querydb/src/main/scala/io/joern/scanners/android/ExternalStorage.scala index 7bbb3199d27f..857f341291c8 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/ExternalStorage.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/ExternalStorage.scala @@ -3,13 +3,13 @@ package io.joern.scanners.android import io.joern.console.* import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import io.joern.macros.QueryMacros.* import io.joern.scanners.* import io.shiftleft.semanticcpg.language.* object ExternalStorage extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve // TODO: improve matching around external storage permissions diff --git a/querydb/src/main/scala/io/joern/scanners/android/Intents.scala b/querydb/src/main/scala/io/joern/scanners/android/Intents.scala index 2b8de93a286e..20f506048be1 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/Intents.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/Intents.scala @@ -1,15 +1,15 @@ package io.joern.scanners.android -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* object Intents extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve @q diff --git a/querydb/src/main/scala/io/joern/scanners/android/JavaScriptInterface.scala b/querydb/src/main/scala/io/joern/scanners/android/JavaScriptInterface.scala index 41834d9b4ae0..0c01df9a7c4d 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/JavaScriptInterface.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/JavaScriptInterface.scala @@ -3,13 +3,13 @@ package io.joern.scanners.android import io.joern.console.* import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import io.joern.macros.QueryMacros.* import io.joern.scanners.* import io.shiftleft.semanticcpg.language.* object JavaScriptInterface extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve // TODO: take into account network_security_config diff --git a/querydb/src/main/scala/io/joern/scanners/android/RootDetection.scala b/querydb/src/main/scala/io/joern/scanners/android/RootDetection.scala index 4586fcccb094..d586a0426670 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/RootDetection.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/RootDetection.scala @@ -1,15 +1,15 @@ package io.joern.scanners.android -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* object RootDetection extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve @q diff --git a/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala b/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala index c4b1be1f536f..b2d9e501050d 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala @@ -3,13 +3,13 @@ package io.joern.scanners.android import io.joern.console.* import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import io.joern.macros.QueryMacros.* import io.joern.scanners.* import io.shiftleft.semanticcpg.language.* object UnprotectedAppParts extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve @q diff --git a/querydb/src/main/scala/io/joern/scanners/android/UnsafeReflection.scala b/querydb/src/main/scala/io/joern/scanners/android/UnsafeReflection.scala index a9c3ec5c3d5b..e4f76b6f74b2 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/UnsafeReflection.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/UnsafeReflection.scala @@ -1,14 +1,14 @@ package io.joern.scanners.android -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* object UnsafeReflection extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve // todo: support `build.gradle.kts` diff --git a/querydb/src/main/scala/io/joern/scanners/c/CopyLoops.scala b/querydb/src/main/scala/io/joern/scanners/c/CopyLoops.scala index 7b00fb2a9c5a..4495ece033a7 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/CopyLoops.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/CopyLoops.scala @@ -1,9 +1,9 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ +import io.joern.scanners.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* object CopyLoops extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/CredentialDrop.scala b/querydb/src/main/scala/io/joern/scanners/c/CredentialDrop.scala index 966cc3d79022..c8d9cb07c5db 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/CredentialDrop.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/CredentialDrop.scala @@ -1,9 +1,9 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.joern.console._ -import io.shiftleft.semanticcpg.language._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.joern.console.* +import io.shiftleft.semanticcpg.language.* +import io.joern.macros.QueryMacros.* object CredentialDrop extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/DangerousFunctions.scala b/querydb/src/main/scala/io/joern/scanners/c/DangerousFunctions.scala index 9ef4d463a9ee..cd54e9a76625 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/DangerousFunctions.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/DangerousFunctions.scala @@ -1,9 +1,9 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.joern.console._ -import io.shiftleft.semanticcpg.language._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.joern.console.* +import io.shiftleft.semanticcpg.language.* +import io.joern.macros.QueryMacros.* object DangerousFunctions extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala b/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala index e7e58ca2933c..14207f3bebd7 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala @@ -1,16 +1,16 @@ package io.joern.scanners.c -import io.joern.scanners._ +import io.joern.scanners.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ -import io.joern.console._ -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* +import io.joern.console.* +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* object HeapBasedOverflow extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve /** Find calls to malloc where the first argument contains an arithmetic expression, the allocated buffer flows into diff --git a/querydb/src/main/scala/io/joern/scanners/c/IntegerTruncations.scala b/querydb/src/main/scala/io/joern/scanners/c/IntegerTruncations.scala index 3f5bfe035b2b..405bd68b7ebb 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/IntegerTruncations.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/IntegerTruncations.scala @@ -1,9 +1,9 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.shiftleft.semanticcpg.language._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.shiftleft.semanticcpg.language.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* object IntegerTruncations extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/Metrics.scala b/querydb/src/main/scala/io/joern/scanners/c/Metrics.scala index 7cc521612c18..96acd023b041 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/Metrics.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/Metrics.scala @@ -1,9 +1,9 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.joern.console._ -import io.shiftleft.semanticcpg.language._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.joern.console.* +import io.shiftleft.semanticcpg.language.* +import io.joern.macros.QueryMacros.* object Metrics extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/MissingLengthCheck.scala b/querydb/src/main/scala/io/joern/scanners/c/MissingLengthCheck.scala index ed6b7d17dd92..f82855e5866f 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/MissingLengthCheck.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/MissingLengthCheck.scala @@ -1,14 +1,14 @@ package io.joern.scanners.c import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.shiftleft.codepropertygraph.generated.nodes import io.joern.dataflowengineoss.queryengine.EngineContext -import io.shiftleft.semanticcpg.language._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language.operatorextension._ -import QueryLangExtensions._ +import io.shiftleft.semanticcpg.language.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.operatorextension.* +import QueryLangExtensions.* object MissingLengthCheck extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala b/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala index 99bbc63a5b3d..36da5781b0ec 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala @@ -1,16 +1,16 @@ package io.joern.scanners.c import io.joern.scanners.{Crew, QueryTags} -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ -import io.joern.console._ +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.macros.QueryMacros._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* object NullTermination extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve @q diff --git a/querydb/src/main/scala/io/joern/scanners/c/RetvalChecks.scala b/querydb/src/main/scala/io/joern/scanners/c/RetvalChecks.scala index ca0c9e3eaf7d..479ea0c060f8 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/RetvalChecks.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/RetvalChecks.scala @@ -1,10 +1,10 @@ package io.joern.scanners.c import io.joern.scanners.{Crew, QueryTags} -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import QueryLangExtensions._ +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import QueryLangExtensions.* object RetvalChecks extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/SignedLeftShift.scala b/querydb/src/main/scala/io/joern/scanners/c/SignedLeftShift.scala index 232da036f815..7730e7236342 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/SignedLeftShift.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/SignedLeftShift.scala @@ -1,10 +1,10 @@ package io.joern.scanners.c -import io.joern.scanners._ +import io.joern.scanners.* import io.shiftleft.codepropertygraph.generated.Operators -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* object SignedLeftShift extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/c/SocketApi.scala b/querydb/src/main/scala/io/joern/scanners/c/SocketApi.scala index 9fe1d3901450..db5cc5171738 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/SocketApi.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/SocketApi.scala @@ -1,11 +1,11 @@ package io.joern.scanners.c import io.joern.scanners.{Crew, QueryTags} -import io.joern.console._ +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import QueryLangExtensions._ +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import QueryLangExtensions.* object SocketApi extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/ghidra/DangerousFunctions.scala b/querydb/src/main/scala/io/joern/scanners/ghidra/DangerousFunctions.scala index 16216093a627..0ccebc7be7ef 100644 --- a/querydb/src/main/scala/io/joern/scanners/ghidra/DangerousFunctions.scala +++ b/querydb/src/main/scala/io/joern/scanners/ghidra/DangerousFunctions.scala @@ -1,9 +1,9 @@ package io.joern.scanners.ghidra -import io.joern.scanners._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ +import io.joern.scanners.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* object DangerousFunctions extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala b/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala index 21310c543181..3c47d454a90c 100644 --- a/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala +++ b/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala @@ -1,10 +1,10 @@ package io.joern.scanners.ghidra -import io.joern.scanners._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ -import io.joern.dataflowengineoss.language._ +import io.joern.scanners.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext object UserInputIntoDangerousFunctions extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/java/CrossSiteScripting.scala b/querydb/src/main/scala/io/joern/scanners/java/CrossSiteScripting.scala index e7265c367e12..66cfe2fe7f28 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/CrossSiteScripting.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/CrossSiteScripting.scala @@ -1,10 +1,10 @@ package io.joern.scanners.java -import io.joern.scanners._ -import io.shiftleft.semanticcpg.language._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.joern.dataflowengineoss.language._ +import io.joern.scanners.* +import io.shiftleft.semanticcpg.language.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext object CrossSiteScripting extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala b/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala index 1e43586657f0..5b96efcabfbb 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala @@ -1,10 +1,10 @@ package io.joern.scanners.java -import io.joern.scanners._ -import io.shiftleft.semanticcpg.language._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.joern.dataflowengineoss.language._ +import io.joern.scanners.* +import io.shiftleft.semanticcpg.language.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext /** @see diff --git a/querydb/src/main/scala/io/joern/scanners/java/DangerousFunctions.scala b/querydb/src/main/scala/io/joern/scanners/java/DangerousFunctions.scala index cdfabc11b03f..ccee98ccbf3a 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/DangerousFunctions.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/DangerousFunctions.scala @@ -1,9 +1,9 @@ package io.joern.scanners.java -import io.joern.scanners._ -import io.shiftleft.semanticcpg.language._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.shiftleft.semanticcpg.language.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* object DangerousFunctions extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala b/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala index 67aebe735261..fa987cfd722e 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala @@ -1,10 +1,10 @@ package io.joern.scanners.java -import io.joern.scanners._ -import io.shiftleft.semanticcpg.language._ -import io.joern.console._ -import io.joern.macros.QueryMacros._ -import io.joern.dataflowengineoss.language._ +import io.joern.scanners.* +import io.shiftleft.semanticcpg.language.* +import io.joern.console.* +import io.joern.macros.QueryMacros.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext // The queries are tied to springframework diff --git a/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkCommunication.scala b/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkCommunication.scala index 2f31e6c2800d..64cff884c102 100644 --- a/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkCommunication.scala +++ b/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkCommunication.scala @@ -1,16 +1,16 @@ package io.joern.scanners.kotlin -import io.joern.scanners._ -import io.joern.console._ +import io.joern.console.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.dataflowengineoss.language._ -import io.joern.macros.QueryMacros._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.macros.QueryMacros.* +import io.joern.scanners.* import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* object NetworkCommunication extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve // todo: improve by including trust managers created via `object` expressions diff --git a/querydb/src/main/scala/io/joern/scanners/kotlin/PathTraversals.scala b/querydb/src/main/scala/io/joern/scanners/kotlin/PathTraversals.scala index fe64c33c1203..97a08137614d 100644 --- a/querydb/src/main/scala/io/joern/scanners/kotlin/PathTraversals.scala +++ b/querydb/src/main/scala/io/joern/scanners/kotlin/PathTraversals.scala @@ -1,15 +1,15 @@ package io.joern.scanners.kotlin -import io.joern.scanners._ -import io.joern.console._ +import io.joern.scanners.* +import io.joern.console.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics -import io.joern.dataflowengineoss.language._ -import io.joern.macros.QueryMacros._ -import io.shiftleft.semanticcpg.language._ +import io.joern.dataflowengineoss.semanticsloader.NoSemantics +import io.joern.dataflowengineoss.language.* +import io.joern.macros.QueryMacros.* +import io.shiftleft.semanticcpg.language.* object PathTraversals extends QueryBundle { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) implicit val resolver: ICallResolver = NoResolve @q diff --git a/querydb/src/main/scala/io/joern/scanners/php/SQLInjection.scala b/querydb/src/main/scala/io/joern/scanners/php/SQLInjection.scala index 0829f18301c0..7fbaa232ad3d 100644 --- a/querydb/src/main/scala/io/joern/scanners/php/SQLInjection.scala +++ b/querydb/src/main/scala/io/joern/scanners/php/SQLInjection.scala @@ -1,12 +1,12 @@ package io.joern.scanners.php -import io.joern.console._ -import io.joern.dataflowengineoss.language._ +import io.joern.console.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.macros.QueryMacros._ -import io.joern.scanners._ +import io.joern.macros.QueryMacros.* +import io.joern.scanners.* import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* object SQLInjection extends QueryBundle { diff --git a/querydb/src/main/scala/io/joern/scanners/php/ShellExec.scala b/querydb/src/main/scala/io/joern/scanners/php/ShellExec.scala index 923538116f84..cbf2854c4c56 100644 --- a/querydb/src/main/scala/io/joern/scanners/php/ShellExec.scala +++ b/querydb/src/main/scala/io/joern/scanners/php/ShellExec.scala @@ -1,12 +1,12 @@ package io.joern.scanners.php -import io.joern.console._ -import io.joern.dataflowengineoss.language._ +import io.joern.console.* +import io.joern.dataflowengineoss.language.* import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.macros.QueryMacros._ -import io.joern.scanners._ +import io.joern.macros.QueryMacros.* +import io.joern.scanners.* import io.shiftleft.codepropertygraph.generated.Operators -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* object ShellExec extends QueryBundle { diff --git a/querydb/src/test/scala/io/joern/scanners/android/RootDetectionTests.scala b/querydb/src/test/scala/io/joern/scanners/android/RootDetectionTests.scala index c8280bdabd37..e4a0cebea4e0 100644 --- a/querydb/src/test/scala/io/joern/scanners/android/RootDetectionTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/android/RootDetectionTests.scala @@ -1,11 +1,11 @@ package io.joern.scanners.android import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.semanticsloader.NoSemantics import io.joern.suites.KotlinQueryTestSuite class RootDetectionTests extends KotlinQueryTestSuite(RootDetection) { - implicit val engineContext: EngineContext = EngineContext(Semantics.empty) + implicit val engineContext: EngineContext = EngineContext(NoSemantics) "the `rootDetectionViaFileChecks` query" when { "should match on all multi-file positive examples" in { diff --git a/querydb/src/test/scala/io/joern/scanners/android/UnprotectedAppPartsTests.scala b/querydb/src/test/scala/io/joern/scanners/android/UnprotectedAppPartsTests.scala index 81d807064d2f..c013457a12fa 100644 --- a/querydb/src/test/scala/io/joern/scanners/android/UnprotectedAppPartsTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/android/UnprotectedAppPartsTests.scala @@ -1,9 +1,9 @@ package io.joern.scanners.android -import io.joern.console.scan._ +import io.joern.console.scan.* import io.shiftleft.codepropertygraph.generated.nodes.CfgNode import io.joern.suites.KotlinQueryTestSuite -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class UnprotectedAppPartsTests extends KotlinQueryTestSuite(UnprotectedAppParts) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/CopyLoopTests.scala b/querydb/src/test/scala/io/joern/scanners/c/CopyLoopTests.scala index 4c20bd6d4791..82520686a738 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/CopyLoopTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/CopyLoopTests.scala @@ -2,8 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.shiftleft.semanticcpg.language._ -import io.joern.console.scan._ +import io.shiftleft.semanticcpg.language.* +import io.joern.console.scan.* class CopyLoopTests extends CQueryTestSuite(CopyLoops) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala b/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala index b583fc4ddf61..e3bedcba6b1e 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/HeapBasedOverflowTests.scala @@ -2,7 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.console.scan._ +import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class HeapBasedOverflowTests extends CQueryTestSuite(HeapBasedOverflow) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/IntegerTruncationsTests.scala b/querydb/src/test/scala/io/joern/scanners/c/IntegerTruncationsTests.scala index b4fac1fd3913..9e3b5d177c99 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/IntegerTruncationsTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/IntegerTruncationsTests.scala @@ -2,8 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.shiftleft.semanticcpg.language._ -import io.joern.console.scan._ +import io.shiftleft.semanticcpg.language.* +import io.joern.console.scan.* class IntegerTruncationsTests extends CQueryTestSuite(IntegerTruncations) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala b/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala index 91d0dd61d4eb..a741cea32590 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/MetricsTests.scala @@ -2,7 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.console.scan._ +import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class MetricsTests extends CQueryTestSuite(Metrics) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/NullTerminationTests.scala b/querydb/src/test/scala/io/joern/scanners/c/NullTerminationTests.scala index 139c01859aec..327618fd92f8 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/NullTerminationTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/NullTerminationTests.scala @@ -2,8 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.shiftleft.semanticcpg.language._ -import io.joern.console.scan._ +import io.shiftleft.semanticcpg.language.* +import io.joern.console.scan.* class NullTerminationTests extends CQueryTestSuite(NullTermination) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/QueryWithReachableBy.scala b/querydb/src/test/scala/io/joern/scanners/c/QueryWithReachableBy.scala index 6a6d21b311df..6eff6c970a4f 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/QueryWithReachableBy.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/QueryWithReachableBy.scala @@ -1,10 +1,10 @@ package io.joern.scanners.c -import io.joern.scanners._ -import io.joern.console._ -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ -import io.joern.macros.QueryMacros._ +import io.joern.scanners.* +import io.joern.console.* +import io.joern.dataflowengineoss.language.* +import io.shiftleft.semanticcpg.language.* +import io.joern.macros.QueryMacros.* import io.joern.dataflowengineoss.queryengine.EngineContext /** Just to make sure that we support reachableBy queries, which did not work before diff --git a/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreePostUsage.scala b/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreePostUsage.scala index 568bb515c515..ed2611ddacbe 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreePostUsage.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreePostUsage.scala @@ -2,8 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.console.scan._ -import io.shiftleft.semanticcpg.language._ +import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class UseAfterFreePostUsage extends CQueryTestSuite(UseAfterFree) { diff --git a/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreeReturnTests.scala b/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreeReturnTests.scala index 3ae07cc654f9..190be8f9bd57 100644 --- a/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreeReturnTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/c/UseAfterFreeReturnTests.scala @@ -2,8 +2,8 @@ package io.joern.scanners.c import io.joern.suites.CQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.console.scan._ -import io.shiftleft.semanticcpg.language._ +import io.joern.console.scan.* +import io.shiftleft.semanticcpg.language.* class UseAfterFreeReturnTests extends CQueryTestSuite(UseAfterFree) { diff --git a/querydb/src/test/scala/io/joern/scanners/kotlin/NetworkProtocolsTests.scala b/querydb/src/test/scala/io/joern/scanners/kotlin/NetworkProtocolsTests.scala index 3c0bbfa01176..ae41b9d9d210 100644 --- a/querydb/src/test/scala/io/joern/scanners/kotlin/NetworkProtocolsTests.scala +++ b/querydb/src/test/scala/io/joern/scanners/kotlin/NetworkProtocolsTests.scala @@ -1,9 +1,9 @@ package io.joern.scanners.kotlin -import io.joern.console.scan._ +import io.joern.console.scan.* import io.joern.suites.KotlinQueryTestSuite import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class NetworkProtocolsTests extends KotlinQueryTestSuite(NetworkProtocols) { "should find calls relevant to insecure network protocol usage" in { diff --git a/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala b/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala index 9fc370a5284f..f35909393c30 100644 --- a/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala @@ -2,7 +2,7 @@ package io.joern.suites import io.joern.console.QueryDatabase import org.scalatest.wordspec.AnyWordSpec -import org.scalatest.matchers.should.Matchers._ +import org.scalatest.matchers.should.Matchers.* class AllBundlesTestSuite extends AnyWordSpec { val argumentProvider = new QDBArgumentProvider(3) diff --git a/querydb/src/test/scala/io/joern/suites/AndroidQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/AndroidQueryTestSuite.scala index 4be9b05268b7..f40e1d3f7529 100644 --- a/querydb/src/test/scala/io/joern/suites/AndroidQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/AndroidQueryTestSuite.scala @@ -1,12 +1,12 @@ package io.joern.suites -import io.joern.console.scan._ +import io.joern.console.scan.* import io.joern.console.{CodeSnippet, Query, QueryBundle} import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.util.QueryUtil import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.ConfigFile -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AndroidQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) extends KotlinCode2CpgFixture(withOssDataflow = true, withDefaultJars = true) { diff --git a/querydb/src/test/scala/io/joern/suites/CQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/CQueryTestSuite.scala index 93e5d618c08c..97e29744fd44 100644 --- a/querydb/src/test/scala/io/joern/suites/CQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/CQueryTestSuite.scala @@ -2,12 +2,12 @@ package io.joern.suites import io.joern.util.QueryUtil import io.shiftleft.codepropertygraph.generated.nodes -import io.joern.console.scan._ +import io.joern.console.scan.* import io.joern.console.QueryBundle import io.joern.console.Query import io.joern.c2cpg.testfixtures.DataFlowCodeToCpgSuite import io.joern.x2cpg.testfixtures.TestCpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class CQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) extends DataFlowCodeToCpgSuite { diff --git a/querydb/src/test/scala/io/joern/suites/GhidraQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/GhidraQueryTestSuite.scala index 1a6d1cc5eadc..81ea14d8af67 100644 --- a/querydb/src/test/scala/io/joern/suites/GhidraQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/GhidraQueryTestSuite.scala @@ -1,12 +1,12 @@ package io.joern.suites import io.joern.console.QueryBundle -import io.joern.console.scan._ +import io.joern.console.scan.* import io.joern.ghidra2cpg.fixtures.DataFlowBinToCpgSuite import io.joern.util.QueryUtil import io.shiftleft.codepropertygraph.generated.nodes import io.joern.console.Query -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.utils.ProjectRoot class GhidraQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) extends DataFlowBinToCpgSuite { diff --git a/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala index 3407c2837ee8..00711ed73b9b 100644 --- a/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/JavaQueryTestSuite.scala @@ -1,12 +1,13 @@ package io.joern.suites -import io.joern.console.scan._ +import io.joern.console.scan.* import io.joern.console.{CodeSnippet, Query, QueryBundle} import io.joern.javasrc2cpg.testfixtures.JavaSrcCode2CpgFixture import io.joern.util.QueryUtil import io.joern.x2cpg.testfixtures.TestCpg import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Literal, Method, StoredNode} +import io.shiftleft.semanticcpg.language.* class JavaQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) extends JavaSrcCode2CpgFixture(withOssDataflow = true) { diff --git a/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala b/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala index 01c102334903..469c1517b919 100644 --- a/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/KotlinQueryTestSuite.scala @@ -6,7 +6,8 @@ import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.joern.x2cpg.testfixtures.TestCpg import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} -import io.joern.console.scan._ +import io.shiftleft.semanticcpg.language.* +import io.joern.console.scan.* import io.shiftleft.utils.ProjectRoot class KotlinQueryTestSuite[QB <: QueryBundle](val queryBundle: QB) diff --git a/querydb/src/test/scala/io/joern/suites/QDBArgumentProvider.scala b/querydb/src/test/scala/io/joern/suites/QDBArgumentProvider.scala index 0f8946a59d9e..7c19ed88d228 100644 --- a/querydb/src/test/scala/io/joern/suites/QDBArgumentProvider.scala +++ b/querydb/src/test/scala/io/joern/suites/QDBArgumentProvider.scala @@ -2,7 +2,7 @@ package io.joern.suites import io.joern.console.DefaultArgumentProvider import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.{Parser, Semantics} +import io.joern.dataflowengineoss.semanticsloader.{FullNameSemanticsParser, Semantics} import java.nio.file.Paths diff --git a/semanticcpg/build.sbt b/semanticcpg/build.sbt index ea2590fc0a0d..8c2c66a7e0b5 100644 --- a/semanticcpg/build.sbt +++ b/semanticcpg/build.sbt @@ -1,11 +1,12 @@ name := "semanticcpg" libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, - "org.json4s" %% "json4s-native" % Versions.json4s, - "org.apache.commons" % "commons-text" % Versions.commonsText, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, + "org.json4s" %% "json4s-native" % Versions.json4s, + "org.scala-lang.modules" %% "scala-xml" % "2.2.0", + "org.apache.commons" % "commons-text" % Versions.commonsText, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala index d10dd2108cf0..7a62e226f138 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/Overlays.scala @@ -1,9 +1,9 @@ package io.shiftleft.semanticcpg -import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder} import io.shiftleft.codepropertygraph.generated.Properties import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* object Overlays { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala index f1af0f8d8cd3..2f6ff9fd4c27 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/accesspath/TrackedBase.scala @@ -1,6 +1,7 @@ package io.shiftleft.semanticcpg.accesspath -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* trait TrackedBase case class TrackedNamedVariable(name: String) extends TrackedBase diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/AstGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/AstGenerator.scala index f68afc41afcb..52a649f52ffd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/AstGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/AstGenerator.scala @@ -3,7 +3,7 @@ package io.shiftleft.semanticcpg.dotgenerator import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, MethodParameterOut} import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class AstGenerator { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CallGraphGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CallGraphGenerator.scala index d1aecf60bb0e..dfee4db69a23 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CallGraphGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CallGraphGenerator.scala @@ -3,7 +3,7 @@ package io.shiftleft.semanticcpg.dotgenerator import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Method, StoredNode} import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import scala.collection.mutable diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala index 2507bcf9e3d6..9223d7d98218 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala @@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.StoredNode import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.Edge -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* class CdgGenerator extends CfgGenerator { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala index 45999fbe46e8..067ce1302792 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CfgGenerator.scala @@ -1,10 +1,9 @@ package io.shiftleft.semanticcpg.dotgenerator import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} -import io.shiftleft.semanticcpg.language._ -import overflowdb.Node +import io.shiftleft.semanticcpg.language.* class CfgGenerator { @@ -44,12 +43,12 @@ class CfgGenerator { protected def expand(v: StoredNode): Iterator[Edge] = v._cfgOut.map(node => Edge(v, node, edgeType = edgeType)) - private def isConditionInControlStructure(v: Node): Boolean = v match { + private def isConditionInControlStructure(v: StoredNode): Boolean = v match { case id: Identifier => id.astParent.isControlStructure case _ => false } - private def cfgNodeShouldBeDisplayed(v: Node): Boolean = + private def cfgNodeShouldBeDisplayed(v: StoredNode): Boolean = isConditionInControlStructure(v) || !(v.isInstanceOf[Literal] || v.isInstanceOf[Identifier] || diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala index 6ee27691b66f..a7429d8e86b2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala @@ -1,5 +1,6 @@ package io.shiftleft.semanticcpg.dotgenerator +import flatgraph.Accessors import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -70,7 +71,10 @@ object DotSerializer { } private def stringRepr(vertex: StoredNode): String = { - val maybeLineNo: Optional[AnyRef] = vertex.propertyOption(PropertyNames.LINE_NUMBER) + // TODO MP after the initial flatgraph migration (where we want to maintain semantics as far as + // possible) this might become `vertex.property(Properties.LineNumber)` which derives to `Option[Int]` + val lineNoMaybe = vertex.propertyOption[Int](PropertyNames.LINE_NUMBER) + StringEscapeUtils.escapeHtml4(vertex match { case call: Call => (call.name, limit(call.code)).toString case contrl: ControlStructure => (contrl.label, contrl.controlStructureType, contrl.code).toString @@ -87,7 +91,7 @@ object DotSerializer { case typeDecl: TypeDecl => (typeDecl.label, typeDecl.name).toString() case member: Member => (member.label, member.name).toString() case _ => "" - }) + (if (maybeLineNo.isPresent) s"${maybeLineNo.get()}" else "") + }) + lineNoMaybe.map(lineNo => s"$lineNo").getOrElse("") } private def toCfgNode(node: StoredNode): CfgNode = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/TypeHierarchyGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/TypeHierarchyGenerator.scala index 25891f6e7c48..2ec4454d901f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/TypeHierarchyGenerator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/TypeHierarchyGenerator.scala @@ -3,7 +3,7 @@ package io.shiftleft.semanticcpg.dotgenerator import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{StoredNode, Type, TypeDecl} import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import scala.collection.mutable diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala index 86944d6da608..6292879b1b0d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/AccessPathHandling.scala @@ -1,8 +1,9 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.{Operators, Properties, PropertyNames} -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.accesspath._ +import io.shiftleft.codepropertygraph.generated.{Operators, Properties} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.accesspath.* +import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.IteratorHasAsScala @@ -42,8 +43,9 @@ object AccessPathHandling { .collect { case node: Literal => ConstantAccess(node.code) case node: Identifier => ConstantAccess(node.name) - case other if other.propertyOption(PropertyNames.NAME).isPresent => - logger.warn(s"unexpected/deprecated node encountered: $other with properties: ${other.propertiesMap()}") + case other if other.propertyOption(Properties.Name).isDefined => + val properties = other.propertiesMap + logger.warn(s"unexpected/deprecated node encountered: $other with properties: $properties") ConstantAccess(other.property(Properties.Name)) } .getOrElse(VariableAccess) :: tail diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala index 0a00dd9c5586..008e87131dbb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala @@ -1,8 +1,8 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import overflowdb.traversal._ import scala.annotation.tailrec @@ -64,7 +64,7 @@ object LocationCreator { @tailrec private def findVertex(node: StoredNode, instanceCheck: StoredNode => Boolean): Option[StoredNode] = - node._astIn.nextOption() match { + node._astIn.iterator.nextOption() match { case Some(head) if instanceCheck(head) => Some(head) case Some(head) => findVertex(head, instanceCheck) case None => None diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala index 2a4384e9c7e1..dbcce1aa4807 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes.NewNode -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder trait HasStoreMethod { def store()(implicit diffBuilder: DiffGraphBuilder): Unit diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala index 9248f9e72b1d..44b52836c0ac 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, NewTagNodePair, StoredNode} -import overflowdb.BatchedUpdate.DiffGraphBuilder +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder class NewTagNodePairTraversal(traversal: Iterator[NewTagNodePair]) extends HasStoreMethod { override def store()(implicit diffGraph: DiffGraphBuilder): Unit = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala index 6c7c4024c26b..f0712ce1f87d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeExtensionFinder.scala @@ -1,29 +1,8 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.{ - Call, - Identifier, - Literal, - Local, - Method, - MethodParameterIn, - MethodParameterOut, - MethodRef, - MethodReturn, - StoredNode -} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.nodemethods.{ - CallMethods, - IdentifierMethods, - LiteralMethods, - LocalMethods, - MethodMethods, - MethodParameterInMethods, - MethodParameterOutMethods, - MethodRefMethods, - MethodReturnMethods -} +import io.shiftleft.semanticcpg.language.nodemethods.* trait NodeExtensionFinder { def apply(n: StoredNode): Option[NodeExtension] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala index 71971d654828..337714e94255 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeSteps.scala @@ -1,18 +1,16 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.codedumper.CodeDumper -import overflowdb.Node -import overflowdb.traversal._ -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} /** Steps for all node types * * This is the base class for all steps defined on */ -@help.Traversal(elementType = classOf[StoredNode]) +@Traversal(elementType = classOf[StoredNode]) class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) extends AnyVal { @Doc( @@ -23,15 +21,16 @@ class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) exten |the file node that represents that source file. |""" ) - def file: Iterator[File] = - traversal - .choose(_.label) { - case NodeTypes.NAMESPACE => _.in(EdgeTypes.REF).out(EdgeTypes.SOURCE_FILE) - case NodeTypes.COMMENT => _.in(EdgeTypes.AST).hasLabel(NodeTypes.FILE) - case _ => - _.repeat(_.coalesce(_.out(EdgeTypes.SOURCE_FILE), _.in(EdgeTypes.AST)))(_.until(_.hasLabel(NodeTypes.FILE))) - } - .cast[File] + def file: Iterator[File] = { + traversal.flatMap { + case namespace: Namespace => + namespace.refIn.sourceFileOut + case comment: Comment => + comment.astIn + case node => + Iterator(node).repeat(_.coalesce(_._sourceFileOut, _._astIn))(_.until(_.hasLabel(File.Label))).cast[File] + } + } @Doc( info = "Location, including filename and line number", @@ -84,11 +83,6 @@ class NodeSteps[NodeType <: StoredNode](val traversal: Iterator[NodeType]) exten }.l } - /* follow the incoming edges of the given type as long as possible */ - protected def walkIn(edgeType: String): Iterator[Node] = - traversal - .repeat(_.in(edgeType))(_.until(_.in(edgeType).countTrav.filter(_ == 0))) - @Doc( info = "Tag node with `tagName`", longInfo = """ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala index daa8f1b82e23..4f45f3f3bea3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NodeTypeStarters.scala @@ -1,318 +1,92 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} -import overflowdb._ -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc -import overflowdb.traversal.{InitialTraversal, TraversalSource} +import io.shiftleft.codepropertygraph.generated.help.{Doc, TraversalSource} +import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters.IteratorHasAsScala +/** Starting point for a new traversal, e.g. + * - `cpg.method`, `cpg.call` etc. - these are generated by the flatgraph codegenerator and automatically inherited + * - `cpg.method.name` + */ +@TraversalSource +class NodeTypeStarters(cpg: Cpg) { -@help.TraversalSource -class NodeTypeStarters(cpg: Cpg) extends TraversalSource(cpg.graph) { - - /** Traverse to all nodes. - */ - @Doc(info = "All nodes of the graph") - override def all: Traversal[StoredNode] = - cpg.graph.nodes.asScala.cast[StoredNode] - - /** Traverse to all annotations - */ - def annotation: Traversal[Annotation] = - InitialTraversal.from[Annotation](cpg.graph, NodeTypes.ANNOTATION) - - /** Traverse to all arguments passed to methods - */ + /** Traverse to all arguments passed to methods */ @Doc(info = "All arguments (actual parameters)") - def argument: Traversal[Expression] = - call.argument + def argument: Iterator[Expression] = + cpg.call.argument - /** Shorthand for `cpg.argument.code(code)` - */ - def argument(code: String): Traversal[Expression] = - argument.code(code) + /** Shorthand for `cpg.argument.code(code)` */ + def argument(code: String): Iterator[Expression] = + cpg.argument.code(code) @Doc(info = "All breaks (`ControlStructure` nodes)") - def break: Traversal[ControlStructure] = - controlStructure.isBreak - - /** Traverse to all call sites - */ - @Doc(info = "All call sites") - def call: Traversal[Call] = - InitialTraversal.from[Call](cpg.graph, NodeTypes.CALL) - - /** Shorthand for `cpg.call.name(name)` - */ - def call(name: String): Traversal[Call] = - call.name(name) - - /** Traverse to all comments in source-based CPGs. - */ - @Doc(info = "All comments in source-based CPGs") - def comment: Traversal[Comment] = - InitialTraversal.from[Comment](cpg.graph, NodeTypes.COMMENT) - - /** Shorthand for `cpg.comment.code(code)` - */ - def comment(code: String): Traversal[Comment] = - comment.has(Properties.Code -> code) - - /** Traverse to all config files - */ - @Doc(info = "All config files") - def configFile: Traversal[ConfigFile] = - InitialTraversal.from[ConfigFile](cpg.graph, NodeTypes.CONFIG_FILE) - - /** Shorthand for `cpg.configFile.name(name)` - */ - def configFile(name: String): Traversal[ConfigFile] = - configFile.name(name) - - /** Traverse to all dependencies - */ - @Doc(info = "All dependencies") - def dependency: Traversal[Dependency] = - InitialTraversal.from[Dependency](cpg.graph, NodeTypes.DEPENDENCY) - - /** Shorthand for `cpg.dependency.name(name)` - */ - def dependency(name: String): Traversal[Dependency] = - dependency.name(name) - - @Doc(info = "All control structures (source-based frontends)") - def controlStructure: Traversal[ControlStructure] = - InitialTraversal.from[ControlStructure](cpg.graph, NodeTypes.CONTROL_STRUCTURE) + def break: Iterator[ControlStructure] = + cpg.controlStructure.isBreak @Doc(info = "All continues (`ControlStructure` nodes)") - def continue: Traversal[ControlStructure] = - controlStructure.isContinue + def continue: Iterator[ControlStructure] = + cpg.controlStructure.isContinue @Doc(info = "All do blocks (`ControlStructure` nodes)") - def doBlock: Traversal[ControlStructure] = - controlStructure.isDo + def doBlock: Iterator[ControlStructure] = + cpg.controlStructure.isDo @Doc(info = "All else blocks (`ControlStructure` nodes)") - def elseBlock: Traversal[ControlStructure] = - controlStructure.isElse + def elseBlock: Iterator[ControlStructure] = + cpg.controlStructure.isElse @Doc(info = "All throws (`ControlStructure` nodes)") - def throws: Traversal[ControlStructure] = - controlStructure.isThrow - - /** Traverse to all source files - */ - @Doc(info = "All source files") - def file: Traversal[File] = - InitialTraversal.from[File](cpg.graph, NodeTypes.FILE) - - /** Shorthand for `cpg.file.name(name)` - */ - def file(name: String): Traversal[File] = - file.name(name) + def throws: Iterator[ControlStructure] = + cpg.controlStructure.isThrow @Doc(info = "All for blocks (`ControlStructure` nodes)") - def forBlock: Traversal[ControlStructure] = - controlStructure.isFor + def forBlock: Iterator[ControlStructure] = + cpg.controlStructure.isFor @Doc(info = "All gotos (`ControlStructure` nodes)") - def goto: Traversal[ControlStructure] = - controlStructure.isGoto - - /** Traverse to all identifiers, e.g., occurrences of local variables or class members in method bodies. - */ - @Doc(info = "All identifier usages") - def identifier: Traversal[Identifier] = - InitialTraversal.from[Identifier](cpg.graph, NodeTypes.IDENTIFIER) - - /** Shorthand for `cpg.identifier.name(name)` - */ - def identifier(name: String): Traversal[Identifier] = - identifier.name(name) + def goto: Iterator[ControlStructure] = + cpg.controlStructure.isGoto @Doc(info = "All if blocks (`ControlStructure` nodes)") - def ifBlock: Traversal[ControlStructure] = - controlStructure.isIf - - /** Traverse to all jump targets - */ - @Doc(info = "All jump targets, i.e., labels") - def jumpTarget: Traversal[JumpTarget] = - InitialTraversal.from[JumpTarget](cpg.graph, NodeTypes.JUMP_TARGET) - - /** Traverse to all local variable declarations - */ - @Doc(info = "All local variables") - def local: Traversal[Local] = - InitialTraversal.from[Local](cpg.graph, NodeTypes.LOCAL) - - /** Shorthand for `cpg.local.name` - */ - def local(name: String): Traversal[Local] = - local.name(name) - - /** Traverse to all literals (constant strings and numbers provided directly in the code). - */ - @Doc(info = "All literals, e.g., numbers or strings") - def literal: Traversal[Literal] = - InitialTraversal.from[Literal](cpg.graph, NodeTypes.LITERAL) - - /** Shorthand for `cpg.literal.code(code)` - */ - def literal(code: String): Traversal[Literal] = - literal.code(code) - - /** Traverse to all methods - */ - @Doc(info = "All methods") - def method: Traversal[Method] = - InitialTraversal.from[Method](cpg.graph, NodeTypes.METHOD) - - /** Shorthand for `cpg.method.name(name)` - */ - @Doc(info = "All methods with a name that matches the given pattern") - def method(namePattern: String): Traversal[Method] = - method.name(namePattern) - - /** Traverse to all formal return parameters - */ - @Doc(info = "All formal return parameters") - def methodReturn: Traversal[MethodReturn] = - InitialTraversal.from[MethodReturn](cpg.graph, NodeTypes.METHOD_RETURN) - - /** Traverse to all class members - */ - @Doc(info = "All members of complex types (e.g., classes/structures)") - def member: Traversal[Member] = - InitialTraversal.from[Member](cpg.graph, NodeTypes.MEMBER) - - /** Shorthand for `cpg.member.name(name)` - */ - def member(name: String): Traversal[Member] = - member.name(name) - - /** Traverse to all meta data entries - */ - @Doc(info = "Meta data blocks for graph") - def metaData: Traversal[MetaData] = - InitialTraversal.from[MetaData](cpg.graph, NodeTypes.META_DATA) - - /** Traverse to all method references - */ - @Doc(info = "All method references") - def methodRef: Traversal[MethodRef] = - InitialTraversal.from[MethodRef](cpg.graph, NodeTypes.METHOD_REF) - - /** Shorthand for `cpg.methodRef.filter(_.referencedMethod.name(name))` - */ - def methodRef(name: String): Traversal[MethodRef] = - methodRef.where(_.referencedMethod.name(name)) - - /** Traverse to all namespaces, e.g., packages in Java. - */ - @Doc(info = "All namespaces") - def namespace: Traversal[Namespace] = - InitialTraversal.from[Namespace](cpg.graph, NodeTypes.NAMESPACE) - - /** Shorthand for `cpg.namespace.name(name)` - */ - def namespace(name: String): Traversal[Namespace] = - namespace.name(name) - - /** Traverse to all namespace blocks, e.g., packages in Java. - */ - def namespaceBlock: Traversal[NamespaceBlock] = - InitialTraversal.from[NamespaceBlock](cpg.graph, NodeTypes.NAMESPACE_BLOCK) - - /** Shorthand for `cpg.namespaceBlock.name(name)` - */ - def namespaceBlock(name: String): Traversal[NamespaceBlock] = - namespaceBlock.name(name) - - /** Traverse to all input parameters - */ + def ifBlock: Iterator[ControlStructure] = + cpg.controlStructure.isIf + + /** Shorthand for `cpg.methodRef.where(_.referencedMethod.name(name))` + * + * Note re API design: this step was supposed to be called `methodRef(name: String)`, but due to limitations in + * Scala's implicit resolution (and the setup of our implicit steps) we have to disambiguate it from `.methodRef` by + * name. + * + * More precisely: Scala's implicit resolution reports 'ambiguous implicits' if two methods with the same name but + * different parameters are defined in two different (implicitly reachable) classes. The `.methodRef` step is defined + * in `generated.CpgNodeStarter`. This step (filter by name) doesn't get generated by the codegen because it's more + * complex than the other 'filter by primary key' starter steps. + */ + def methodRefWithName(name: String): Iterator[MethodRef] = + cpg.methodRef.where(_.referencedMethod.name(name)) + + /** Traverse to all input parameters */ @Doc(info = "All parameters") - def parameter: Traversal[MethodParameterIn] = - InitialTraversal.from[MethodParameterIn](cpg.graph, NodeTypes.METHOD_PARAMETER_IN) + def parameter: Iterator[MethodParameterIn] = + cpg.methodParameterIn - /** Shorthand for `cpg.parameter.name(name)` - */ - def parameter(name: String): Traversal[MethodParameterIn] = + /** Shorthand for `cpg.parameter.name(name)` */ + def parameter(name: String): Iterator[MethodParameterIn] = parameter.name(name) - /** Traverse to all return expressions - */ - @Doc(info = "All actual return parameters") - def ret: Traversal[Return] = - InitialTraversal.from[Return](cpg.graph, NodeTypes.RETURN) - - /** Shorthand for `returns.code(code)` - */ - def ret(code: String): Traversal[Return] = - ret.code(code) - - @Doc(info = "All imports") - def imports: Traversal[Import] = - InitialTraversal.from[Import](cpg.graph, NodeTypes.IMPORT) - @Doc(info = "All switch blocks (`ControlStructure` nodes)") - def switchBlock: Traversal[ControlStructure] = - controlStructure.isSwitch + def switchBlock: Iterator[ControlStructure] = + cpg.controlStructure.isSwitch @Doc(info = "All try blocks (`ControlStructure` nodes)") - def tryBlock: Traversal[ControlStructure] = - controlStructure.isTry - - /** Traverse to all types, e.g., Set - */ - @Doc(info = "All used types") - def typ: Traversal[Type] = - InitialTraversal.from[Type](cpg.graph, NodeTypes.TYPE) - - /** Shorthand for `cpg.typ.name(name)` - */ - @Doc(info = "All used types with given name") - def typ(name: String): Traversal[Type] = - typ.name(name) - - /** Traverse to all declarations, e.g., Set - */ - @Doc(info = "All declarations of types") - def typeDecl: Traversal[TypeDecl] = - InitialTraversal.from[TypeDecl](cpg.graph, NodeTypes.TYPE_DECL) - - /** Shorthand for cpg.typeDecl.name(name) - */ - def typeDecl(name: String): Traversal[TypeDecl] = - typeDecl.name(name) - - /** Traverse to all tags - */ - @Doc(info = "All tags") - def tag: Traversal[Tag] = - InitialTraversal.from[Tag](cpg.graph, NodeTypes.TAG) - - @Doc(info = "All tags with given name") - def tag(name: String): Traversal[Tag] = - tag.name(name) - - /** Traverse to all template DOM nodes - */ - @Doc(info = "All template DOM nodes") - def templateDom: Traversal[TemplateDom] = - InitialTraversal.from[TemplateDom](cpg.graph, NodeTypes.TEMPLATE_DOM) - - /** Traverse to all type references - */ - @Doc(info = "All type references") - def typeRef: Traversal[TypeRef] = - InitialTraversal.from[TypeRef](cpg.graph, NodeTypes.TYPE_REF) + def tryBlock: Iterator[ControlStructure] = + cpg.controlStructure.isTry @Doc(info = "All while blocks (`ControlStructure` nodes)") - def whileBlock: Traversal[ControlStructure] = - controlStructure.isWhile + def whileBlock: Iterator[ControlStructure] = + cpg.controlStructure.isWhile } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala index 0194d4dc571b..e6efabae3f28 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Show.scala @@ -1,9 +1,8 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.NewNode -import overflowdb.Node +import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, NewNode, StoredNode} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* /** Typeclass for (pretty) printing an object */ @@ -18,20 +17,20 @@ object Show { override def apply(a: Any): String = a match { case node: NewNode => val label = node.label - val properties = propsToString(node.properties.toList) + val properties = propsToString(node.properties) s"($label): $properties" - case node: Node => + case node: StoredNode => val label = node.label val id = node.id().toString - val properties = propsToString(node.propertiesMap.asScala.toList) + val properties = propsToString(node.properties) s"($label,$id): $properties" case other => other.toString } - private def propsToString(keyValues: List[(String, Any)]): String = { - keyValues + private def propsToString(properties: Map[String, Any]): String = { + properties .filter(_._2.toString.nonEmpty) .sortBy(_._1) .map { case (key, value) => s"$key: $value" } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala index 4aa0fce401d3..e24170db71a0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala @@ -1,9 +1,9 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes.AbstractNode +import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, StoredNode} import org.json4s.native.Serialization.{write, writePretty} import org.json4s.{CustomSerializer, Extraction, Formats} -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import replpp.Colors import replpp.Operators.* @@ -14,6 +14,7 @@ import scala.jdk.CollectionConverters.* /** Base class for our DSL These are the base steps available in all steps of the query language. There are no * constraints on the element types, unlike e.g. [[NodeSteps]] */ +@Traversal(elementType = classOf[AnyRef]) class Steps[A](val traversal: Iterator[A]) extends AnyVal { /** Execute the traversal and convert it to a mutable buffer @@ -81,14 +82,19 @@ class Steps[A](val traversal: Iterator[A]) extends AnyVal { object Steps { private lazy val nodeSerializer = new CustomSerializer[AbstractNode](implicit format => ( - { case _ => ??? }, - { case node: (AbstractNode & Product) => - val elementMap = (0 until node.productArity).map { i => + { case _ => ??? }, // deserializer not required for now + { case node: AbstractNode => + val elementMap = Map.newBuilder[String, Any] + (0 until node.productArity).foreach { i => val label = node.productElementName(i) val element = node.productElement(i) - label -> element - }.toMap + ("_label" -> node.label) - Extraction.decompose(elementMap) + elementMap.addOne(label -> element) + } + elementMap.addOne("_label" -> node.label) + if (node.isInstanceOf[StoredNode]) { + elementMap.addOne("_id" -> node.asInstanceOf[StoredNode].id()) + } + Extraction.decompose(elementMap.result()) } ) ) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala index 936cf1439df9..c2a8d6e5efc9 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/android/ConfigFileTraversal.scala @@ -1,9 +1,10 @@ package io.shiftleft.semanticcpg.language.android import io.joern.semanticcpg.utils.SecureXmlParsing -import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.codepropertygraph.generated.nodes.ConfigFile +import io.shiftleft.semanticcpg.language.* -class ConfigFileTraversal(val traversal: Iterator[nodes.ConfigFile]) extends AnyVal { +class ConfigFileTraversal(val traversal: Iterator[ConfigFile]) extends AnyVal { def usesCleartextTraffic = traversal .filter(_.name.endsWith(Constants.androidManifestXml)) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala index 6b627ae47e06..29f08c4af74e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/callgraphextension/MethodTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.callgraphextension -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Method]) class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { /** Intended for internal use! Traverse to direct and transitive callers of the method. diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala index aa10a624c79f..80f3622f27ae 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/AstNodeDot.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.dotextension import io.shiftleft.codepropertygraph.generated.nodes.AstNode import io.shiftleft.semanticcpg.dotgenerator.DotAstGenerator -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* class AstNodeDot[NodeType <: AstNode](val traversal: Iterator[NodeType]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala index 107b19037495..84d1bdf4647d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/dotextension/CfgNodeDot.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.dotextension import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.semanticcpg.dotgenerator.{DotCdgGenerator, DotCfgGenerator} -import overflowdb.traversal.* +import io.shiftleft.semanticcpg.language.* class CfgNodeDot(val traversal: Iterator[Method]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala index 4400d389dcd9..1e6b4d6dead8 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/importresolver/ResolvedImportAsTagTraversal.scala @@ -1,18 +1,17 @@ package io.shiftleft.semanticcpg.language.importresolver +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Declaration, Member, Tag} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc class ResolvedImportAsTagExt(node: Tag) extends AnyVal { - @Doc(info = "Parses this tag as an EvaluatedImport class") def _toEvaluatedImport: Option[EvaluatedImport] = EvaluatedImport.tagToEvaluatedImport(node) - @Doc(info = "If this tag represents a resolved import, will attempt to find the CPG entities this refers to") def resolvedEntity: Iterator[AstNode] = { - val cpg = Cpg(node.graph()) + val cpg = Cpg(node.graph) node._toEvaluatedImport.iterator .collectAll[ResolvedImport] .flatMap { @@ -25,9 +24,9 @@ class ResolvedImportAsTagExt(node: Tag) extends AnyVal { } .iterator } - } +@Traversal(elementType = classOf[Tag]) class ResolvedImportAsTagTraversal(steps: Iterator[Tag]) extends AnyVal { @Doc(info = "Parses these tags as EvaluatedImport classes") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala index c115d84a3883..741f650ee6ba 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableAsNodeTraversal.scala @@ -1,12 +1,14 @@ package io.shiftleft.semanticcpg.language.modulevariable +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{Cpg, Operators} import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.modulevariable.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Local]) class ModuleVariableAsLocalTraversal(traversal: Iterator[Local]) extends AnyVal { @Doc(info = "Locals representing module variables") @@ -16,6 +18,7 @@ class ModuleVariableAsLocalTraversal(traversal: Iterator[Local]) extends AnyVal } +@Traversal(elementType = classOf[Identifier]) class ModuleVariableAsIdentifierTraversal(traversal: Iterator[Identifier]) extends AnyVal { @Doc(info = "Identifiers representing module variables") @@ -25,12 +28,13 @@ class ModuleVariableAsIdentifierTraversal(traversal: Iterator[Identifier]) exten } +@Traversal(elementType = classOf[FieldIdentifier]) class ModuleVariableAsFieldIdentifierTraversal(traversal: Iterator[FieldIdentifier]) extends AnyVal { @Doc(info = "Field identifiers representing module variables") def moduleVariables: Iterator[OpNodes.ModuleVariable] = { traversal.flatMap { fieldIdentifier => - Cpg(fieldIdentifier.graph()).method + Cpg(fieldIdentifier.graph).method .fullNameExact(fieldIdentifier.inFieldAccess.argument(1).isIdentifier.typeFullName.toSeq*) .isModule .local @@ -40,13 +44,14 @@ class ModuleVariableAsFieldIdentifierTraversal(traversal: Iterator[FieldIdentifi } +@Traversal(elementType = classOf[Member]) class ModuleVariableAsMemberTraversal(traversal: Iterator[Member]) extends AnyVal { @Doc(info = "Members representing module variables") def moduleVariables: Iterator[OpNodes.ModuleVariable] = { val members = traversal.toList lazy val memberNames = members.name.toSeq - members.headOption.map(m => Cpg(m.graph())) match + members.headOption.map(m => Cpg(m.graph)) match case Some(cpg) => cpg.method .fullNameExact(members.typeDecl.fullName.toSeq*) @@ -57,6 +62,7 @@ class ModuleVariableAsMemberTraversal(traversal: Iterator[Member]) extends AnyVa } +@Traversal(elementType = classOf[Expression]) class ModuleVariableAsExpressionTraversal(traversal: Iterator[Expression]) extends AnyVal { @Doc(info = "Expression nodes representing module variables") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala index 1c58587093c2..23e12cd0fe32 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/ModuleVariableTraversal.scala @@ -1,11 +1,13 @@ package io.shiftleft.semanticcpg.language.modulevariable +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Local]) class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) extends AnyVal { @Doc(info = "All assignments where the module variables in this traversal are the target across the program") @@ -32,7 +34,7 @@ class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) exten ) def references: Iterator[Identifier | FieldIdentifier] = { val variables = traversal.toList - variables.headOption.map(node => Cpg(node.graph())) match + variables.headOption.map(node => Cpg(node.graph)) match case Some(cpg) => val modules = cpg.method.isModule.l val variableNames = variables.name.toSet @@ -78,7 +80,7 @@ class ModuleVariableTraversal(traversal: Iterator[OpNodes.ModuleVariable]) exten val variables = traversal.toList lazy val moduleNames = variables.method.isModule.fullName.dedup.toSeq lazy val variableNames = variables.name.toSeq - variables.headOption.map(node => Cpg(node.graph())) match + variables.headOption.map(node => Cpg(node.graph)) match case Some(cpg) => cpg.typeDecl.fullNameExact(moduleNames*).member.nameExact(variableNames*) case None => Iterator.empty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala index b3462fd1807e..da34b086d2af 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/NodeTypeStarters.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.modulevariable +import io.shiftleft.codepropertygraph.generated.help.{Doc, TraversalSource} import io.shiftleft.codepropertygraph.generated.Cpg -import overflowdb.traversal.help.{Doc, TraversalSource} import io.shiftleft.semanticcpg.language.* @TraversalSource diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala index dd442178f8af..2c43a8bf567c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/OpNodes.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.modulevariable -import io.shiftleft.codepropertygraph.generated.nodes.{Block, Local, Member, StaticType} +import io.shiftleft.codepropertygraph.generated.nodes.{Local, StaticType} trait ModuleVariableT object OpNodes { @@ -8,7 +8,6 @@ object OpNodes { /** Represents a module-level global variable. This kind of node behaves like both a local variable and a field access * and is common in languages such as Python/JavaScript. */ - type ModuleVariable = Local & StaticType[ModuleVariableT] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala index fb80d7092fa2..f70927b3dea2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableAsNodeMethods.scala @@ -2,11 +2,11 @@ package io.shiftleft.semanticcpg.language.modulevariable.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, Local, Member} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc class ModuleVariableAsLocalMethods(node: Local) extends AnyVal { - @Doc(info = "If this local is declared on the module-defining method level") + /** If this local is declared on the module-defining method level */ def isModuleVariable: Boolean = node.method.isModule.nonEmpty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala index ee2a84c2e86d..5a13de925796 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/modulevariable/nodemethods/ModuleVariableMethods.scala @@ -6,19 +6,19 @@ import io.shiftleft.semanticcpg.language.modulevariable.OpNodes import io.shiftleft.semanticcpg.language.operatorextension.OpNodes as OpExtNodes import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.importresolver.{ResolvedMember, ResolvedTypeDecl} -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc class ModuleVariableMethods(node: OpNodes.ModuleVariable) extends AnyVal { - @Doc(info = "References of this module variable across the codebase, as either identifiers or field identifiers") + /** References of this module variable across the codebase, as either identifiers or field identifiers */ def references: Iterator[Identifier | FieldIdentifier] = node.start.references - @Doc(info = "The module members being referenced in the respective module type declaration") + /** The module members being referenced in the respective module type declaration */ def referencingMembers: Iterator[Member] = { - Cpg(node.graph()).typeDecl.fullNameExact(node.method.fullName.toSeq*).member.nameExact(node.name) + Cpg(node.graph).typeDecl.fullNameExact(node.method.fullName.toSeq*).member.nameExact(node.name) } - @Doc(info = "Returns the assignments where the module variable is the target (LHS)") + /** Returns the assignments where the module variable is the target (LHS) */ def definitions: Iterator[OpExtNodes.Assignment] = node.start.definitions } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala index 14ad5766be64..c23ff1a463d5 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/AstNodeMethods.scala @@ -56,7 +56,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { val additionalDepth = if (p(node)) { 1 } else { 0 } - val childDepths = node.astChildren.map(_.depth(p)).l + val childDepths = astChildren.map(_.depth(p)).l additionalDepth + (if (childDepths.isEmpty) { 0 } else { @@ -70,7 +70,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { /** Direct children of node in the AST. Siblings are ordered by their `order` fields */ def astChildren: Iterator[AstNode] = - node._astOut.cast[AstNode].sortBy(_.order).iterator + node._astOut.cast[AstNode].toSeq.sortBy(_.order).iterator /** Siblings of this node in the AST, ordered by their `order` fields */ @@ -108,8 +108,7 @@ class AstNodeMethods(val node: AstNode) extends AnyVal with NodeExtension { case member: Member => member case node: MethodParameterIn => node.method - case node: MethodParameterOut => - node.method.methodReturn + case node: MethodParameterOut => node.method.methodReturn case node: Call if MemberAccess.isGenericMemberAccessName(node.name) => parentExpansion(node) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala index b124dcbd0c2e..0d228ce4be7c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CallMethods.scala @@ -6,20 +6,27 @@ import io.shiftleft.semanticcpg.NodeExtension import io.shiftleft.semanticcpg.language.* class CallMethods(val node: Call) extends AnyVal with NodeExtension with HasLocation { + + def isStatic: Boolean = + node.dispatchType == "STATIC_DISPATCH" + + def isDynamic: Boolean = + node.dispatchType == "DYNAMIC_DISPATCH" + def receiver: Iterator[Expression] = - node.receiverOut + node.receiverOut.collectAll[Expression] def arguments(index: Int): Iterator[Expression] = - node._argumentOut - .collect { - case expr: Expression if expr.argumentIndex == index => expr - } + node._argumentOut.collect { + case expr: Expression if expr.argumentIndex == index => expr + } + // TODO define as named step in the schema def argument: Iterator[Expression] = node._argumentOut.collectAll[Expression] def argument(index: Int): Expression = - arguments(index).head + arguments(index).next def argumentOption(index: Int): Option[Expression] = node._argumentOut.collectFirst { @@ -32,7 +39,6 @@ class CallMethods(val node: Call) extends AnyVal with NodeExtension with HasLoca node.astChildren.isBlock.maxByOption(_.order).iterator.expressionDown } - override def location: NewLocation = { + override def location: NewLocation = LocationCreator(node, node.code, node.label, node.lineNumber, node.method) - } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala index 08ee7d72a7e6..fa14297ad531 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala @@ -11,9 +11,8 @@ class CfgNodeMethods(val node: CfgNode) extends AnyVal with NodeExtension { /** Successors in the CFG */ - def cfgNext: Iterator[CfgNode] = { + def cfgNext: Iterator[CfgNode] = Iterator.single(node).cfgNext - } /** Maps each node in the traversal to a traversal returning its n successors. */ @@ -31,9 +30,8 @@ class CfgNodeMethods(val node: CfgNode) extends AnyVal with NodeExtension { /** Predecessors in the CFG */ - def cfgPrev: Iterator[CfgNode] = { + def cfgPrev: Iterator[CfgNode] = Iterator.single(node).cfgPrev - } /** Recursively determine all nodes on which this CFG node is control-dependent. */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala index e04ff421b06a..134f42ae3397 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/IdentifierMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Declaration, Identifier, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator, *} +import io.shiftleft.semanticcpg.language.* class IdentifierMethods(val identifier: Identifier) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala index 3bbc9892b5bc..8962c206ff4a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LiteralMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{Literal, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator, _} +import io.shiftleft.semanticcpg.language.* class LiteralMethods(val literal: Literal) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala index 9bfa5eac78bc..94abc5458ecd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/LocalMethods.scala @@ -6,7 +6,7 @@ import io.shiftleft.semanticcpg.language.* class LocalMethods(val local: Local) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { - LocationCreator(local, local.name, local.label, local.lineNumber, local.method.head) + LocationCreator(local, local.name, local.label, local.lineNumber, method.head) } /** The method hosting this local variable diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala index 52a2034ae28f..7141f8aa8af0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodMethods.scala @@ -37,14 +37,14 @@ class MethodMethods(val method: Method) extends AnyVal with NodeExtension with H /** List of CFG nodes in reverse post order */ def reversePostOrder: Iterator[CfgNode] = { - def expand(x: CfgNode) = { x.cfgNext.iterator } + def expand(x: CfgNode) = x.cfgNext.iterator NodeOrdering.reverseNodeList(NodeOrdering.postOrderNumbering(method, expand).toList).iterator } /** List of CFG nodes in post order */ def postOrder: Iterator[CfgNode] = { - def expand(x: CfgNode) = { x.cfgNext.iterator } + def expand(x: CfgNode) = x.cfgNext.iterator NodeOrdering.nodeList(NodeOrdering.postOrderNumbering(method, expand).toList).iterator } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala index 1f2a0c5420cc..1b39a89145c3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterInMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterIn, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodParameterInMethods(val paramIn: MethodParameterIn) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala index 851c5949abbb..29471342f6ca 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodParameterOutMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterOut, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodParameterOutMethods(val paramOut: MethodParameterOut) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala index fc971fff51c9..03ea62a6dd3a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodRefMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.{MethodRef, NewLocation} import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language.{HasLocation, LocationCreator} +import io.shiftleft.semanticcpg.language.* class MethodRefMethods(val methodRef: MethodRef) extends AnyVal with NodeExtension with HasLocation { override def location: NewLocation = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala index 4ea458735b3b..8d7494c63773 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/MethodReturnMethods.scala @@ -19,5 +19,6 @@ class MethodReturnMethods(val node: MethodReturn) extends AnyVal with NodeExtens callsites.collectAll[Call] } + // TODO define in schema as named step def typ: Iterator[Type] = node.evalTypeOut } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala index 600fe2c1299a..25325e1faccb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/NodeMethods.scala @@ -1,11 +1,10 @@ package io.shiftleft.semanticcpg.language.nodemethods -import io.shiftleft.codepropertygraph.generated.nodes.{NewLocation, StoredNode} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.NodeExtension -import io.shiftleft.semanticcpg.language._ -import overflowdb.NodeOrDetachedNode +import io.shiftleft.semanticcpg.language.* -class NodeMethods(val node: NodeOrDetachedNode) extends AnyVal with NodeExtension { +class NodeMethods(val node: AbstractNode) extends AnyVal with NodeExtension { def location(implicit finder: NodeExtensionFinder): NewLocation = node match { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala index cc3b25194d16..4cc8d1d7d840 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/ArrayAccessTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension -import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Identifier} -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Expression, Identifier} +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Call]) class ArrayAccessTraversal(val traversal: Iterator[OpNodes.ArrayAccess]) extends AnyVal { @Doc(info = "The expression representing the array") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala index f7751bedab93..31cf82c39a6b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/AssignmentTraversal.scala @@ -1,11 +1,12 @@ package io.shiftleft.semanticcpg.language.operatorextension +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Expression} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[nodes.Call]) +@Traversal(elementType = classOf[Call]) class AssignmentTraversal(val traversal: Iterator[OpNodes.Assignment]) extends AnyVal { @Doc(info = "Left-hand sides of assignments") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala index 6bb06f0ce627..2a0a535afce9 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/FieldAccessTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension -import io.shiftleft.codepropertygraph.generated.nodes.{FieldIdentifier, Member, TypeDecl} +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, FieldIdentifier, Member, TypeDecl} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Call]) class FieldAccessTraversal(val traversal: Iterator[OpNodes.FieldAccess]) extends AnyVal { @Doc(info = "Attempts to resolve the type declaration for this field access") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala index b3ef1b0ce155..5f63f72efef3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/Implicits.scala @@ -2,22 +2,27 @@ package io.shiftleft.semanticcpg.language.operatorextension import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression} -import io.shiftleft.semanticcpg.language.operatorextension.nodemethods._ +import io.shiftleft.semanticcpg.language.operatorextension.nodemethods.* trait Implicits { + implicit def toNodeTypeStartersOperatorExtension(cpg: Cpg): NodeTypeStarters = new NodeTypeStarters(cpg) implicit def toArrayAccessExt(arrayAccess: OpNodes.ArrayAccess): ArrayAccessMethods = new ArrayAccessMethods(arrayAccess) + implicit def toArrayAccessTrav(steps: Iterator[OpNodes.ArrayAccess]): ArrayAccessTraversal = new ArrayAccessTraversal(steps) implicit def toFieldAccessExt(fieldAccess: OpNodes.FieldAccess): FieldAccessMethods = new FieldAccessMethods(fieldAccess) + implicit def toFieldAccessTrav(steps: Iterator[OpNodes.FieldAccess]): FieldAccessTraversal = new FieldAccessTraversal(steps) - implicit def toAssignmentExt(assignment: OpNodes.Assignment): AssignmentMethods = new AssignmentMethods(assignment) + implicit def toAssignmentExt(assignment: OpNodes.Assignment): AssignmentMethods = + new AssignmentMethods(assignment) + implicit def toAssignmentTrav(steps: Iterator[OpNodes.Assignment]): AssignmentTraversal = new AssignmentTraversal(steps) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala index 5ddf05c29036..c012b035b506 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/NodeTypeStarters.scala @@ -1,11 +1,10 @@ package io.shiftleft.semanticcpg.language.operatorextension +import io.shiftleft.codepropertygraph.generated.help.{Doc, TraversalSource} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.{Doc, TraversalSource} -/** Steps that allow traversing from `cpg` to operators. - */ +/** Steps that allow traversing from `cpg` to operators. */ @TraversalSource class NodeTypeStarters(cpg: Cpg) { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala index d5c61829abb7..ab9ccd10a6af 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/OpAstNodeTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.AstNode import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[AstNode]) class OpAstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { @Doc(info = "Any assignments that this node is a part of (traverse up)") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala index 3a17608b98e3..4e8b700c8ae8 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/TargetTraversal.scala @@ -1,9 +1,11 @@ package io.shiftleft.semanticcpg.language.operatorextension +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.Expression import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc +@Traversal(elementType = classOf[Expression]) class TargetTraversal(val traversal: Iterator[Expression]) extends AnyVal { @Doc( diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala index 8675591639b8..578c429f54ed 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/ArrayAccessMethods.scala @@ -22,7 +22,7 @@ class ArrayAccessMethods(val arrayAccess: OpNodes.ArrayAccess) extends AnyVal { } def simpleName: Iterator[String] = { - arrayAccess.array match { + array match { case id: Identifier => Iterator.single(id.name) case _ => Iterator.empty } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/AssignmentMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/AssignmentMethods.scala index 4faa072246f2..d470dbe92a12 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/AssignmentMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/AssignmentMethods.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.operatorextension.nodemethods import io.shiftleft.codepropertygraph.generated.nodes.Expression -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes class AssignmentMethods(val assignment: OpNodes.Assignment) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/TargetMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/TargetMethods.scala index 1a301475a3eb..dc633f06599b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/TargetMethods.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/operatorextension/nodemethods/TargetMethods.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.operatorextension.nodemethods import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Expression} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.{OpNodes, allArrayAccessTypes} class TargetMethods(val expr: Expression) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala index 1917298f0a5f..cb36c6d7deb3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/package.scala @@ -1,8 +1,9 @@ package io.shiftleft.semanticcpg +import flatgraph.help.DocSearchPackages +import io.shiftleft.codepropertygraph.generated import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.traversal.NodeTraversalImplicits import io.shiftleft.semanticcpg.language.bindingextension.{ MethodTraversal as BindingMethodTraversal, TypeDeclTraversal as BindingTypeDeclTraversal @@ -10,16 +11,11 @@ import io.shiftleft.semanticcpg.language.bindingextension.{ import io.shiftleft.semanticcpg.language.callgraphextension.{CallTraversal, MethodTraversal} import io.shiftleft.semanticcpg.language.dotextension.{AstNodeDot, CfgNodeDot, InterproceduralNodeDot} import io.shiftleft.semanticcpg.language.nodemethods.* -import io.shiftleft.semanticcpg.language.types.expressions.generalizations.{ - AstNodeTraversal, - CfgNodeTraversal, - DeclarationTraversal, - ExpressionTraversal -} +import io.shiftleft.semanticcpg.language.types.expressions.generalizations.* import io.shiftleft.semanticcpg.language.types.expressions.{CallTraversal as OriginalCall, *} import io.shiftleft.semanticcpg.language.types.propertyaccessors.* import io.shiftleft.semanticcpg.language.types.structure.{MethodTraversal as OriginalMethod, *} -import overflowdb.NodeOrDetachedNode +import io.shiftleft.semanticcpg.language.types.structure.* /** Language for traversing the code property graph * @@ -27,20 +23,19 @@ import overflowdb.NodeOrDetachedNode * `steps` package, e.g. `Steps` */ package object language - extends operatorextension.Implicits + extends generated.language + with operatorextension.Implicits with modulevariable.Implicits with importresolver.Implicits - with LowPrioImplicits - with NodeTraversalImplicits { + with LowPrioImplicits { // Implicit conversions from generated node types. We use these to add methods // to generated node types. + implicit def cfgNodeToAstNode(node: CfgNode): AstNodeMethods = new AstNodeMethods(node) + implicit def toExtendedNode(node: AbstractNode): NodeMethods = new NodeMethods(node) + implicit def toExtendedStoredNode(node: StoredNode): StoredNodeMethods = new StoredNodeMethods(node) + implicit def toAstNodeMethods(node: AstNode): AstNodeMethods = new AstNodeMethods(node) + implicit def toExpressionMethods(node: Expression): ExpressionMethods = new ExpressionMethods(node) - implicit def cfgNodeToAsNode(node: CfgNode): AstNodeMethods = new AstNodeMethods(node) - implicit def toExtendedNode(node: NodeOrDetachedNode): NodeMethods = new NodeMethods(node) - implicit def toExtendedStoredNode(node: StoredNode): StoredNodeMethods = new StoredNodeMethods(node) - implicit def toAstNodeMethods(node: AstNode): AstNodeMethods = new AstNodeMethods(node) - implicit def toCfgNodeMethods(node: CfgNode): CfgNodeMethods = new CfgNodeMethods(node) - implicit def toExpressionMethods(node: Expression): ExpressionMethods = new ExpressionMethods(node) implicit def toMethodMethods(node: Method): MethodMethods = new MethodMethods(node) implicit def toMethodReturnMethods(node: MethodReturn): MethodReturnMethods = new MethodReturnMethods(node) implicit def toCallMethods(node: Call): CallMethods = new CallMethods(node) @@ -68,8 +63,7 @@ package object language implicit def iterOnceToTypeDeclTrav[A <: TypeDecl](a: IterableOnce[A]): TypeDeclTraversal = new TypeDeclTraversal(a.iterator) - implicit def iterOnceToOriginalCallTrav[A <: Call](a: IterableOnce[A]): OriginalCall = - new OriginalCall(a.iterator) + implicit def iterOnceToOriginalCallTrav(traversal: IterableOnce[Call]): OriginalCall = new OriginalCall(traversal) implicit def singleToControlStructureTrav[A <: ControlStructure](a: A): ControlStructureTraversal = new ControlStructureTraversal(Iterator.single(a)) @@ -110,8 +104,6 @@ package object language implicit def iterOnceToMethodParameterInTrav[A <: MethodParameterIn](a: IterableOnce[A]): MethodParameterTraversal = new MethodParameterTraversal(a.iterator) - implicit def singleToMethodParameterOutTrav[A <: MethodParameterOut](a: A): MethodParameterOutTraversal = - new MethodParameterOutTraversal(Iterator.single(a)) implicit def iterOnceToMethodParameterOutTrav[A <: MethodParameterOut]( a: IterableOnce[A] ): MethodParameterOutTraversal = @@ -163,11 +155,6 @@ package object language implicit def iterOnceToBindingTypeDeclTrav[A <: TypeDecl](a: IterableOnce[A]): BindingTypeDeclTraversal = new BindingTypeDeclTraversal(a.iterator) - implicit def singleToAstNodeDot[A <: AstNode](a: A): AstNodeDot[A] = - new AstNodeDot(Iterator.single(a)) - implicit def iterOnceToAstNodeDot[A <: AstNode](a: IterableOnce[A]): AstNodeDot[A] = - new AstNodeDot(a.iterator) - implicit def singleToCfgNodeDot[A <: Method](a: A): CfgNodeDot = new CfgNodeDot(Iterator.single(a)) implicit def iterOnceToCfgNodeDot[A <: Method](a: IterableOnce[A]): CfgNodeDot = @@ -268,11 +255,29 @@ package object language implicit def toExpression[A <: Expression](a: IterableOnce[A]): ExpressionTraversal[A] = new ExpressionTraversal[A](a.iterator) + + object NonStandardImplicits { + + // note: this causes problems because MethodParameterOut has an `index` property and the `MethodParameterOutTraversal` defines an `index` step... + implicit def singleToMethodParameterOutTrav[A <: MethodParameterOut](a: A): MethodParameterOutTraversal = + new MethodParameterOutTraversal(Iterator.single(a)) + + } } -trait LowPrioImplicits extends overflowdb.traversal.Implicits { - implicit def singleToCfgNodeTraversal[A <: CfgNode](a: A): CfgNodeTraversal[A] = - new CfgNodeTraversal[A](Iterator.single(a)) +trait LowPrioImplicits { + implicit val docSearchPackages: DocSearchPackages = + Cpg.defaultDocSearchPackage + .withAdditionalPackage("io.joern") + .withAdditionalPackage("io.shiftleft") + + implicit def singleToAstNodeDot[A <: AstNode](a: A): AstNodeDot[A] = + new AstNodeDot(Iterator.single(a)) + implicit def iterOnceToAstNodeDot[A <: AstNode](a: IterableOnce[A]): AstNodeDot[A] = + new AstNodeDot(a.iterator) + + implicit def toCfgNodeMethods(node: CfgNode): CfgNodeMethods = new CfgNodeMethods(node) + implicit def iterOnceToCfgNodeTraversal[A <: CfgNode](a: IterableOnce[A]): CfgNodeTraversal[A] = new CfgNodeTraversal[A](a.iterator) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala index 33e47f74f375..5fc9724a0c34 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/CallTraversal.scala @@ -5,19 +5,16 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment import io.shiftleft.semanticcpg.language.operatorextension.allAssignmentTypes -/** A call site - */ +/** A call site. */ class CallTraversal(val traversal: Iterator[Call]) extends AnyVal { - /** Only statically dispatched calls - */ + /** Only statically dispatched calls */ def isStatic: Iterator[Call] = - traversal.dispatchType("STATIC_DISPATCH") + traversal.filter(_.isStatic) - /** Only dynamically dispatched calls - */ + /** Only dynamically dispatched calls */ def isDynamic: Iterator[Call] = - traversal.dispatchType("DYNAMIC_DISPATCH") + traversal.filter(_.isDynamic) /** Only assignment calls */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala index f117e19b91e9..831e6358b5bf 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/ControlStructureTraversal.scala @@ -1,15 +1,17 @@ package io.shiftleft.semanticcpg.language.types.expressions +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, ControlStructure, Expression} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Properties} +import io.shiftleft.codepropertygraph.generated.ControlStructureTypes import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc object ControlStructureTraversal { val secondChildIndex = 2 val thirdChildIndex = 3 } +@Traversal(elementType = classOf[ControlStructure]) class ControlStructureTraversal(val traversal: Iterator[ControlStructure]) extends AnyVal { import ControlStructureTraversal.* @@ -23,11 +25,11 @@ class ControlStructureTraversal(val traversal: Iterator[ControlStructure]) exten @Doc(info = "Sub tree taken when condition evaluates to true") def whenTrue: Iterator[AstNode] = - traversal.out.has(Properties.Order, secondChildIndex: Int).cast[AstNode] + traversal.out.collectAll[AstNode].order(secondChildIndex) @Doc(info = "Sub tree taken when condition evaluates to false") def whenFalse: Iterator[AstNode] = - traversal.out.has(Properties.Order, thirdChildIndex).cast[AstNode] + traversal.out.collectAll[AstNode].order(thirdChildIndex) @Doc(info = "Only `Try` control structures") def isTry: Iterator[ControlStructure] = diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala index 8b11e5635c21..b9c6ea9f7e69 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/IdentifierTraversal.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.expressions import io.shiftleft.codepropertygraph.generated.nodes.{Declaration, Identifier} -import io.shiftleft.semanticcpg.language.toTraversalSugarExt +import io.shiftleft.semanticcpg.language.* /** An identifier, e.g., an instance of a local variable, or a temporary variable */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 33bfa563f290..7a66de691219 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -1,20 +1,18 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[AstNode]) +@Traversal(elementType = classOf[AstNode]) class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { /** Nodes of the AST rooted in this node, including the node itself. */ @Doc(info = "All nodes of the abstract syntax tree") - def ast: Iterator[AstNode] = { - traversal.repeat(_.out(EdgeTypes.AST))(_.emit).cast[AstNode] - } + def ast: Iterator[AstNode] = + traversal.repeat(_._astOut)(_.emit).cast[AstNode] /** All nodes of the abstract syntax tree rooted in this node, which match `predicate`. Equivalent of `match` in the * original CPG paper. @@ -38,7 +36,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Nodes of the AST rooted in this node, minus the node itself */ def astMinusRoot: Iterator[AstNode] = - traversal.repeat(_.out(EdgeTypes.AST))(_.emitAllButFirst).cast[AstNode] + traversal.repeat(_._astOut)(_.emitAllButFirst).cast[AstNode] /** Direct children of node in the AST. Siblings are ordered by their `order` fields */ @@ -48,7 +46,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Parent AST node */ def astParent: Iterator[AstNode] = - traversal.in(EdgeTypes.AST).cast[AstNode] + traversal._astIn.cast[AstNode] /** Siblings of this node in the AST, ordered by their `order` fields */ @@ -58,7 +56,7 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Traverses up the AST and returns the first block node. */ def parentBlock: Iterator[Block] = - traversal.repeat(_.in(EdgeTypes.AST))(_.emit.until(_.hasLabel(NodeTypes.BLOCK))).collectAll[Block] + traversal.repeat(_._astIn)(_.emit.until(_.hasLabel(Block.Label))).collectAll[Block] /** Nodes of the AST obtained by expanding AST edges backwards until the method root is reached */ @@ -72,24 +70,26 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal /** Nodes of the AST obtained by expanding AST edges backwards until `root` or the method root is reached */ - def inAst(root: AstNode): Iterator[AstNode] = + def inAst(root: AstNode): Iterator[AstNode] = { traversal - .repeat(_.in(EdgeTypes.AST))( + .repeat(_._astIn)( _.emit - .until(_.or(_.hasLabel(NodeTypes.METHOD), _.filter(n => root != null && root == n))) + .until(_.or(_.hasLabel(Method.Label), _.filter(n => root != null && root == n))) ) .cast[AstNode] + } /** Nodes of the AST obtained by expanding AST edges backwards until `root` or the method root is reached, minus this * node */ - def inAstMinusLeaf(root: AstNode): Iterator[AstNode] = + def inAstMinusLeaf(root: AstNode): Iterator[AstNode] = { traversal - .repeat(_.in(EdgeTypes.AST))( + .repeat(_._astIn)( _.emitAllButFirst - .until(_.or(_.hasLabel(NodeTypes.METHOD), _.filter(n => root != null && root == n))) + .until(_.or(_.hasLabel(Method.Label), _.filter(n => root != null && root == n))) ) .cast[AstNode] + } /** Traverse only to those AST nodes that are also control flow graph nodes */ @@ -208,10 +208,11 @@ class AstNodeTraversal[A <: AstNode](val traversal: Iterator[A]) extends AnyVal def isTypeDecl: Iterator[TypeDecl] = traversal.collectAll[TypeDecl] - def walkAstUntilReaching(labels: List[String]): Iterator[StoredNode] = + def walkAstUntilReaching(labels: List[String]): Iterator[StoredNode] = { traversal - .repeat(_.out(EdgeTypes.AST))(_.emitAllButFirst.until(_.hasLabel(labels*))) + .repeat(_._astOut)(_.emitAllButFirst.until(_.hasLabel(labels*))) .dedup .cast[StoredNode] + } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala index 3a3d22f7840f..99843e52b96f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversal.scala @@ -1,11 +1,12 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.neighboraccessors.Lang.* +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[CfgNode]) +@Traversal(elementType = classOf[CfgNode]) class CfgNodeTraversal[A <: CfgNode](val traversal: Iterator[A]) extends AnyVal { /** Textual representation of CFG node @@ -21,7 +22,6 @@ class CfgNodeTraversal[A <: CfgNode](val traversal: Iterator[A]) extends AnyVal /** Traverse to next expression in CFG. */ - @Doc(info = "Nodes directly reachable via outgoing CFG edges") def cfgNext: Iterator[CfgNode] = traversal._cfgOut diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala index f1075dad19c3..be7e4c5566cc 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/DeclarationTraversal.scala @@ -1,28 +1,27 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help -/** A declaration, such as a local or parameter. - */ -@help.Traversal(elementType = classOf[Declaration]) +/** A declaration, such as a local or parameter. */ +@Traversal(elementType = classOf[Declaration]) class DeclarationTraversal[NodeType <: Declaration](val traversal: Iterator[NodeType]) extends AnyVal { - /** The closure binding node referenced by this declaration - */ + /** The closure binding node referenced by this declaration */ + @Doc(info = "The closure binding node referenced by this declaration") def closureBinding: Iterator[ClosureBinding] = traversal.flatMap(_._refIn).collectAll[ClosureBinding] - /** Methods that capture this declaration - */ + /** Methods that capture this declaration */ + @Doc(info = "Methods that capture this declaration") def capturedByMethodRef: Iterator[MethodRef] = closureBinding.flatMap(_._captureIn).collectAll[MethodRef] - /** Types that capture this declaration - */ + /** Types that capture this declaration */ + @Doc(info = "Types that capture this declaration") def capturedByTypeRef: Iterator[TypeRef] = closureBinding.flatMap(_._captureIn).collectAll[TypeRef] - /** The parent method. - */ + /** The parent method. */ + @Doc(info = "The parent method.") def method: Iterator[Method] = traversal.flatMap { case x: Local => x.method case x: MethodParameterIn => x.method diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala index 741f69202624..c3e22ff0dfc7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversal.scala @@ -59,8 +59,8 @@ class ExpressionTraversal[NodeType <: Expression](val traversal: Iterator[NodeTy */ def method: Iterator[Method] = traversal._containsIn - .flatMap { - case x: Method => x.start + .map { + case x: Method => x case x: TypeDecl => x.astParent } .collectAll[Method] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala index 679b220aafaa..76b7d1672f61 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/propertyaccessors/ModifierAccessors.scala @@ -1,10 +1,8 @@ package io.shiftleft.semanticcpg.language.types.propertyaccessors import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Modifier} -import io.shiftleft.codepropertygraph.generated.traversal.toModifierTraversalExtGen +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.* class ModifierAccessors[A <: AstNode](val traversal: Iterator[A]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala index a597e668bf1f..104ebab2ea2d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/AnnotationTraversal.scala @@ -1,34 +1,34 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes -import overflowdb.traversal._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* /** An (Java-) annotation, e.g., @Test. */ -class AnnotationTraversal(val traversal: Iterator[nodes.Annotation]) extends AnyVal { +class AnnotationTraversal(val traversal: Iterator[Annotation]) extends AnyVal { /** Traverse to parameter assignments */ - def parameterAssign: Iterator[nodes.AnnotationParameterAssign] = + def parameterAssign: Iterator[AnnotationParameterAssign] = traversal.flatMap(_._annotationParameterAssignViaAstOut) /** Traverse to methods annotated with this annotation. */ - def method: Iterator[nodes.Method] = + def method: Iterator[Method] = traversal.flatMap(_._methodViaAstIn) /** Traverse to type declarations annotated by this annotation */ - def typeDecl: Iterator[nodes.TypeDecl] = + def typeDecl: Iterator[TypeDecl] = traversal.flatMap(_._typeDeclViaAstIn) /** Traverse to member annotated by this annotation */ - def member: Iterator[nodes.Member] = + def member: Iterator[Member] = traversal.flatMap(_._memberViaAstIn) /** Traverse to parameter annotated by this annotation */ - def parameter: Iterator[nodes.MethodParameterIn] = + def parameter: Iterator[MethodParameterIn] = traversal.flatMap(_._methodParameterInViaAstIn) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala index 36309ff44325..93dbb4f4ce87 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/DependencyTraversal.scala @@ -1,9 +1,9 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes.Import -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -class DependencyTraversal(val traversal: Iterator[nodes.Dependency]) extends AnyVal { - def imports: Iterator[Import] = traversal.in(EdgeTypes.IMPORTS).cast[Import] +class DependencyTraversal(val traversal: Iterator[Dependency]) extends AnyVal { + def imports: Iterator[Import] = + traversal.importsIn } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/FileTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/FileTraversal.scala index c9aec5674e6f..f520f09db339 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/FileTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/FileTraversal.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* /** A compilation unit diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala index e5131658039a..e46ff2b324e6 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/ImportTraversal.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Import, NamespaceBlock} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* class ImportTraversal(val traversal: Iterator[Import]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 26c73041412c..5319bfc24365 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language.* @@ -13,8 +13,8 @@ class LocalTraversal(val traversal: Iterator[Local]) extends AnyVal { def method: Iterator[Method] = { // TODO The following line of code is here for backwards compatibility. // Use the lower commented out line once not required anymore. - traversal.repeat(_.in(EdgeTypes.AST))(_.until(_.hasLabel(NodeTypes.METHOD))).cast[Method] - // definingBlock.method + traversal.repeat(_._astIn)(_.until(_.hasLabel(Method.Label))).cast[Method] +// definingBlock.method } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala index d83a7062a256..5f45484923ea 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTraversal.scala @@ -1,7 +1,6 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Member} +import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, Call, Member} import io.shiftleft.semanticcpg.language.* /** A member variable of a class/type. @@ -10,7 +9,7 @@ class MemberTraversal(val traversal: Iterator[Member]) extends AnyVal { /** Traverse to annotations of member */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) /** Places where diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala index abb9778a1fd6..3ef123059d5f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterOutTraversal.scala @@ -3,11 +3,12 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters.* - class MethodParameterOutTraversal(val traversal: Iterator[MethodParameterOut]) extends AnyVal { - def paramIn: Iterator[MethodParameterIn] = traversal.flatMap(_.parameterLinkIn.headOption) + def paramIn: Iterator[MethodParameterIn] = { + // TODO define a named step in schema + traversal.flatMap(_.parameterLinkIn.collectAll[MethodParameterIn]) + } /* method parameter indexes are based, i.e. first parameter has index (that's how java2cpg generates it) */ def index(num: Int): Iterator[MethodParameterOut] = @@ -27,9 +28,10 @@ class MethodParameterOutTraversal(val traversal: Iterator[MethodParameterOut]) e for { paramOut <- traversal method = paramOut.method - call <- method.callIn - arg <- call.argumentOut.collectAll[Expression] - if paramOut.parameterLinkIn.index.headOption.contains(arg.argumentIndex) + call <- method._callIn + arg <- call._argumentOut.collectAll[Expression] + // TODO define 'parameterLinkIn' as named step in schema + if paramOut.parameterLinkIn.collectAll[MethodParameterIn].index.headOption.contains(arg.argumentIndex) } yield arg } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala index 2f2af5ea8302..8d5a74d56ba3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTraversal.scala @@ -1,33 +1,32 @@ package io.shiftleft.semanticcpg.language.types.structure +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help import scala.jdk.CollectionConverters.* -/** Formal method input parameter - */ -@help.Traversal(elementType = classOf[MethodParameterIn]) +/** Formal method input parameter */ +@Traversal(elementType = classOf[MethodParameterIn]) class MethodParameterTraversal(val traversal: Iterator[MethodParameterIn]) extends AnyVal { - /** Traverse to parameter annotations - */ + /** Traverse to parameter annotations */ + @Doc(info = "Traverse to parameter annotations") def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) - /** Traverse to all parameters with index greater or equal than `num` - */ + /** Traverse to all parameters with index greater or equal than `num` */ + @Doc(info = "Traverse to all parameters with index greater or equal than `num`") def indexFrom(num: Int): Iterator[MethodParameterIn] = traversal.filter(_.index >= num) - /** Traverse to all parameters with index smaller or equal than `num` - */ + /** Traverse to all parameters with index smaller or equal than `num` */ + @Doc(info = "Traverse to all parameters with index smaller or equal than `num`") def indexTo(num: Int): Iterator[MethodParameterIn] = traversal.filter(_.index <= num) - /** Traverse to arguments (actual parameters) associated with this formal parameter - */ + /** Traverse to arguments (actual parameters) associated with this formal parameter */ + @Doc(info = "Traverse to arguments (actual parameters) associated with this formal parameter") def argument(implicit callResolver: ICallResolver): Iterator[Expression] = for { paramIn <- traversal diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala index a96b5fb9baf7..fb1dc0660ea1 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodReturnTraversal.scala @@ -1,16 +1,16 @@ package io.shiftleft.semanticcpg.language.types.structure +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc -@help.Traversal(elementType = classOf[MethodReturn]) +@Traversal(elementType = classOf[MethodReturn]) class MethodReturnTraversal(val traversal: Iterator[MethodReturn]) extends AnyVal { @Doc(info = "traverse to parent method") def method: Iterator[Method] = - traversal.flatMap(_._methodViaAstIn) + traversal._methodViaAstIn def returnUser(implicit callResolver: ICallResolver): Iterator[Call] = traversal.flatMap(_.returnUser) @@ -19,11 +19,11 @@ class MethodReturnTraversal(val traversal: Iterator[MethodReturn]) extends AnyVa */ @Doc(info = "traverse to last expressions in CFG (can be multiple)") def cfgLast: Iterator[CfgNode] = - traversal.flatMap(_.cfgIn) + traversal.cfgIn /** Traverse to return type */ @Doc(info = "traverse to return type") def typ: Iterator[Type] = - traversal.flatMap(_.evalTypeOut) + traversal.evalTypeOut } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala index ef31a200c906..1e58aca4a55c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala @@ -1,15 +1,14 @@ package io.shiftleft.semanticcpg.language.types.structure +import io.shiftleft.codepropertygraph.generated.help.{Doc, Traversal} import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* -import overflowdb.* -import overflowdb.traversal.help -import overflowdb.traversal.help.Doc +import io.shiftleft.codepropertygraph.generated.help.Doc /** A method, function, or procedure */ -@help.Traversal(elementType = classOf[Method]) +@Traversal(elementType = classOf[Method]) class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { /** Traverse to annotations of method @@ -164,7 +163,8 @@ class MethodTraversal(val traversal: Iterator[Method]) extends AnyVal { // some language frontends don't have a TYPE_DECL for a METHOD case Some(namespaceBlock: NamespaceBlock) => namespaceBlock.start // other language frontends always embed their method in a TYPE_DECL - case _ => m.definingTypeDecl.namespaceBlock + case _ => + m.definingTypeDecl.iterator.namespaceBlock } } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala index 7f95dee3285a..c6aaecf5b1a3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceBlockTraversal.scala @@ -5,16 +5,17 @@ import io.shiftleft.semanticcpg.language.* class NamespaceBlockTraversal(val traversal: Iterator[NamespaceBlock]) extends AnyVal { - /** Namespaces for namespace blocks. + /** Namespaces for namespace blocks. TODO define a name in the schema */ def namespace: Iterator[Namespace] = - traversal.flatMap(_.refOut) + traversal.flatMap(_._namespaceViaRefOut) - /** The type declarations defined in this namespace + /** The type declarations defined in this namespace TODO define a name in the schema */ def typeDecl: Iterator[TypeDecl] = traversal.flatMap(_._typeDeclViaAstOut) + // TODO define a name in the schema def method: Iterator[Method] = traversal.flatMap(_._methodViaAstOut) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala index c636047e2dff..3509c5f6a058 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTraversal.scala @@ -10,12 +10,12 @@ class NamespaceTraversal(val traversal: Iterator[Namespace]) extends AnyVal { /** The type declarations defined in this namespace */ def typeDecl: Iterator[TypeDecl] = - traversal.flatMap(_.refIn).flatMap(_._typeDeclViaAstOut) + traversal.refIn.astOut.collectAll[TypeDecl] /** Methods defined in this namespace */ def method: Iterator[Method] = - traversal.flatMap(_.refIn).flatMap(_._methodViaAstOut) + traversal.refIn.astOut.collectAll[Method] /** External namespaces - any namespaces which contain one or more external type. */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala index 0df69620c7a0..ea97c42e32cb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeDeclTraversal.scala @@ -1,6 +1,5 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -11,13 +10,13 @@ class TypeDeclTraversal(val traversal: Iterator[TypeDecl]) extends AnyVal { /** Annotations of the type declaration */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.flatMap(_._annotationViaAstOut) /** Types referencing to this type declaration. */ def referencingType: Iterator[Type] = - traversal.flatMap(_.refIn) + traversal.refIn /** Namespace in which this type declaration is defined */ @@ -57,7 +56,7 @@ class TypeDeclTraversal(val traversal: Iterator[TypeDecl]) extends AnyVal { /** Direct and transitive base type declaration. */ def derivedTypeDeclTransitive: Iterator[TypeDecl] = - traversal.repeat(_.derivedTypeDecl)(_.emitAllButFirst) + traversal.repeat(_.derivedTypeDecl)(_.emitAllButFirst.dedup) /** Direct base type declaration. */ @@ -67,7 +66,7 @@ class TypeDeclTraversal(val traversal: Iterator[TypeDecl]) extends AnyVal { /** Direct and transitive base type declaration. */ def baseTypeDeclTransitive: Iterator[TypeDecl] = - traversal.repeat(_.baseTypeDecl)(_.emitAllButFirst) + traversal.repeat(_.baseTypeDecl)(_.emitAllButFirst.dedup) /** Traverse to alias type declarations. */ @@ -105,7 +104,7 @@ class TypeDeclTraversal(val traversal: Iterator[TypeDecl]) extends AnyVal { /** Direct and transitive alias type declarations. */ def aliasTypeDeclTransitive: Iterator[TypeDecl] = - traversal.repeat(_.aliasTypeDecl)(_.emitAllButFirst) + traversal.repeat(_.aliasTypeDecl)(_.emitAllButFirst.dedup) def content: Iterator[String] = { traversal.flatMap(contentOnSingle) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala index ea2bf4e0e536..de589cac3d91 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTraversal.scala @@ -1,6 +1,5 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -8,7 +7,7 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Annotations of the corresponding type declaration. */ - def annotation: Iterator[nodes.Annotation] = + def annotation: Iterator[Annotation] = traversal.referencedTypeDecl.annotation /** Namespaces in which the corresponding type declaration is defined. @@ -44,7 +43,7 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Direct and transitive base types of the corresponding type declaration. */ def baseTypeTransitive: Iterator[Type] = - traversal.repeat(_.baseType)(_.emitAllButFirst) + traversal.repeat(_.baseType)(_.emitAllButFirst.dedup) /** Direct derived types. */ @@ -54,12 +53,12 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Direct and transitive derived types. */ def derivedTypeTransitive: Iterator[Type] = - traversal.repeat(_.derivedType)(_.emitAllButFirst) + traversal.repeat(_.derivedType)(_.emitAllButFirst.dedup) /** Type declarations which derive from this type. */ def derivedTypeDecl: Iterator[TypeDecl] = - traversal.flatMap(_.inheritsFromIn) + traversal.inheritsFromIn /** Direct alias types. */ @@ -69,26 +68,27 @@ class TypeTraversal(val traversal: Iterator[Type]) extends AnyVal { /** Direct and transitive alias types. */ def aliasTypeTransitive: Iterator[Type] = - traversal.repeat(_.aliasType)(_.emitAllButFirst) + traversal.repeat(_.aliasType)(_.emitAllButFirst.dedup) def localOfType: Iterator[Local] = - traversal.flatMap(_._localViaEvalTypeIn) + traversal._localViaEvalTypeIn def memberOfType: Iterator[Member] = - traversal.flatMap(_.evalTypeIn).collectAll[Member] + traversal.evalTypeIn.collectAll[Member] @deprecated("Please use `parameterOfType`") def parameter: Iterator[MethodParameterIn] = parameterOfType def parameterOfType: Iterator[MethodParameterIn] = - traversal.flatMap(_.evalTypeIn).collectAll[MethodParameterIn] + traversal.evalTypeIn.collectAll[MethodParameterIn] def methodReturnOfType: Iterator[MethodReturn] = - traversal.flatMap(_.evalTypeIn).collectAll[MethodReturn] + traversal.evalTypeIn.collectAll[MethodReturn] def expressionOfType: Iterator[Expression] = expression + // TODO define in schema def expression: Iterator[Expression] = - traversal.flatMap(_.evalTypeIn).collectAll[Expression] + traversal.evalTypeIn.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/layers/LayerCreator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/layers/LayerCreator.scala index 582eabb8eba4..64aa3f553d22 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/layers/LayerCreator.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/layers/LayerCreator.scala @@ -1,9 +1,6 @@ package io.shiftleft.semanticcpg.layers -import better.files.File -import io.shiftleft.SerializedCpg import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.passes.CpgPassBase import io.shiftleft.semanticcpg.Overlays import org.slf4j.{Logger, LoggerFactory} @@ -36,19 +33,6 @@ abstract class LayerCreator { } } - protected def initSerializedCpg(outputDir: Option[String], passName: String, index: Int = 0): SerializedCpg = { - outputDir match { - case Some(dir) => new SerializedCpg((File(dir) / s"${index}_$passName").path.toAbsolutePath.toString) - case None => new SerializedCpg() - } - } - - protected def runPass(pass: CpgPassBase, context: LayerCreatorContext, index: Int = 0): Unit = { - val serializedCpg = initSerializedCpg(context.outputDir, pass.name, index) - pass.createApplySerializeAndStore(serializedCpg) - serializedCpg.close() - } - def create(context: LayerCreatorContext): Unit } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala index 9b0f25175138..a427ab3d9678 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/package.scala @@ -1,6 +1,6 @@ package io.shiftleft -import overflowdb.traversal.help.Table.AvailableWidthProvider +import flatgraph.help.Table.AvailableWidthProvider /** Domain specific language for querying code property graphs * diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala index 3c5a80dabddc..b523d8893fd3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/DummyNode.scala @@ -1,52 +1,7 @@ package io.shiftleft.semanticcpg.testing import io.shiftleft.codepropertygraph.generated.nodes.StoredNode -import overflowdb.{Edge, Node, Property, PropertyKey} -import java.util - -/** mixin trait for test nodes */ trait DummyNodeImpl extends StoredNode { - // Members declared in overflowdb.Element - def graph(): overflowdb.Graph = ??? - def property[A](x$1: overflowdb.PropertyKey[A]): A = ??? - def property(x$1: String): Object = ??? - def propertyKeys(): java.util.Set[String] = ??? - def propertiesMap(): java.util.Map[String, Object] = ??? - def propertyOption(x$1: String): java.util.Optional[Object] = ??? - def propertyOption[A](x$1: overflowdb.PropertyKey[A]): java.util.Optional[A] = ??? - override def addEdgeImpl(label: String, inNode: Node, keyValues: Any*): Edge = ??? - override def addEdgeImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Edge = ??? - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: Any*): Unit = ??? - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Unit = ??? - override def setPropertyImpl(key: String, value: Any): Unit = ??? - override def setPropertyImpl[A](key: PropertyKey[A], value: A): Unit = ??? - override def setPropertyImpl(property: Property[?]): Unit = ??? - override def removePropertyImpl(key: String): Unit = ??? - override def removeImpl(): Unit = ??? - - // Members declared in scala.Equals - def canEqual(that: Any): Boolean = ??? - - def both(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def both(): java.util.Iterator[overflowdb.Node] = ??? - def bothE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def bothE(): java.util.Iterator[overflowdb.Edge] = ??? - def id(): Long = ??? - def in(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def in(): java.util.Iterator[overflowdb.Node] = ??? - def inE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def inE(): java.util.Iterator[overflowdb.Edge] = ??? - def out(x$1: String*): java.util.Iterator[overflowdb.Node] = ??? - def out(): java.util.Iterator[overflowdb.Node] = ??? - def outE(x$1: String*): java.util.Iterator[overflowdb.Edge] = ??? - def outE(): java.util.Iterator[overflowdb.Edge] = ??? - - // Members declared in scala.Product - def productArity: Int = ??? - def productElement(n: Int): Any = ??? - - // Members declared in io.shiftleft.codepropertygraph.generated.nodes.StoredNode - def productElementLabel(n: Int): String = ??? - def valueMap: java.util.Map[String, AnyRef] = ??? + def propertiesMap: java.util.Map[String, Any] = ??? } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala new file mode 100644 index 000000000000..96ba88e2ba73 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/MockCpg.scala @@ -0,0 +1,229 @@ +package io.shiftleft.semanticcpg.testing + +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Languages, ModifierTypes} +import io.shiftleft.passes.CpgPass +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder + +object MockCpg { + + def apply(): MockCpg = new MockCpg + + def apply(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f) +} + +case class MockCpg(cpg: Cpg = Cpg.emptyCpg) { + + def withMetaData(language: String = Languages.C): MockCpg = withMetaData(language, Nil) + + def withMetaData(language: String, overlays: List[String]): MockCpg = { + withCustom { (diffGraph, _) => + diffGraph.addNode(NewMetaData().language(language).overlays(overlays)) + } + } + + def withFile(filename: String, content: Option[String] = None): MockCpg = + withCustom { (graph, _) => + val newFile = NewFile().name(filename) + content.foreach(newFile.content(_)) + graph.addNode(newFile) + } + + def withNamespace(name: String, inFile: Option[String] = None): MockCpg = + withCustom { (graph, _) => + { + val namespaceBlock = NewNamespaceBlock().name(name) + val namespace = NewNamespace().name(name) + graph.addNode(namespaceBlock) + graph.addNode(namespace) + graph.addEdge(namespaceBlock, namespace, EdgeTypes.REF) + if (inFile.isDefined) { + val fileNode = cpg.file(inFile.get).head + graph.addEdge(namespaceBlock, fileNode, EdgeTypes.SOURCE_FILE) + } + } + } + + def withTypeDecl( + name: String, + isExternal: Boolean = false, + inNamespace: Option[String] = None, + inFile: Option[String] = None, + offset: Option[Int] = None, + offsetEnd: Option[Int] = None + ): MockCpg = + withCustom { (graph, _) => + { + val typeNode = NewType().name(name) + val typeDeclNode = NewTypeDecl() + .name(name) + .fullName(name) + .isExternal(isExternal) + + offset.foreach(typeDeclNode.offset(_)) + offsetEnd.foreach(typeDeclNode.offsetEnd(_)) + + val member = NewMember().name("amember") + val modifier = NewModifier().modifierType(ModifierTypes.STATIC) + + graph.addNode(typeDeclNode) + graph.addNode(typeNode) + graph.addNode(member) + graph.addNode(modifier) + graph.addEdge(typeNode, typeDeclNode, EdgeTypes.REF) + graph.addEdge(typeDeclNode, member, EdgeTypes.AST) + graph.addEdge(member, modifier, EdgeTypes.AST) + + if (inNamespace.isDefined) { + val namespaceBlock = cpg.namespaceBlock(inNamespace.get).head + graph.addEdge(namespaceBlock, typeDeclNode, EdgeTypes.AST) + } + if (inFile.isDefined) { + val fileNode = cpg.file(inFile.get).head + graph.addEdge(typeDeclNode, fileNode, EdgeTypes.SOURCE_FILE) + } + } + } + + def withMethod( + name: String, + external: Boolean = false, + inTypeDecl: Option[String] = None, + fileName: String = "", + offset: Option[Int] = None, + offsetEnd: Option[Int] = None + ): MockCpg = + withCustom { (graph, _) => + val retParam = NewMethodReturn().typeFullName("int").order(10) + val param = NewMethodParameterIn().order(1).index(1).name("param1") + val paramType = NewType().name("paramtype") + val paramOut = NewMethodParameterOut().name("param1").order(1) + val method = + NewMethod().isExternal(external).name(name).fullName(name).signature("asignature").filename(fileName) + offset.foreach(method.offset(_)) + offsetEnd.foreach(method.offsetEnd(_)) + val block = NewBlock().typeFullName("int") + val modifier = NewModifier().modifierType("modifiertype") + + graph.addNode(method) + graph.addNode(retParam) + graph.addNode(param) + graph.addNode(paramType) + graph.addNode(paramOut) + graph.addNode(block) + graph.addNode(modifier) + graph.addEdge(method, retParam, EdgeTypes.AST) + graph.addEdge(method, param, EdgeTypes.AST) + graph.addEdge(param, paramOut, EdgeTypes.PARAMETER_LINK) + graph.addEdge(method, block, EdgeTypes.AST) + graph.addEdge(param, paramType, EdgeTypes.EVAL_TYPE) + graph.addEdge(paramOut, paramType, EdgeTypes.EVAL_TYPE) + graph.addEdge(method, modifier, EdgeTypes.AST) + + if (inTypeDecl.isDefined) { + val typeDeclNode = cpg.typeDecl(inTypeDecl.get).head + graph.addEdge(typeDeclNode, method, EdgeTypes.AST) + } + + if (fileName != "") { + val file = cpg.file + .nameExact(fileName) + .headOption + .getOrElse(throw new RuntimeException(s"file with name='$fileName' not found")) + graph.addEdge(method, file, EdgeTypes.SOURCE_FILE) + } + } + + def withTagsOnMethod( + methodName: String, + methodTags: List[(String, String)] = List(), + paramTags: List[(String, String)] = List() + ): MockCpg = + withCustom { (graph, cpg) => + implicit val diffGraph: DiffGraphBuilder = graph + methodTags.foreach { case (k, v) => + cpg.method(methodName).newTagNodePair(k, v).store()(diffGraph) + } + paramTags.foreach { case (k, v) => + cpg.method(methodName).parameter.newTagNodePair(k, v).store()(diffGraph) + } + } + + def withCallInMethod(methodName: String, callName: String, code: Option[String] = None): MockCpg = + withCustom { (graph, cpg) => + val methodNode = cpg.method(methodName).head + val blockNode = methodNode.block + val callNode = NewCall().name(callName).code(code.getOrElse(callName)) + graph.addNode(callNode) + graph.addEdge(blockNode, callNode, EdgeTypes.AST) + graph.addEdge(methodNode, callNode, EdgeTypes.CONTAINS) + } + + def withMethodCall(calledMethod: String, callingMethod: String, code: Option[String] = None): MockCpg = + withCustom { (graph, cpg) => + val callingMethodNode = cpg.method(callingMethod).head + val calledMethodNode = cpg.method(calledMethod).head + val callNode = NewCall().name(calledMethod).code(code.getOrElse(calledMethod)) + graph.addEdge(callNode, calledMethodNode, EdgeTypes.CALL) + graph.addEdge(callingMethodNode, callNode, EdgeTypes.CONTAINS) + } + + def withLocalInMethod(methodName: String, localName: String): MockCpg = + withCustom { (graph, cpg) => + val methodNode = cpg.method(methodName).head + val blockNode = methodNode.block + val typeNode = NewType().name("alocaltype") + val localNode = NewLocal().name(localName).typeFullName("alocaltype") + graph.addNode(localNode) + graph.addNode(typeNode) + graph.addEdge(blockNode, localNode, EdgeTypes.AST) + graph.addEdge(localNode, typeNode, EdgeTypes.EVAL_TYPE) + } + + def withLiteralArgument(callName: String, literalCode: String): MockCpg = { + withCustom { (graph, cpg) => + val callNode = cpg.call(callName).head + val methodNode = callNode.method + val literalNode = NewLiteral().code(literalCode) + val typeDecl = NewTypeDecl() + .name("ATypeDecl") + .fullName("ATypeDecl") + + graph.addNode(typeDecl) + graph.addNode(literalNode) + graph.addEdge(callNode, literalNode, EdgeTypes.AST) + graph.addEdge(methodNode, literalNode, EdgeTypes.CONTAINS) + } + } + + def withIdentifierArgument(callName: String, name: String, index: Int = 1): MockCpg = + withArgument(callName, NewIdentifier().name(name).argumentIndex(index)) + + def withCallArgument(callName: String, callArgName: String, code: String = "", index: Int = 1): MockCpg = + withArgument(callName, NewCall().name(callArgName).code(code).argumentIndex(index)) + + def withArgument(callName: String, newNode: NewNode): MockCpg = withCustom { (graph, cpg) => + val callNode = cpg.call(callName).head + val methodNode = callNode.method + val typeDecl = NewTypeDecl().name("abc") + graph.addEdge(callNode, newNode, EdgeTypes.AST) + graph.addEdge(callNode, newNode, EdgeTypes.ARGUMENT) + graph.addEdge(methodNode, newNode, EdgeTypes.CONTAINS) + graph.addEdge(newNode, typeDecl, EdgeTypes.REF) + graph.addNode(newNode) + } + + def withCustom(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = { + val diffGraph = new DiffGraphBuilder(cpg.graph.schema) + f(diffGraph, cpg) + class MyPass extends CpgPass(cpg) { + override def run(builder: DiffGraphBuilder): Unit = { + builder.absorb(diffGraph) + } + } + new MyPass().createAndApply() + this + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala deleted file mode 100644 index 1207a9efa669..000000000000 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala +++ /dev/null @@ -1,234 +0,0 @@ -package io.shiftleft.semanticcpg - -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Languages, ModifierTypes} -import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ -import overflowdb.BatchedUpdate -import overflowdb.BatchedUpdate.DiffGraphBuilder - -package object testing { - - object MockCpg { - - def apply(): MockCpg = new MockCpg - - def apply(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f) - } - - case class MockCpg(cpg: Cpg = Cpg.empty) { - - def withMetaData(language: String = Languages.C): MockCpg = withMetaData(language, Nil) - - def withMetaData(language: String, overlays: List[String]): MockCpg = { - withCustom { (diffGraph, _) => - diffGraph.addNode(NewMetaData().language(language).overlays(overlays)) - } - } - - def withFile(filename: String, content: Option[String] = None): MockCpg = - withCustom { (graph, _) => - val newFile = NewFile().name(filename) - content.foreach(newFile.content(_)) - graph.addNode(newFile) - } - - def withNamespace(name: String, inFile: Option[String] = None): MockCpg = - withCustom { (graph, _) => - { - val namespaceBlock = NewNamespaceBlock().name(name) - val namespace = NewNamespace().name(name) - graph.addNode(namespaceBlock) - graph.addNode(namespace) - graph.addEdge(namespaceBlock, namespace, EdgeTypes.REF) - if (inFile.isDefined) { - val fileNode = cpg.file.name(inFile.get).head - graph.addEdge(namespaceBlock, fileNode, EdgeTypes.SOURCE_FILE) - } - } - } - - def withTypeDecl( - name: String, - isExternal: Boolean = false, - inNamespace: Option[String] = None, - inFile: Option[String] = None, - offset: Option[Int] = None, - offsetEnd: Option[Int] = None - ): MockCpg = - withCustom { (graph, _) => - { - val typeNode = NewType().name(name) - val typeDeclNode = NewTypeDecl() - .name(name) - .fullName(name) - .isExternal(isExternal) - - offset.foreach(typeDeclNode.offset(_)) - offsetEnd.foreach(typeDeclNode.offsetEnd(_)) - - val member = NewMember().name("amember") - val modifier = NewModifier().modifierType(ModifierTypes.STATIC) - - graph.addNode(typeDeclNode) - graph.addNode(typeNode) - graph.addNode(member) - graph.addNode(modifier) - graph.addEdge(typeNode, typeDeclNode, EdgeTypes.REF) - graph.addEdge(typeDeclNode, member, EdgeTypes.AST) - graph.addEdge(member, modifier, EdgeTypes.AST) - - if (inNamespace.isDefined) { - val namespaceBlock = cpg.namespaceBlock(inNamespace.get).head - graph.addEdge(namespaceBlock, typeDeclNode, EdgeTypes.AST) - } - if (inFile.isDefined) { - val fileNode = cpg.file.name(inFile.get).head - graph.addEdge(typeDeclNode, fileNode, EdgeTypes.SOURCE_FILE) - } - } - } - - def withMethod( - name: String, - external: Boolean = false, - inTypeDecl: Option[String] = None, - fileName: String = "", - offset: Option[Int] = None, - offsetEnd: Option[Int] = None - ): MockCpg = - withCustom { (graph, _) => - val retParam = NewMethodReturn().typeFullName("int").order(10) - val param = NewMethodParameterIn().order(1).index(1).name("param1") - val paramType = NewType().name("paramtype") - val paramOut = NewMethodParameterOut().name("param1").order(1) - val method = - NewMethod().isExternal(external).name(name).fullName(name).signature("asignature").filename(fileName) - offset.foreach(method.offset(_)) - offsetEnd.foreach(method.offsetEnd(_)) - val block = NewBlock().typeFullName("int") - val modifier = NewModifier().modifierType("modifiertype") - - graph.addNode(method) - graph.addNode(retParam) - graph.addNode(param) - graph.addNode(paramType) - graph.addNode(paramOut) - graph.addNode(block) - graph.addNode(modifier) - graph.addEdge(method, retParam, EdgeTypes.AST) - graph.addEdge(method, param, EdgeTypes.AST) - graph.addEdge(param, paramOut, EdgeTypes.PARAMETER_LINK) - graph.addEdge(method, block, EdgeTypes.AST) - graph.addEdge(param, paramType, EdgeTypes.EVAL_TYPE) - graph.addEdge(paramOut, paramType, EdgeTypes.EVAL_TYPE) - graph.addEdge(method, modifier, EdgeTypes.AST) - - if (inTypeDecl.isDefined) { - val typeDeclNode = cpg.typeDecl.name(inTypeDecl.get).head - graph.addEdge(typeDeclNode, method, EdgeTypes.AST) - } - - if (fileName != "") { - val file = cpg.file - .nameExact(fileName) - .headOption - .getOrElse(throw new RuntimeException(s"file with name='$fileName' not found")) - graph.addEdge(method, file, EdgeTypes.SOURCE_FILE) - } - } - - def withTagsOnMethod( - methodName: String, - methodTags: List[(String, String)] = List(), - paramTags: List[(String, String)] = List() - ): MockCpg = - withCustom { (graph, cpg) => - implicit val diffGraph: DiffGraphBuilder = graph - methodTags.foreach { case (k, v) => - cpg.method.name(methodName).newTagNodePair(k, v).store()(diffGraph) - } - paramTags.foreach { case (k, v) => - cpg.method.name(methodName).parameter.newTagNodePair(k, v).store()(diffGraph) - } - } - - def withCallInMethod(methodName: String, callName: String, code: Option[String] = None): MockCpg = - withCustom { (graph, cpg) => - val methodNode = cpg.method.name(methodName).head - val blockNode = methodNode.block - val callNode = NewCall().name(callName).code(code.getOrElse(callName)) - graph.addNode(callNode) - graph.addEdge(blockNode, callNode, EdgeTypes.AST) - graph.addEdge(methodNode, callNode, EdgeTypes.CONTAINS) - } - - def withMethodCall(calledMethod: String, callingMethod: String, code: Option[String] = None): MockCpg = - withCustom { (graph, cpg) => - val callingMethodNode = cpg.method.name(callingMethod).head - val calledMethodNode = cpg.method.name(calledMethod).head - val callNode = NewCall().name(calledMethod).code(code.getOrElse(calledMethod)) - graph.addEdge(callNode, calledMethodNode, EdgeTypes.CALL) - graph.addEdge(callingMethodNode, callNode, EdgeTypes.CONTAINS) - } - - def withLocalInMethod(methodName: String, localName: String): MockCpg = - withCustom { (graph, cpg) => - val methodNode = cpg.method.name(methodName).head - val blockNode = methodNode.block - val typeNode = NewType().name("alocaltype") - val localNode = NewLocal().name(localName).typeFullName("alocaltype") - graph.addNode(localNode) - graph.addNode(typeNode) - graph.addEdge(blockNode, localNode, EdgeTypes.AST) - graph.addEdge(localNode, typeNode, EdgeTypes.EVAL_TYPE) - } - - def withLiteralArgument(callName: String, literalCode: String): MockCpg = { - withCustom { (graph, cpg) => - val callNode = cpg.call.name(callName).head - val methodNode = callNode.method - val literalNode = NewLiteral().code(literalCode) - val typeDecl = NewTypeDecl() - .name("ATypeDecl") - .fullName("ATypeDecl") - - graph.addNode(typeDecl) - graph.addNode(literalNode) - graph.addEdge(callNode, literalNode, EdgeTypes.AST) - graph.addEdge(methodNode, literalNode, EdgeTypes.CONTAINS) - } - } - - def withIdentifierArgument(callName: String, name: String, index: Int = 1): MockCpg = - withArgument(callName, NewIdentifier().name(name).argumentIndex(index)) - - def withCallArgument(callName: String, callArgName: String, code: String = "", index: Int = 1): MockCpg = - withArgument(callName, NewCall().name(callArgName).code(code).argumentIndex(index)) - - def withArgument(callName: String, newNode: NewNode): MockCpg = withCustom { (graph, cpg) => - val callNode = cpg.call.name(callName).head - val methodNode = callNode.method - val typeDecl = NewTypeDecl().name("abc") - graph.addEdge(callNode, newNode, EdgeTypes.AST) - graph.addEdge(callNode, newNode, EdgeTypes.ARGUMENT) - graph.addEdge(methodNode, newNode, EdgeTypes.CONTAINS) - graph.addEdge(newNode, typeDecl, EdgeTypes.REF) - graph.addNode(newNode) - } - - def withCustom(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = { - val diffGraph = Cpg.newDiffGraphBuilder - f(diffGraph, cpg) - class MyPass extends CpgPass(cpg) { - override def run(builder: BatchedUpdate.DiffGraphBuilder): Unit = { - builder.absorb(diffGraph) - } - } - new MyPass().createAndApply() - this - } - } - -} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/utils/Statements.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/utils/Statements.scala index e121e3830b91..78b76d838b71 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/utils/Statements.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/utils/Statements.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.utils import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* object Statements { def countAll(cpg: Cpg): Long = diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala index d47661adee88..da5b1b4e803a 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala @@ -1,15 +1,13 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.{Cpg, DiffGraphBuilder} +import flatgraph.DiffGraphApplier.applyDiff +import io.shiftleft.codepropertygraph.generated.nodes.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.BatchedUpdate.{DiffGraphBuilder, applyDiff} - -import scala.jdk.CollectionConverters._ class NewNodeStepsTest extends AnyWordSpec with Matchers { - import io.shiftleft.semanticcpg.language.NewNodeNodeStepsTest._ + import io.shiftleft.semanticcpg.language.NewNodeNodeStepsTest.* "stores NewNodes" in { implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder @@ -17,9 +15,9 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { val cpg = Cpg.empty new NewNodeSteps(newNode.start).store() - cpg.graph.nodes.toList.size shouldBe 0 + cpg.all.size shouldBe 0 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.nodes.toList.size shouldBe 1 + cpg.all.size shouldBe 1 } "can access the node label" in { @@ -29,17 +27,19 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { "stores containedNodes and connecting edge" when { "embedding a StoredNode and a NewNode" in { - implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder - val cpg = Cpg.empty - val existingContainedNode = cpg.graph.addNode(42L, "MODIFIER").asInstanceOf[StoredNode] - cpg.graph.V().asScala.toSet shouldBe Set(existingContainedNode) + val cpg = Cpg.empty + val newModifier = NewModifier() + applyDiff(cpg.graph, Cpg.newDiffGraphBuilder.addNode(newModifier)) + val existingContainedNode = newModifier.storedRef.get + cpg.graph.allNodes.toSet shouldBe Set(existingContainedNode) - val newContainedNode = newTestNode() - val newNode = newTestNode(evidence = List(existingContainedNode, newContainedNode)) + implicit val diffGraphBuilder: DiffGraphBuilder = Cpg.newDiffGraphBuilder + val newContainedNode = newTestNode() + val newNode = newTestNode(evidence = List(existingContainedNode, newContainedNode)) new NewNodeSteps(newNode.start).store() - cpg.graph.V().asScala.length shouldBe 1 + cpg.all.length shouldBe 1 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.V().asScala.length shouldBe 3 + cpg.all.length shouldBe 3 } "embedding a NewNode recursively" in { @@ -49,9 +49,9 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers { val newContainedNodeL0 = newTestNode(evidence = List(newContainedNodeL1)) val newNode = newTestNode(evidence = List(newContainedNodeL0)) new NewNodeSteps(newNode.start).store() - cpg.graph.V().asScala.size shouldBe 0 + cpg.all.size shouldBe 0 applyDiff(cpg.graph, diffGraphBuilder) - cpg.graph.V().asScala.size shouldBe 3 + cpg.all.size shouldBe 3 } } diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/OverlaysTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/OverlaysTests.scala similarity index 96% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/OverlaysTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/OverlaysTests.scala index f1fcf8098a71..f11d33aed67f 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/OverlaysTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/OverlaysTests.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala index 8e9107fe4412..1d276d2341c6 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/StepsTest.scala @@ -1,18 +1,15 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.Cpg.docSearchPackages import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{NodeTypes, Properties} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg +import flatgraph.help.Table.{AvailableWidthProvider, ConstantWidth} import org.json4s.* import org.json4s.native.JsonMethods.parse import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.traversal.help.Table.{AvailableWidthProvider, ConstantWidth} - -import java.util.Optional -import scala.jdk.CollectionConverters.IteratorHasAsScala class StepsTest extends AnyWordSpec with Matchers { @@ -52,7 +49,7 @@ class StepsTest extends AnyWordSpec with Matchers { val method: Method = cpg.method.head val results: List[Method] = cpg.method.id(method.id).toList results.size shouldBe 1 - results.head.underlying.id + results.head.id } "providing multiple" in { @@ -123,6 +120,9 @@ class StepsTest extends AnyWordSpec with Matchers { val parsed = parse(json).children.head // exactly one result for the above query (parsed \ "_label") shouldBe JString("METHOD") (parsed \ "name") shouldBe JString("foo") + + // id should be defined, but we don't care what number it is + (parsed \ "_id") shouldBe a[JInt] } "operating on NewNode" in { @@ -158,7 +158,7 @@ class StepsTest extends AnyWordSpec with Matchers { val nodeId = mainMethods.head.id val printed = mainMethods.p.head printed.should(startWith(s"""(METHOD,$nodeId):""")) - printed.should(include("IS_EXTERNAL: false")) + printed.should(include("SIGNATURE: asignature")) printed.should(include("FULL_NAME: woo")) } @@ -197,19 +197,19 @@ class StepsTest extends AnyWordSpec with Matchers { "show domain overview" in { val domainStartersHelp = Cpg.empty.help domainStartersHelp should include(".comment") - domainStartersHelp should include("All comments in source-based CPGs") + domainStartersHelp should include("A source code comment") domainStartersHelp should include(".arithmetic") domainStartersHelp should include("All arithmetic operations") } "provide node-specific overview" in { val methodStepsHelp = Cpg.empty.method.help - methodStepsHelp should include("Available steps for Method") + methodStepsHelp should include("Available steps for `Method`") methodStepsHelp should include(".namespace") methodStepsHelp should include(".depth") // from AstNode val methodStepsHelpVerbose = Cpg.empty.method.helpVerbose - methodStepsHelpVerbose should include("traversal name") + methodStepsHelpVerbose should include("implemented in") methodStepsHelpVerbose should include("structure.MethodTraversal") val assignmentStepsHelp = Cpg.empty.assignment.help @@ -283,7 +283,6 @@ class StepsTest extends AnyWordSpec with Matchers { def methodParameterOut = cpg.graph .nodes(NodeTypes.METHOD_PARAMETER_OUT) - .asScala .cast[MethodParameterOut] .name("param1") methodParameterOut.typ.name.head shouldBe "paramtype" @@ -305,7 +304,7 @@ class StepsTest extends AnyWordSpec with Matchers { file.typeDecl.name.head shouldBe "AClass" file.head.typeDecl.name.head shouldBe "AClass" - def block = cpg.graph.nodes(NodeTypes.BLOCK).asScala.cast[Block].typeFullName("int") + def block = cpg.graph.nodes(NodeTypes.BLOCK).cast[Block].typeFullName("int") block.local.name.size shouldBe 1 block.flatMap(_.local.name).size shouldBe 1 @@ -349,13 +348,6 @@ class StepsTest extends AnyWordSpec with Matchers { method.head.modifier.modifierType.toSetMutable shouldBe Set("modifiertype") } - "id starter step" in { - // only verifying what compiles and what doesn't... - // if it compiles, :shipit: - assertCompiles("cpg.id(1).out") - assertDoesNotCompile("cpg.id(1).outV") // `.outV` is only available on Traversal[Edge] - } - "property accessors" in { val cpg = MockCpg().withCustom { (diffGraph, _) => diffGraph @@ -370,21 +362,35 @@ class StepsTest extends AnyWordSpec with Matchers { val (Seq(emptyCall), Seq(callWithProperties)) = cpg.call.l.partition(_.argumentName.isEmpty) - emptyCall.propertyOption(Properties.TypeFullName) shouldBe Optional.of("") - emptyCall.propertyOption(Properties.TypeFullName.name) shouldBe Optional.of("") - emptyCall.propertyOption(Properties.ArgumentName) shouldBe Optional.empty - emptyCall.propertyOption(Properties.ArgumentName.name) shouldBe Optional.empty + // Cardinality.One + emptyCall.property(Properties.TypeFullName) shouldBe "" + emptyCall.propertyOption(Properties.TypeFullName) shouldBe Some("") + emptyCall.propertyOption(Properties.TypeFullName.name) shouldBe Some("") + // Cardinality.ZeroOrOne + emptyCall.property(Properties.ArgumentName) shouldBe None + emptyCall.propertyOption(Properties.ArgumentName) shouldBe None + emptyCall.propertyOption(Properties.ArgumentName.name) shouldBe None + // Cardinality.List // these ones are rather a historic accident it'd be better and more consistent to return `None` here - // we'll defer that change until after the flatgraph port though and just document it for now - emptyCall.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Optional.of(Seq.empty) - emptyCall.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Optional.of(Seq.empty) - - callWithProperties.propertyOption(Properties.TypeFullName) shouldBe Optional.of("aa") - callWithProperties.propertyOption(Properties.TypeFullName.name) shouldBe Optional.of("aa") - callWithProperties.propertyOption(Properties.ArgumentName) shouldBe Optional.of("bb") - callWithProperties.propertyOption(Properties.ArgumentName.name) shouldBe Optional.of("bb") - callWithProperties.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Optional.of(Seq("cc", "dd")) - callWithProperties.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Optional.of(Seq("cc", "dd")) + emptyCall.property(Properties.DynamicTypeHintFullName) shouldBe Seq.empty + emptyCall.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Some(Seq.empty) + emptyCall.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Some(Seq.empty) + + // Cardinality.One + callWithProperties.property(Properties.TypeFullName) shouldBe "aa" + callWithProperties.propertyOption(Properties.TypeFullName) shouldBe Some("aa") + callWithProperties.propertyOption(Properties.TypeFullName.name) shouldBe Some("aa") + + // Cardinality.ZeroOrOne + callWithProperties.property(Properties.ArgumentName) shouldBe Some("bb") + callWithProperties.propertyOption(Properties.ArgumentName) shouldBe Some("bb") + callWithProperties.propertyOption(Properties.ArgumentName.name) shouldBe Some("bb") + + // Cardinality.List + callWithProperties.property(Properties.DynamicTypeHintFullName) shouldBe Seq("cc", "dd") + callWithProperties.propertyOption(Properties.DynamicTypeHintFullName) shouldBe Some(Seq("cc", "dd")) + callWithProperties.propertyOption(Properties.DynamicTypeHintFullName.name) shouldBe Some(Seq("cc", "dd")) } } diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/accesspath/AccessPathTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/accesspath/AccessPathTests.scala similarity index 98% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/accesspath/AccessPathTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/accesspath/AccessPathTests.scala index 3cb15128a159..8a191132d0a8 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/accesspath/AccessPathTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/accesspath/AccessPathTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.accesspath -import io.shiftleft.semanticcpg.accesspath.MatchResult._ -import org.scalatest.matchers.should.Matchers._ +import io.shiftleft.semanticcpg.accesspath.MatchResult.* +import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec class AccessPathTests extends AnyWordSpec { diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/bindingextension/BindingTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/bindingextension/BindingTests.scala index 76db43185dbb..195e5472f2d7 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/bindingextension/BindingTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/bindingextension/BindingTests.scala @@ -1,8 +1,8 @@ package io.shiftleft.semanticcpg.language.bindingextension import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala index fe3c0938d0a6..650914868147 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/operatorextension/OperatorExtensionTests.scala @@ -3,8 +3,7 @@ package io.shiftleft.semanticcpg.language.operatorextension import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Identifier -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.ArrayAccess +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversalTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversalTests.scala index 361bc45d222e..aedde1cb4dad 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversalTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/CfgNodeTraversalTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversalTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversalTests.scala index 1ec5f62bd598..0a9d459c5b48 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversalTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/ExpressionTraversalTests.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/FileTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/FileTests.scala index 3c9a9728c270..c750f85b538a 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/FileTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/FileTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes.{File, Namespace, TypeDecl} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.LoneElement import org.scalatest.matchers.should.Matchers diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTests.scala index a12a3a7edb2e..3972b43d70b5 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MemberTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.ModifierTypes -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTests.scala index d98aa68f5811..5694017e9942 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodParameterTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes.{Method, MethodParameterIn} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala index 908a6a3d8a16..c80baace6241 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala @@ -2,7 +2,7 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Expression, Literal, Method, NamespaceBlock, TypeDecl} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTests.scala index 184c015a0ceb..2e29239b6f0f 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/NamespaceTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.structure -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.testing.MockCpg import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTests.scala index 19442132e951..dedad804af74 100644 --- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTests.scala +++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/TypeTests.scala @@ -70,8 +70,6 @@ class TypeTests extends AnyWordSpec with Matchers { .name(".*Base") .toList - cpg.typeDecl.name(".*Derived").baseTypeDecl.foreach(println) - queryResult.size shouldBe 1 } diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/utils/CountStatementsTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/utils/CountStatementsTests.scala similarity index 100% rename from semanticcpg/src/test/scala/io/shiftleft/semanticcpg/utils/CountStatementsTests.scala rename to semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/utils/CountStatementsTests.scala diff --git a/test-dataflow-slice.sc b/test-dataflow-slice.sc new file mode 100644 index 000000000000..dfeeb9b23cb9 --- /dev/null +++ b/test-dataflow-slice.sc @@ -0,0 +1,17 @@ +import upickle.default.* +import io.shiftleft.utils.IOUtils +import java.nio.file.Path +import io.joern.dataflowengineoss.slicing.{DataFlowSlice, SliceEdge} + +@main def exec(sliceFile: String) = { + val jsonContent = IOUtils.readLinesInFile(Path.of(sliceFile)).mkString + val dataFlowSlice = read[DataFlowSlice](jsonContent) + val nodeMap = dataFlowSlice.nodes.map(n => n.id -> n).toMap + val edges = dataFlowSlice.edges.toList + .map { case SliceEdge(src, dst, _) => + (nodeMap(src).lineNumber, nodeMap(dst).lineNumber) -> List(nodeMap(src).code, nodeMap(dst).code).distinct + } + .sortBy(_._1) + .flatMap(_._2) + println(edges) +} diff --git a/tests/code/javasrc/SliceTest.java b/tests/code/javasrc/SliceTest.java new file mode 100644 index 000000000000..0d2b2dd099fa --- /dev/null +++ b/tests/code/javasrc/SliceTest.java @@ -0,0 +1,16 @@ + + +public class SliceTest { + + public void foo(boolean b) { + String s = new Foo("MALICIOUS"); + if (b) { + s.setFoo("SAFE"); + } + bar(b); + } + + public void bar(String x) { + System.out.println(s); + } +} \ No newline at end of file