diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala index 2333179302cb..273366a8d6a4 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala @@ -5,6 +5,7 @@ import io.joern.x2cpg.astgen.AstGenConfig import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} import io.joern.x2cpg.utils.Environment import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import org.slf4j.LoggerFactory import scopt.OParser @@ -40,16 +41,21 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) with FrontendHTTPServer[Config, CSharpSrc2Cpg] { private val logger = LoggerFactory.getLogger(getClass) + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, csharpsrc2cpg: CSharpSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - if (Environment.pathExists(absPath)) { - csharpsrc2cpg.run(config.withInputPath(absPath)) - } else { - logger.warn(s"Given path '$absPath' does not exist, skipping") + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + if (Environment.pathExists(absPath)) { + csharpsrc2cpg.run(config.withInputPath(absPath)) + } else { + logger.warn(s"Given path '$absPath' does not exist, skipping") + } } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..639704906c97 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala @@ -0,0 +1,78 @@ +package io.joern.csharpsrc2cpg.io + +import better.files.File +import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class CSharp2CpgHTTPServerTests extends CSharpCode2CpgFixture with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("csharp2cpgTestsHttpTest") + val file = dir / "main.cs" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(basicBoilerplate(s"Console.WriteLine($indexStr);")) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.csharpsrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.csharpsrc2cpg.Main.stop() + } + + "Using csharp2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("Main") + cpg.call.code.l shouldBe List("Console.WriteLine()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("Main") + cpg.call.code.l shouldBe List(s"Console.WriteLine($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala index 1dc9312deaad..bd00ad8a77bc 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala @@ -1,11 +1,11 @@ package io.joern.csharpsrc2cpg.io import better.files.File -import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary +import io.joern.csharpsrc2cpg.CSharpSrc2Cpg +import io.joern.csharpsrc2cpg.Config import io.joern.csharpsrc2cpg.passes.AstCreationPass import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture import io.joern.csharpsrc2cpg.utils.DotNetAstGenRunner -import io.joern.csharpsrc2cpg.{CSharpSrc2Cpg, Config} import io.joern.x2cpg.X2Cpg.newEmptyCpg import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.generated.Cpg diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala index 5112f2ccf523..4b3e9a15b0b6 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.gosrc2cpg import io.joern.gosrc2cpg.Frontend.* import io.joern.x2cpg.astgen.AstGenConfig import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -42,10 +43,15 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) with FrontendHTTPServer[Config, GoSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() def run(config: Config, gosrc2cpg: GoSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - gosrc2cpg.run(config.withInputPath(absPath)) + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + gosrc2cpg.run(config.withInputPath(absPath)) + } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..d9e1ac60a2ac --- /dev/null +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala @@ -0,0 +1,84 @@ +package io.joern.go2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class GoSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("gosrc2cpgTestsHttpTest") + val file = dir / "main.go" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |package main + |func main$indexStr() { + | print("Hello World!") + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.gosrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.gosrc2cpg.Main.stop() + } + + "Using gosrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain(s"main$index") + cpg.call.code.l shouldBe List("""print("Hello World!")""") + } + } + } + } + +} diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala index e89fc8d7e883..c7f3068523b2 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala @@ -2,11 +2,16 @@ package io.joern.javasrc2cpg import io.joern.javasrc2cpg.Frontend.* import io.joern.javasrc2cpg.jpastprinter.JavaParserAstPrinter +import io.joern.x2cpg.X2CpgConfig +import io.joern.x2cpg.X2CpgMain import io.joern.x2cpg.frontendspecific.javasrc2cpg -import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} -import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} +import io.joern.x2cpg.passes.frontend.TypeRecoveryParserConfig +import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser +import java.util.concurrent.ExecutorService + /** Command line configuration parameters */ final case class Config( @@ -133,7 +138,9 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) with FrontendHTTPServer[Config, JavaSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() override def main(args: Array[String]): Unit = { // TODO: This is a hack to allow users to use the "--show-env" option without having @@ -146,14 +153,14 @@ object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) { } def run(config: Config, javasrc2Cpg: JavaSrc2Cpg): Unit = { - if (config.showEnv) { - JavaSrc2Cpg.showEnv() - } else if (config.dumpJavaparserAsts) { - JavaParserAstPrinter.printJpAsts(config) - } else { - javasrc2Cpg.run(config) + config match { + case c if c.serverMode => startup() + case c if c.showEnv => JavaSrc2Cpg.showEnv() + case c if c.dumpJavaparserAsts => JavaParserAstPrinter.printJpAsts(c) + case _ => javasrc2Cpg.run(config) } } def getCmdLineParser = cmdLineParser + } diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..773fce3a89c7 --- /dev/null +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala @@ -0,0 +1,85 @@ +package io.joern.javasrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class JavaSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("javasrc2cpgTestsHttpTest") + val file = dir / "Main.java" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |class HelloWorld { + | public static void main(String[] args) { + | System.out.println($indexStr); + | } + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.javasrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.javasrc2cpg.Main.stop() + } + + "Using javasrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("System.out.println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"System.out.println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala index 4d20c13c1e90..98bcf4053e47 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala @@ -2,6 +2,7 @@ package io.joern.kotlin2cpg import io.joern.kotlin2cpg.Frontend.* import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser case class DefaultContentRootJarPath(path: String, isResource: Boolean) @@ -96,8 +97,12 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new Kotlin2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new Kotlin2Cpg()) with FrontendHTTPServer[Config, Kotlin2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, kotlin2cpg: Kotlin2Cpg): Unit = { - kotlin2cpg.run(config) + if (config.serverMode) { startup() } + else { kotlin2cpg.run(config) } } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..2ceaa95affa3 --- /dev/null +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/Kotlin2CpgHTTPServerTests.scala @@ -0,0 +1,84 @@ +package io.joern.kotlin2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class Kotlin2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("kotlin2cpgTestsHttpTest") + val file = dir / "main.kt" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |package mypkg + |fun main(args : Array) { + | println($indexStr) + |} + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.kotlin2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.kotlin2cpg.Main.stop() + } + + "Using kotlin2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("kotlin2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List("println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("kotlin2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l shouldBe List(s"println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala similarity index 94% rename from joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala rename to joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala index 730b34aa9ee4..fd252128c740 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/SourceFilesPickerTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/io/SourceFilesPickerTests.scala @@ -1,7 +1,6 @@ -package io.joern.kotlin2cpg +package io.joern.kotlin2cpg.io import io.joern.kotlin2cpg.files.SourceFilesPicker - import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers import org.scalatest.BeforeAndAfterAll diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala similarity index 93% rename from joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala rename to joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala index 50915958495e..6219f73c175a 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/DefaultRegisteredTypesTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/DefaultRegisteredTypesTests.scala @@ -1,4 +1,4 @@ -package io.joern.kotlin2cpg +package io.joern.kotlin2cpg.types import io.joern.kotlin2cpg.testfixtures.KotlinCode2CpgFixture import io.shiftleft.semanticcpg.language.* diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala index db3ade3308b1..afe2947167a6 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.php2cpg import io.joern.php2cpg.Frontend.* import io.joern.x2cpg.passes.frontend.* import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser /** Command line configuration parameters @@ -51,8 +52,12 @@ object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new Php2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new Php2Cpg()) with FrontendHTTPServer[Config, Php2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, php2Cpg: Php2Cpg): Unit = { - php2Cpg.run(config) + if (config.serverMode) { startup() } + else { php2Cpg.run(config) } } } diff --git a/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..ddadfc9d0622 --- /dev/null +++ b/joern-cli/frontends/php2cpg/src/test/scala/io/joern/php2cpg/io/Php2CpgHTTPServerTests.scala @@ -0,0 +1,79 @@ +package io.joern.php2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class Php2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("php2cpgTestsHttpTest") + val file = dir / "main.php" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse(""""Hello, World!"""") + file.writeText(s""" fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.call.code.l shouldBe List("""print("Hello, World!")""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("php2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.call.code.l shouldBe List(s"print($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala index 2ab54ea2c507..7bf13ec760c1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala +++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Main.scala @@ -3,6 +3,7 @@ package io.joern.pysrc2cpg import io.joern.pysrc2cpg.Frontend.cmdLineParser import io.joern.x2cpg.X2CpgMain import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -38,9 +39,15 @@ private object Frontend { } } -object NewMain extends X2CpgMain(cmdLineParser, new Py2CpgOnFileSystem())(new Py2CpgOnFileSystemConfig()) { +object NewMain + extends X2CpgMain(cmdLineParser, new Py2CpgOnFileSystem())(Py2CpgOnFileSystemConfig()) + with FrontendHTTPServer[Py2CpgOnFileSystemConfig, Py2CpgOnFileSystem] { + + override protected def newDefaultConfig(): Py2CpgOnFileSystemConfig = Py2CpgOnFileSystemConfig() + def run(config: Py2CpgOnFileSystemConfig, frontend: Py2CpgOnFileSystem): Unit = { - frontend.run(config) + if (config.serverMode) { startup() } + else { frontend.run(config) } } def getCmdLineParser = cmdLineParser diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala index 3861a8fb9f43..28cce1f5d986 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssertCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala index dfd12f7901db..2811650be190 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AssignCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala index fb009aa002a0..21d6ca62a217 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/AttributeCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala index 32965a724e30..7b0917042e1e 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BinOpCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala index d9e6ce6e5163..c210a453b33c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BoolOpCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala index ab82b9b8b083..3dae7d13a396 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BuiltinIdentifierTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala index 1238e6ed089b..4aafd1d33add 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/BytesLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala index 503becf26f54..3337cc5bf97d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CallCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.semanticcpg.language.* diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala index aa85ab8bbd06..5740d5ae4947 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ClassCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class ClassCpgTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala index 7fc6fcc9955f..3550933f5e3f 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/CompareCpgTests.scala @@ -1,8 +1,8 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala index ccda1f361591..ed60926e22c7 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ContentCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class ContentCpgTests extends PySrc2CpgFixture() { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala index 5fd9dbd26c11..a0b02eeed10e 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DeleteCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala index c1698709dd6c..2e996ab2c924 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/DictCpgTests.scala @@ -1,10 +1,8 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.{Py2CpgTestContext, PySrc2CpgFixture} -import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec class DictCpgTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala index 6f7e592a7a48..586b04fe7dd2 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FormatStringCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala index c818c5c53431..d9dde2aa24f2 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.codepropertygraph.generated.ModifierTypes import io.shiftleft.codepropertygraph.generated.nodes.Call diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala index c9e0512606a4..6647fe7454d1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IfCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, nodes} import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala index 122cccd6d386..4d066ba21c45 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ImportCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala index 5ae76bdb79f7..d3c94e23307d 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/IntLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala index 378c47532e30..b02fd1a9c3cc 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ListCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala index 5e1171d10070..448cd17a4f9c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MemberCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.nodes.Member import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala index 0fa7609e8490..8476473d02a1 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/MethodCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala index ced1d8d5ce99..a35deb9282e9 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ModuleFunctionCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala index 32adc0bc7394..9510fd3d5ac7 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/PatternMatchingTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.semanticcpg.language.* diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala index 48b154ac8d5e..32c14bcb75e9 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/RaiseCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.Operators -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala index 031df1583f3c..1b17c2a3b039 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/ReturnCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala index d23fa0e133af..10f518a9a70a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SetCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala index ece135da41e0..f48c201a8aba 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SliceCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class SliceCpgTests extends PySrc2CpgFixture() { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala index 3f8aa59df120..0b90acbbfa41 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StarredCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala index 2cf2a22b2cda..9a3455c7abd3 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StrLiteralCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.joern.x2cpg.frontendspecific.pysrc2cpg.Constants import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala index c3e730932133..feb7aff51640 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/StringExpressionListCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.DispatchTypes -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala index e5186682a86d..adf4531c24ea 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/SubscriptCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala index 103473b7bf72..92a0f02ba0a8 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/TryCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala index 182b0fbfff05..f98cb73ca464 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/UnaryOpCpgTests.scala @@ -1,7 +1,7 @@ package io.joern.pysrc2cpg.cpg +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} -import io.joern.pysrc2cpg.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec import org.scalatest.matchers.should.Matchers diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala index bcd4dc12a3c7..a476f60601a0 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/VariableReferencingCpgTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.codepropertygraph.generated.EvaluationStrategies import io.shiftleft.semanticcpg.language.* import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala index f06eee5e617b..b2c771250c7a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/WhileCpgTests.scala @@ -1,5 +1,6 @@ package io.joern.pysrc2cpg.cpg -import io.joern.pysrc2cpg.Py2CpgTestContext + +import io.joern.pysrc2cpg.testfixtures.Py2CpgTestContext import io.shiftleft.semanticcpg.language.* import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, nodes} import org.scalatest.freespec.AnyFreeSpec diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala index f82fe60b8483..89539650b9d3 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/dataflow/DataFlowTests.scala @@ -10,7 +10,7 @@ import io.joern.dataflowengineoss.semanticsloader.{ NoSemantics, PassThroughMapping } -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Literal, Member, Method} import io.shiftleft.semanticcpg.language.* diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..732674ecdeb0 --- /dev/null +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/io/PySrc2CpgHTTPServerTests.scala @@ -0,0 +1,82 @@ +package io.joern.pysrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class PySrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("pysrc2cpgTestsHttpTest") + val file = dir / "main.py" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |def main(): + | print($indexStr) + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.pysrc2cpg.NewMain.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.pysrc2cpg.NewMain.stop() + } + + "Using pysrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("pysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("print()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("pysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"print($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala index 798b35fa5788..c8c993f5d995 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ConfigPassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class ConfigPassTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala index af54b4d96685..8db531070350 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/DynamicTypeHintFullNamePassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala index 506a5be71027..8cb67c3aee6a 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/ImportsPassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* class ImportsPassTests extends PySrc2CpgFixture(withOssDataflow = false) { diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala index 43d91d177418..10611836281c 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/InheritanceFullNamePassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.shiftleft.semanticcpg.language.* import java.io.File diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala index b5198b3cee8a..a564993e2571 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala @@ -1,6 +1,6 @@ package io.joern.pysrc2cpg.passes -import io.joern.pysrc2cpg.PySrc2CpgFixture +import io.joern.pysrc2cpg.testfixtures.PySrc2CpgFixture import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Member} diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala similarity index 95% rename from joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala rename to joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala index 6b3facb4d231..ff1526453a45 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/Py2CpgTestContext.scala @@ -1,5 +1,6 @@ -package io.joern.pysrc2cpg +package io.joern.pysrc2cpg.testfixtures +import io.joern.pysrc2cpg.Py2Cpg import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.X2Cpg.defaultOverlayCreators import io.shiftleft.codepropertygraph.generated.Cpg diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala similarity index 66% rename from joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala rename to joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala index 0d9e982eabfb..177f456fde34 100644 --- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala +++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/testfixtures/PySrc2CpgFixture.scala @@ -1,26 +1,26 @@ -package io.joern.pysrc2cpg +package io.joern.pysrc2cpg.testfixtures import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.layers.dataflows.* -import io.joern.dataflowengineoss.queryengine.EngineContext -import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} -import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, SemanticTestCpg} -import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.frontendspecific.pysrc2cpg.{ - DynamicTypeHintFullNamePass, - ImportsPass, - PythonImportResolverPass, - PythonInheritanceNamePass, - PythonTypeHintCallLinker, - PythonTypeRecoveryPassGenerator -} +import io.joern.dataflowengineoss.semanticsloader.Semantics +import io.joern.dataflowengineoss.testfixtures.SemanticCpgTestFixture +import io.joern.dataflowengineoss.testfixtures.SemanticTestCpg +import io.joern.pysrc2cpg.Py2CpgOnFileSystem +import io.joern.pysrc2cpg.Py2CpgOnFileSystemConfig +import io.joern.x2cpg.frontendspecific.pysrc2cpg.DynamicTypeHintFullNamePass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.ImportsPass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonImportResolverPass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonInheritanceNamePass +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonTypeHintCallLinker +import io.joern.x2cpg.frontendspecific.pysrc2cpg.PythonTypeRecoveryPassGenerator import io.joern.x2cpg.passes.base.AstLinkerPass import io.joern.x2cpg.passes.callgraph.NaiveCallLinker -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg} +import io.joern.x2cpg.testfixtures.Code2CpgFixture +import io.joern.x2cpg.testfixtures.DefaultTestCpg +import io.joern.x2cpg.testfixtures.LanguageFrontend import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import io.shiftleft.semanticcpg.layers.LayerCreatorContext +import io.shiftleft.semanticcpg.language.ICallResolver +import io.shiftleft.semanticcpg.language.NoResolve trait PythonFrontend extends LanguageFrontend { override val fileSuffix: String = ".py" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala index 26fa95e9eab8..1db133b9c32a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/Main.scala @@ -1,9 +1,13 @@ package io.joern.rubysrc2cpg import io.joern.rubysrc2cpg.Frontend.* -import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} +import io.joern.x2cpg.DependencyDownloadConfig +import io.joern.x2cpg.X2CpgConfig +import io.joern.x2cpg.X2CpgMain +import io.joern.x2cpg.passes.frontend.TypeRecoveryParserConfig +import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.joern.x2cpg.typestub.TypeStubConfig -import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser final case class Config( @@ -79,8 +83,12 @@ private object Frontend { } } -object Main extends X2CpgMain(cmdLineParser, new RubySrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new RubySrc2Cpg()) with FrontendHTTPServer[Config, RubySrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() + def run(config: Config, rubySrc2Cpg: RubySrc2Cpg): Unit = { - rubySrc2Cpg.run(config) + if (config.serverMode) { startup() } + else { rubySrc2Cpg.run(config) } } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..5538dd30b360 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/io/RubySrc2CpgHTTPServerTests.scala @@ -0,0 +1,83 @@ +package io.joern.rubysrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class RubySrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("rubysrc2cpgTestsHttpTest") + val file = dir / "main.rb" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse(""""Hello, World!"""") + file.writeText(s""" + |def main + | puts $indexStr + |end + |""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.rubysrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.rubysrc2cpg.Main.stop() + } + + "Using rubysrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("rubysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("""puts "Hello, World!"""") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("rubysrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"puts $index") + } + } + } + } + +} diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala index b9d071df2273..2d2d37dd6257 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/Main.scala @@ -4,6 +4,7 @@ import io.joern.swiftsrc2cpg.Frontend.* import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig} import io.joern.x2cpg.utils.Environment import io.joern.x2cpg.{X2CpgConfig, X2CpgMain} +import io.joern.x2cpg.utils.server.FrontendHTTPServer import scopt.OParser import java.nio.file.Paths @@ -34,14 +35,19 @@ object Frontend { } -object Main extends X2CpgMain(cmdLineParser, new SwiftSrc2Cpg()) { +object Main extends X2CpgMain(cmdLineParser, new SwiftSrc2Cpg()) with FrontendHTTPServer[Config, SwiftSrc2Cpg] { + + override protected def newDefaultConfig(): Config = Config() def run(config: Config, swiftsrc2cpg: SwiftSrc2Cpg): Unit = { - val absPath = Paths.get(config.inputPath).toAbsolutePath.toString - if (Environment.pathExists(absPath)) { - swiftsrc2cpg.run(config.withInputPath(absPath)) - } else { - System.exit(1) + if (config.serverMode) { startup() } + else { + val absPath = Paths.get(config.inputPath).toAbsolutePath.toString + if (Environment.pathExists(absPath)) { + swiftsrc2cpg.run(config.withInputPath(absPath)) + } else { + System.exit(1) + } } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala new file mode 100644 index 000000000000..fccf57f5b5e8 --- /dev/null +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/SwiftSrc2CpgHTTPServerTests.scala @@ -0,0 +1,82 @@ +package io.joern.swiftsrc2cpg.io + +import better.files.File +import io.joern.x2cpg.utils.server.FrontendHTTPClient +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader +import io.shiftleft.semanticcpg.language.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import scala.collection.parallel.CollectionConverters.RangeIsParallelizable +import scala.util.Failure +import scala.util.Success + +class SwiftSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private var port: Int = -1 + + private def newProjectUnderTest(index: Option[Int] = None): File = { + val dir = File.newTemporaryDirectory("swiftsrc2cpgTestsHttpTest") + val file = dir / "main.swift" + file.createIfNotExists(createParents = true) + val indexStr = index.map(_.toString).getOrElse("") + file.writeText(s""" + |func main() { + | println($indexStr) + |}""".stripMargin) + file.deleteOnExit() + dir.deleteOnExit() + } + + override def beforeAll(): Unit = { + // Start server + port = io.joern.swiftsrc2cpg.Main.startup() + } + + override def afterAll(): Unit = { + // Stop server + io.joern.swiftsrc2cpg.Main.stop() + } + + "Using swiftsrc2cpg in server mode" should { + "build CPGs correctly (single test)" in { + val cpgOutFile = File.newTemporaryFile("swiftsrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest() + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain("println()") + } + } + + "build CPGs correctly (multi-threaded test)" in { + (0 until 10).par.foreach { index => + val cpgOutFile = File.newTemporaryFile("swiftsrc2cpg.bin") + cpgOutFile.deleteOnExit() + val projectUnderTest = newProjectUnderTest(Some(index)) + val input = projectUnderTest.path.toAbsolutePath.toString + val output = cpgOutFile.toString + val client = FrontendHTTPClient(port) + val req = client.buildRequest(Array(s"input=$input", s"output=$output")) + client.sendRequest(req) match { + case Failure(exception) => fail(exception.getMessage) + case Success(out) => + out shouldBe output + val cpg = CpgLoader.load(output) + cpg.method.name.l should contain("main") + cpg.call.code.l should contain(s"println($index)") + } + } + } + } + +} diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala index 37cbe4d1e8b1..31259124391e 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/server/FrontendHTTPServer.scala @@ -21,13 +21,13 @@ import scala.util.Try object FrontendHTTPServer { /** ExecutorService for single-threaded execution. */ - private lazy val SingleThreadExecutor: ExecutorService = Executors.newSingleThreadExecutor() + def singleThreadExecutor(): ExecutorService = Executors.newSingleThreadExecutor() /** ExecutorService for cached thread pool execution. */ - private lazy val CachedThreadPoolExecutor: ExecutorService = Executors.newCachedThreadPool() + def cachedThreadPoolExecutor(): ExecutorService = Executors.newCachedThreadPool() /** Default ExecutorService used by `FrontendHTTPServer`. */ - private val DefaultExecutor: ExecutorService = CachedThreadPoolExecutor + def defaultExecutor(): ExecutorService = cachedThreadPoolExecutor() } @@ -64,7 +64,7 @@ trait FrontendHTTPServer[T <: X2CpgConfig[T], X <: X2CpgFrontend[T]] { this: X2C * This can be overridden to switch between single-threaded and multi-threaded execution. By default, it uses the * cached thread pool executor from `FrontendHTTPServer`. */ - protected val executor: ExecutorService = FrontendHTTPServer.DefaultExecutor + protected val executor: ExecutorService = FrontendHTTPServer.defaultExecutor() /** Handler for HTTP requests, providing functionality to handle specific routes. * @@ -123,6 +123,7 @@ trait FrontendHTTPServer[T <: X2CpgConfig[T], X <: X2CpgFrontend[T]] { this: X2C */ def stop(): Unit = { underlyingServer.foreach { server => + executor.shutdown() server.stop() logger.debug("Server stopped.") }