[frontends] Added server mode to more frontends

joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/Main.scala

@@ -5,6 +5,7 @@ import io.joern.x2cpg.astgen.AstGenConfig
 import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig}
 import io.joern.x2cpg.utils.Environment
 import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain}
+import io.joern.x2cpg.utils.server.FrontendHTTPServer
 import org.slf4j.LoggerFactory
 import scopt.OParser
 
@@ -40,16 +41,21 @@ object Frontend {
 
 }
 
-object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) {
+object Main extends X2CpgMain(cmdLineParser, new CSharpSrc2Cpg()) with FrontendHTTPServer[Config, CSharpSrc2Cpg] {
 
  private val logger = LoggerFactory.getLogger(getClass)
 
+ override protected def newDefaultConfig(): Config = Config()
+
  def run(config: Config, csharpsrc2cpg: CSharpSrc2Cpg): Unit = {
- val absPath = Paths.get(config.inputPath).toAbsolutePath.toString
- if (Environment.pathExists(absPath)) {
- csharpsrc2cpg.run(config.withInputPath(absPath))
- } else {
- logger.warn(s"Given path '$absPath' does not exist, skipping")
+ if (config.serverMode) { startup() }
+ else {
+ val absPath = Paths.get(config.inputPath).toAbsolutePath.toString
+ if (Environment.pathExists(absPath)) {
+ csharpsrc2cpg.run(config.withInputPath(absPath))
+ } else {
+ logger.warn(s"Given path '$absPath' does not exist, skipping")
+ }
  }
  }
 

joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/CSharp2CpgHTTPServerTests.scala

@@ -0,0 +1,78 @@
+package io.joern.csharpsrc2cpg.io
+
+import better.files.File
+import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture
+import io.joern.x2cpg.utils.server.FrontendHTTPClient
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
+import io.shiftleft.semanticcpg.language.*
+import org.scalatest.BeforeAndAfterAll
+
+import scala.collection.parallel.CollectionConverters.RangeIsParallelizable
+import scala.util.Failure
+import scala.util.Success
+
+class CSharp2CpgHTTPServerTests extends CSharpCode2CpgFixture with BeforeAndAfterAll {
+
+ private var port: Int = -1
+
+ private def newProjectUnderTest(index: Option[Int] = None): File = {
+ val dir = File.newTemporaryDirectory("csharp2cpgTestsHttpTest")
+ val file = dir / "main.cs"
+ file.createIfNotExists(createParents = true)
+ val indexStr = index.map(_.toString).getOrElse("")
+ file.writeText(basicBoilerplate(s"Console.WriteLine($indexStr);"))
+ file.deleteOnExit()
+ dir.deleteOnExit()
+ }
+
+ override def beforeAll(): Unit = {
+ // Start server
+ port = io.joern.csharpsrc2cpg.Main.startup()
+ }
+
+ override def afterAll(): Unit = {
+ // Stop server
+ io.joern.csharpsrc2cpg.Main.stop()
+ }
+
+ "Using csharp2cpg in server mode" should {
+ "build CPGs correctly (single test)" in {
+ val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest()
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain("Main")
+ cpg.call.code.l shouldBe List("Console.WriteLine()")
+ }
+ }
+
+ "build CPGs correctly (multi-threaded test)" in {
+ (0 until 10).par.foreach { index =>
+ val cpgOutFile = File.newTemporaryFile("csharp2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest(Some(index))
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain("Main")
+ cpg.call.code.l shouldBe List(s"Console.WriteLine($index)")
+ }
+ }
+ }
+ }
+
+}

joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/ProjectParseTests.scala

@@ -1,11 +1,11 @@
 package io.joern.csharpsrc2cpg.io
 
 import better.files.File
-import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary
+import io.joern.csharpsrc2cpg.CSharpSrc2Cpg
+import io.joern.csharpsrc2cpg.Config
 import io.joern.csharpsrc2cpg.passes.AstCreationPass
 import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture
 import io.joern.csharpsrc2cpg.utils.DotNetAstGenRunner
-import io.joern.csharpsrc2cpg.{CSharpSrc2Cpg, Config}
 import io.joern.x2cpg.X2Cpg.newEmptyCpg
 import io.joern.x2cpg.utils.Report
 import io.shiftleft.codepropertygraph.generated.Cpg

joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/Main.scala

@@ -3,6 +3,7 @@ package io.joern.gosrc2cpg
 import io.joern.gosrc2cpg.Frontend.*
 import io.joern.x2cpg.astgen.AstGenConfig
 import io.joern.x2cpg.{X2CpgConfig, X2CpgMain}
+import io.joern.x2cpg.utils.server.FrontendHTTPServer
 import scopt.OParser
 
 import java.nio.file.Paths
@@ -42,10 +43,15 @@ object Frontend {
 
 }
 
-object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) {
+object Main extends X2CpgMain(cmdLineParser, new GoSrc2Cpg()) with FrontendHTTPServer[Config, GoSrc2Cpg] {
+
+ override protected def newDefaultConfig(): Config = Config()
 
  def run(config: Config, gosrc2cpg: GoSrc2Cpg): Unit = {
- val absPath = Paths.get(config.inputPath).toAbsolutePath.toString
- gosrc2cpg.run(config.withInputPath(absPath))
+ if (config.serverMode) { startup() }
+ else {
+ val absPath = Paths.get(config.inputPath).toAbsolutePath.toString
+ gosrc2cpg.run(config.withInputPath(absPath))
+ }
  }
 }

joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/io/GoSrc2CpgHTTPServerTests.scala

@@ -0,0 +1,84 @@
+package io.joern.go2cpg.io
+
+import better.files.File
+import io.joern.x2cpg.utils.server.FrontendHTTPClient
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
+import io.shiftleft.semanticcpg.language.*
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+import scala.collection.parallel.CollectionConverters.RangeIsParallelizable
+import scala.util.Failure
+import scala.util.Success
+
+class GoSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
+
+ private var port: Int = -1
+
+ private def newProjectUnderTest(index: Option[Int] = None): File = {
+ val dir = File.newTemporaryDirectory("gosrc2cpgTestsHttpTest")
+ val file = dir / "main.go"
+ file.createIfNotExists(createParents = true)
+ val indexStr = index.map(_.toString).getOrElse("")
+ file.writeText(s"""
+ |package main
+ |func main$indexStr() {
+ | print("Hello World!")
+ |}
+ |""".stripMargin)
+ file.deleteOnExit()
+ dir.deleteOnExit()
+ }
+
+ override def beforeAll(): Unit = {
+ // Start server
+ port = io.joern.gosrc2cpg.Main.startup()
+ }
+
+ override def afterAll(): Unit = {
+ // Stop server
+ io.joern.gosrc2cpg.Main.stop()
+ }
+
+ "Using gosrc2cpg in server mode" should {
+ "build CPGs correctly (single test)" in {
+ val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest()
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain("main")
+ cpg.call.code.l shouldBe List("""print("Hello World!")""")
+ }
+ }
+
+ "build CPGs correctly (multi-threaded test)" in {
+ (0 until 10).par.foreach { index =>
+ val cpgOutFile = File.newTemporaryFile("gosrc2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest(Some(index))
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain(s"main$index")
+ cpg.call.code.l shouldBe List("""print("Hello World!")""")
+ }
+ }
+ }
+ }
+
+}

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/Main.scala

@@ -2,11 +2,16 @@ package io.joern.javasrc2cpg
 
 import io.joern.javasrc2cpg.Frontend.*
 import io.joern.javasrc2cpg.jpastprinter.JavaParserAstPrinter
+import io.joern.x2cpg.X2CpgConfig
+import io.joern.x2cpg.X2CpgMain
 import io.joern.x2cpg.frontendspecific.javasrc2cpg
-import io.joern.x2cpg.{X2CpgConfig, X2CpgMain}
-import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig}
+import io.joern.x2cpg.passes.frontend.TypeRecoveryParserConfig
+import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
+import io.joern.x2cpg.utils.server.FrontendHTTPServer
 import scopt.OParser
 
+import java.util.concurrent.ExecutorService
+
 /** Command line configuration parameters
  */
 final case class Config(
@@ -133,7 +138,9 @@ private object Frontend {
  }
 }
 
-object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) {
+object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) with FrontendHTTPServer[Config, JavaSrc2Cpg] {
+
+ override protected def newDefaultConfig(): Config = Config()
 
  override def main(args: Array[String]): Unit = {
  // TODO: This is a hack to allow users to use the "--show-env" option without having
@@ -146,14 +153,14 @@ object Main extends X2CpgMain(cmdLineParser, new JavaSrc2Cpg()) {
  }
 
  def run(config: Config, javasrc2Cpg: JavaSrc2Cpg): Unit = {
- if (config.showEnv) {
- JavaSrc2Cpg.showEnv()
- } else if (config.dumpJavaparserAsts) {
- JavaParserAstPrinter.printJpAsts(config)
- } else {
- javasrc2Cpg.run(config)
+ config match {
+ case c if c.serverMode => startup()
+ case c if c.showEnv => JavaSrc2Cpg.showEnv()
+ case c if c.dumpJavaparserAsts => JavaParserAstPrinter.printJpAsts(c)
+ case _ => javasrc2Cpg.run(config)
  }
  }
 
  def getCmdLineParser = cmdLineParser
+
 }

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/io/JavaSrc2CpgHTTPServerTests.scala

@@ -0,0 +1,85 @@
+package io.joern.javasrc2cpg.io
+
+import better.files.File
+import io.joern.x2cpg.utils.server.FrontendHTTPClient
+import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
+import io.shiftleft.semanticcpg.language.*
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+import scala.collection.parallel.CollectionConverters.RangeIsParallelizable
+import scala.util.Failure
+import scala.util.Success
+
+class JavaSrc2CpgHTTPServerTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
+
+ private var port: Int = -1
+
+ private def newProjectUnderTest(index: Option[Int] = None): File = {
+ val dir = File.newTemporaryDirectory("javasrc2cpgTestsHttpTest")
+ val file = dir / "Main.java"
+ file.createIfNotExists(createParents = true)
+ val indexStr = index.map(_.toString).getOrElse("")
+ file.writeText(s"""
+ |class HelloWorld {
+ | public static void main(String[] args) {
+ | System.out.println($indexStr);
+ | }
+ |}
+ |""".stripMargin)
+ file.deleteOnExit()
+ dir.deleteOnExit()
+ }
+
+ override def beforeAll(): Unit = {
+ // Start server
+ port = io.joern.javasrc2cpg.Main.startup()
+ }
+
+ override def afterAll(): Unit = {
+ // Stop server
+ io.joern.javasrc2cpg.Main.stop()
+ }
+
+ "Using javasrc2cpg in server mode" should {
+ "build CPGs correctly (single test)" in {
+ val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest()
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain("main")
+ cpg.call.code.l should contain("System.out.println()")
+ }
+ }
+
+ "build CPGs correctly (multi-threaded test)" in {
+ (0 until 10).par.foreach { index =>
+ val cpgOutFile = File.newTemporaryFile("javasrc2cpg.bin")
+ cpgOutFile.deleteOnExit()
+ val projectUnderTest = newProjectUnderTest(Some(index))
+ val input = projectUnderTest.path.toAbsolutePath.toString
+ val output = cpgOutFile.toString
+ val client = FrontendHTTPClient(port)
+ val req = client.buildRequest(Array(s"input=$input", s"output=$output"))
+ client.sendRequest(req) match {
+ case Failure(exception) => fail(exception.getMessage)
+ case Success(out) =>
+ out shouldBe output
+ val cpg = CpgLoader.load(output)
+ cpg.method.name.l should contain("main")
+ cpg.call.code.l should contain(s"System.out.println($index)")
+ }
+ }
+ }
+ }
+
+}