diff --git a/build.sbt b/build.sbt index c74f95905b1f..88a537196206 100644 --- a/build.sbt +++ b/build.sbt @@ -48,9 +48,9 @@ lazy val aggregatedProjects: Seq[ProjectReference] = Seq( ) ThisBuild / libraryDependencies ++= Seq( - "org.slf4j" % "slf4j-api" % "2.0.7", - "org.apache.logging.log4j" % "log4j-slf4j2-impl" % "2.20.0" % Optional, - "org.apache.logging.log4j" % "log4j-core" % "2.20.0" % Optional + "org.slf4j" % "slf4j-api" % Versions.slf4j, + "org.apache.logging.log4j" % "log4j-slf4j2-impl" % Versions.log4j % Optional, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j % Optional // `Optional` means "not transitive", but still included in "stage/lib" ) @@ -110,6 +110,6 @@ publish / skip := true // don't publish the root project // Avoids running root tasks on the benchmarks project lazy val root = project .in(file(".")) - .aggregate(aggregatedProjects*) + .aggregate(aggregatedProjects *) ThisBuild / Test / packageBin / publishArtifact := true diff --git a/console/build.sbt b/console/build.sbt index 5a5ad8c0223e..4bb3338584ae 100644 --- a/console/build.sbt +++ b/console/build.sbt @@ -2,14 +2,6 @@ name := "console" enablePlugins(JavaAppPackaging) -val ScoptVersion = "4.1.0" -val CaskVersion = "0.9.2" -val CirceVersion = "0.14.6" -val ZeroturnaroundVersion = "1.17" -val OsLibVersion = "0.9.3" -val PprintVersion = "0.8.1" -val CommonsLangVersion = "3.14.0" - dependsOn( Projects.semanticcpg, Projects.macros, @@ -25,15 +17,13 @@ dependsOn( libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "com.michaelpollmeier" %% "scala-repl-pp-server" % Versions.scalaReplPP, - "com.github.scopt" %% "scopt" % ScoptVersion, - "org.typelevel" %% "cats-effect" % Versions.cats, - "io.circe" %% "circe-generic" % CirceVersion, - "io.circe" %% "circe-parser" % CirceVersion, - "org.zeroturnaround" % "zt-zip" % ZeroturnaroundVersion, - "com.lihaoyi" %% "os-lib" % OsLibVersion, - "com.lihaoyi" %% "pprint" % PprintVersion, - "com.lihaoyi" %% "cask" % CaskVersion, - "org.apache.commons" % "commons-lang3" % CommonsLangVersion, + "com.github.scopt" %% "scopt" % Versions.scopt, + "org.typelevel" %% "cats-effect" % Versions.catsEffect, + "org.zeroturnaround" % "zt-zip" % Versions.zeroTurnaround, + "com.lihaoyi" %% "os-lib" % Versions.osLib, + "com.lihaoyi" %% "pprint" % Versions.pPrint, + "com.lihaoyi" %% "cask" % Versions.cask, + "org.apache.commons" % "commons-lang3" % Versions.commonsLang, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/dataflowengineoss/build.sbt b/dataflowengineoss/build.sbt index 1820abbbfd86..02b1c18410a6 100644 --- a/dataflowengineoss/build.sbt +++ b/dataflowengineoss/build.sbt @@ -5,11 +5,10 @@ dependsOn(Projects.semanticcpg, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( "org.antlr" % "antlr4" % Versions.antlr, "org.antlr" % "antlr4-runtime" % Versions.antlr, - "io.circe" %% "circe-core" % Versions.circe, - "io.circe" %% "circe-generic" % Versions.circe, - "io.circe" %% "circe-parser" % Versions.circe, + "com.lihaoyi" %% "upickle" % Versions.upickle, + "com.lihaoyi" %% "ujson" % Versions.upickle, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4" + "org.scala-lang.modules" %% "scala-parallel-collections" % Versions.scalaParallel ) enablePlugins(Antlr4Plugin) diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala index 873071150a08..5e387803a228 100644 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/slicing/package.scala @@ -1,22 +1,18 @@ package io.joern.dataflowengineoss import better.files.File -import io.circe.{Decoder, Encoder, HCursor, Json} import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import overflowdb.PropertyKey +import upickle.default.* import java.util.concurrent.{ExecutorService, Executors} import java.util.regex.Pattern package object slicing { - import cats.syntax.functor.* - import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} - import io.circe.syntax.EncoderOps - trait BaseConfig[T <: BaseConfig[T]] { var inputPath: File = File("cpg.bin") @@ -150,14 +146,11 @@ package object slicing { * @param edges * a map linking nodes with their edges. */ - case class DataFlowSlice(nodes: Set[SliceNode], edges: Set[SliceEdge]) extends ProgramSlice { - def toJson: String = this.asJson.toString() + case class DataFlowSlice(nodes: Set[SliceNode], edges: Set[SliceEdge]) extends ProgramSlice derives ReadWriter { - def toJsonPretty: String = this.asJson.spaces2 - } + def toJson: String = write(this) - implicit val encodeDataFlowSlice: Encoder[DataFlowSlice] = Encoder.instance { case DataFlowSlice(nodes, edges) => - Json.obj("nodes" -> nodes.asJson, "edges" -> edges.asJson) + def toJsonPretty: String = write(this, indent = 2, sortKeys = true) } case class SliceNode( @@ -170,28 +163,9 @@ package object slicing { parentFile: String = "", lineNumber: Option[Integer] = None, columnNumber: Option[Integer] = None - ) - - implicit val encodeSliceNode: Encoder[SliceNode] = Encoder.instance { - case SliceNode(id, label, name, code, typeFullName, parentMethod, parentFile, lineNumber, columnNumber) => - Json.obj( - "id" -> id.asJson, - "label" -> label.asJson, - "name" -> name.asJson, - "code" -> code.asJson, - "typeFullName" -> typeFullName.asJson, - "parentMethod" -> parentMethod.asJson, - "parentFile" -> parentFile.asJson, - "lineNumber" -> lineNumber.asJson, - "columnNumber" -> columnNumber.asJson - ) - } + ) derives ReadWriter - case class SliceEdge(src: Long, dst: Long, label: String) - - implicit val encodeSliceEdge: Encoder[SliceEdge] = Encoder.instance { case SliceEdge(src, dst, label) => - Json.obj("src" -> src.asJson, "dst" -> dst.asJson, "label" -> label.asJson) - } + case class SliceEdge(src: Long, dst: Long, label: String) derives ReadWriter /** A usage slice of an object at the start of its definition until its final usage. * @@ -209,7 +183,7 @@ package object slicing { definedBy: Option[DefComponent], invokedCalls: List[ObservedCall], argToCalls: List[ObservedCallWithArgPos] - ) { + ) derives ReadWriter { override def toString: String = s"{tgt: $targetObj${definedBy.map(p => s" = $p").getOrElse("")}, " + s"inv: [${invokedCalls.mkString(",")}], " + @@ -217,21 +191,6 @@ package object slicing { s"}" } - implicit val decodeObjectUsageSlice: Decoder[ObjectUsageSlice] = - (c: HCursor) => - for { - x <- c.downField("targetObj").as[DefComponent] - p <- c.downField("definedBy").as[Option[DefComponent]] - r <- c.downField("invokedCalls").as[List[ObservedCall]] - a <- c.downField("argToCalls").as[List[ObservedCallWithArgPos]] - } yield { - ObjectUsageSlice(x, p, r, a) - } - implicit val encodeObjectUsageSlice: Encoder[ObjectUsageSlice] = - Encoder.instance { case ObjectUsageSlice(c, p, r, a) => - Json.obj("targetObj" -> c.asJson, "definedBy" -> p.asJson, "invokedCalls" -> r.asJson, "argToCalls" -> a.asJson) - } - /** Packages the object usage slices along with the method source code. * * @param code @@ -250,31 +209,7 @@ package object slicing { slices: Set[ObjectUsageSlice], lineNumber: Option[Int] = None, columnNumber: Option[Int] = None - ) - - implicit val decodeMethodUsageSlice: Decoder[MethodUsageSlice] = - (c: HCursor) => - for { - code <- c.downField("code").as[String] - fn <- c.downField("fullName").as[String] - fln <- c.downField("fileName").as[String] - ss <- c.downField("slices").as[Set[ObjectUsageSlice]] - lin <- c.downField("lineNumber").as[Option[Int]] - col <- c.downField("columnNumber").as[Option[Int]] - } yield { - MethodUsageSlice(code, fn, fln, ss, lin, col) - } - implicit val encodeMethodUsageSlice: Encoder[MethodUsageSlice] = - Encoder.instance { case MethodUsageSlice(a, b, c, d, e, f) => - Json.obj( - "code" -> a.asJson, - "fullName" -> b.asJson, - "fileName" -> c.asJson, - "slices" -> d.asJson, - "lineNumber" -> e.asJson, - "columnNumber" -> f.asJson - ) - } + ) derives ReadWriter /** Represents a source of data-generation, i.e., where data is defined and can be assigned to some variable or used * in an argument. @@ -303,9 +238,7 @@ package object slicing { columnNumber: Option[Int] = None, label: String = "LOCAL" ) extends DefComponent - - implicit val localDefDecoder: Decoder[LocalDef] = deriveDecoder[LocalDef] - implicit val localDefEncoder: Encoder[LocalDef] = deriveEncoder[LocalDef] + derives ReadWriter /** Represents a literal. */ @@ -316,9 +249,7 @@ package object slicing { columnNumber: Option[Int] = None, label: String = "LITERAL" ) extends DefComponent - - implicit val literalDefDecoder: Decoder[LiteralDef] = deriveDecoder[LiteralDef] - implicit val literalDefEncoder: Encoder[LiteralDef] = deriveEncoder[LiteralDef] + derives ReadWriter /** Represents data introduced via a parameter. * @@ -332,13 +263,11 @@ package object slicing { lineNumber: Option[Int] = None, columnNumber: Option[Int] = None, label: String = "PARAM" - ) extends DefComponent { + ) extends DefComponent + derives ReadWriter { override def toString: String = super.toString + s" @ pos #$position" } - implicit val paramDefDecoder: Decoder[ParamDef] = deriveDecoder[ParamDef] - implicit val paramDefEncoder: Encoder[ParamDef] = deriveEncoder[ParamDef] - /** Represents data introduced by the return value of a call. * * @param resolvedMethod @@ -351,14 +280,12 @@ package object slicing { lineNumber: Option[Int] = None, columnNumber: Option[Int] = None, label: String = "CALL" - ) extends DefComponent { + ) extends DefComponent + derives ReadWriter { override def toString: String = super.toString + resolvedMethod.map(s => s" @ $s").getOrElse("") } - implicit val callDefDecoder: Decoder[CallDef] = deriveDecoder[CallDef] - implicit val callDefEncoder: Encoder[CallDef] = deriveEncoder[CallDef] - - /** Representds data introduced by an unhandled data structure. + /** Represents data introduced by an unhandled data structure. */ case class UnknownDef( name: String, @@ -367,29 +294,29 @@ package object slicing { columnNumber: Option[Int] = None, label: String = "UNKNOWN" ) extends DefComponent - - implicit val unknownDefDecoder: Decoder[UnknownDef] = deriveDecoder[UnknownDef] - implicit val unknownDefEncoder: Encoder[UnknownDef] = deriveEncoder[UnknownDef] + derives ReadWriter // The following encoders make sure the object does follow ClassName: { properties ... } format but instead // is just { properties }. This makes it less automatically serializable but we have `label` to encode classes. - implicit val encodeDefComponent: Encoder[DefComponent] = Encoder.instance { - case local @ LocalDef(_, _, _, _, _) => local.asJson - case literal @ LiteralDef(_, _, _, _, _) => literal.asJson - case call @ CallDef(_, _, _, _, _, _) => call.asJson - case param @ ParamDef(_, _, _, _, _, _) => param.asJson - case unknown @ UnknownDef(_, _, _, _, _) => unknown.asJson - } - - implicit val decodeDefComponent: Decoder[DefComponent] = - List[Decoder[DefComponent]]( - Decoder[LocalDef].widen, - Decoder[LiteralDef].widen, - Decoder[CallDef].widen, - Decoder[ParamDef].widen, - Decoder[UnknownDef].widen - ).reduceLeft(_ or _) + implicit val defComponentRw: ReadWriter[DefComponent] = readwriter[ujson.Value].bimap[DefComponent]( + { + case local: LocalDef => write(local) + case literal: LiteralDef => write(literal) + case call: CallDef => write(call) + case param: ParamDef => write(param) + case unknown: UnknownDef => write(unknown) + }, + json => + json("label").strOpt match { + case Some("LOCAL") => read[LocalDef](json) + case Some("LITERAL") => read[LiteralDef](json) + case Some("CALL") => read[CallDef](json) + case Some("PARAM") => read[ParamDef](json) + case Some("UNKNOWN") => read[UnknownDef](json) + case _ => throw new RuntimeException(s"Unable to deserialize the given `DefComponent`: $json") + } + ) object DefComponent { @@ -471,30 +398,7 @@ package object slicing { lineNumber: Option[Int] = None, columnNumber: Option[Int] = None ) extends UsedCall(callName, resolvedMethod, paramTypes, returnType, lineNumber, columnNumber) - - implicit val decodeObservedCall: Decoder[ObservedCall] = - (c: HCursor) => - for { - x <- c.downField("callName").as[String] - m <- c.downField("resolvedMethod").as[Option[String]] - p <- c.downField("paramTypes").as[List[String]] - r <- c.downField("returnType").as[String] - lin <- c.downField("lineNumber").as[Option[Int]] - col <- c.downField("columnNumber").as[Option[Int]] - } yield { - ObservedCall(x, m, p, r, lin, col) - } - implicit val encodeObservedCall: Encoder[ObservedCall] = - Encoder.instance { case ObservedCall(c, m, p, r, lin, col) => - Json.obj( - "callName" -> c.asJson, - "resolvedMethod" -> m.asJson, - "paramTypes" -> p.asJson, - "returnType" -> r.asJson, - "lineNumber" -> lin.asJson, - "columnNumber" -> col.asJson - ) - } + derives ReadWriter /** Extends observed call with a specific argument in mind. * @@ -529,53 +433,38 @@ package object slicing { ) } - implicit val decodeObservedCallWithArgPos: Decoder[ObservedCallWithArgPos] = - (c: HCursor) => - for { - x <- c.downField("callName").as[String] - m <- c.downField("resolvedMethod").as[Option[String]] - p <- c.downField("paramTypes").as[List[String]] - r <- c.downField("returnType").as[String] - lin <- c.downField("lineNumber").as[Option[Int]] - col <- c.downField("columnNumber").as[Option[Int]] - } yield { - val pos = c.downField("position").as[Int] match { - case Left(_) => - c.downField("position").as[String] match { - case Left(err) => - throw new RuntimeException( - "Unable to decode `position` as the field is neither a string nor an integer", - err - ) - case Right(argName) => Left(argName) - } - case Right(argIdx) => Right(argIdx) + implicit val observedCallWithArgPosRw: ReadWriter[ObservedCallWithArgPos] = + readwriter[ujson.Value].bimap[ObservedCallWithArgPos]( + x => { + val position = x.position match { + case Left(str) => ujson.Str(str) + case Right(num) => ujson.Num(num) } - ObservedCallWithArgPos(x, m, p, r, pos, lin, col) + ujson.Obj( + "callName" -> x.callName, + "resolvedMethod" -> x.resolvedMethod, + "paramTypes" -> x.paramTypes, + "returnType" -> x.returnType, + "lineNumber" -> x.lineNumber, + "columnNumber" -> x.columnNumber, + "position" -> position + ) + }, + json => { + val position = + if (json("position").strOpt.isDefined) Left(json("position").str) + else Right(json("position").num.toInt) + ObservedCallWithArgPos( + json("callName").str, + read[Option[String]](json("resolvedMethod")), + read[List[String]](json("paramTypes")), + json("returnType").str, + position, + read[Option[Int]](json("lineNumber")), + read[Option[Int]](json("columnNumber")) + ) } - implicit val encodeObservedCallWithArgPos: Encoder[ObservedCallWithArgPos] = - Encoder.instance { case ObservedCallWithArgPos(c, m, p, r, a, lin, col) => - Json.obj( - "callName" -> c.asJson, - "resolvedMethod" -> m.asJson, - "paramTypes" -> p.asJson, - "returnType" -> r.asJson, - "position" -> (a match { - case Left(argName) => argName.asJson - case Right(argIdx) => argIdx.asJson - }), - "lineNumber" -> lin.asJson, - "columnNumber" -> col.asJson - ) - } - - implicit val encodeUsedCall: Encoder[UsedCall] = Encoder.instance { - case oc @ ObservedCall(_, _, _, _, _, _) => oc.asJson - case oca @ ObservedCallWithArgPos(_, _, _, _, _, _, _) => oca.asJson - } - - implicit val decodeUsedCall: Decoder[UsedCall] = - List[Decoder[UsedCall]](Decoder[ObservedCall].widen, Decoder[ObservedCallWithArgPos].widen).reduceLeft(_ or _) + ) /** Describes types defined within the application. * @@ -593,31 +482,7 @@ package object slicing { fileName: String = "", lineNumber: Option[Int] = None, columnNumber: Option[Int] = None - ) - - implicit val decodeUserDefinedType: Decoder[UserDefinedType] = - (c: HCursor) => - for { - n <- c.downField("name").as[String] - f <- c.downField("fields").as[List[LocalDef]] - p <- c.downField("procedures").as[List[ObservedCall]] - fn <- c.downField("fileName").as[String] - lin <- c.downField("lineNumber").as[Option[Int]] - col <- c.downField("columnNumber").as[Option[Int]] - } yield { - UserDefinedType(n, f, p, fn, lin, col) - } - implicit val encodeUserDefinedType: Encoder[UserDefinedType] = - Encoder.instance { case UserDefinedType(n, f, p, fn, lin, col) => - Json.obj( - "name" -> n.asJson, - "fields" -> f.asJson, - "procedures" -> p.asJson, - "fileName" -> fn.asJson, - "lineNumber" -> lin.asJson, - "columnNumber" -> col.asJson - ) - } + ) derives ReadWriter /** The program usage slices and UDTs. * @@ -627,22 +492,11 @@ package object slicing { * the UDTs. */ case class ProgramUsageSlice(objectSlices: List[MethodUsageSlice], userDefinedTypes: List[UserDefinedType]) - extends ProgramSlice { + extends ProgramSlice derives ReadWriter { - def toJson: String = this.asJson.toString() + def toJson: String = upickle.default.write(this) - def toJsonPretty: String = this.asJson.spaces2 + def toJsonPretty: String = upickle.default.write(this, indent = 2, sortKeys = true) } - implicit val decodeProgramUsageSlice: Decoder[ProgramUsageSlice] = - (c: HCursor) => - for { - o <- c.downField("objectSlices").as[List[MethodUsageSlice]] - u <- c.downField("userDefinedTypes").as[List[UserDefinedType]] - } yield { - ProgramUsageSlice(o, u) - } - implicit val encodeProgramUsageSlice: Encoder[ProgramUsageSlice] = Encoder.instance { - case ProgramUsageSlice(os, udts) => Json.obj("objectSlices" -> os.asJson, "userDefinedTypes" -> udts.asJson) - } } diff --git a/joern-cli/build.sbt b/joern-cli/build.sbt index 7a1f19de94bb..88b17ef355bd 100644 --- a/joern-cli/build.sbt +++ b/joern-cli/build.sbt @@ -5,8 +5,9 @@ dependsOn(Projects.console, Projects.console % "test->test", Projects.dataflowen libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "com.lihaoyi" %% "requests" % Versions.requests, - "com.github.scopt" %% "scopt" % "4.1.0", - "org.reflections" % "reflections" % "0.10.2", + "com.lihaoyi" %% "upickle" % Versions.upickle, + "com.github.scopt" %% "scopt" % Versions.scopt, + "org.reflections" % "reflections" % Versions.reflection, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/c2cpg/build.sbt b/joern-cli/frontends/c2cpg/build.sbt index bedf789e36da..23bf0bd4c7dc 100644 --- a/joern-cli/frontends/c2cpg/build.sbt +++ b/joern-cli/frontends/c2cpg/build.sbt @@ -7,12 +7,12 @@ dependsOn( ) libraryDependencies ++= Seq( - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", - "org.eclipse.platform" % "org.eclipse.core.resources" % "3.20.100", - "org.eclipse.platform" % "org.eclipse.text" % "3.14.0", + "org.scala-lang.modules" %% "scala-parallel-collections" % Versions.scalaParallel, + "org.eclipse.platform" % "org.eclipse.core.resources" % Versions.eclipseCore, + "org.eclipse.platform" % "org.eclipse.text" % Versions.eclipseText, // see note in readme re self-publishing cdt-core - "io.joern" % "eclipse-cdt-core" % "8.4.0.202401242025", - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "io.joern" % "eclipse-cdt-core" % Versions.eclipseCdt, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) dependencyOverrides ++= Seq( diff --git a/joern-cli/frontends/csharpsrc2cpg/build.sbt b/joern-cli/frontends/csharpsrc2cpg/build.sbt index 29e967406e88..8fd9324f79cc 100644 --- a/joern-cli/frontends/csharpsrc2cpg/build.sbt +++ b/joern-cli/frontends/csharpsrc2cpg/build.sbt @@ -21,7 +21,6 @@ astGenVersion := appProperties.value.getString("csharpsrc2cpg.dotnetastgen_versi libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.0", "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/ghidra2cpg/build.sbt b/joern-cli/frontends/ghidra2cpg/build.sbt index 9fe9d9041976..2a8dba0adc58 100644 --- a/joern-cli/frontends/ghidra2cpg/build.sbt +++ b/joern-cli/frontends/ghidra2cpg/build.sbt @@ -3,9 +3,9 @@ name := "ghidra2cpg" dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.joern" % "ghidra" % "11.0_PUBLIC_20231222-2", - "com.github.scopt" %% "scopt" % "4.1.0", - "commons-io" % "commons-io" % "2.16.0", + "io.joern" % "ghidra" % Versions.ghidra, + "com.github.scopt" %% "scopt" % Versions.scopt, + "commons-io" % "commons-io" % Versions.commonsIo, "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "io.shiftleft" %% "codepropertygraph-protos" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt index 5feaff4377b0..0db7ea5024f0 100644 --- a/joern-cli/frontends/gosrc2cpg/build.sbt +++ b/joern-cli/frontends/gosrc2cpg/build.sbt @@ -1,7 +1,8 @@ +import com.typesafe.config.{Config, ConfigFactory} +import versionsort.VersionHelper + import scala.sys.process.stringToProcess import scala.util.Try -import versionsort.VersionHelper -import com.typesafe.config.{Config, ConfigFactory} name := "gosrc2cpg" @@ -10,11 +11,7 @@ dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, - "com.lihaoyi" %% "os-lib" % "0.9.3", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.0", - "io.circe" %% "circe-core" % Versions.circe, - "io.circe" %% "circe-generic" % Versions.circe, - "io.circe" %% "circe-parser" % Versions.circe + "com.lihaoyi" %% "os-lib" % Versions.osLib ) scalacOptions ++= Seq( diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala index 414db4f34c4f..5d458be064cf 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/model/GoMod.scala @@ -1,10 +1,10 @@ package io.joern.gosrc2cpg.model -import io.circe.Decoder.Result -import io.circe.{Decoder, HCursor} import io.joern.gosrc2cpg.Config import io.joern.gosrc2cpg.utils.UtilityConstants.fileSeparateorPattern +import upickle.default.* +import java.util import java.util.Set import java.util.concurrent.ConcurrentSkipListSet import scala.util.control.Breaks.* @@ -63,7 +63,31 @@ class GoModHelper(config: Option[Config] = None, meta: Option[GoMod] = None) { } } -case class GoMod(fileFullPath: String, module: GoModModule, dependencies: List[GoModDependency]) +case class GoMod( + @upickle.implicits.key("node_filename") fileFullPath: String = "", + @upickle.implicits.key("Module") module: GoModModule, + @upickle.implicits.key("dependencies") dependencies: List[GoModDependency] = List.empty +) derives ReadWriter + +implicit val goModModuleRw: ReadWriter[GoModModule] = readwriter[ujson.Value].bimap[GoModModule]( + x => + ujson.Obj( + "Name" -> x.name, + "node_line_no" -> x.lineNo.getOrElse(-1), + "node_col_no" -> x.colNo.getOrElse(-1), + "node_line_no_end" -> x.endLineNo.getOrElse(-1), + "node_col_no_end" -> x.endColNo.getOrElse(-1) + ), + json => + GoModModule( + name = json("Name").strOpt.getOrElse(""), + lineNo = json("node_line_no").numOpt.map(_.toInt), + colNo = json("node_col_no").numOpt.map(_.toInt), + endLineNo = json("node_line_no_end").numOpt.map(_.toInt), + endColNo = json("node_col_no_end").numOpt.map(_.toInt) + ) +) + case class GoModModule( name: String, lineNo: Option[Int] = None, @@ -71,73 +95,48 @@ case class GoModModule( endLineNo: Option[Int] = None, endColNo: Option[Int] = None ) + +implicit val goModDependencyRw: ReadWriter[GoModDependency] = readwriter[ujson.Value].bimap[GoModDependency]( + x => + ujson.Obj( + "Module" -> x.module, + "Version" -> x.version, + "Indirect" -> x.indirect, + "node_line_no" -> x.lineNo.getOrElse(-1), + "node_col_no" -> x.colNo.getOrElse(-1), + "node_line_no_end" -> x.endLineNo.getOrElse(-1), + "node_col_no_end" -> x.endColNo.getOrElse(-1) + ), + json => + GoModDependency( + module = json("Module").strOpt.getOrElse(""), + version = json("Version").strOpt.getOrElse(""), + indirect = json("Indirect").boolOpt.getOrElse(false), + lineNo = json("node_line_no").numOpt.map(_.toInt), + colNo = json("node_col_no").numOpt.map(_.toInt), + endLineNo = json("node_line_no_end").numOpt.map(_.toInt), + endColNo = json("node_col_no_end").numOpt.map(_.toInt) + ) +) + case class GoModDependency( module: String, version: String, - indirect: Boolean, - var beingUsed: Boolean, + indirect: Boolean = false, + var beingUsed: Boolean = false, lineNo: Option[Int] = None, colNo: Option[Int] = None, endLineNo: Option[Int] = None, endColNo: Option[Int] = None, - usedPackages: Set[String] = new ConcurrentSkipListSet[String]() + usedPackages: util.Set[String] = new ConcurrentSkipListSet[String]() ) -object CirceEnDe { - implicit val decoderModModule: Decoder[GoModModule] = new Decoder[GoModModule] { - override def apply(c: HCursor): Result[GoModModule] = { - val name = c.downField("Name").as[String] - val lineNo = c.downField("node_line_no").as[Int] - val endLineNo = c.downField("node_line_no_end").as[Int] - val colNo = c.downField("node_col_no").as[Int] - val endColNo = c.downField("node_col_no_end").as[Int] - Right( - GoModModule( - name = name.getOrElse(""), - lineNo = lineNo.toOption, - colNo = colNo.toOption, - endLineNo = endLineNo.toOption, - endColNo = endColNo.toOption - ) - ) - } - } - implicit val decoderModDependency: Decoder[GoModDependency] = new Decoder[GoModDependency] { - override def apply(c: HCursor): Result[GoModDependency] = { - val module = c.downField("Module").as[String] - val version = c.downField("Version").as[String] - val indirect = c.downField("Indirect").as[Boolean] - val lineNo = c.downField("node_line_no").as[Int] - val endLineNo = c.downField("node_line_no_end").as[Int] - val colNo = c.downField("node_col_no").as[Int] - val endColNo = c.downField("node_col_no_end").as[Int] - Right( - GoModDependency( - module = module.getOrElse(""), - version = version.getOrElse(""), - indirect = indirect.getOrElse(false), - beingUsed = false, - lineNo = lineNo.toOption, - colNo = colNo.toOption, - endLineNo = endLineNo.toOption, - endColNo = endColNo.toOption - ) - ) - } - } +implicit val javaSetRw: ReadWriter[util.Set[String]] = { + import scala.jdk.CollectionConverters.* - implicit val decoderModMetadata: Decoder[GoMod] = new Decoder[GoMod] { - override def apply(c: HCursor): Result[GoMod] = { - val fileName = c.downField("node_filename").as[String] - val module = c.downField("Module").as[GoModModule] - val dependencies = c.downField("dependencies").as[List[GoModDependency]] - Right( - GoMod( - fileFullPath = fileName.getOrElse(""), - module = module.getOrElse(null), - dependencies = dependencies.getOrElse(List[GoModDependency]()) - ) - ) - } - } + readwriter[ujson.Value] + .bimap[util.Set[String]]( + x => ujson.Arr(x.asScala.map(ujson.Str.apply).toSeq*), + json => json.arr.map(_.str).toSet.asJava + ) } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/GoAstJsonParser.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/GoAstJsonParser.scala index ff15127bb236..683b6983a0bd 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/GoAstJsonParser.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/parser/GoAstJsonParser.scala @@ -1,15 +1,14 @@ package io.joern.gosrc2cpg.parser -import io.circe.parser.* -import io.joern.gosrc2cpg.model.CirceEnDe.decoderModMetadata import io.joern.gosrc2cpg.model.GoMod import io.joern.x2cpg.astgen.ParserResult import io.shiftleft.utils.IOUtils import org.slf4j.LoggerFactory import ujson.Value.Value +import upickle.default.* import java.nio.file.{Path, Paths} -import scala.util.Try +import scala.util.{Try, Success, Failure} object GoAstJsonParser { @@ -26,11 +25,11 @@ object GoAstJsonParser { def readModFile(file: Path): Option[GoMod] = { val jsonContent = IOUtils.readLinesInFile(file).mkString - (decode[GoMod](jsonContent) match { - case Left(error) => - logger.warn(s"Error decoding JSON - '${file.toString}': $error") - Left(error) - case x => x - }).toOption + Try(read[GoMod](jsonContent)) match { + case Failure(error) => + logger.warn(s"Error decoding JSON - '${file.toString}'", error) + None + case Success(x) => Some(x) + } } } diff --git a/joern-cli/frontends/javasrc2cpg/build.sbt b/joern-cli/frontends/javasrc2cpg/build.sbt index 291fb186fd44..130b79c3c0cc 100644 --- a/joern-cli/frontends/javasrc2cpg/build.sbt +++ b/joern-cli/frontends/javasrc2cpg/build.sbt @@ -4,13 +4,13 @@ dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.github.javaparser" % "javaparser-symbol-solver-core" % "3.25.9", + "com.github.javaparser" % "javaparser-symbol-solver-core" % Versions.javaParser, "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, - "org.projectlombok" % "lombok" % "1.18.32", - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", - "org.scala-lang.modules" %% "scala-parser-combinators" % "2.3.0", - "net.lingala.zip4j" % "zip4j" % "2.11.5" + "org.projectlombok" % "lombok" % Versions.lombok, + "org.scala-lang.modules" %% "scala-parallel-collections" % Versions.scalaParallel, + "org.scala-lang.modules" %% "scala-parser-combinators" % Versions.scalaParserCombinators, + "net.lingala.zip4j" % "zip4j" % Versions.zip4j ) enablePlugins(JavaAppPackaging, LauncherJarPlugin) diff --git a/joern-cli/frontends/jimple2cpg/build.sbt b/joern-cli/frontends/jimple2cpg/build.sbt index a506fc986ae9..2ea5dbe57b93 100644 --- a/joern-cli/frontends/jimple2cpg/build.sbt +++ b/joern-cli/frontends/jimple2cpg/build.sbt @@ -3,10 +3,11 @@ name := "jimple2cpg" dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.soot-oss" % "soot" % "4.4.1", - "org.scalatest" %% "scalatest" % Versions.scalatest % Test, - "org.benf" % "cfr" % "0.152", + "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "org.soot-oss" % "soot" % Versions.soot, + "org.typelevel" %% "cats-core" % Versions.catsCore, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test, + "org.benf" % "cfr" % Versions.cfr ) enablePlugins(JavaAppPackaging, LauncherJarPlugin) diff --git a/joern-cli/frontends/jssrc2cpg/build.sbt b/joern-cli/frontends/jssrc2cpg/build.sbt index 51a72f9ff1bf..7ba016d9f494 100644 --- a/joern-cli/frontends/jssrc2cpg/build.sbt +++ b/joern-cli/frontends/jssrc2cpg/build.sbt @@ -19,7 +19,6 @@ astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version") libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.0", "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala index 0502a4e36200..88a8a00bc987 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/PackageJsonParser.scala @@ -1,16 +1,14 @@ package io.joern.jssrc2cpg.utils -import java.nio.file.{Path, Paths} -import org.slf4j.LoggerFactory -import com.fasterxml.jackson.databind.ObjectMapper import io.shiftleft.utils.IOUtils import org.apache.commons.lang3.StringUtils +import org.slf4j.LoggerFactory +import upickle.default.read +import java.nio.file.{Path, Paths} import scala.collection.concurrent.TrieMap -import scala.util.Try -import scala.jdk.CollectionConverters._ -import scala.util.Failure -import scala.util.Success +import scala.jdk.CollectionConverters.* +import scala.util.{Failure, Success, Try} object PackageJsonParser { private val logger = LoggerFactory.getLogger(PackageJsonParser.getClass) @@ -43,38 +41,35 @@ object PackageJsonParser { val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PackageJsonLockFilename)) val lockDeps = Try { - val content = IOUtils.readEntireFile(lockDepsPath) - val objectMapper = new ObjectMapper - val packageJson = objectMapper.readTree(content) + val content = IOUtils.readEntireFile(lockDepsPath) + val packageJson = read[ujson.Obj](content) var depToVersion = Map.empty[String, String] - val dependencyIt = Option(packageJson.get("dependencies")) - .map(_.fields().asScala) - .getOrElse(Iterator.empty) - dependencyIt.foreach { entry => - val depName = entry.getKey - val versionNode = entry.getValue.get("version") - if (versionNode != null) { - depToVersion = depToVersion.updated(depName, versionNode.asText()) - } + val dependencyIt = packageJson.value.get("dependencies").map(_.obj).getOrElse(Map.empty[String, ujson.Value]) + dependencyIt.foreach { + case (depName, value @ ujson.Str(version)) => + depToVersion = depToVersion.updated(depName, version) + case (depName, value @ ujson.Obj(obj)) => + obj.get("version").foreach { version => + depToVersion = depToVersion.updated(depName, version.str) + } + case (depName, value) => + logger.warn(s"Unexpected version structure for dependency $depName: ${value.getClass}") } depToVersion }.toOption // lazy val because we only evaluate this in case no package lock file is available. lazy val deps = Try { - val content = IOUtils.readEntireFile(depsPath) - val objectMapper = new ObjectMapper - val packageJson = objectMapper.readTree(content) + val content = IOUtils.readEntireFile(depsPath) + val packageJson = read[ujson.Obj](content) var depToVersion = Map.empty[String, String] ProjectDependencies .foreach { dependency => - val dependencyIt = Option(packageJson.get(dependency)) - .map(_.fields().asScala) - .getOrElse(Iterator.empty) - dependencyIt.foreach { entry => - depToVersion = depToVersion.updated(entry.getKey, entry.getValue.asText()) + val dependencyIt = packageJson.value.get(dependency).map(_.obj).getOrElse(Map.empty[String, ujson.Value]) + dependencyIt.foreach { case (key, value) => + depToVersion = depToVersion.updated(key, value.str) } } depToVersion diff --git a/joern-cli/frontends/kotlin2cpg/build.sbt b/joern-cli/frontends/kotlin2cpg/build.sbt index fe2ca1d45b84..83032b7c6024 100644 --- a/joern-cli/frontends/kotlin2cpg/build.sbt +++ b/joern-cli/frontends/kotlin2cpg/build.sbt @@ -11,7 +11,7 @@ dependsOn( libraryDependencies ++= Seq( "com.lihaoyi" %% "requests" % Versions.requests, "com.lihaoyi" %% "ujson" % Versions.upickle, - "com.squareup.tools.build" % "maven-archeologist" % "0.0.10", + "com.squareup.tools.build" % "maven-archeologist" % Versions.mavenArcheologist, "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling, "org.jetbrains.kotlin" % "kotlin-stdlib-jdk8" % kotlinVersion, diff --git a/joern-cli/frontends/php2cpg/build.sbt b/joern-cli/frontends/php2cpg/build.sbt index 8f97499dd5e7..d5649619bb86 100644 --- a/joern-cli/frontends/php2cpg/build.sbt +++ b/joern-cli/frontends/php2cpg/build.sbt @@ -16,8 +16,7 @@ libraryDependencies ++= Seq( "com.lihaoyi" %% "upickle" % Versions.upickle, "com.lihaoyi" %% "ujson" % Versions.upickle, "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test, - "io.circe" %% "circe-core" % Versions.circe + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) lazy val phpParseInstallTask = taskKey[Unit]("Install PHP-Parse using PHP Composer") diff --git a/joern-cli/frontends/pysrc2cpg/build.sbt b/joern-cli/frontends/pysrc2cpg/build.sbt index cb9497f777e0..8e42aabbf429 100644 --- a/joern-cli/frontends/pysrc2cpg/build.sbt +++ b/joern-cli/frontends/pysrc2cpg/build.sbt @@ -4,7 +4,7 @@ dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", + "org.scala-lang.modules" %% "scala-parallel-collections" % Versions.scalaParallel, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt index 518bded8fb8c..a9b2c4d79553 100644 --- a/joern-cli/frontends/rubysrc2cpg/build.sbt +++ b/joern-cli/frontends/rubysrc2cpg/build.sbt @@ -4,7 +4,7 @@ dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "org.apache.commons" % "commons-compress" % "1.26.1", // For unpacking Gems with `--download-dependencies` + "org.apache.commons" % "commons-compress" % Versions.commonsCompress, // For unpacking Gems with `--download-dependencies` "org.scalatest" %% "scalatest" % Versions.scalatest % Test, "org.antlr" % "antlr4-runtime" % Versions.antlr ) diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala index dca6df625b86..e1deaf7bbfc7 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala @@ -1,6 +1,5 @@ package io.joern.joerncli -import io.circe.Json import io.joern.joerncli.CpgBasedTool.exitIfInvalid import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames @@ -36,8 +35,7 @@ class BagOfPropertiesForNodes extends EmbeddingGenerator[AstNode, (String, Strin override def hash(label: String): String = label override def vectorToString(vector: Map[(String, String), Double]): String = { - val jsonObj = Json.fromFields(vector.keys.map { case (k, v) => (k, Json.fromString(v)) }) - jsonObj.toString + ujson.write(vector.keys.map { case (k, v) => (k, ujson.Str(v)) }) } } diff --git a/project/Versions.scala b/project/Versions.scala index 419d75bc1da9..cdecda3e3195 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -3,17 +3,46 @@ object Versions { val cpg = parseVersion("cpgVersion") // Dont upgrade antlr to 4.10 or above since those versions require java 11 or higher which // causes problems upstreams. - val antlr = "4.7.2" - val scalatest = "3.2.18" - val cats = "3.5.4" - val json4s = "4.0.7" - val gradleTooling = "8.3" - val circe = "0.14.6" - val requests = "0.8.0" - val upickle = "3.3.0" - val scalaReplPP = "0.1.85" + val antlr = "4.7.2" + val cask = "0.9.2" + val catsCore = "2.10.0" + val catsEffect = "3.5.4" + val cfr = "0.152" + val commonsCompress = "1.26.1" + val commonsIo = "2.16.0" + val commonsLang = "3.14.0" + val commonsText = "1.12.0" + val eclipseCdt = "8.4.0.202401242025" + val eclipseCore = "3.20.100" + val eclipseText = "3.14.0" + val ghidra = "11.0_PUBLIC_20231222-2" + val gradleTooling = "8.3" + val jacksonDatabind = "2.17.0" + val javaParser = "3.25.9" + val json4s = "4.0.7" + val lombok = "1.18.32" + val mavenArcheologist = "0.0.10" + val pPrint = "0.8.1" + val reflection = "0.10.2" + val requests = "0.8.0" + val scalaParallel = "1.0.4" + val scalaParserCombinators = "2.3.0" + val scalaReplPP = "0.1.85" + val scalatest = "3.2.18" + val scopt = "4.1.0" + val soot = "4.4.1" + val slf4j = "2.0.7" + val log4j = "2.20.0" + val upickle = "3.3.0" + val zeroTurnaround = "1.17" + + // Shared with `projects/meta-build.sbt`, which needs to be updated there directly + val betterFiles = "3.9.2" + val javaCc = "7.0.12" + val osLib = "0.9.3" val typeSafeConfig = "1.4.3" val versionSort = "1.0.11" + val zip4j = "2.11.5" private def parseVersion(key: String): String = { val versionRegexp = s""".*val $key[ ]+=[ ]?"(.*?)"""".r diff --git a/project/meta-build.sbt b/project/meta-build.sbt index cefb44f73413..190b1077c539 100644 --- a/project/meta-build.sbt +++ b/project/meta-build.sbt @@ -1,8 +1,8 @@ libraryDependencies ++= Seq( - "com.typesafe" % "config" % "1.4.2", + "com.typesafe" % "config" % "1.4.3", "com.michaelpollmeier" % "versionsort" % "1.0.11", "net.lingala.zip4j" % "zip4j" % "2.11.5", "com.github.pathikrit" %% "better-files" % "3.9.2", "net.java.dev.javacc" % "javacc" % "7.0.12", - "com.lihaoyi" %% "os-lib" % "0.9.1" + "com.lihaoyi" %% "os-lib" % "0.9.3" ) diff --git a/semanticcpg/build.sbt b/semanticcpg/build.sbt index 322ea417b282..ea2590fc0a0d 100644 --- a/semanticcpg/build.sbt +++ b/semanticcpg/build.sbt @@ -4,7 +4,7 @@ libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "com.michaelpollmeier" %% "scala-repl-pp" % Versions.scalaReplPP, "org.json4s" %% "json4s-native" % Versions.json4s, - "org.apache.commons" % "commons-text" % "1.12.0", + "org.apache.commons" % "commons-text" % Versions.commonsText, "org.scalatest" %% "scalatest" % Versions.scalatest % Test )