Skip to content

Commit

Permalink
[rubysrc2cpg] add --useDeprecatedFrontend flag (joernio#3755)
Browse files Browse the repository at this point in the history
* [rubysrc2cpg] add `--useDeprecatedFrontend` flag

* Provide `rubyDeprecated` from `importCode`
  • Loading branch information
xavierpinho authored Oct 26, 2023
1 parent a7d95db commit 2088b60
Show file tree
Hide file tree
Showing 94 changed files with 315 additions and 244 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import java.nio.file.Path
import scala.util.{Failure, Success, Try}

class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Reporting {
import io.joern.console.Console._
import io.joern.console.Console.*

private val config = console.config
private val workspace = console.workspace
Expand Down Expand Up @@ -60,10 +60,11 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep
def csharp: Frontend = new BinaryFrontend("csharp", Languages.CSHARP, "C# Source Frontend (Roslyn)")
def llvm: Frontend = new BinaryFrontend("llvm", Languages.LLVM, "LLVM Bitcode Frontend")
def php: SourceBasedFrontend = new SourceBasedFrontend("php", Languages.PHP, "PHP source frontend", "php")
def ruby: SourceBasedFrontend = new SourceBasedFrontend("ruby", Languages.RUBYSRC, "Ruby source frontend", "rb")
def ruby: SourceBasedFrontend = new RubyFrontend("Ruby source frontend", false)
def rubyDeprecated: SourceBasedFrontend = new RubyFrontend("Ruby source deprecated frontend", true)

private def allFrontends: List[Frontend] =
List(c, cpp, ghidra, kotlin, java, jvm, javascript, jssrc, golang, llvm, php, python, csharp, ruby)
List(c, cpp, ghidra, kotlin, java, jvm, javascript, jssrc, golang, llvm, php, python, csharp, ruby, rubyDeprecated)

// this is only abstract to force people adding frontends to make a decision whether the frontend consumes binaries or source
abstract class Frontend(val name: String, val language: String, val description: String = "") {
Expand Down Expand Up @@ -100,6 +101,31 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep
}
}
}

/** Only a wrapper so as to more easily pick the deprecated variant without having to provide the
* `--useDeprecatedFrontend` flag each time.
*
* @param useDeprecatedFrontend
* If set, will invoke the frontend with the `--useDeprecatedFrontend` flag
*/
private class RubyFrontend(description: String, useDeprecatedFrontend: Boolean = false)
extends SourceBasedFrontend("ruby", Languages.RUBYSRC, description, "rb") {
private val deprecatedFlag = "--useDeprecatedFrontend"

private def addDeprecatedFlagIfNeeded(args: List[String]): List[String] = {
Option.when(useDeprecatedFrontend && !args.contains(deprecatedFlag))(deprecatedFlag).toList ++ args
}

override def cpgGeneratorForLanguage(
language: String,
config: FrontendConfig,
rootPath: Path,
args: List[String]
): Option[CpgGenerator] = {
super.cpgGeneratorForLanguage(language, config, rootPath, addDeprecatedFlagIfNeeded(args))
}
}

class CFrontend(name: String, extension: String = "c")
extends SourceBasedFrontend(name, Languages.NEWC, "Eclipse CDT Based Frontend for C/C++", extension)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.joern.console.cpgcreation

import io.joern.console.FrontendConfig
import io.joern.rubysrc2cpg.RubySrc2Cpg
import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg}
import io.shiftleft.codepropertygraph.Cpg

import java.nio.file.Path
Expand All @@ -21,7 +21,10 @@ case class RubyCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgG
override def isJvmBased = true

override def applyPostProcessingPasses(cpg: Cpg): Cpg = {
RubySrc2Cpg.postProcessingPasses(cpg).foreach(_.createAndApply())
// TODO: here we need a Ruby-specific `Config`, which we shall build from the existing `FrontendConfig`. We only
// care for `--useDeprecatedFrontend` though, for now. Nevertheless, there should be a better way of handling this.
val rubyConfig = Config().withUseDeprecatedFrontend(config.cmdLineParams.exists(_ == "--useDeprecatedFrontend"))
RubySrc2Cpg.postProcessingPasses(cpg, rubyConfig).foreach(_.createAndApply())
cpg
}

Expand Down
7 changes: 3 additions & 4 deletions joern-cli/frontends/rubysrc2cpg/build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ libraryDependencies ++= Seq(

enablePlugins(JavaAppPackaging, LauncherJarPlugin, Antlr4Plugin)

Antlr4 / antlr4PackageName := Some("io.joern.rubysrc2cpg.parser")
Antlr4 / antlr4Version := Versions.antlr
Antlr4 / antlr4GenVisitor := true
Antlr4 / javaSource := (Compile / sourceManaged).value
Antlr4 / antlr4Version := Versions.antlr
Antlr4 / antlr4GenVisitor := true
Antlr4 / javaSource := (Compile / sourceManaged).value
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ lexer grammar RubyLexer;
// Auxiliary tokens and features
// --------------------------------------------------------

@header {
package io.joern.rubysrc2cpg.deprecated.parser;
}

tokens {
STRING_INTERPOLATION_END,
REGULAR_EXPRESSION_INTERPOLATION_END,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
parser grammar RubyParser;

@header {
package io.joern.rubysrc2cpg.deprecated.parser;
}

options {
tokenVocab = RubyLexer;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery}
import io.joern.x2cpg.{X2CpgConfig, X2CpgMain}
import scopt.OParser

final case class Config(enableDependencyDownload: Boolean = false, antlrCacheMemLimit: Double = 0.6d)
extends X2CpgConfig[Config]
final case class Config(
enableDependencyDownload: Boolean = false,
antlrCacheMemLimit: Double = 0.6d,
useDeprecatedFrontend: Boolean = false
) extends X2CpgConfig[Config]
with TypeRecoveryParserConfig[Config] {

def withEnableDependencyDownload(value: Boolean): Config = {
Expand All @@ -16,6 +19,10 @@ final case class Config(enableDependencyDownload: Boolean = false, antlrCacheMem
def withAntlrCacheMemoryLimit(value: Double): Config = {
copy(antlrCacheMemLimit = value).withInheritedFields(this)
}

def withUseDeprecatedFrontend(value: Boolean): Config = {
copy(useDeprecatedFrontend = value).withInheritedFields(this)
}
}

private object Frontend {
Expand Down Expand Up @@ -43,6 +50,9 @@ private object Frontend {
success
}
.text("sets the heap usage threshold at which the ANTLR DFA cache is cleared during parsing (default 0.6)"),
opt[Unit]("useDeprecatedFrontend")
.action((_, c) => c.withUseDeprecatedFrontend(true))
.text("uses the original (but deprecated) Ruby frontend (default false)"),
XTypeRecovery.parserOptions
)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package io.joern.rubysrc2cpg

import better.files.File
import io.joern.rubysrc2cpg.astcreation.ResourceManagedParser
import io.joern.rubysrc2cpg.passes.*
import io.joern.rubysrc2cpg.utils.PackageTable
import io.joern.rubysrc2cpg.deprecated.astcreation.ResourceManagedParser
import io.joern.rubysrc2cpg.deprecated.passes.*
import io.joern.rubysrc2cpg.deprecated.utils.PackageTable
import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
import io.joern.x2cpg.X2CpgFrontend
import io.joern.x2cpg.datastructures.Global
Expand All @@ -28,23 +28,34 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] {
withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) =>
new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply()
new ConfigFileCreationPass(cpg).createAndApply()
Using.resource(new ResourceManagedParser(config.antlrCacheMemLimit)) { parser =>
if (config.enableDependencyDownload && !scala.util.Properties.isWin) {
val tempDir = File.newTemporaryDirectory()
try {
downloadDependency(config.inputPath, tempDir.toString())
new AstPackagePass(cpg, tempDir.toString(), global, parser, RubySrc2Cpg.packageTableInfo, config.inputPath)(
config.schemaValidation
).createAndApply()
} finally {
tempDir.delete()
}
}
if (config.useDeprecatedFrontend) {
deprecatedCreateCpgAction(cpg, config)
} else {
newCreateCpgAction(cpg, config)
}
}
}

val astCreationPass = new AstCreationPass(cpg, global, parser, RubySrc2Cpg.packageTableInfo, config)
astCreationPass.createAndApply()
TypeNodePass.withRegisteredTypes(astCreationPass.allUsedTypes(), cpg).createAndApply()
private def newCreateCpgAction(cpg: Cpg, config: Config): Unit = {
// TODO
}

private def deprecatedCreateCpgAction(cpg: Cpg, config: Config): Unit = {
Using.resource(new ResourceManagedParser(config.antlrCacheMemLimit)) { parser =>
if (config.enableDependencyDownload && !scala.util.Properties.isWin) {
val tempDir = File.newTemporaryDirectory()
try {
downloadDependency(config.inputPath, tempDir.toString())
new AstPackagePass(cpg, tempDir.toString(), global, parser, RubySrc2Cpg.packageTableInfo, config.inputPath)(
config.schemaValidation
).createAndApply()
} finally {
tempDir.delete()
}
}
val astCreationPass = new AstCreationPass(cpg, global, parser, RubySrc2Cpg.packageTableInfo, config)
astCreationPass.createAndApply()
TypeNodePass.withRegisteredTypes(astCreationPass.allUsedTypes(), cpg).createAndApply()
}
}

Expand All @@ -71,18 +82,23 @@ object RubySrc2Cpg {

val packageTableInfo = new PackageTable()

def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] =
List(
// TODO commented below two passes, as waiting on Dependency download PR to get merged
new IdentifierToCallPass(cpg),
new ImportResolverPass(cpg, packageTableInfo),
new RubyTypeRecoveryPass(cpg),
new RubyTypeHintCallLinker(cpg),
new NaiveCallLinker(cpg),
def postProcessingPasses(cpg: Cpg, config: Config): List[CpgPassBase] = {
if (config.useDeprecatedFrontend) {
List(
// TODO commented below two passes, as waiting on Dependency download PR to get merged
new IdentifierToCallPass(cpg),
new ImportResolverPass(cpg, packageTableInfo),
new RubyTypeRecoveryPass(cpg),
new RubyTypeHintCallLinker(cpg),
new NaiveCallLinker(cpg),

// Some of passes above create new methods, so, we
// need to run the ASTLinkerPass one more time
new AstLinkerPass(cpg)
)
// Some of passes above create new methods, so, we
// need to run the ASTLinkerPass one more time
new AstLinkerPass(cpg)
)
} else {
List()
}
}

}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.{RubyLexer, RubyLexerPostProcessor, RubyParser}
import io.joern.rubysrc2cpg.deprecated.parser.{RubyLexer, RubyLexerPostProcessor, RubyParser}
import org.antlr.v4.runtime.*
import org.antlr.v4.runtime.atn.ATN
import org.antlr.v4.runtime.dfa.DFA
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package io.joern.rubysrc2cpg.astcreation
import io.joern.rubysrc2cpg.parser.RubyParser
import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.utils.PackageContext
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.deprecated.parser.RubyParser
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.rubysrc2cpg.deprecated.utils.PackageContext
import io.joern.x2cpg.Ast.storeInDiffGraph
import io.joern.x2cpg.Defines.DynamicCallUnknownFullName
import io.joern.x2cpg.X2Cpg.stripQuotes
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.joern.rubysrc2cpg.astcreation
import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines as RubyDefines
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines as RubyDefines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators, nodes}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.ControlStructureTypes
import io.shiftleft.codepropertygraph.generated.nodes.NewControlStructure
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.x2cpg.Ast
import io.shiftleft.codepropertygraph.generated.nodes.{NewJumpTarget, NewLiteral}
import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.rubysrc2cpg.deprecated.passes.Defines.*
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, NewCall, NewIdentifier}
import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.utils.PackageContext
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.rubysrc2cpg.deprecated.utils.PackageContext
import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines}
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.HereDocHandling
import io.joern.rubysrc2cpg.parser.RubyParser.{HereDocArgumentContext, HereDocLiteralContext}
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.deprecated.parser.HereDocHandling
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.{HereDocArgumentContext, HereDocLiteralContext}
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewIdentifier, NewLiteral}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser
import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.passes.Defines.getBuiltInType
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.rubysrc2cpg.deprecated.passes.Defines.getBuiltInType
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.NewCall
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import better.files.File
import io.joern.rubysrc2cpg.parser.RubyParser.*
import io.joern.rubysrc2cpg.passes.Defines
import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.x2cpg.Defines.DynamicCallUnknownFullName
import io.joern.x2cpg.Imports.createImportNodeAndLink
import io.joern.x2cpg.X2Cpg.stripQuotes
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
package io.joern.rubysrc2cpg.astcreation

import io.joern.rubysrc2cpg.parser.RubyParser.{
BodyStatementContext,
ClassDefinitionPrimaryContext,
ClassOrModuleReferenceContext,
ModuleDefinitionPrimaryContext,
ScopedConstantReferenceContext
}
import io.joern.rubysrc2cpg.passes.Defines
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.rubysrc2cpg.deprecated.parser.RubyParser.*
import io.joern.rubysrc2cpg.deprecated.passes.Defines
import io.joern.x2cpg.utils.*
import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines}
import io.shiftleft.codepropertygraph.generated.nodes.*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package io.joern.rubysrc2cpg.astcreation
package io.joern.rubysrc2cpg.deprecated.astcreation

import io.joern.x2cpg.datastructures.Scope
import io.shiftleft.codepropertygraph.generated.EdgeTypes
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package io.joern.rubysrc2cpg.parser
package io.joern.rubysrc2cpg.deprecated.parser

import org.antlr.v4.runtime.ParserRuleContext
import org.antlr.v4.runtime.tree.TerminalNode
Expand Down
Loading

0 comments on commit 2088b60

Please sign in to comment.