Cleanups (#200)

* Light cleanups to prepare for overflowdb-only generator

* Upgrade CPG

* Use `KeyPool` from `codepropertygraph`

* Pass keyPool to all users instead of calling global `IdPool`

build.sbt

@@ -3,7 +3,7 @@ organization := "io.shiftleft"
 scalaVersion := "2.13.1"
 enablePlugins(GitVersioning)
 
-val cpgVersion = "0.11.276"
+val cpgVersion = "0.11.307"
 val antlrVersion = "4.7.2"
 
 libraryDependencies ++= Seq(

src/main/java/io/shiftleft/fuzzyc2cpg/ast/AstNode.java

@@ -1,6 +1,5 @@
 package io.shiftleft.fuzzyc2cpg.ast;
 
-import io.shiftleft.fuzzyc2cpg.ast.expressions.Expression;
 import io.shiftleft.fuzzyc2cpg.ast.walking.ASTNodeVisitor;
 
 import java.util.ArrayList;
@@ -119,10 +118,6 @@ public void setCodeStr(String aCodeStr) {
  code = aCodeStr;
  }
 
- public String getLocationString() {
- return this.location.toString();
- }
-
  public CodeLocation getLocation() {
  return this.location;
  }
@@ -135,20 +130,9 @@ public String getTypeAsString() {
  return this.getClass().getSimpleName();
  }
 
- public String getFullTypeName() {
- return this.getClass().getName();
- }
-
 
  /* special methods */
 
- public String getOperatorCode() {
- if (Expression.class.isAssignableFrom(this.getClass())) {
- return ((Expression) this).getOperator();
- }
- return null;
- }
-
  public void accept(ASTNodeVisitor visitor) {
  visitor.visit(this);
  }

src/main/java/io/shiftleft/fuzzyc2cpg/ast/declarations/ClassDefStatement.java

@@ -1,6 +1,5 @@
 package io.shiftleft.fuzzyc2cpg.ast.declarations;
 
-import io.shiftleft.fuzzyc2cpg.CommonParser;
 import io.shiftleft.fuzzyc2cpg.ast.AstNode;
 import io.shiftleft.fuzzyc2cpg.ast.DummyIdentifierNode;
 import io.shiftleft.fuzzyc2cpg.ast.expressions.Identifier;

src/main/java/io/shiftleft/fuzzyc2cpg/output/inmemory/OutputModule.java

@@ -23,9 +23,7 @@ public Cpg getInternalGraph() {
  }
 
  @Override
- public void setOutputIdentifier(String identifier) {
-
- }
+ public void setOutputIdentifier(String identifier) { }
 
  @Override
  public void persistCpg(CpgStruct.Builder cpg) {
@@ -35,10 +33,6 @@ public void persistCpg(CpgStruct.Builder cpg) {
  }
 
  public void persist() {
- constructTinkerGraphFromCpg();
- }
-
- private void constructTinkerGraphFromCpg() {
  CpgStruct.Builder mergedBuilder = CpgStruct.newBuilder();
 
  cpgBuilders.forEach(builder -> {
@@ -48,7 +42,6 @@ private void constructTinkerGraphFromCpg() {
  List<CpgStruct> list = new LinkedList<>();
  list.add(mergedBuilder.build());
  cpg = ProtoCpgLoader.loadFromListOfProtos(list, OdbConfig.withoutOverflow());
-
  }
 
 }

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModule.java

@@ -17,12 +17,10 @@
 import java.util.concurrent.ThreadLocalRandom;
 
 public class OutputModule implements CpgOutputModule {
- private Logger logger = LoggerFactory.getLogger(getClass());
- private static String ProtoSuffix = ".bin";
+ private final Logger logger = LoggerFactory.getLogger(getClass());
 
  private final Path protoTempDir;
-
- private boolean writeToDisk;
+ private final boolean writeToDisk;
 
  private String outputIdentifier;
 
@@ -94,6 +92,7 @@ private String generateOutputFilename(int postfix) {
  hasher.putUnencodedChars(outputIdentifier);
  hasher.putInt(postfix);
 
- return protoTempDir.toString() + File.separator + hasher.hash() + ProtoSuffix;
+ String protoSuffix = ".bin";
+ return protoTempDir.toString() + File.separator + hasher.hash() + protoSuffix;
  }
 }

src/main/java/io/shiftleft/fuzzyc2cpg/output/protobuf/OutputModuleFactory.java

@@ -2,7 +2,6 @@
 
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModule;
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;

src/main/java/io/shiftleft/fuzzyc2cpg/parser/AntlrParserDriver.java

@@ -4,13 +4,13 @@
 import io.shiftleft.fuzzyc2cpg.ast.AstNode;
 import io.shiftleft.fuzzyc2cpg.ast.AstNodeBuilder;
 import io.shiftleft.fuzzyc2cpg.ast.logical.statements.CompoundStatement;
+import io.shiftleft.passes.KeyPool;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Stack;
 import java.util.function.Consumer;
 
-import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory;
 import io.shiftleft.proto.cpg.Cpg;
 import jdk.nashorn.internal.runtime.ParserException;
 import org.antlr.v4.runtime.*;
@@ -41,6 +41,7 @@ abstract public class AntlrParserDriver {
  private List<AntlrParserDriverObserver> observers = new ArrayList<>();
  private Cpg.CpgStruct.Builder cpg;
  private Cpg.CpgStruct.Node fileNode;
+ private KeyPool keyPool;
 
  public AntlrParserDriver() {
  super();
@@ -50,6 +51,10 @@ public void setCpg(Cpg.CpgStruct.Builder cpg) {
  this.cpg = cpg;
  }
 
+ public void setKeyPool(KeyPool keyPool) {
+ this.keyPool = keyPool;
+ }
+
  public void setFileNode(Cpg.CpgStruct.Node fileNode) {
  this.fileNode = fileNode;
  }
@@ -84,6 +89,7 @@ private void handleHiddenTokens(String filename) {
  // We can add to `CPG` here
 
  Cpg.CpgStruct.Node commentNode = Utils.newNode(Cpg.CpgStruct.Node.NodeType.COMMENT)
+ .setKey(keyPool.next())
  .addProperty(Cpg.CpgStruct.Node.Property.newBuilder()
  .setName(Cpg.NodePropertyName.LINE_NUMBER)
  .setValue(Cpg.PropertyValue.newBuilder().setIntValue(line)))

src/main/java/io/shiftleft/fuzzyc2cpg/parser/modules/CModuleParserTreeListener.java

@@ -1,10 +1,7 @@
 package io.shiftleft.fuzzyc2cpg.parser.modules;
 
-import java.util.Iterator;
 import java.util.List;
 
-import io.shiftleft.fuzzyc2cpg.ast.expressions.Identifier;
-import io.shiftleft.fuzzyc2cpg.parser.AstNodeFactory;
 import org.antlr.v4.runtime.ParserRuleContext;
 
 import io.shiftleft.fuzzyc2cpg.ModuleBaseListener;

src/main/scala/io/shiftleft/fuzzyc2cpg/AstVisitor.scala

@@ -12,11 +12,15 @@ import io.shiftleft.fuzzyc2cpg.astnew.AstToCpgConverter
 import io.shiftleft.fuzzyc2cpg.cfg.AstToCfgConverter
 import io.shiftleft.fuzzyc2cpg.output.CpgOutputModuleFactory
 import io.shiftleft.fuzzyc2cpg.parser.AntlrParserDriverObserver
+import io.shiftleft.passes.KeyPool
 import io.shiftleft.proto.cpg.Cpg.CpgStruct
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node
 import org.antlr.v4.runtime.ParserRuleContext
 
-class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgStruct.Builder, astParentNode: Node)
+class AstVisitor(outputModuleFactory: CpgOutputModuleFactory,
+ structureCpg: CpgStruct.Builder,
+ astParentNode: Node,
+ keyPool: KeyPool)
  extends ASTNodeVisitor
  with AntlrParserDriverObserver {
  private var fileNameOption = Option.empty[String]
@@ -31,7 +35,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
  outputModule.setOutputIdentifier(outputIdentifier)
 
  val bodyCpg = CpgStruct.newBuilder()
- val cpgAdapter = new ProtoCpgAdapter(bodyCpg)
+ val cpgAdapter = new ProtoCpgAdapter(bodyCpg, keyPool)
  val astToCpgConverter =
  new AstToCpgConverter(astParentNode, cpgAdapter)
  astToCpgConverter.convert(functionDef)
@@ -56,7 +60,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
  * Callback triggered for every class/struct
  * */
  override def visit(classDefStatement: ClassDefStatement): Unit = {
- val cpgAdapter = new ProtoCpgAdapter(structureCpg)
+ val cpgAdapter = new ProtoCpgAdapter(structureCpg, keyPool)
  val astToCpgConverter =
  new AstToCpgConverter(astParentNode, cpgAdapter)
  astToCpgConverter.convert(classDefStatement)
@@ -66,7 +70,7 @@ class AstVisitor(outputModuleFactory: CpgOutputModuleFactory, structureCpg: CpgS
  * Callback triggered for every global identifier declaration
  * */
  override def visit(identifierDeclStmt: IdentifierDeclStatement): Unit = {
- val cpgAdapter = new ProtoCpgAdapter(structureCpg)
+ val cpgAdapter = new ProtoCpgAdapter(structureCpg, keyPool)
  val astToCpgConverter =
  new AstToCpgConverter(astParentNode, cpgAdapter)
  astToCpgConverter.convert(identifierDeclStmt)