From beedbe1bb1c6dc69af9e321d3c710b73a6662069 Mon Sep 17 00:00:00 2001 From: ferriarnus <61201275+ferriarnus@users.noreply.github.com> Date: Wed, 30 Oct 2024 00:13:44 +0100 Subject: [PATCH 01/10] first test --- src/main/java/org/taumc/glsl/Main.java | 28 +- src/main/java/org/taumc/glsl/Transformer.java | 540 +++++++++++++++--- .../org/taumc/glsl/TransformerCollector.java | 78 +++ .../org/taumc/glsl/TransformerRemover.java | 78 +++ 4 files changed, 646 insertions(+), 78 deletions(-) create mode 100644 src/main/java/org/taumc/glsl/TransformerCollector.java create mode 100644 src/main/java/org/taumc/glsl/TransformerRemover.java diff --git a/src/main/java/org/taumc/glsl/Main.java b/src/main/java/org/taumc/glsl/Main.java index 0f7de6c..aa8297b 100644 --- a/src/main/java/org/taumc/glsl/Main.java +++ b/src/main/java/org/taumc/glsl/Main.java @@ -30,21 +30,23 @@ public static void main(String[] args) throws Exception { parser.setBuildParseTree(true); var translationUnit = parser.translation_unit(); - Util.injectVariable(translationUnit, "in ivec2 a_LightCoord2;"); - Util.rename(translationUnit, "a_Color", "rewritten_Color"); + Transformer transformer = new Transformer(translationUnit); + + transformer.injectVariable("in ivec2 a_LightCoord2;"); + transformer.rename("a_Color", "rewritten_Color"); Set found = new HashSet<>(); - Util.renameArray(translationUnit, "test", "replaced", found); + transformer.renameArray("test", "replaced", found); //Util.removeVariable(translationUnit, "_vert_tex_diffuse_coord"); - Util.removeVariable(translationUnit, "testing"); - Util.removeConstAssignment(translationUnit); - Util.renameFunctionCall(translationUnit, "_vert_init", "newCall"); - Util.renameAndWrapShadow(translationUnit, "function", "wrapped"); - Util.prependMain(translationUnit, "injected = 5;"); - Util.replaceExpression(translationUnit, "vartest", "unint(5)"); - Util.removeUnusedFunctions(translationUnit); - Util.rewriteStructArrays(translationUnit); - Util.renameFunctionCall(translationUnit, "texture2D", "texture"); - Util.replaceExpression(translationUnit, "gl_TextureMatrix[0]", "mat4(1.0f)"); + transformer.removeVariable("testing"); + transformer.removeConstAssignment(); + transformer.renameFunctionCall("_vert_init", "newCall"); + transformer.renameAndWrapShadow("function", "wrapped"); + transformer.prependMain("injected = 5;"); + transformer.replaceExpression("vartest", "unint(5)"); + transformer.removeUnusedFunctions(); + transformer.rewriteStructArrays(); + transformer.renameFunctionCall("texture2D", "texture"); + transformer.replaceExpression("gl_TextureMatrix[0]", "mat4(1.0f)"); System.out.println(getFormattedShader(translationUnit)); diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 232e3db..9af8588 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -1,9 +1,12 @@ package org.taumc.glsl; +import com.ibm.icu.impl.CollectionSet; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.TerminalNode; import org.taumc.glsl.grammar.GLSLLexer; @@ -19,9 +22,34 @@ public class Transformer { private final GLSLParser.Translation_unitContext root; + private final List functionDefinitions; + private final List postfixExpressions; + private final List assignmentExpressions; + private final List variableIdentifiers; + private final List parameterDeclarations; + private final List typelessDeclarations; + private final List functionPrototypes; + private final List binaryExpressions; + private final List storageQualifiers; + private final List structDeclarations; + private final List singleDeclarations; + private final List textures; public Transformer(GLSLParser.Translation_unitContext root) { this.root = root; + this.functionDefinitions = new ArrayList<>(); + this.postfixExpressions = new ArrayList<>(); + this.assignmentExpressions = new ArrayList<>(); + this.variableIdentifiers = new ArrayList<>(); + this.parameterDeclarations = new ArrayList<>(); + this.typelessDeclarations = new ArrayList<>(); + this.functionPrototypes = new ArrayList<>(); + this.binaryExpressions = new ArrayList<>(); + this.storageQualifiers = new ArrayList<>(); + this.structDeclarations = new ArrayList<>(); + this.singleDeclarations = new ArrayList<>(); + this.textures = new ArrayList<>(); + ParseTreeWalker.DEFAULT.walk(new TransformerCollector(this), root); } /** @@ -33,15 +61,32 @@ public void injectVariable(String code) { GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); var insert = parser.external_declaration(); - AtomicReference ref = new AtomicReference<>(); - ParseTreeWalker.DEFAULT.walk(new VariableInjector(ref), root); + ParserRuleContext left = null; + if (!storageQualifiers.isEmpty()) { + var parent = storageQualifiers.get(0).getParent(); + while (!(parent instanceof GLSLParser.External_declarationContext)) { + if (parent.getParent() == null) { + break; + } + parent = parent.getParent(); + } + if (parent instanceof GLSLParser.External_declarationContext list) { + left = list; + } + } + + if (left == null){ + if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { + left = list; + } + } - var left = ref.get(); if (left != null) { var parent = left.getParent(); int i = parent.children.indexOf(left); parent.children.add(i, insert); insert.setParent(parent); + scanNode(insert); } } @@ -50,15 +95,17 @@ public void injectFunction(String code) { GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); var insert = parser.external_declaration(); - AtomicReference ref = new AtomicReference<>(); - ParseTreeWalker.DEFAULT.walk(new FunctionInjector(ref), root); + ParserRuleContext left = null; + if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { + left = list; + } - var left = ref.get(); if (left != null) { var parent = left.getParent(); int i = parent.children.indexOf(left); parent.children.add(i, insert); insert.setParent(parent); + scanNode(insert); } } @@ -67,62 +114,219 @@ public void rename(String oldName, String newName) { } public void rename(Map names) { - ParseTreeWalker.DEFAULT.walk(new Renamer(names), root); + List nodes = new ArrayList<>(); + nodes.addAll(typelessDeclarations.stream().map(GLSLParser.Typeless_declarationContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(variableIdentifiers.stream().map(GLSLParser.Variable_identifierContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(functionPrototypes.stream().map(GLSLParser.Function_prototypeContext::IDENTIFIER).collect(Collectors.toList())); + for (var node : nodes) { + Token token = node.getSymbol(); + if(token instanceof CommonToken cToken) { + String replacement = names.get(cToken.getText()); + if(replacement != null) { + cToken.setText(replacement); + } + } + } } public void replaceExpression(String oldCode, String newCode) { - ParseTreeWalker.DEFAULT.walk(new ReplaceExpression(oldCode, newCode), root); + GLSLLexer oldLexer = new GLSLLexer(CharStreams.fromString(oldCode)); + GLSLParser oldParser = new GLSLParser(new CommonTokenStream(oldLexer)); + GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); + GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); + var oldExpression = oldParser.binary_expression(); + for (var ctx : binaryExpressions) { + if (ctx.getText().equals(oldExpression.getText())) { + replaceNode(ctx, newParser.binary_expression()); + } else if (ctx.getText().startsWith(oldExpression.getText())) { + if (ctx.unary_expression() != null) { + if (ctx.unary_expression().postfix_expression() != null) { + if (ctx.unary_expression().postfix_expression().postfix_expression() != null) { + var postfix = ctx.unary_expression().postfix_expression().postfix_expression(); + while (postfix.getText().startsWith(oldExpression.getText()) && !postfix.getText().equals(oldExpression.getText())) { + if (postfix.postfix_expression() != null) { + postfix = postfix.postfix_expression(); + } else { + break; + } + } + if (postfix.getText().equals(oldExpression.getText())) { + replaceNode(postfix, newParser.unary_expression()); + } + } + } + } + } + } + } + + private void replaceNode(ParserRuleContext oldNode, ParserRuleContext newNode) { + scanNode(newNode); + newNode.parent = oldNode.getParent(); + int i = oldNode.getParent().children.indexOf(oldNode); + oldNode.getParent().children.set(i, newNode); + removeNode(oldNode); + } + + private void removeNode(ParserRuleContext node) { + ParseTreeWalker.DEFAULT.walk(new TransformerRemover(this), node); + } + + private void scanNode(ParserRuleContext node) { + ParseTreeWalker.DEFAULT.walk(new TransformerCollector(this), node); } public void prependMain(String code) { - ParseTreeWalker.DEFAULT.walk(new PrependFunction("main", code), root); + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + var insert = parser.statement(); + for (var ctx : functionDefinitions) { + if (ctx.function_prototype().IDENTIFIER().getText().equals("main")) { + ctx.compound_statement_no_new_scope().statement_list().children.add(0, insert); + scanNode(insert); + } + } } public void removeVariable(String code) { - AtomicReference top = new AtomicReference<>(); - ParseTreeWalker.DEFAULT.walk(new RemoveVariable(code, top), root); - if (top.get() != null) { - if (top.get().getParent() instanceof GLSLParser.Init_declarator_listContext listContext) { - int i = listContext.children.indexOf(top.get()); - listContext.children.remove(i-1); - listContext.children.remove(i-1); - } else if (top.get().parent instanceof GLSLParser.Single_declarationContext singleContext) { - singleContext.getParent().getParent().getParent().getParent().children.remove(singleContext.getParent().getParent().getParent()); + ParserRuleContext typeless = null; + for (var ctx : typelessDeclarations) { + if (ctx.IDENTIFIER() != null) { + Token token = ctx.IDENTIFIER().getSymbol(); + if(token instanceof CommonToken cToken) { + if(code.equals(cToken.getText())) { + if (ctx.getParent() instanceof GLSLParser.Single_declarationContext) { + if (ctx.getParent().getParent() instanceof GLSLParser.Init_declarator_listContext list) { + if (!list.typeless_declaration().isEmpty()) { + var entry = list.typeless_declaration(0); + cToken.setText(entry.getText()); + typeless = entry; + break; + } else { + typeless = ctx; + } + } + } + else if (ctx.getParent() instanceof GLSLParser.Init_declarator_listContext) { + typeless = ctx; + break; + } + } + } } } + + if (typeless == null) { + return; + } + + if (typeless.getParent() instanceof GLSLParser.Init_declarator_listContext listContext) { + int i = listContext.children.indexOf(typeless); + removeNode((ParserRuleContext) listContext.children.remove(i-1)); + removeNode((ParserRuleContext) listContext.children.remove(i-1)); + } else if (typeless.parent instanceof GLSLParser.Single_declarationContext singleContext) { + ParserRuleContext node = singleContext.getParent().getParent().getParent(); + node.getParent().children.remove(node); + removeNode(node); + } } public int findType(String code) { - AtomicInteger type = new AtomicInteger(); - FastTreeWalker.walk(new TypeFinder(code, type), root); - return type.get(); + for (var ctx : singleDeclarations) { + if (ctx.typeless_declaration() == null) { + return 0; + } + if (ctx.typeless_declaration().IDENTIFIER().getSymbol() instanceof CommonToken cToken) { + if (cToken.getText().equals(code)) { + if (ctx.fully_specified_type().type_specifier().type_specifier_nonarray().getChild(0) instanceof TerminalNode node) { + if (node.getSymbol() instanceof CommonToken t) { + return t.getType(); + } + } + } + } + if (ctx.getParent() instanceof GLSLParser.Init_declarator_listContext listContext) { + for (var entry : listContext.typeless_declaration()) { + if (entry.IDENTIFIER().getSymbol() instanceof CommonToken cToken) { + if (cToken.getText().equals(code)) { + if (ctx.fully_specified_type().type_specifier().type_specifier_nonarray().getChild(0) instanceof TerminalNode node) { + if (node.getSymbol() instanceof CommonToken t) { + return t.getType(); + } + } + } + } + } + } + } + return 0; } public void appendMain(String code) { - ParseTreeWalker.DEFAULT.walk(new AppendFunction("main", code), root); + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + var insert = parser.statement(); + for (var ctx : functionDefinitions) { + if (ctx.function_prototype().IDENTIFIER().getText().equals("main")) { + ctx.compound_statement_no_new_scope().statement_list().children.add(insert); + scanNode(insert); + } + } } public boolean containsCall(String name) { - AtomicBoolean atomicBoolean = new AtomicBoolean(false); - FastTreeWalker.walk(new IdentifierCollector(id -> { - if(id.equals(name)) { - atomicBoolean.set(true); - return false; - } else { - return true; + for (var ctx : variableIdentifiers) { + if (ctx.IDENTIFIER().getSymbol() instanceof CommonToken cToken) { + if (cToken.getText().equals(name)) { + return true; + } } - }), root); - return atomicBoolean.get(); + } + return false; } public boolean hasVariable(String name) { - AtomicBoolean atomicBoolean = new AtomicBoolean(false); - ParseTreeWalker.DEFAULT.walk(new HasVariable(name, atomicBoolean), root); - return atomicBoolean.get(); + List nodes = new ArrayList<>(); + nodes.addAll(typelessDeclarations.stream().map(GLSLParser.Typeless_declarationContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(functionPrototypes.stream().map(GLSLParser.Function_prototypeContext::IDENTIFIER).collect(Collectors.toList())); + for (var node : nodes) { + Token token = node.getSymbol(); + if(token instanceof CommonToken cToken) { + if (name.equals(cToken.getText())) { + return true; + } + } + } + return false; } public void renameArray(String oldName, String newName, Set found) { - ParseTreeWalker.DEFAULT.walk(new ArrayExpressionRewriteListener(Collections.singletonMap(oldName, newName), found), root); + renameArray(Collections.singletonMap(oldName, newName), found); + } + + public void renameArray(Map replacements, Set found) { + for (var ctx : postfixExpressions) { + if (ctx.postfix_expression() == null) { + return; + } + if (ctx.postfix_expression().primary_expression() == null) { + return; + } + if (ctx.postfix_expression().primary_expression().variable_identifier() == null) { + return; + } + if (ctx.postfix_expression().primary_expression().variable_identifier().IDENTIFIER().getSymbol() instanceof CommonToken cToken) { + String replacement = replacements.get(cToken.getText()); + if(replacement != null && ctx.integer_expression() != null) { + String i = ctx.integer_expression().getText(); + found.add(Integer.parseInt(i)); + cToken.setText(replacement + i); + ctx.removeLastChild(); + ctx.removeLastChild(); + ctx.removeLastChild(); + } + } + } } /** @@ -139,35 +343,54 @@ public void renameFunctionCall(String oldName, String newName) { } public void renameFunctionCall(Map names) { - ParseTreeWalker.DEFAULT.walk(new ExpressionRenamer(names), root); + List nodes = new ArrayList<>(); + nodes.addAll(typelessDeclarations.stream().map(GLSLParser.Typeless_declarationContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(variableIdentifiers.stream().map(GLSLParser.Variable_identifierContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(textures.stream().map(c -> { + if (c.TEXTURE2D() != null) { + return c.TEXTURE2D(); + } else { + return c.TEXTURE3D(); + } + }).collect(Collectors.toList())); + for (var node : nodes) { + Token token = node.getSymbol(); + if(token instanceof CommonToken cToken) { + String replacement = names.get(cToken.getText()); + if(replacement != null) { + cToken.setText(replacement); + } + } + } } public void renameAndWrapShadow(String oldName, String newName) { - ParseTreeWalker.DEFAULT.walk( new FunctionCallWrapper(oldName, "vec4"), root); + for (var ctx : postfixExpressions) { + String function = ctx.getText(); + if (ctx.function_call_parameters() != null && function.startsWith(oldName + "(")) { + function = "vec4" + "(" + function + ")"; + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(function)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + var def = parser.postfix_expression(); + replaceNode(ctx, def); + } + } renameFunctionCall(oldName, newName); } - public List collectFunctions(GLSLParser.Translation_unitContext root) { - List result = new ArrayList<>(); - ParseTreeWalker.DEFAULT.walk(new FunctionCollector(result), root); - return result; - } - public void removeFunction(String name) { - AtomicReference context = new AtomicReference<>(); - FastTreeWalker.walk(new FunctionRemover(name, context), root); - if (context.get() != null) { - context.get().getParent().children.remove(context.get()); + for (var ctx : functionPrototypes) { + if (ctx.IDENTIFIER().getText().equals(name)) { + if (ctx.getParent().getParent() instanceof GLSLParser.External_declarationContext declarationContext) { + removeNode(declarationContext); + } + } } } public void removeUnusedFunctions() { - List result = collectFunctions(root); - Set usedIdentifiers = new HashSet<>(); - FastTreeWalker.walk(new IdentifierCollector(id -> { - usedIdentifiers.add(id); - return true; - }), root); + List result = functionPrototypes.stream().map(c -> c.IDENTIFIER().getText()).collect(Collectors.toList()); + List usedIdentifiers = variableIdentifiers.stream().map(c -> c.IDENTIFIER().getText()).collect(Collectors.toList()); List functionsToRemove = result.stream().filter(name -> !usedIdentifiers.contains(name) && !name.equals("main")).collect(Collectors.toList()); // TODO - remove all the functions in one walk of the tree for (String name : functionsToRemove) { @@ -177,13 +400,67 @@ public void removeUnusedFunctions() { public Map> findConstParameter() { Map> functions = new HashMap<>(); - ParseTreeWalker.DEFAULT.walk(new ConstParameterFinder(functions), root); + for (var ctx : parameterDeclarations) { + GLSLParser.Type_qualifierContext typeQualifierContext = ctx.type_qualifier(); + if (typeQualifierContext == null) { + continue; + } + GLSLParser.Single_type_qualifierContext singleTypeQualifierContext = typeQualifierContext.single_type_qualifier(0); + if (singleTypeQualifierContext == null) { + continue; + } + GLSLParser.Storage_qualifierContext storageQualifierContext = singleTypeQualifierContext.storage_qualifier(); + if (storageQualifierContext != null && storageQualifierContext.CONST() != null) { + if (ctx.getParent().getParent() instanceof GLSLParser.Function_prototypeContext proto) { + if (functions.containsKey(proto.IDENTIFIER().getText())) { + functions.get(proto.IDENTIFIER().getText()).add(ctx.parameter_declarator().IDENTIFIER().getText()); + + } else { + List params = new ArrayList<>(); + params.add(ctx.parameter_declarator().IDENTIFIER().getText()); + functions.put(proto.IDENTIFIER().getText(), params); + } + } + } + } return functions; } public void removeConstAssignment(Map> functions) { for (Map.Entry> entry : functions.entrySet()) { - ParseTreeWalker.DEFAULT.walk(new ConstAssignmentRemover(entry.getKey(), entry.getValue()), root); + for (var ctx : variableIdentifiers) { + if (entry.getValue().contains(ctx.IDENTIFIER().getText())) { + var parent = ctx.getParent(); + while (!(parent instanceof GLSLParser.Function_definitionContext definitionContext)) { + parent = parent.getParent(); + if (parent == null) { + return; + } + } + + if (definitionContext.function_prototype().IDENTIFIER().getText().equals(entry.getKey())) { + parent = ctx.getParent(); + while (!(parent instanceof GLSLParser.Single_declarationContext declarationContext)) { + parent = parent.getParent(); + if (parent == null) { + return; + } + } + + entry.getValue().add(declarationContext.typeless_declaration().IDENTIFIER().getText()); + + GLSLParser.Type_qualifierContext typeQualifierContext = declarationContext.fully_specified_type().type_qualifier(); + if (typeQualifierContext == null || typeQualifierContext.single_type_qualifier(0) == null) { + return; + } + GLSLParser.Storage_qualifierContext storageQualifierContext = typeQualifierContext.single_type_qualifier(0).storage_qualifier(); + if (storageQualifierContext != null && storageQualifierContext.CONST() != null) { + declarationContext.fully_specified_type().children.remove(typeQualifierContext); + removeNode(typeQualifierContext); + } + } + } + } } } @@ -206,25 +483,158 @@ public void makeOutDeclaration(GLSLParser.Single_declarationContext inDeclaratio } public Map findQualifiers(int type) { - Map result = new HashMap<>(); - ParseTreeWalker.DEFAULT.walk(new QualifierFinder(type, result), root); - return result; + Map nodes = new HashMap<>(); + mainLoop: + for (var ctx : storageQualifiers) { + if (ctx.children.get(0) instanceof TerminalNode node && + node.getSymbol() instanceof CommonToken token && + token.getType() == type) { + var parent = node.getParent(); + while (!(parent instanceof GLSLParser.Single_declarationContext declarationContext)) { + parent = parent.getParent(); + if (parent == null) { + continue mainLoop; + } + } + + nodes.put(declarationContext.typeless_declaration().IDENTIFIER().getText(), declarationContext); + + if (declarationContext.getParent() instanceof GLSLParser.Init_declarator_listContext listContext) { + if (listContext.typeless_declaration() != null) { + for (var entry : listContext.typeless_declaration()) { + nodes.put(entry.IDENTIFIER().getText(), declarationContext); + } + } + } + } + } + return nodes; } public boolean hasAssigment(String name) { - AtomicBoolean atomicBoolean = new AtomicBoolean(false); - ParseTreeWalker.DEFAULT.walk(new AssigmentChecker(name, atomicBoolean), root); - return atomicBoolean.get(); + for (var ctx : assignmentExpressions) { + if (ctx.unary_expression() != null && (ctx.unary_expression().getText().startsWith(name) || ctx.unary_expression().getText().startsWith(name + "."))) { + return true; + } + } + return false; } public void rewriteStructArrays() { - ParseTreeWalker.DEFAULT.walk(new StructArrayRewriter(), root); + for (var ctx : structDeclarations) { + if (ctx.type_specifier().array_specifier() != null) { + var array_specifier = ctx.type_specifier().array_specifier(); + if (array_specifier.dimension().get(0).constant_expression() != null) { + return; + } + ctx.type_specifier().children.remove(array_specifier); + for (var entry : ctx.struct_declarator_list().struct_declarator()) { + array_specifier.setParent(entry); + entry.children.add(array_specifier); + } + } + } } public List collectStorage() { - List tokens = new ArrayList<>(); - ParseTreeWalker.DEFAULT.walk(new StorageCollector(tokens), root); - return tokens; + return storageQualifiers.stream().filter(ctx -> ctx.getChild(0) instanceof TerminalNode).map(ctx -> (TerminalNode) ctx.getChild(0)).collect(Collectors.toList()); + } + + //Add + public void addFunctionDefinition(GLSLParser.Function_definitionContext ctx) { + this.functionDefinitions.add(ctx); + } + + public void addPostfix(GLSLParser.Postfix_expressionContext ctx) { + this.postfixExpressions.add(ctx); + } + + public void addAssignment(GLSLParser.Assignment_expressionContext ctx) { + this.assignmentExpressions.add(ctx); + } + + public void addVariable(GLSLParser.Variable_identifierContext ctx) { + this.variableIdentifiers.add(ctx); + } + + public void addParameter(GLSLParser.Parameter_declarationContext ctx) { + this.parameterDeclarations.add(ctx); + } + + public void addFunctionPrototype(GLSLParser.Function_prototypeContext ctx) { + this.functionPrototypes.add(ctx); + } + + public void addTypeless(GLSLParser.Typeless_declarationContext ctx) { + this.typelessDeclarations.add(ctx); + } + + public void addBinary(GLSLParser.Binary_expressionContext ctx) { + this.binaryExpressions.add(ctx); + } + + public void addStorage(GLSLParser.Storage_qualifierContext ctx) { + this.storageQualifiers.add(ctx); + } + + public void addStruct(GLSLParser.Struct_declarationContext ctx) { + this.structDeclarations.add(ctx); + } + + public void addSingle(GLSLParser.Single_declarationContext ctx) { + this.singleDeclarations.add(ctx); + } + + public void addTexture(GLSLParser.Type_specifier_nonarrayContext ctx) { + this.textures.add(ctx); + } + + //Remove + public void removeFunctionDefinition(GLSLParser.Function_definitionContext ctx) { + functionDefinitions.remove(ctx); + } + + public void removeFunctionPrototype(GLSLParser.Function_prototypeContext ctx) { + functionPrototypes.remove(ctx); + } + + public void removeParameter(GLSLParser.Parameter_declarationContext ctx) { + parameterDeclarations.remove(ctx); + } + + public void removeTypeless(GLSLParser.Typeless_declarationContext ctx) { + typelessDeclarations.remove(ctx); + } + + public void removeTexture(GLSLParser.Type_specifier_nonarrayContext ctx) { + textures.remove(ctx); + } + + public void removePostfix(GLSLParser.Postfix_expressionContext ctx) { + postfixExpressions.remove(ctx); + } + + public void removeAssignment(GLSLParser.Assignment_expressionContext ctx) { + assignmentExpressions.remove(ctx); + } + + public void removeBinary(GLSLParser.Binary_expressionContext ctx) { + binaryExpressions.remove(ctx); } + public void removeStorage(GLSLParser.Storage_qualifierContext ctx) { + storageQualifiers.remove(ctx); + } + + public void removeStruct(GLSLParser.Struct_declarationContext ctx) { + structDeclarations.remove(ctx); + } + + public void removeSingle(GLSLParser.Single_declarationContext ctx) { + singleDeclarations.remove(ctx); + } + + public void removeVariables(GLSLParser.Variable_identifierContext ctx) { + variableIdentifiers.remove(ctx); + } } diff --git a/src/main/java/org/taumc/glsl/TransformerCollector.java b/src/main/java/org/taumc/glsl/TransformerCollector.java new file mode 100644 index 0000000..dfa8ef3 --- /dev/null +++ b/src/main/java/org/taumc/glsl/TransformerCollector.java @@ -0,0 +1,78 @@ +package org.taumc.glsl; + +import org.taumc.glsl.grammar.GLSLParser; +import org.taumc.glsl.grammar.GLSLParserBaseListener; + +public class TransformerCollector extends GLSLParserBaseListener { + + private final Transformer transformer; + + public TransformerCollector(Transformer transformer) { + this.transformer = transformer; + } + + @Override + public void enterFunction_definition(GLSLParser.Function_definitionContext ctx) { + this.transformer.addFunctionDefinition(ctx); + } + + @Override + public void enterFunction_prototype(GLSLParser.Function_prototypeContext ctx) { + this.transformer.addFunctionPrototype(ctx); + } + + @Override + public void enterParameter_declaration(GLSLParser.Parameter_declarationContext ctx) { + this.transformer.addParameter(ctx); + } + + @Override + public void enterVariable_identifier(GLSLParser.Variable_identifierContext ctx) { + this.transformer.addVariable(ctx); + } + + @Override + public void enterTypeless_declaration(GLSLParser.Typeless_declarationContext ctx) { + this.transformer.addTypeless(ctx); + } + + @Override + public void enterType_specifier_nonarray(GLSLParser.Type_specifier_nonarrayContext ctx) { + if (ctx.TEXTURE2D() != null) { + this.transformer.addTexture(ctx); + } + if (ctx.TEXTURE3D() != null) { + this.transformer.addTexture(ctx); + } + } + + @Override + public void enterPostfix_expression(GLSLParser.Postfix_expressionContext ctx) { + this.transformer.addPostfix(ctx); + } + + @Override + public void exitAssignment_expression(GLSLParser.Assignment_expressionContext ctx) { + this.transformer.addAssignment(ctx); + } + + @Override + public void enterBinary_expression(GLSLParser.Binary_expressionContext ctx) { + this.transformer.addBinary(ctx); + } + + @Override + public void enterStorage_qualifier(GLSLParser.Storage_qualifierContext ctx) { + this.transformer.addStorage(ctx); + } + + @Override + public void enterStruct_declaration(GLSLParser.Struct_declarationContext ctx) { + this.transformer.addStruct(ctx); + } + + @Override + public void enterSingle_declaration(GLSLParser.Single_declarationContext ctx) { + this.transformer.addSingle(ctx); + } +} diff --git a/src/main/java/org/taumc/glsl/TransformerRemover.java b/src/main/java/org/taumc/glsl/TransformerRemover.java new file mode 100644 index 0000000..246f5e6 --- /dev/null +++ b/src/main/java/org/taumc/glsl/TransformerRemover.java @@ -0,0 +1,78 @@ +package org.taumc.glsl; + +import org.taumc.glsl.grammar.GLSLParser; +import org.taumc.glsl.grammar.GLSLParserBaseListener; + +public class TransformerRemover extends GLSLParserBaseListener { + + private final Transformer transformer; + + public TransformerRemover(Transformer transformer) { + this.transformer = transformer; + } + + @Override + public void enterFunction_definition(GLSLParser.Function_definitionContext ctx) { + this.transformer.removeFunctionDefinition(ctx); + } + + @Override + public void enterFunction_prototype(GLSLParser.Function_prototypeContext ctx) { + this.transformer.removeFunctionPrototype(ctx); + } + + @Override + public void enterParameter_declaration(GLSLParser.Parameter_declarationContext ctx) { + this.transformer.removeParameter(ctx); + } + + @Override + public void enterVariable_identifier(GLSLParser.Variable_identifierContext ctx) { + this.transformer.removeVariables(ctx); + } + + @Override + public void enterTypeless_declaration(GLSLParser.Typeless_declarationContext ctx) { + this.transformer.removeTypeless(ctx); + } + + @Override + public void enterType_specifier_nonarray(GLSLParser.Type_specifier_nonarrayContext ctx) { + if (ctx.TEXTURE2D() != null) { + this.transformer.removeTexture(ctx); + } + if (ctx.TEXTURE3D() != null) { + this.transformer.removeTexture(ctx); + } + } + + @Override + public void enterPostfix_expression(GLSLParser.Postfix_expressionContext ctx) { + this.transformer.removePostfix(ctx); + } + + @Override + public void exitAssignment_expression(GLSLParser.Assignment_expressionContext ctx) { + this.transformer.removeAssignment(ctx); + } + + @Override + public void enterBinary_expression(GLSLParser.Binary_expressionContext ctx) { + this.transformer.removeBinary(ctx); + } + + @Override + public void enterStorage_qualifier(GLSLParser.Storage_qualifierContext ctx) { + this.transformer.removeStorage(ctx); + } + + @Override + public void enterStruct_declaration(GLSLParser.Struct_declarationContext ctx) { + this.transformer.removeStruct(ctx); + } + + @Override + public void enterSingle_declaration(GLSLParser.Single_declarationContext ctx) { + this.transformer.removeSingle(ctx); + } +} From 8348da2f2b16cd15cda1c91f0fa65a3ac6126680 Mon Sep 17 00:00:00 2001 From: ferriarnus <61201275+ferriarnus@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:08:36 +0100 Subject: [PATCH 02/10] last changes --- .../java/org/taumc/glsl/InjectorPoint.java | 40 +++++++++ src/main/java/org/taumc/glsl/Transformer.java | 83 +++++++++++-------- 2 files changed, 90 insertions(+), 33 deletions(-) create mode 100644 src/main/java/org/taumc/glsl/InjectorPoint.java diff --git a/src/main/java/org/taumc/glsl/InjectorPoint.java b/src/main/java/org/taumc/glsl/InjectorPoint.java new file mode 100644 index 0000000..ac448a3 --- /dev/null +++ b/src/main/java/org/taumc/glsl/InjectorPoint.java @@ -0,0 +1,40 @@ +package org.taumc.glsl; + +import org.taumc.glsl.grammar.GLSLParser; +import org.taumc.glsl.grammar.GLSLParserBaseListener; + +import java.util.concurrent.atomic.AtomicReference; + +public class InjectorPoint extends GLSLParserBaseListener { + + private final Transformer transformer; + private final boolean function; + + public InjectorPoint(Transformer transformer, boolean function) { + this.transformer = transformer; + this.function = function; + } + + @Override + public void enterFunction_definition(GLSLParser.Function_definitionContext ctx) { + if (ctx.getParent() instanceof GLSLParser.External_declarationContext list) { + transformer.function = list; + transformer.variable = list; + } + } + + @Override + public void enterStorage_qualifier(GLSLParser.Storage_qualifierContext ctx) { + if (function) { + return; + } + var parent = ctx.getParent(); + while (!(parent instanceof GLSLParser.External_declarationContext list)) { + if (parent.getParent() == null) { + return; + } + parent = parent.getParent(); + } + transformer.variable = list; + } +} diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 9af8588..8bccbe0 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -34,6 +34,8 @@ public class Transformer { private final List structDeclarations; private final List singleDeclarations; private final List textures; + public GLSLParser.External_declarationContext variable = null; + public GLSLParser.External_declarationContext function = null; public Transformer(GLSLParser.Translation_unitContext root) { this.root = root; @@ -61,32 +63,34 @@ public void injectVariable(String code) { GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); var insert = parser.external_declaration(); - ParserRuleContext left = null; - if (!storageQualifiers.isEmpty()) { - var parent = storageQualifiers.get(0).getParent(); - while (!(parent instanceof GLSLParser.External_declarationContext)) { - if (parent.getParent() == null) { - break; + if (variable == null) { + if (!storageQualifiers.isEmpty()) { + var parent = storageQualifiers.get(0).getParent(); + while (!(parent instanceof GLSLParser.External_declarationContext)) { + if (parent.getParent() == null) { + break; + } + parent = parent.getParent(); + } + if (parent instanceof GLSLParser.External_declarationContext list) { + variable = list; } - parent = parent.getParent(); - } - if (parent instanceof GLSLParser.External_declarationContext list) { - left = list; } } - if (left == null){ + if (variable == null){ if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { - left = list; + variable = list; } } - if (left != null) { - var parent = left.getParent(); - int i = parent.children.indexOf(left); + if (variable != null) { + var parent = variable.getParent(); + int i = parent.children.indexOf(variable); parent.children.add(i, insert); insert.setParent(parent); scanNode(insert); + ParseTreeWalker.DEFAULT.walk(new InjectorPoint(this, false), insert); } } @@ -95,17 +99,19 @@ public void injectFunction(String code) { GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); var insert = parser.external_declaration(); - ParserRuleContext left = null; - if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { - left = list; + if (function == null) { + if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { + function = list; + } } - if (left != null) { - var parent = left.getParent(); - int i = parent.children.indexOf(left); + if (function != null) { + var parent = function.getParent(); + int i = parent.children.indexOf(function); parent.children.add(i, insert); insert.setParent(parent); scanNode(insert); + ParseTreeWalker.DEFAULT.walk(new InjectorPoint(this, true), insert); } } @@ -132,11 +138,12 @@ public void rename(Map names) { public void replaceExpression(String oldCode, String newCode) { GLSLLexer oldLexer = new GLSLLexer(CharStreams.fromString(oldCode)); GLSLParser oldParser = new GLSLParser(new CommonTokenStream(oldLexer)); - GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); - GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); var oldExpression = oldParser.binary_expression(); - for (var ctx : binaryExpressions) { + var binaryExpression = new ArrayList<>(binaryExpressions); + for (var ctx : binaryExpression) { if (ctx.getText().equals(oldExpression.getText())) { + GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); + GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); replaceNode(ctx, newParser.binary_expression()); } else if (ctx.getText().startsWith(oldExpression.getText())) { if (ctx.unary_expression() != null) { @@ -151,6 +158,8 @@ public void replaceExpression(String oldCode, String newCode) { } } if (postfix.getText().equals(oldExpression.getText())) { + GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); + GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); replaceNode(postfix, newParser.unary_expression()); } } @@ -190,7 +199,8 @@ public void prependMain(String code) { public void removeVariable(String code) { ParserRuleContext typeless = null; - for (var ctx : typelessDeclarations) { + var typelessDeclaration = new ArrayList<>(this.typelessDeclarations); + for (var ctx : typelessDeclaration) { if (ctx.IDENTIFIER() != null) { Token token = ctx.IDENTIFIER().getSymbol(); if(token instanceof CommonToken cToken) { @@ -222,8 +232,12 @@ else if (ctx.getParent() instanceof GLSLParser.Init_declarator_listContext) { if (typeless.getParent() instanceof GLSLParser.Init_declarator_listContext listContext) { int i = listContext.children.indexOf(typeless); - removeNode((ParserRuleContext) listContext.children.remove(i-1)); - removeNode((ParserRuleContext) listContext.children.remove(i-1)); + if (listContext.children.remove(i-1) instanceof ParserRuleContext context) { + removeNode(context); + } + if (listContext.children.remove(i-1) instanceof ParserRuleContext context) { + removeNode(context); + } } else if (typeless.parent instanceof GLSLParser.Single_declarationContext singleContext) { ParserRuleContext node = singleContext.getParent().getParent().getParent(); node.getParent().children.remove(node); @@ -307,13 +321,13 @@ public void renameArray(String oldName, String newName, Set found) { public void renameArray(Map replacements, Set found) { for (var ctx : postfixExpressions) { if (ctx.postfix_expression() == null) { - return; + continue; } if (ctx.postfix_expression().primary_expression() == null) { - return; + continue; } if (ctx.postfix_expression().primary_expression().variable_identifier() == null) { - return; + continue; } if (ctx.postfix_expression().primary_expression().variable_identifier().IDENTIFIER().getSymbol() instanceof CommonToken cToken) { String replacement = replacements.get(cToken.getText()); @@ -344,7 +358,7 @@ public void renameFunctionCall(String oldName, String newName) { public void renameFunctionCall(Map names) { List nodes = new ArrayList<>(); - nodes.addAll(typelessDeclarations.stream().map(GLSLParser.Typeless_declarationContext::IDENTIFIER).collect(Collectors.toList())); + nodes.addAll(functionPrototypes.stream().map(GLSLParser.Function_prototypeContext::IDENTIFIER).collect(Collectors.toList())); nodes.addAll(variableIdentifiers.stream().map(GLSLParser.Variable_identifierContext::IDENTIFIER).collect(Collectors.toList())); nodes.addAll(textures.stream().map(c -> { if (c.TEXTURE2D() != null) { @@ -365,7 +379,8 @@ public void renameFunctionCall(Map names) { } public void renameAndWrapShadow(String oldName, String newName) { - for (var ctx : postfixExpressions) { + var postfixExpression = new ArrayList<>(postfixExpressions); + for (var ctx : postfixExpression) { String function = ctx.getText(); if (ctx.function_call_parameters() != null && function.startsWith(oldName + "(")) { function = "vec4" + "(" + function + ")"; @@ -379,9 +394,11 @@ public void renameAndWrapShadow(String oldName, String newName) { } public void removeFunction(String name) { - for (var ctx : functionPrototypes) { + var functionPrototype = new ArrayList<>(functionPrototypes); + for (var ctx : functionPrototype) { if (ctx.IDENTIFIER().getText().equals(name)) { if (ctx.getParent().getParent() instanceof GLSLParser.External_declarationContext declarationContext) { + declarationContext.getParent().children.remove(declarationContext); removeNode(declarationContext); } } From 55c86226ae3fd0b9dcb24fc32f15d877f81c2b83 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:42:13 -0400 Subject: [PATCH 03/10] Reduce calls to getText --- src/main/java/org/taumc/glsl/Transformer.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 8bccbe0..5e9bdc6 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -139,25 +139,27 @@ public void replaceExpression(String oldCode, String newCode) { GLSLLexer oldLexer = new GLSLLexer(CharStreams.fromString(oldCode)); GLSLParser oldParser = new GLSLParser(new CommonTokenStream(oldLexer)); var oldExpression = oldParser.binary_expression(); + String oldText = oldExpression.getText(); var binaryExpression = new ArrayList<>(binaryExpressions); for (var ctx : binaryExpression) { - if (ctx.getText().equals(oldExpression.getText())) { + String ctxText = ctx.getText(); + if (ctxText.equals(oldText)) { GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); replaceNode(ctx, newParser.binary_expression()); - } else if (ctx.getText().startsWith(oldExpression.getText())) { + } else if (ctxText.startsWith(oldText)) { if (ctx.unary_expression() != null) { if (ctx.unary_expression().postfix_expression() != null) { if (ctx.unary_expression().postfix_expression().postfix_expression() != null) { var postfix = ctx.unary_expression().postfix_expression().postfix_expression(); - while (postfix.getText().startsWith(oldExpression.getText()) && !postfix.getText().equals(oldExpression.getText())) { + while (postfix.getText().startsWith(oldText) && !postfix.getText().equals(oldText)) { if (postfix.postfix_expression() != null) { postfix = postfix.postfix_expression(); } else { break; } } - if (postfix.getText().equals(oldExpression.getText())) { + if (postfix.getText().equals(oldText)) { GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); replaceNode(postfix, newParser.unary_expression()); From 0a1e8e46349663fa96cc093086749d4ab8391155 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:14:44 -0400 Subject: [PATCH 04/10] Add injectAtEnd --- src/main/java/org/taumc/glsl/Transformer.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 5e9bdc6..91192b9 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -115,6 +115,15 @@ public void injectFunction(String code) { } } + public void injectAtEnd(String code) { + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + var insert = parser.external_declaration(); + + root.children.add(insert); + scanNode(insert); + } + public void rename(String oldName, String newName) { rename(Collections.singletonMap(oldName, newName)); } From 4670375429072aa926a333dd7cb525e6a6bc54e1 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:47:02 -0400 Subject: [PATCH 05/10] Cache text for binary expressions --- src/main/java/org/taumc/glsl/Transformer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 91192b9..fd0f036 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -1,5 +1,6 @@ package org.taumc.glsl; +import com.github.bsideup.jabel.Desugar; import com.ibm.icu.impl.CollectionSet; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonToken; @@ -29,7 +30,7 @@ public class Transformer { private final List parameterDeclarations; private final List typelessDeclarations; private final List functionPrototypes; - private final List binaryExpressions; + private final List binaryExpressions; private final List storageQualifiers; private final List structDeclarations; private final List singleDeclarations; @@ -37,6 +38,9 @@ public class Transformer { public GLSLParser.External_declarationContext variable = null; public GLSLParser.External_declarationContext function = null; + @Desugar + private record CachedBinaryExpression(GLSLParser.Binary_expressionContext expr, String exprText) {} + public Transformer(GLSLParser.Translation_unitContext root) { this.root = root; this.functionDefinitions = new ArrayList<>(); @@ -149,9 +153,10 @@ public void replaceExpression(String oldCode, String newCode) { GLSLParser oldParser = new GLSLParser(new CommonTokenStream(oldLexer)); var oldExpression = oldParser.binary_expression(); String oldText = oldExpression.getText(); - var binaryExpression = new ArrayList<>(binaryExpressions); - for (var ctx : binaryExpression) { - String ctxText = ctx.getText(); + var exprList = new ArrayList<>(binaryExpressions); + for (var cachedExpr : exprList) { + String ctxText = cachedExpr.exprText(); + var ctx = cachedExpr.expr(); if (ctxText.equals(oldText)) { GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); @@ -598,7 +603,7 @@ public void addTypeless(GLSLParser.Typeless_declarationContext ctx) { } public void addBinary(GLSLParser.Binary_expressionContext ctx) { - this.binaryExpressions.add(ctx); + this.binaryExpressions.add(new CachedBinaryExpression(ctx, ctx.getText())); } public void addStorage(GLSLParser.Storage_qualifierContext ctx) { From b22c7a9c9be0acf79dd669b72d7ef96226e4c3b0 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 18:51:52 -0400 Subject: [PATCH 06/10] Add proper shader parsing logic Old logic was using sluggish ANTLR prediction mode, which slowed everything down --- .../java/org/taumc/glsl/ShaderParser.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/main/java/org/taumc/glsl/ShaderParser.java diff --git a/src/main/java/org/taumc/glsl/ShaderParser.java b/src/main/java/org/taumc/glsl/ShaderParser.java new file mode 100644 index 0000000..d619316 --- /dev/null +++ b/src/main/java/org/taumc/glsl/ShaderParser.java @@ -0,0 +1,57 @@ +package org.taumc.glsl; + +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.BufferedTokenStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ConsoleErrorListener; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.taumc.glsl.grammar.GLSLLexer; +import org.taumc.glsl.grammar.GLSLParser; +import org.taumc.glsl.grammar.GLSLPreParser; + +public class ShaderParser { + public record ParsedShader(GLSLPreParser.Translation_unitContext pre, GLSLParser.Translation_unitContext full) {} + + private static void configureNoError(Parser parser) { + parser.setErrorHandler(new BailErrorStrategy()); + parser.removeErrorListeners(); + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + } + + private static void configureError(Parser parser) { + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.addErrorListener(ConsoleErrorListener.INSTANCE); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + } + + private static ParsedShader tryParse(GLSLLexer lexer, GLSLPreParser preParser, GLSLParser parser) { + var pre = preParser.translation_unit(); + lexer.reset(); + parser.reset(); + var full = parser.translation_unit(); + return new ParsedShader(pre, full); + } + + public static ParsedShader parseShader(String shader) { + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(shader)); + GLSLPreParser preParser = new GLSLPreParser(new BufferedTokenStream(lexer)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + preParser.setBuildParseTree(true); + parser.setBuildParseTree(true); + configureNoError(parser); + configureNoError(preParser); + try { + return tryParse(lexer, preParser, parser); + } catch (Exception e) { + lexer.reset(); + parser.reset(); + preParser.reset(); + configureError(parser); + configureError(preParser); + return tryParse(lexer, preParser, parser); + } + } +} From 7d53a8c8f8ec28fac8ee3eba21d5d34957d24f9c Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 19:32:37 -0400 Subject: [PATCH 07/10] Faster parsing --- .../java/org/taumc/glsl/ShaderParser.java | 19 ++++++ src/main/java/org/taumc/glsl/Transformer.java | 62 +++++++------------ 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/taumc/glsl/ShaderParser.java b/src/main/java/org/taumc/glsl/ShaderParser.java index d619316..df48cd7 100644 --- a/src/main/java/org/taumc/glsl/ShaderParser.java +++ b/src/main/java/org/taumc/glsl/ShaderParser.java @@ -1,5 +1,6 @@ package org.taumc.glsl; +import com.github.bsideup.jabel.Desugar; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.BufferedTokenStream; import org.antlr.v4.runtime.CharStreams; @@ -12,7 +13,10 @@ import org.taumc.glsl.grammar.GLSLParser; import org.taumc.glsl.grammar.GLSLPreParser; +import java.util.function.Function; + public class ShaderParser { + @Desugar public record ParsedShader(GLSLPreParser.Translation_unitContext pre, GLSLParser.Translation_unitContext full) {} private static void configureNoError(Parser parser) { @@ -54,4 +58,19 @@ public static ParsedShader parseShader(String shader) { return tryParse(lexer, preParser, parser); } } + + public static T parseSnippet(String codeSnippet, Function parseLogic) { + GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(codeSnippet)); + GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); + parser.setBuildParseTree(true); + configureNoError(parser); + try { + return parseLogic.apply(parser); + } catch (Exception e) { + lexer.reset(); + parser.reset(); + configureError(parser); + return parseLogic.apply(parser); + } + } } diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index fd0f036..d765613 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -10,7 +10,6 @@ import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.TerminalNode; -import org.taumc.glsl.grammar.GLSLLexer; import org.taumc.glsl.grammar.GLSLParser; import java.util.*; @@ -24,13 +23,13 @@ public class Transformer { private final GLSLParser.Translation_unitContext root; private final List functionDefinitions; - private final List postfixExpressions; + private final List> postfixExpressions; private final List assignmentExpressions; private final List variableIdentifiers; private final List parameterDeclarations; private final List typelessDeclarations; private final List functionPrototypes; - private final List binaryExpressions; + private final List> binaryExpressions; private final List storageQualifiers; private final List structDeclarations; private final List singleDeclarations; @@ -39,7 +38,7 @@ public class Transformer { public GLSLParser.External_declarationContext function = null; @Desugar - private record CachedBinaryExpression(GLSLParser.Binary_expressionContext expr, String exprText) {} + private record CachedTextExpression(T expr, String exprText) {} public Transformer(GLSLParser.Translation_unitContext root) { this.root = root; @@ -63,9 +62,7 @@ public Transformer(GLSLParser.Translation_unitContext root) { * @param code the to inject code */ public void injectVariable(String code) { - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var insert = parser.external_declaration(); + var insert = ShaderParser.parseSnippet(code, GLSLParser::external_declaration); if (variable == null) { if (!storageQualifiers.isEmpty()) { @@ -99,9 +96,7 @@ public void injectVariable(String code) { } public void injectFunction(String code) { - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var insert = parser.external_declaration(); + var insert = ShaderParser.parseSnippet(code, GLSLParser::external_declaration); if (function == null) { if (functionDefinitions.get(0).getParent() instanceof GLSLParser.External_declarationContext list) { @@ -120,9 +115,7 @@ public void injectFunction(String code) { } public void injectAtEnd(String code) { - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var insert = parser.external_declaration(); + var insert = ShaderParser.parseSnippet(code, GLSLParser::external_declaration); root.children.add(insert); scanNode(insert); @@ -149,18 +142,14 @@ public void rename(Map names) { } public void replaceExpression(String oldCode, String newCode) { - GLSLLexer oldLexer = new GLSLLexer(CharStreams.fromString(oldCode)); - GLSLParser oldParser = new GLSLParser(new CommonTokenStream(oldLexer)); - var oldExpression = oldParser.binary_expression(); + var oldExpression = ShaderParser.parseSnippet(oldCode, GLSLParser::binary_expression); String oldText = oldExpression.getText(); var exprList = new ArrayList<>(binaryExpressions); for (var cachedExpr : exprList) { String ctxText = cachedExpr.exprText(); var ctx = cachedExpr.expr(); if (ctxText.equals(oldText)) { - GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); - GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); - replaceNode(ctx, newParser.binary_expression()); + replaceNode(ctx, ShaderParser.parseSnippet(newCode, GLSLParser::binary_expression)); } else if (ctxText.startsWith(oldText)) { if (ctx.unary_expression() != null) { if (ctx.unary_expression().postfix_expression() != null) { @@ -174,9 +163,7 @@ public void replaceExpression(String oldCode, String newCode) { } } if (postfix.getText().equals(oldText)) { - GLSLLexer newLexer = new GLSLLexer(CharStreams.fromString(newCode)); - GLSLParser newParser = new GLSLParser(new CommonTokenStream(newLexer)); - replaceNode(postfix, newParser.unary_expression()); + replaceNode(postfix, ShaderParser.parseSnippet(newCode, GLSLParser::unary_expression)); } } } @@ -202,9 +189,7 @@ private void scanNode(ParserRuleContext node) { } public void prependMain(String code) { - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var insert = parser.statement(); + var insert = ShaderParser.parseSnippet(code, GLSLParser::statement); for (var ctx : functionDefinitions) { if (ctx.function_prototype().IDENTIFIER().getText().equals("main")) { ctx.compound_statement_no_new_scope().statement_list().children.add(0, insert); @@ -293,9 +278,7 @@ public int findType(String code) { } public void appendMain(String code) { - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(code)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var insert = parser.statement(); + var insert = ShaderParser.parseSnippet(code, GLSLParser::statement); for (var ctx : functionDefinitions) { if (ctx.function_prototype().IDENTIFIER().getText().equals("main")) { ctx.compound_statement_no_new_scope().statement_list().children.add(insert); @@ -335,7 +318,8 @@ public void renameArray(String oldName, String newName, Set found) { } public void renameArray(Map replacements, Set found) { - for (var ctx : postfixExpressions) { + for (var cachedExpr : postfixExpressions) { + var ctx = cachedExpr.expr(); if (ctx.postfix_expression() == null) { continue; } @@ -396,14 +380,12 @@ public void renameFunctionCall(Map names) { public void renameAndWrapShadow(String oldName, String newName) { var postfixExpression = new ArrayList<>(postfixExpressions); - for (var ctx : postfixExpression) { - String function = ctx.getText(); - if (ctx.function_call_parameters() != null && function.startsWith(oldName + "(")) { + for (var cachedExpr : postfixExpression) { + String function = cachedExpr.expr().getText(); + if (cachedExpr.expr().function_call_parameters() != null && function.startsWith(oldName + "(")) { function = "vec4" + "(" + function + ")"; - GLSLLexer lexer = new GLSLLexer(CharStreams.fromString(function)); - GLSLParser parser = new GLSLParser(new CommonTokenStream(lexer)); - var def = parser.postfix_expression(); - replaceNode(ctx, def); + var def = ShaderParser.parseSnippet(function, GLSLParser::postfix_expression); + replaceNode(cachedExpr.expr(), def); } } renameFunctionCall(oldName, newName); @@ -579,7 +561,7 @@ public void addFunctionDefinition(GLSLParser.Function_definitionContext ctx) { } public void addPostfix(GLSLParser.Postfix_expressionContext ctx) { - this.postfixExpressions.add(ctx); + this.postfixExpressions.add(new CachedTextExpression<>(ctx, ctx.getText())); } public void addAssignment(GLSLParser.Assignment_expressionContext ctx) { @@ -603,7 +585,7 @@ public void addTypeless(GLSLParser.Typeless_declarationContext ctx) { } public void addBinary(GLSLParser.Binary_expressionContext ctx) { - this.binaryExpressions.add(new CachedBinaryExpression(ctx, ctx.getText())); + this.binaryExpressions.add(new CachedTextExpression<>(ctx, ctx.getText())); } public void addStorage(GLSLParser.Storage_qualifierContext ctx) { @@ -644,7 +626,7 @@ public void removeTexture(GLSLParser.Type_specifier_nonarrayContext ctx) { } public void removePostfix(GLSLParser.Postfix_expressionContext ctx) { - postfixExpressions.remove(ctx); + postfixExpressions.remove(new CachedTextExpression<>(ctx, ctx.getText())); } public void removeAssignment(GLSLParser.Assignment_expressionContext ctx) { @@ -652,7 +634,7 @@ public void removeAssignment(GLSLParser.Assignment_expressionContext ctx) { } public void removeBinary(GLSLParser.Binary_expressionContext ctx) { - binaryExpressions.remove(ctx); + binaryExpressions.remove(new CachedTextExpression<>(ctx, ctx.getText())); } public void removeStorage(GLSLParser.Storage_qualifierContext ctx) { From 791677883672bf45d19e9575f8266dfda2f24f01 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 19:34:52 -0400 Subject: [PATCH 08/10] Small optimization --- src/main/java/org/taumc/glsl/Transformer.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index d765613..e8668c4 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -38,7 +38,16 @@ public class Transformer { public GLSLParser.External_declarationContext function = null; @Desugar - private record CachedTextExpression(T expr, String exprText) {} + private record CachedTextExpression(T expr, String exprText) { + @Override + public boolean equals(Object obj) { + if(obj instanceof CachedTextExpression cachedExpr) { + return cachedExpr.expr == this.expr; + } else { + return false; + } + } + } public Transformer(GLSLParser.Translation_unitContext root) { this.root = root; From 8ccc08171d822bf9df335dc023a1edf01a8f6601 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 19:40:39 -0400 Subject: [PATCH 09/10] Tweak --- src/main/java/org/taumc/glsl/Transformer.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index e8668c4..2341751 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -47,6 +47,11 @@ public boolean equals(Object obj) { return false; } } + + @Override + public int hashCode() { + return System.identityHashCode(expr); + } } public Transformer(GLSLParser.Translation_unitContext root) { @@ -635,7 +640,7 @@ public void removeTexture(GLSLParser.Type_specifier_nonarrayContext ctx) { } public void removePostfix(GLSLParser.Postfix_expressionContext ctx) { - postfixExpressions.remove(new CachedTextExpression<>(ctx, ctx.getText())); + postfixExpressions.remove(new CachedTextExpression<>(ctx, null)); } public void removeAssignment(GLSLParser.Assignment_expressionContext ctx) { @@ -643,7 +648,7 @@ public void removeAssignment(GLSLParser.Assignment_expressionContext ctx) { } public void removeBinary(GLSLParser.Binary_expressionContext ctx) { - binaryExpressions.remove(new CachedTextExpression<>(ctx, ctx.getText())); + binaryExpressions.remove(new CachedTextExpression<>(ctx, null)); } public void removeStorage(GLSLParser.Storage_qualifierContext ctx) { From 77854d02759c7cc43220cc0fe375c36c9337fd00 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Fri, 1 Nov 2024 19:44:15 -0400 Subject: [PATCH 10/10] Actually use cached text --- src/main/java/org/taumc/glsl/Transformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/taumc/glsl/Transformer.java b/src/main/java/org/taumc/glsl/Transformer.java index 2341751..d958d76 100644 --- a/src/main/java/org/taumc/glsl/Transformer.java +++ b/src/main/java/org/taumc/glsl/Transformer.java @@ -395,7 +395,7 @@ public void renameFunctionCall(Map names) { public void renameAndWrapShadow(String oldName, String newName) { var postfixExpression = new ArrayList<>(postfixExpressions); for (var cachedExpr : postfixExpression) { - String function = cachedExpr.expr().getText(); + String function = cachedExpr.exprText(); if (cachedExpr.expr().function_call_parameters() != null && function.startsWith(oldName + "(")) { function = "vec4" + "(" + function + ")"; var def = ShaderParser.parseSnippet(function, GLSLParser::postfix_expression);