diff --git a/README.md b/README.md index 2e26d6d..89eb10b 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,13 @@ Using gradle one can include the hcl4j dependency like so: ```groovy dependencies { - compile "com.bertramlabs.plugins:hcl4j:0.7.7" + compile "com.bertramlabs.plugins:hcl4j:0.8.0" } ``` ## What's New +* **0.8.0** HCL For Loop Tuples now evaluated. * **0.7.7** HCL Periods in attribute names referenced from Unicodes ID_Continue, fixed. * **0.7.6** SLF4j 1.7.36 upgrade to reduce CVE's * **0.7.5** For Tuple with boolean conditionals as value expression did not work before @@ -62,6 +63,6 @@ For More Information on the HCL Syntax Please see the project page: ## Things to be Done -* for tuples and objects are processed but not evaluated +* for loop objects are processed but not evaluated * add more method definitions for base terraform functions * add handlers for data lookup modules diff --git a/build.gradle b/build.gradle index 06208b8..8c2e4dc 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ ext { group = 'com.bertramlabs.plugins' -version = '0.7.7' +version = '0.8.0' ext.isReleaseVersion = !version.endsWith("SNAPSHOT") sourceCompatibility = "1.8" diff --git a/src/main/java/com/bertramlabs/plugins/hcl4j/HCLParser.java b/src/main/java/com/bertramlabs/plugins/hcl4j/HCLParser.java index 577d8a5..597dd42 100644 --- a/src/main/java/com/bertramlabs/plugins/hcl4j/HCLParser.java +++ b/src/main/java/com/bertramlabs/plugins/hcl4j/HCLParser.java @@ -703,6 +703,9 @@ private Object processSymbolPass2(Object val, Map mapPosition) th } private Object processSymbol(Symbol symbol, Map mapPosition) throws HCLParserException { + return processSymbol(symbol,mapPosition,null); + } + private Object processSymbol(Symbol symbol, Map mapPosition, Map stackVars) throws HCLParserException { if(symbol instanceof HCLBlock) { HCLBlock block = (HCLBlock)symbol; @@ -736,7 +739,7 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro } if(symbol.getChildren() != null) { for(Symbol child : block.getChildren()) { - processSymbol(child,mapPosition); + processSymbol(child,mapPosition,stackVars); } } return mapPosition; @@ -765,8 +768,8 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro } else if(symbol instanceof PrimitiveType) { return symbol; } else if(symbol instanceof EvalSymbol) { - return processEvaluation((EvalSymbol) symbol,null); - } else if(symbol instanceof HCLAttribute || symbol instanceof GroupedExpression) { + return processEvaluation((EvalSymbol) symbol,null,stackVars); + } else if(symbol instanceof HCLAttribute || symbol instanceof GroupedExpression || symbol instanceof StringInterpolatedExpression) { Map nestedMap = new LinkedHashMap<>(); if(symbol.getChildren().size() > 0) { Object results = null; @@ -776,14 +779,14 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro switch(child.getName()) { case "+": if(results instanceof String) { - Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap); + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); if(rightResult != null) { results = (String)results + rightResult.toString(); } else { //TODO: Exception? } } else if(results instanceof Double) { - Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap); + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); if(rightResult != null && rightResult instanceof Double) { results = (Double)results + (Double)rightResult; } else { @@ -794,7 +797,7 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro break; case "-": if(results instanceof Double) { - Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap); + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); if(rightResult != null && rightResult instanceof Double) { results = (Double)results - (Double)rightResult; } else { @@ -804,7 +807,7 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro break; case "/": if(results instanceof Double) { - Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap); + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); if(rightResult != null && rightResult instanceof Double) { results = (Double)results / (Double)rightResult; } else { @@ -814,7 +817,7 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro break; case "*": if(results instanceof Double) { - Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap); + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); if(rightResult != null && rightResult instanceof Double) { results = (Double)results * (Double)rightResult; } else { @@ -822,13 +825,83 @@ private Object processSymbol(Symbol symbol, Map mapPosition) thro } } break; + case "?": + //if left side of result is false we need to skip between the ? and the : + if(results instanceof Boolean && !((Boolean) results) || results == null ) { + //skip children until ":" operator + Symbol nextElement = symbol.getChildren().get(++x); + while(!(nextElement instanceof Operator) && nextElement.getName() != ":") { + nextElement = symbol.getChildren().get(++x); + } + } + break; + case ":": + //if we got to a colon operator then we need to skip everything after it as its processed with the ? operator above + x = symbol.getChildren().size(); + break; + case "!=": + Object compareResult2 = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(compareResult2 != results && (compareResult2 == null || !compareResult2.equals(results))) { + results = true; + } else { + results = false; + } + break; + case "==": + Object compareResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(compareResult == results || (compareResult != null && compareResult.equals(results))) { + results = true; + } else { + results = false; + } + break; + case ">": + if(results instanceof Double) { + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(rightResult != null && rightResult instanceof Double) { + results = ((Double)results > (Double)rightResult); + } else { + //TODO: Exception? + } + } + break; + case ">=": + if(results instanceof Double) { + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(rightResult != null && rightResult instanceof Double) { + results = ((Double)results >= (Double)rightResult); + } else { + //TODO: Exception? + } + } + break; + case "<": + if(results instanceof Double) { + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(rightResult != null && rightResult instanceof Double) { + results = ((Double)results < (Double)rightResult); + } else { + //TODO: Exception? + } + } + break; + case "<=": + if(results instanceof Double) { + Object rightResult = processSymbol(symbol.getChildren().get(++x),nestedMap,stackVars); + if(rightResult != null && rightResult instanceof Double) { + results = ((Double)results <= (Double)rightResult); + } else { + //TODO: Exception? + } + } + break; } } else { - results = processSymbol(symbol.getChildren().get(0),nestedMap); + results = processSymbol(symbol.getChildren().get(0),nestedMap,stackVars); } } - if(symbol instanceof GroupedExpression) { + if(symbol instanceof GroupedExpression || symbol instanceof StringInterpolatedExpression) { return results; } else { mapPosition.put(symbol.getName(),results); @@ -894,6 +967,80 @@ protected Object evaluateFunctionCall(String functionName,Function functionSymbo } } + + protected Object evaluateComputedTuple(ComputedTuple thisTuple) throws HCLParserException { +// System.out.println("Evaluating Tuple"); + ArrayList computedResult = new ArrayList(); + Variable indexVariable = null; + Variable valueVariable = null; + if(thisTuple.getVariables().size() == 2) { + //variables with index + + indexVariable = thisTuple.getVariables().get(0); + valueVariable = thisTuple.getVariables().get(1); + } else if(thisTuple.getVariables().size() == 1) { + valueVariable = thisTuple.getVariables().get(0); + } else { + log.warn("Computed Tuple loop found with too many variables at line {}",thisTuple.getLine()); + return null; + } + ForConditional tupleConditional = null; + for(Symbol child : thisTuple.getChildren()) { + if(child instanceof ForConditional) { + tupleConditional = (ForConditional) child; + thisTuple.getChildren().remove(child); + break; + } + } + Object elementResult = null; + Boolean forSourceFound=false; + GroupedExpression sourceExpression = new GroupedExpression(0,0,0); + GroupedExpression iteratorExpression = new GroupedExpression(0,0,0); + for(Symbol child : thisTuple.getChildren()) { + if(child instanceof ForSource) { + forSourceFound = true; + continue; + } + if(forSourceFound) { + iteratorExpression.appendChild(child); + } else { + sourceExpression.appendChild(child); + } + + } + if(sourceExpression.getChildren().size() > 0 && iteratorExpression.getChildren().size() > 0) { + elementResult = processSymbol(sourceExpression,result); + if(elementResult instanceof List) { + List elementResultList = (List)elementResult; + System.out.println("Iterating over list"); + for(int counter=0;counter < elementResultList.size();counter++){ + Map stackVars = new LinkedHashMap<>(); + if(indexVariable != null) { + stackVars.put(indexVariable.getName(),counter); + } + stackVars.put(valueVariable.getName(),elementResultList.get(counter)); + if(tupleConditional != null) { + System.out.println("Conditional Tuple Found"); + Object conditionalResult = processSymbol(tupleConditional ,null,stackVars); + System.out.println("Conditional Result: " + conditionalResult); + if(conditionalResult == null || ((conditionalResult instanceof Boolean) && (Boolean) conditionalResult == false)) { + System.out.println("Does not match"); + continue; + } else { + System.out.println("Matches"); + } + } + computedResult.add(processSymbol(iteratorExpression,null,stackVars)); + System.out.println("Computed Result: " + computedResult); + } + } else { + log.error("Error Processing Tuple from source not a List"); + return computedResult; + } + } + return computedResult; + } + protected Map evaluateDataLookup(String lookupName,String name,Map properties) throws HCLParserException { if(dataLookupRegistry.get(lookupName) != null) { try { @@ -917,8 +1064,14 @@ protected Map evaluateDataLookup(String lookupName,String name,Ma //TODO: DO we throw a method missing exception at some point return null; } + + } + protected Object processEvaluation(EvalSymbol evalSymbol, Object context) throws HCLParserException{ + return processEvaluation(evalSymbol,context,null); + } + protected Object processEvaluation(EvalSymbol evalSymbol, Object context,Map stackVars) throws HCLParserException{ Boolean variableLookup = false; Boolean dataLookup = false; String dataLookupName = null; @@ -945,7 +1098,14 @@ protected Object processEvaluation(EvalSymbol evalSymbol, Object context) throws context = dataLookups; break; default: - context = result.get(child.getName()); + if(stackVars != null) { + context = stackVars.get(child.getName()); + if(context == null) { + context = result.get(child.getName()); + } + } else { + context = result.get(child.getName()); + } } } else if(context != null){ @@ -1054,8 +1214,24 @@ else if(evalSymbol instanceof Function) { } else { return null; } + } else if(evalSymbol instanceof ComputedTuple) { + //time to actually implemented a tuple loop + ComputedTuple thisTuple = (ComputedTuple) evalSymbol; + return evaluateComputedTuple(thisTuple); + } + else if(evalSymbol instanceof Variable) { + System.out.println("Evaluating Variable: " + evalSymbol.getName()); + if(stackVars != null) { + context = stackVars.get(evalSymbol.getName()); + if(context == null) { + context = result.get(evalSymbol.getName()); + } + } else { + context = result.get(evalSymbol.getName()); + } + return context; } - else if(evalSymbol instanceof Variable || evalSymbol instanceof ComputedTuple || evalSymbol instanceof ComputedObject) { + else if(evalSymbol instanceof ComputedObject) { return evalSymbol; } else { return null; diff --git a/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForConditional.java b/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForConditional.java index 514b461..2b11c06 100644 --- a/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForConditional.java +++ b/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForConditional.java @@ -1,8 +1,8 @@ package com.bertramlabs.plugins.hcl4j.RuntimeSymbols; -public class ForConditional extends EvalSymbol{ +public class ForConditional extends GroupedExpression{ public ForConditional(Integer line, Integer column,Integer position) { - super(null,line,column,position); + super(line,column,position); } public String getSymbolName() { diff --git a/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForSource.java b/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForSource.java new file mode 100644 index 0000000..26091f9 --- /dev/null +++ b/src/main/java/com/bertramlabs/plugins/hcl4j/RuntimeSymbols/ForSource.java @@ -0,0 +1,11 @@ +package com.bertramlabs.plugins.hcl4j.RuntimeSymbols; + +public class ForSource extends EvalSymbol{ + public ForSource(Integer line, Integer column,Integer position) { + super(null,line,column,position); + } + + public String getSymbolName() { + return "ForSource"; + } +} diff --git a/src/main/jflex/com/bertramlabs/plugins/hcl4j/HCLLexer.jflex b/src/main/jflex/com/bertramlabs/plugins/hcl4j/HCLLexer.jflex index 6b7383e..f685804 100644 --- a/src/main/jflex/com/bertramlabs/plugins/hcl4j/HCLLexer.jflex +++ b/src/main/jflex/com/bertramlabs/plugins/hcl4j/HCLLexer.jflex @@ -110,15 +110,7 @@ HCLParserException attribute = currentAttribute; } - private void startMap() { - HCLMap currentAttribute = new HCLMap(yyline,yycolumn,yychar); - if(currentBlock == null) { - elementStack.add(currentAttribute); - } else { - currentBlock.appendChild(currentAttribute); - } - currentBlock = currentAttribute; - } + private void startInterpolatedString() { StringInterpolatedExpression currentAttribute; @@ -359,11 +351,11 @@ HCLBlock = {HCLAttributeName} {HCLBlockAttribute}* "{" [^]* "}" | {HCLAttributeN HCLBlockAttribute = {WhiteSpaceOpt} "\"" {HCLDoubleStringCharacters} "\"" {WhiteSpaceOpt} | {WhiteSpace} "\'" {HCLSingleStringCharacters} "\'" {WhiteSpaceOpt} | {WhiteSpace} {HCLAttributeName} {WhiteSpaceOpt} -HCLAttribute = {HCLAttributeName} {WhiteSpaceOpt} "=" | {HCLQuotedPropertyName} {WhiteSpaceOpt} "=" +HCLAttribute = {HCLAttributeName} {WhiteSpaceOpt} [=:] | {HCLQuotedPropertyName} {WhiteSpaceOpt} [=:] MapKeyDef = {MapKey} ":" MapKey = {HCLAttributeName} | "\"" {HCLDoubleStringCharacters} "\"" -MapBlockStart = "{" {WhiteSpaceNLOpt} {MapKeyDef} + HCLDoubleStringCharacters = {HCLDoubleStringCharacter}* HCLSingleStringCharacters = {HCLSingleStringCharacter}* @@ -500,34 +492,16 @@ AssignmentExpression = [^] { \" {yybegin(STRINGDOUBLE); stringAttributeName = true ;string.setLength(0);} {HCLAttributeName} {startAttribute(yytext());} - \= {yybegin(HCLATTRIBUTEVALUE); } + \= {yybegin(HCLATTRIBUTEVALUE); } + \: {yybegin(HCLATTRIBUTEVALUE); } + \, { /*ignore*/ } /* whitespace */ {WhiteSpace} { /* ignore */ } } - { - - {MapKeyDef} { yypushback(yylength()); yybegin(HCLMAPKEY); } - , { /* should probably process this but due to simplicity we dont need to */ } - \} { exitAttribute(true); } - {WhiteSpace} { /* ignore */ } -} - { -{MapKey} { yybegin(HCLMAPKEY); yypushback(yylength()); } -":" { startAttribute(currentMapKey); currentMapKey = null ; yybegin(HCLATTRIBUTEVALUE); } -{Comment} { /* ignore */ } -{WhiteSpace} { /* ignore */ } -} - - { - \" {yybegin(STRINGDOUBLE); string.setLength(0); fromMapKey = true; } - {HCLAttributeName} { currentMapKey = yytext() ; yybegin(HCLMAPKEYDEF);} - {WhiteSpace} { /* ignore */ } -} - { [^,\]\r\n\ \t] { yypushback(yylength()); yybegin(HCLATTRIBUTEVALUE); } \] { exitAttribute(true); } @@ -540,7 +514,7 @@ AssignmentExpression = [^] { {ForExpr} { yybegin(FORLOOPEXPRESSION); yypushback(yylength()); } \[ { startArray();/* process an array */ } - {MapBlockStart} { startMap(); yypushback(yylength()-1) ; yybegin(HCLMAP);} + \{ { blockNames = new ArrayList(); blockNames.add(currentBlock.getName()); curleyBraceCounter++ ; hclBlock(blockNames) ; blockNames = null ; attribute = null ; yybegin(HCLINBLOCK); } \" {if(currentBlock instanceof StringInterpolatedExpression) {startNestedStringExpression();} yybegin(STRINGDOUBLE); string.setLength(0); } {MLineModifierStart} {yybegin(MULTILINESTRING) ; isMultiLineFirstNewLine = true ;isMultilineModified = true; string.setLength(0) ; endOfMultiLineSymbol = yytext().substring(3);} @@ -620,7 +594,7 @@ AssignmentExpression = [^] } { - : { yybegin(HCLATTRIBUTEVALUE); } + : { ((ComputedTuple)currentBlock).appendChild(new ForSource(yyline,yycolumn,yychar)) ; yybegin(HCLATTRIBUTEVALUE); } [\]] { exitAttribute(true); } {IfPrimitive} { startForConditional(); } {Conditional} { yybegin(HCLATTRIBUTEVALUE); yypushback(yylength()); } diff --git a/src/test/groovy/com/bertramlabs/plugins/hcl4j/HCLParserSpec.groovy b/src/test/groovy/com/bertramlabs/plugins/hcl4j/HCLParserSpec.groovy index 63801f5..7763edb 100644 --- a/src/test/groovy/com/bertramlabs/plugins/hcl4j/HCLParserSpec.groovy +++ b/src/test/groovy/com/bertramlabs/plugins/hcl4j/HCLParserSpec.groovy @@ -200,6 +200,29 @@ resource xxx "images" { } + + void "should handle test value with colon" () { + given: + def hcl = ''' +variable "variable1"{ + default = [ + { + key = "someKey" + attributes = { + "attribute1" = ":valueWithColon" + } + } + ] + }""" +''' + HCLParser parser = new HCLParser(); + when: + def results = parser.parse(hcl) + then: + results.variable.variable1.default[0].attributes.attribute1 == ":valueWithColon" + } + + void "should handle Map parsing"() { given: @@ -1115,10 +1138,61 @@ resource "aws_launch_configuration" "web" { results.resource["aws_launch_configuration"]["web"]?.user_data != null } + void "it should handle simple for tuples"() { + given: + def hcl = ''' +locals { +swagger_path_method_parameters = [for my_value in [0,1,2,3]: my_value + 1 ] +} +''' + HCLParser parser = new HCLParser(); + when: + def results = parser.parse(hcl) + println results + then: + results.locals["swagger_path_method_parameters"] == [1,2,3,4] + } + + + void "it should handle conditional for tuples"() { + given: + def hcl = ''' +locals { +swagger_path_method_parameters = [for my_value in [0,1,2,3]: (my_value + 1) if my_value != 3 ] +} +''' + HCLParser parser = new HCLParser(); + when: + def results = parser.parse(hcl) + println results + then: + results.locals["swagger_path_method_parameters"] == [1,2,3] + } + + + void "it should handle simple for tuples with index"() { + given: + def hcl = ''' +locals { +swagger_path_method_parameters = [for i,my_value in [0,1,2,3]: "${i}:${my_value + 1}" ] +} +''' + HCLParser parser = new HCLParser(); + when: + def results = parser.parse(hcl) + println results + then: + results.locals["swagger_path_method_parameters"] == ["0:1.0","1:2.0","2:3.0","3:4.0"] + } + void "it should handle nested for tuples"() { given: def hcl = ''' locals { + swagger_path_methods_split = [["a","b"],["c","d"]] + json_data = { + paths = {a: {b: true}, c:{d: false}} + } swagger_path_method_parameters = [for my_value in local.swagger_path_methods_split: [for method_name, method_value in local.json_data["paths"][my_value[0]][my_value[1]]: format("%s:::%s:::%s", my_value[0], my_value[1], method_name) ] @@ -1131,7 +1205,28 @@ swagger_path_method_parameters = [for my_value in local.swagger_path_methods_spl println results then: results.locals["swagger_path_method_parameters"] != null + } + + void "it should handle doubly nested for tuples"() { + given: + def hcl = ''' +locals { + swagger_path_methods_split = [["a","b"],["c","d"]] + json_data = { + paths = {a: {b: [1,2]}, c:{d: [3,4]]}} + } +swagger_path_method_parameters = [for my_value in local.swagger_path_methods_split: + [for val2 in local.json_data.paths[my_value[0]][my_value[1]] : val2] + ] +} +''' + HCLParser parser = new HCLParser(); + when: + def results = parser.parse(hcl) + println results + then: + results.locals["swagger_path_method_parameters"] != null } void "it should handle sample tf from wu"() {