From 13e7242dd3bca9f542ae40a02b87288fc264d62f Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez Date: Sun, 3 Aug 2014 18:17:19 -0500 Subject: [PATCH] Allow various types to interpolate into string --- src/bkl/expr.py | 71 ++++++++++++++++++----- src/bkl/interpreter/builder.py | 2 + src/bkl/interpreter/simplify.py | 6 +- src/bkl/parser/Bakefile.g | 3 +- src/bkl/parser/BakefileQuotedString.g | 2 +- src/bkl/parser/ast.py | 63 ++++++++++---------- src/bkl/vartypes.py | 5 +- tests/test_parsing/templates/template.ast | 2 +- tests/test_parsing/vars/quoted.ast | 23 +++++++- tests/test_parsing/vars/quoted.bkl | 4 ++ tests/test_parsing/vars/quoted.model | 3 + 11 files changed, 131 insertions(+), 53 deletions(-) diff --git a/src/bkl/expr.py b/src/bkl/expr.py index 5a4d3c83..7a1c640c 100644 --- a/src/bkl/expr.py +++ b/src/bkl/expr.py @@ -55,7 +55,7 @@ class Expr(object): """ def __init__(self, pos=None): self.pos = pos - + def is_const(self): """ Returns true if the expression is constant, i.e. can be evaluated @@ -96,6 +96,12 @@ def as_py(self): """ raise NotImplementedError + def to_string(self): + """ + Return the expression as a Python string. Has same semantics :meth:`as_py()`. + """ + raise NotImplementedError + def as_symbolic(self): """ Returns the value as a symbolic representation, see SymbolicFormatter. @@ -133,6 +139,9 @@ def __init__(self, value, pos=None): def as_py(self): return self.value + def to_string(self): + return str(self.value) + def __nonzero__(self): return bool(self.value) @@ -151,6 +160,9 @@ def __init__(self, items, pos=None): def as_py(self): return [ i.as_py() for i in self.items ] + def to_string(self): + return " ".join([i.to_string() for i in self.items]) + def __nonzero__(self): return bool(self.items) @@ -175,9 +187,12 @@ def __init__(self, items, pos=None): self.items = items def as_py(self): - items = (i.as_py() for i in self.items) + items = (i.to_string() for i in self.items) return "".join(i for i in items if i is not None) + def to_string(self): + return self.as_py() + def __nonzero__(self): for i in self.items: if i: @@ -195,6 +210,9 @@ class NullExpr(Expr): def as_py(self): return None + def to_string(self): + return "" + def __nonzero__(self): return False @@ -225,6 +243,9 @@ def __init__(self, var, pos=None): def as_py(self): raise NonConstError(self) + def to_string(self): + self.as_py() + def __str__(self): return "${%s}" % self.var @@ -251,6 +272,9 @@ def __init__(self, var, context, pos=None): def as_py(self): return self.get_value().as_py() + def to_string(self): + return self.get_value().to_string() + def get_value(self): """ Returns value of the referenced variable. Throws an exception if @@ -280,6 +304,14 @@ def __str__(self): return "$(%s)" % self.var +class ReferenceAsStringExpr(ReferenceExpr): + """ + Reference to a variable inside of a string. Has the same attributes as :class:bkl.expr.ReferenceExpr. + """ + def as_py(self): + return self.get_value().to_string() + + class BoolValueExpr(Expr): """ Constant boolean value, i.e. true or false. @@ -295,6 +327,9 @@ def __init__(self, value, pos=None): def as_py(self): return self.value + def to_string(self): + return str(self) + def __nonzero__(self): return self.value @@ -336,7 +371,7 @@ def __init__(self, operator, left, right=None, pos=None): self.operator = operator self.left = left self.right = right - + def has_bool_operands(self): """ Returns true if the operator is such that it requires boolean operands @@ -367,6 +402,9 @@ def as_py(self): else: assert False, "invalid BoolExpr operator" + def to_string(self): + return BoolValueExpr(self.as_py()).to_string() + def __nonzero__(self): left = bool(self.left) right = bool(self.right) @@ -412,6 +450,9 @@ def __init__(self, cond, yes, no, pos=None): def as_py(self): return self.get_value().as_py() + def to_string(self): + return self.get_value().to_string() + def get_value(self): """ Returns value of the conditional expression, i.e. either @@ -493,6 +534,9 @@ def as_py(self): # with explicit anchor: return "%s/%s" % (self.anchor, "/".join(x.as_py() for x in self.components)) + def to_string(self): + return self.as_py() + def __nonzero__(self): return bool(self.components) @@ -623,16 +667,17 @@ class Visitor(object): def __init__(self): self._dispatch = { - NullExpr : self.null, - LiteralExpr : self.literal, - ListExpr : self.list, - ConcatExpr : self.concat, - ReferenceExpr : self.reference, - PlaceholderExpr : self.placeholder, - PathExpr : self.path, - BoolValueExpr : self.bool_value, - BoolExpr : self.bool, - IfExpr : self.if_, + NullExpr : self.null, + LiteralExpr : self.literal, + ListExpr : self.list, + ConcatExpr : self.concat, + ReferenceExpr : self.reference, + ReferenceAsStringExpr : self.reference, + PlaceholderExpr : self.placeholder, + PathExpr : self.path, + BoolValueExpr : self.bool_value, + BoolExpr : self.bool, + IfExpr : self.if_, } def visit(self, e): diff --git a/src/bkl/interpreter/builder.py b/src/bkl/interpreter/builder.py index b3e53c09..0119b225 100644 --- a/src/bkl/interpreter/builder.py +++ b/src/bkl/interpreter/builder.py @@ -483,6 +483,8 @@ def _build_expression(self, ast): e = BoolValueExpr(ast.value) elif t is VarReferenceNode: e = ReferenceExpr(ast.var, self.context) + elif t is VarRefAsStrNode: + e = ReferenceAsStringExpr(ast.var, self.context) elif t is ListNode: items = [self._build_expression(e) for e in ast.values] e = ListExpr(items) diff --git a/src/bkl/interpreter/simplify.py b/src/bkl/interpreter/simplify.py index be4b27b5..fb122b3f 100644 --- a/src/bkl/interpreter/simplify.py +++ b/src/bkl/interpreter/simplify.py @@ -102,7 +102,7 @@ def bool(self, e): if left is e.left and right is e.right: return e else: - if (isinstance(left, NullExpr) and + if (isinstance(left, NullExpr) and (right is None or isinstance(right, NullExpr))): return NullExpr(pos=e.pos) else: @@ -174,9 +174,9 @@ def bool(self, e): if left is not None and right is not None: assert (left or right) == False return BoolValueExpr(False, pos=e.pos) - elif op == BoolExpr.EQUAL: + elif op == BoolExpr.EQUAL: return BoolValueExpr(e.left.as_py() == e.right.as_py()) - elif op == BoolExpr.NOT_EQUAL: + elif op == BoolExpr.NOT_EQUAL: return BoolValueExpr(e.left.as_py() != e.right.as_py()) except NonConstError: pass diff --git a/src/bkl/parser/Bakefile.g b/src/bkl/parser/Bakefile.g index 86428cad..c0c68709 100644 --- a/src/bkl/parser/Bakefile.g +++ b/src/bkl/parser/Bakefile.g @@ -42,6 +42,7 @@ tokens { BOOLVAL; PATH_ANCHOR; VAR_REFERENCE; + VAR_REFERENCE_ASSTRING; LIST_OR_CONCAT; LIST; CONCAT; @@ -225,7 +226,7 @@ expr_atom // ("foo $(bar)") -- the former is a single value, the latter is a list. This // grammar, however, does *NOT* differentiate between these two cases, that is // done in the bkl.parser.ast._TreeAdaptor.rulePostProcessing() in Python code. -// +// // FIXME: It would be better to do it here, with backtrack=true and validating // predicates to build it directly, but bugs in ANTLR 3.4's Python // binding prevent it from working at the moment. diff --git a/src/bkl/parser/BakefileQuotedString.g b/src/bkl/parser/BakefileQuotedString.g index 44f45908..e2fe0e3d 100644 --- a/src/bkl/parser/BakefileQuotedString.g +++ b/src/bkl/parser/BakefileQuotedString.g @@ -51,7 +51,7 @@ quoted_string_component ; var_reference - : REF_OPEN identifier REF_CLOSE -> ^(VAR_REFERENCE identifier) + : REF_OPEN identifier REF_CLOSE -> ^(VAR_REFERENCE_ASSTRING identifier) ; literal_text: t=ANY_TEXT -> LITERAL[$t, self.unescape($t, $t.text)]; diff --git a/src/bkl/parser/ast.py b/src/bkl/parser/ast.py index d3d02251..fb5ad91d 100644 --- a/src/bkl/parser/ast.py +++ b/src/bkl/parser/ast.py @@ -174,6 +174,8 @@ class VarReferenceNode(Node): var = property(lambda self: self.children[0].text, doc="Referenced variable") +class VarRefAsStrNode(VarReferenceNode): + """Variable interpolating into a string.""" class AssignmentNode(Node): """Assignment of value to a variable.""" @@ -306,35 +308,36 @@ def __init__(self, filename): # mapping of token types to AST node classes TOKENS_MAP = { - BakefileParser.NIL : NilNode, - BakefileParser.PROGRAM : RootNode, - BakefileParser.LITERAL : LiteralNode, - BakefileParser.BOOLVAL : BoolvalNode, - BakefileParser.PATH_ANCHOR : PathAnchorNode, - BakefileParser.ID : IdNode, - BakefileParser.LIST : ListNode, - BakefileParser.CONCAT : ConcatNode, - BakefileParser.LIST_OR_CONCAT : Node, # post-processed below - BakefileParser.VAR_REFERENCE : VarReferenceNode, - BakefileParser.ASSIGN : AssignmentNode, - BakefileParser.APPEND : AppendNode, - BakefileParser.LVALUE : LvalueNode, - BakefileParser.FILES_LIST : FilesListNode, - BakefileParser.TARGET : TargetNode, - BakefileParser.IF : IfNode, - BakefileParser.OR : OrNode, - BakefileParser.AND : AndNode, - BakefileParser.NOT : NotNode, - BakefileParser.EQUAL : EqualNode, - BakefileParser.NOT_EQUAL : NotEqualNode, - BakefileParser.SUBMODULE : SubmoduleNode, - BakefileParser.IMPORT : ImportNode, - BakefileParser.PLUGIN : PluginNode, - BakefileParser.SRCDIR : SrcdirNode, - BakefileParser.BASE_LIST : BaseListNode, - BakefileParser.CONFIGURATION : ConfigurationNode, - BakefileParser.SETTING : SettingNode, - BakefileParser.TEMPLATE : TemplateNode, + BakefileParser.NIL : NilNode, + BakefileParser.PROGRAM : RootNode, + BakefileParser.LITERAL : LiteralNode, + BakefileParser.BOOLVAL : BoolvalNode, + BakefileParser.PATH_ANCHOR : PathAnchorNode, + BakefileParser.ID : IdNode, + BakefileParser.LIST : ListNode, + BakefileParser.CONCAT : ConcatNode, + BakefileParser.LIST_OR_CONCAT : Node, # post-processed below + BakefileParser.VAR_REFERENCE : VarReferenceNode, + BakefileParser.VAR_REFERENCE_ASSTRING : VarRefAsStrNode, + BakefileParser.ASSIGN : AssignmentNode, + BakefileParser.APPEND : AppendNode, + BakefileParser.LVALUE : LvalueNode, + BakefileParser.FILES_LIST : FilesListNode, + BakefileParser.TARGET : TargetNode, + BakefileParser.IF : IfNode, + BakefileParser.OR : OrNode, + BakefileParser.AND : AndNode, + BakefileParser.NOT : NotNode, + BakefileParser.EQUAL : EqualNode, + BakefileParser.NOT_EQUAL : NotEqualNode, + BakefileParser.SUBMODULE : SubmoduleNode, + BakefileParser.IMPORT : ImportNode, + BakefileParser.PLUGIN : PluginNode, + BakefileParser.SRCDIR : SrcdirNode, + BakefileParser.BASE_LIST : BaseListNode, + BakefileParser.CONFIGURATION : ConfigurationNode, + BakefileParser.SETTING : SettingNode, + BakefileParser.TEMPLATE : TemplateNode, } def createWithPayload(self, payload): @@ -360,7 +363,7 @@ def filter_list_or_concat(self, node): FIXME: It would be better to do it here, with backtrack=true and validating predicates to build it directly, but bugs in - ANTLR 3.4's Python binding prevent it from working at the moment. + ANTLR 3.4's Python binding prevent it from working at the moment. """ concats = [] children = node.children diff --git a/src/bkl/vartypes.py b/src/bkl/vartypes.py index fe8f3bf8..1856018c 100644 --- a/src/bkl/vartypes.py +++ b/src/bkl/vartypes.py @@ -163,9 +163,8 @@ class StringType(Type): def _validate_impl(self, e): if isinstance(e, expr.ConcatExpr): - # concatenation of strings is a string - for x in e.items: - self.validate(x) + # all types can be concatenated into string + return True elif (not isinstance(e, expr.LiteralExpr) and # paths etc. can be used as strings too not isinstance(e, expr.BoolExpr) and diff --git a/tests/test_parsing/templates/template.ast b/tests/test_parsing/templates/template.ast index af11ec30..24ec72f1 100644 --- a/tests/test_parsing/templates/template.ast +++ b/tests/test_parsing/templates/template.ast @@ -7,7 +7,7 @@ RootNode IdNode defines ConcatNode LiteralNode "COMPONENT="" - VarReferenceNode + VarRefAsStrNode IdNode id LiteralNode """ TemplateNode diff --git a/tests/test_parsing/vars/quoted.ast b/tests/test_parsing/vars/quoted.ast index 8dd4d21d..2d9b18ef 100644 --- a/tests/test_parsing/vars/quoted.ast +++ b/tests/test_parsing/vars/quoted.ast @@ -22,6 +22,27 @@ RootNode IdNode double ConcatNode LiteralNode "evaluated " - VarReferenceNode + VarRefAsStrNode IdNode bar LiteralNode " here (as in shell)..." + AssignmentNode + LvalueNode + IdNode a + ListNode + LiteralNode "1" + LiteralNode "2" + LiteralNode "3" + AssignmentNode + LvalueNode + IdNode b + BoolvalNode True + AssignmentNode + LvalueNode + IdNode r + ConcatNode + LiteralNode "!!" + VarRefAsStrNode + IdNode a + LiteralNode " " + VarRefAsStrNode + IdNode b diff --git a/tests/test_parsing/vars/quoted.bkl b/tests/test_parsing/vars/quoted.bkl index 9bbd78d3..3680c33c 100644 --- a/tests/test_parsing/vars/quoted.bkl +++ b/tests/test_parsing/vars/quoted.bkl @@ -10,3 +10,7 @@ zar = "this is called \"escaping\""; single = 'no eval done $(bar) here'; double = "evaluated $(bar) here (as in shell)..."; +// Also test interpolation of various types +a = 1 2 3; +b = true; +r = "!!$(a) $(b)"; diff --git a/tests/test_parsing/vars/quoted.model b/tests/test_parsing/vars/quoted.model index 846621be..1f896249 100644 --- a/tests/test_parsing/vars/quoted.model +++ b/tests/test_parsing/vars/quoted.model @@ -5,6 +5,9 @@ module { zar = this is called "escaping" single = no eval done $(bar) here double = evaluated 1 2 3 here (as in shell)... + a = [1, 2, 3] + b = true + r = !!$(a) true } targets { }