Skip to content

Commit

Permalink
Refine newline handling further for the Python3 target
Browse files Browse the repository at this point in the history 
Signed-off-by: Phil Elson <[email protected]>
  • Loading branch information
@pelson
pelson committed Nov 21, 2024
1 parent bb10cd6 commit ac3fe8b
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 73 deletions.
48 changes: 19 additions & 29 deletions runtime/Python3/src/antlr4/xpath/XPathLexer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,37 +7,27 @@

def serializedATN():
return [
4, 0, 8, 50, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7,
3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 1, 0, 1, 0, 1,
0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 5, 4, 29, 8,
4, 10, 4, 12, 4, 32, 9, 4, 1, 4, 1, 4, 1, 5, 1, 5, 3, 5, 38, 8,
5, 1, 6, 1, 6, 1, 7, 1, 7, 5, 7, 44, 8, 7, 10, 7, 12, 7, 47, 9,
7, 1, 7, 1, 7, 1, 45, 0, 8, 1, 3, 3, 4, 5, 5, 7, 6, 9, 7, 11, 0,
13, 0, 15, 8, 1, 0, 2, 5, 0, 48, 57, 95, 95, 183, 183, 768, 879,
8255, 8256, 13, 0, 65, 90, 97, 122, 192, 214, 216, 246, 248, 767,
880, 893, 895, 8191, 8204, 8205, 8304, 8591, 11264, 12271, 12289,
55295, 63744, 64975, 65008, 65533, 50, 0, 1, 1, 0, 0, 0, 0, 3, 1,
0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0,
15, 1, 0, 0, 0, 1, 17, 1, 0, 0, 0, 3, 20, 1, 0, 0, 0, 5, 22, 1,
0, 0, 0, 7, 24, 1, 0, 0, 0, 9, 26, 1, 0, 0, 0, 11, 37, 1, 0, 0,
0, 13, 39, 1, 0, 0, 0, 15, 41, 1, 0, 0, 0, 17, 18, 5, 47, 0, 0,
18, 19, 5, 47, 0, 0, 19, 2, 1, 0, 0, 0, 20, 21, 5, 47, 0, 0, 21,
4, 1, 0, 0, 0, 22, 23, 5, 42, 0, 0, 23, 6, 1, 0, 0, 0, 24, 25, 5,
33, 0, 0, 25, 8, 1, 0, 0, 0, 26, 30, 3, 13, 6, 0, 27, 29, 3, 11,
5, 0, 28, 27, 1, 0, 0, 0, 29, 32, 1, 0, 0, 0, 30, 28, 1, 0, 0, 0,
30, 31, 1, 0, 0, 0, 31, 33, 1, 0, 0, 0, 32, 30, 1, 0, 0, 0, 33,
34, 6, 4, 0, 0, 34, 10, 1, 0, 0, 0, 35, 38, 3, 13, 6, 0, 36, 38,
7, 0, 0, 0, 37, 35, 1, 0, 0, 0, 37, 36, 1, 0, 0, 0, 38, 12, 1, 0,
0, 0, 39, 40, 7, 1, 0, 0, 40, 14, 1, 0, 0, 0, 41, 45, 5, 39, 0,
0, 42, 44, 9, 0, 0, 0, 43, 42, 1, 0, 0, 0, 44, 47, 1, 0, 0, 0, 45,
46, 1, 0, 0, 0, 45, 43, 1, 0, 0, 0, 46, 48, 1, 0, 0, 0, 47, 45,
1, 0, 0, 0, 48, 49, 5, 39, 0, 0, 49, 16, 1, 0, 0, 0, 4, 0, 30, 37,
45, 1, 1, 4, 0
]
4,0,8,50,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,6,2,
7,7,7,1,0,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,4,1,4,5,4,29,8,4,10,4,12,4,
32,9,4,1,4,1,4,1,5,1,5,3,5,38,8,5,1,6,1,6,1,7,1,7,5,7,44,8,7,10,7,12,7,
47,9,7,1,7,1,7,1,45,0,8,1,3,3,4,5,5,7,6,9,7,11,0,13,0,15,8,1,0,2,5,0,48,
57,95,95,183,183,768,879,8255,8256,13,0,65,90,97,122,192,214,216,246,248,
767,880,893,895,8191,8204,8205,8304,8591,11264,12271,12289,55295,63744,
64975,65008,65533,50,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,
9,1,0,0,0,0,15,1,0,0,0,1,17,1,0,0,0,3,20,1,0,0,0,5,22,1,0,0,0,7,24,1,0,
0,0,9,26,1,0,0,0,11,37,1,0,0,0,13,39,1,0,0,0,15,41,1,0,0,0,17,18,5,47,0,
0,18,19,5,47,0,0,19,2,1,0,0,0,20,21,5,47,0,0,21,4,1,0,0,0,22,23,5,42,0,
0,23,6,1,0,0,0,24,25,5,33,0,0,25,8,1,0,0,0,26,30,3,13,6,0,27,29,3,11,5,
0,28,27,1,0,0,0,29,32,1,0,0,0,30,28,1,0,0,0,30,31,1,0,0,0,31,33,1,0,0,0,
32,30,1,0,0,0,33,34,6,4,0,0,34,10,1,0,0,0,35,38,3,13,6,0,36,38,7,0,0,0,
37,35,1,0,0,0,37,36,1,0,0,0,38,12,1,0,0,0,39,40,7,1,0,0,40,14,1,0,0,0,41,
45,5,39,0,0,42,44,9,0,0,0,43,42,1,0,0,0,44,47,1,0,0,0,45,46,1,0,0,0,45,
43,1,0,0,0,46,48,1,0,0,0,47,45,1,0,0,0,48,49,5,39,0,0,49,16,1,0,0,0,4,0,
30,37,45,1,1,4,0
] # fmt: skip


class XPathLexer(Lexer):

atn = ATNDeserializer().deserialize(serializedATN())

decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)]
Expand Down Expand Up @@ -95,7 +85,7 @@ class XPathLexer(Lexer):

def __init__(
self,
input = None,
input=None,
output: typing.TextIO = sys.stdout,
) -> None:
super().__init__(input, output)
Expand Down
73 changes: 29 additions & 44 deletions tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import typing

from antlr4 import *


<namedActions.header>
<parser>

Expand All @@ -69,16 +70,17 @@ else:
from <file.parserName> import <file.parserName>
<header>

# This class defines a complete listener for a parse tree produced by <file.parserName>.

class <file.grammarName>Listener(ParseTreeListener):
# This class defines a complete listener for a parse tree produced by <file.parserName>.

<file.listenerNames:{lname |
# Enter a parse tree produced by <file.parserName>#<lname>.
def enter<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context) -> None:
# Enter a parse tree produced by <file.parserName>#<lname>.
pass

# Exit a parse tree produced by <file.parserName>#<lname>.
def exit<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context) -> None:
# Exit a parse tree produced by <file.parserName>#<lname>.
pass

}; separator="\n">
Expand All @@ -97,15 +99,14 @@ else:
from <file.parserName> import <file.parserName>
<header>

# This class defines a complete generic visitor for a parse tree produced by <file.parserName>.

class <file.grammarName>Visitor(ParseTreeVisitor):
# This class defines a complete generic visitor for a parse tree produced by <file.parserName>.

<file.visitorNames:{lname |
# Visit a parse tree produced by <file.parserName>#<lname>.
def visit<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context):
# Visit a parse tree produced by <file.parserName>#<lname>.
return self.visitChildren(ctx)

}; separator="\n">

del <file.parserName>
Expand Down Expand Up @@ -169,7 +170,6 @@ class <parser.name>(<if(superClass)><superClass><else>Parser<endif>):

<funcs; separator="\n">


<if(sempredFuncs)>
def sempred(self, localctx: RuleContext, ruleIndex: int, predIndex: int):
if self._predicates is None:
Expand All @@ -185,8 +185,6 @@ class <parser.name>(<if(superClass)><superClass><else>Parser<endif>):
<sempredFuncs.values; separator="\n">
<endif>



>>

dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= <<
Expand Down Expand Up @@ -252,7 +250,7 @@ def <r.name>_action(self, localctx: <r.ctxType>, actionIndex: int) -> None:
<elseif(rest(actions))>
elif actionIndex == <index>:
<actions.(index)>
<endif> }; separator="\n\n">
<endif> }; separator="\n">
>>

/* This generates a private method since the predIndex is generated, making an
Expand All @@ -262,23 +260,19 @@ RuleSempredFunction(r, actions) ::= <<
def <r.name>_sempred(self, localctx: <r.ctxType>, predIndex: int) -> int:
<actions:{index|
<if(first(actions))>
if predIndex == <index>:
return <actions.(index)>
if predIndex == <index>:
return <actions.(index)>
<elseif(rest(actions))>
elif predIndex == <index>:
return <actions.(index)>
<endif> }; separator="\n">
elif predIndex == <index>:
return <actions.(index)>
<endif> }; separator="">

>>

RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble,exceptions) ::= <<

<ruleCtx>

<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">

def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.type)>: <a.type><endif>}>):

localctx = <parser.name>.<currentRule.ctxType>(
self,
self._ctx,
Expand Down Expand Up @@ -309,10 +303,8 @@ def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.
LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
namedActions,finallyAction,postamble) ::=
<<

<ruleCtx>
<altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">

def <currentRule.escapedName>(self, _p: int = 0<if(currentRule.args)>, <args:{a | , <a>}><endif>) -> <parser.name>.<currentRule.ctxType>:
_parentctx = self._ctx
_parentState = self.state
Expand Down Expand Up @@ -360,7 +352,6 @@ token = self._input.LA(1)
pass}; separator="\nel">
else:
<error>

>>

LL1OptionalBlock(choice, alts, error) ::= <<
Expand All @@ -380,7 +371,6 @@ self._errHandler.sync(self)
<preamble; separator="\n">
if <expr>:
<alts; separator="\n">

<!else if ( !(<followExpr>) ) <error>!>
>>

Expand All @@ -394,7 +384,6 @@ while <loopExpr>:
self.state = <choice.loopBackStateNumber>
self._errHandler.sync(self)
<iteration>

>>

LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
Expand All @@ -408,7 +397,6 @@ while True:
<iteration>
if not (<loopExpr>):
break

>>

// LL(*) stuff
Expand All @@ -424,7 +412,6 @@ if la_ == <i>:
<alt>
pass
}; separator="\nel">

>>

OptionalBlock(choice, alts, error) ::= <<
Expand All @@ -435,21 +422,19 @@ la_ = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
if la_ == <i><if(!choice.ast.greedy)>+1<endif>:
<alt>
}; separator="\nel">

>>

StarBlock(choice, alts, sync, iteration) ::= <<
self.state = <choice.stateNumber>
self._errHandler.sync(self)
_alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
while _alt!=<choice.exitAlt> and _alt != ATN.INVALID_ALT_NUMBER:
while _alt != <choice.exitAlt> and _alt != ATN.INVALID_ALT_NUMBER:
if _alt == 1<if(!choice.ast.greedy)> + 1<endif>:
<iteration>
<alts> <! should only be one !>
self.state = <choice.loopBackStateNumber>
self._errHandler.sync(self)
_alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)

>>

PlusBlock(choice, alts, error) ::= <<
Expand All @@ -466,7 +451,6 @@ if _alt == <i><if(!choice.ast.greedy)> + 1<endif>:
self.state = <choice.loopBackStateNumber> <! loopback/exit decision !>
self._errHandler.sync(self)
_alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)

>>

Sync(s) ::= "sync(<s.expecting.name>)"
Expand Down Expand Up @@ -525,7 +509,7 @@ CommonSetStuff(m, expr, capture, invert) ::= <<
self.state = <m.stateNumber>
<if(m.labels)><m.labels:{l | <labelref(l)> = }>self._input.LT(1)<endif>
<capture>
<if(invert)>if <m.varName> \<= 0 or <expr><else>if not(<expr>)<endif>:
<if(invert)>if <m.varName> \<= 0 or <expr><else>if not (<expr>)<endif>:
<if(m.labels)><m.labels:{l | <labelref(l)> = }><else> <endif>self._errHandler.recoverInline(self)
else:
self._errHandler.reportMatch(self)
Expand Down Expand Up @@ -606,21 +590,23 @@ SetNonLocalAttr(s, rhsChunks) ::= "self.getInvokingContext(<s.ruleIndex>).<s.es

AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"

TokenDecl(t) ::= "self.<t.escapedName> = None # <TokenLabelType()>"
TokenTypeDecl(t) ::= "self.<t.escapedName> = 0 # <TokenLabelType()> type"
TokenListDecl(t) ::= "self.<t.escapedName> = list() # of <TokenLabelType()>s"
RuleContextDecl(r) ::= "self.<r.escapedName> = None # <r.ctxName>"
RuleContextListDecl(rdecl) ::= "self.<rdecl.escapedName> = list() # of <rdecl.ctxName>s"
TokenDecl(t) ::= "self.<t.escapedName> = None # <TokenLabelType()>"
TokenTypeDecl(t) ::= "self.<t.escapedName> = 0 # <TokenLabelType()> type"
TokenListDecl(t) ::= "self.<t.escapedName> = list() # of <TokenLabelType()>s"
RuleContextDecl(r) ::= "self.<r.escapedName> = None # <r.ctxName>"
RuleContextListDecl(rdecl) ::= "self.<rdecl.escapedName> = list() # of <rdecl.ctxName>s"

ContextTokenGetterDecl(t) ::= <<
def <t.escapedName>(self):
return self.getToken(<parser.name>.<t.name>, 0)

>>

// should never be called
ContextTokenListGetterDecl(t) ::= <<
def <t.name>_list(self):
return self.getTokens(<parser.name>.<t.name>)

>>

ContextTokenListIndexedGetterDecl(t) ::= <<
Expand All @@ -629,6 +615,7 @@ def <t.escapedName>(self, i: typing.Optional[int] = None):
return self.getTokens(<parser.name>.<t.name>)
else:
return self.getToken(<parser.name>.<t.name>, i)

>>

ContextRuleGetterDecl(r) ::= <<
Expand Down Expand Up @@ -683,8 +670,7 @@ class <struct.escapedName>(<if(contextSuperClass)><contextSuperClass><else>Parse
<attrs:{a | <a>}; separator="\n">
<struct.ctorAttrs:{a | self.<a.escapedName> = <a.escapedName>}; separator="\n">

<getters:{g | <g>}; separator="\n\n">

<getters:{g | <g>}; separator="\n">
def getRuleIndex(self):
return <parser.name>.RULE_<struct.derivedFromName>

Expand All @@ -695,6 +681,7 @@ class <struct.escapedName>(<if(contextSuperClass)><contextSuperClass><else>Parse

<endif>
<dispatchMethods; separator="\n">

<extensionMembers; separator="\n">

>>
Expand All @@ -719,7 +706,7 @@ class <struct.escapedName>(<currentRule.name; format="cap">Context):

ListenerDispatchMethod(method) ::= <<
def <if(method.isEnter)>enter<else>exit<endif>Rule(self, listener: ParseTreeListener):
if hasattr( listener, "<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">" ):
if hasattr(listener, "<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">"):
listener.<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(self)

>>
Expand All @@ -730,7 +717,6 @@ def accept(self, visitor: ParseTreeVisitor):
return visitor.visit<struct.derivedFromName; format="cap">(self)
else:
return visitor.visitChildren(self)

>>

AttributeDecl(d) ::= "self.<d.escapedName> = <if(d.initValue)><d.initValue><else>None<endif>"
Expand Down Expand Up @@ -807,7 +793,6 @@ else:


class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):

atn = ATNDeserializer().deserialize(serializedATN())

decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)]
Expand Down Expand Up @@ -852,7 +837,7 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):

def __init__(
self,
input = None,
input=None,
output: typing.TextIO = sys.stdout,
) -> None:
super().__init__(input, output)
Expand All @@ -875,8 +860,8 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):
SerializedATN(model) ::= <<
def serializedATN():
return [
<model.serialized: {s | <s>}; separator=", ", wrap>
]
<model.serialized: {s | <s>}; separator=",", wrap>
] # fmt: skip
>>

/** Using a type to init value map, try to init a type; if not in table
Expand Down
2 changes: 2 additions & 0 deletions tool/src/org/antlr/v4/codegen/target/Python3Target.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ public class Python3Target extends Target {

public Python3Target(CodeGenerator gen) {
super(gen);
// note that lineWidth isn't a hard upper limit.
gen.lineWidth = 79;
}

@Override
Expand Down

0 comments on commit ac3fe8b

Please sign in to comment.