Merge pull request #439 from jjttjj/interactive-reads

Interactive reads

convex-core/src/main/antlr4/convex/core/lang/reader/antlr/Convex.g4

@@ -92,18 +92,10 @@ BOOL : 'true' | 'false' ;
 
 DOUBLE:
   (DIGITS | SIGNED_DIGITS) DOUBLE_TAIL;
-
-fragment  
-DOUBLE_TAIL:
-  DECIMAL EPART | DECIMAL | EPART;
 
-fragment  
-DECIMAL:
-  '.' DIGITS;
-
-fragment 
-EPART:
-  [eE] (DIGITS | SIGNED_DIGITS);  
+fragment
+DOUBLE_TAIL:
+  [.eE] [-0-9.eE]*;
 
 DIGITS:
   [0-9]+;
@@ -148,7 +140,7 @@ QUOTING: '\'' | '`' | '~' | '~@';
 
 
 KEYWORD:
-   ':' NAME;
+   ':' NAME?;
 
 SYMBOL
     : NAME
@@ -203,6 +195,6 @@ fragment
 COMMENT: ';' ~[\r\n]* ;
 
 TRASH
-    : ( WS | COMMENT ) -> channel(HIDDEN)
+    : ( WS | COMMENT ) -> skip
     ;
 

convex-core/src/main/java/convex/core/lang/Reader.java

@@ -55,7 +55,8 @@ public static AList<ACell> readAll(String source) {
 	}
 
 	/**
-	 * Parses an expression and returns a form as an Object
+	 * Parses an expression and returns a form as an Object, consuming the
+	 * entire source
 	 * 
 	 * @param source Reader instance to get expression from
 	 * @return Parsed form (may be nil)
@@ -75,4 +76,16 @@ public static <R extends ACell> R read(String source) {
 		return (R) AntlrReader.read(source);
 	}
 
+    	/**
+	 * Parses an expression and returns a form as an Object. Leaves
+	 * remaining input on the source.
+	 * 
+	 * @param source PushbackReader instance to get expression from
+	 * @return Parsed form (may be nil)
+	 */
+	public static ACell readOne(java.io.PushbackReader source) throws IOException {
+		return AntlrReader.readOne(source);
+	}
+
+
 }

convex-core/src/main/java/convex/core/lang/reader/AntlrReader.java

@@ -6,7 +6,14 @@
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.CommonTokenFactory;
+import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.UnbufferedCharStream;
+import org.antlr.v4.runtime.UnbufferedTokenStream;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.IntStream;
 import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
@@ -228,7 +235,11 @@ public void enterDoubleValue(DoubleValueContext ctx) {
 		@Override
 		public void exitDoubleValue(DoubleValueContext ctx) {
 			String s=ctx.getText();
-			push( CVMDouble.parse(s));
+			try {
+				push( CVMDouble.parse(s));
+			} catch (NumberFormatException x) {
+				throw new ParseException("Unparseable double value: "+s,x);
+			}
 		}
 
 		@Override
@@ -435,31 +446,31 @@ public void exitSingleForm(SingleFormContext ctx) {
 	public static ACell read(String s) {
 		return read(CharStreams.fromString(s));
 	}
-	
+
 	public static ACell read(java.io.Reader r) throws IOException {
 		return read(CharStreams.fromReader(r));
 	}
-	
+
 	public static ACell read(CharStream cs) {
 		ConvexLexer lexer=new ConvexLexer(cs);
 		lexer.removeErrorListeners();
 		CommonTokenStream tokens = new CommonTokenStream(lexer);
 		ConvexParser parser = new ConvexParser(tokens);
 		parser.removeErrorListeners();
-		
+
 		ParseTree tree = parser.singleForm();
-		
+
 		CRListener visitor=new CRListener();
 		ParseTreeWalker.DEFAULT.walk(visitor, tree);
-		
+
 		ArrayList<ACell> top=visitor.popList();
 		if (top.size()!=1) {
 			throw new ParseException("Bad parse output: "+top);
 		}
-		
+
 		return top.get(0);
 	}
-	
+
 	public static AList<ACell> readAll(String source) {
 		return readAll(CharStreams.fromString(source));
 	}
@@ -471,12 +482,101 @@ public static AList<ACell> readAll(CharStream cs) {
 		ConvexParser parser = new ConvexParser(tokens);
 		parser.removeErrorListeners();
 		ParseTree tree = parser.forms();
-		
+
 		CRListener visitor=new CRListener();
 		ParseTreeWalker.DEFAULT.walk(visitor, tree);
-		
+
 		ArrayList<ACell> top=visitor.popList();
 		return Lists.create(top);
 	}
 
+	public static ACell readOne(java.io.PushbackReader r) {
+		try {
+			CharStream cs = new InteractiveCharStream(r);
+			ConvexLexer lexer=new ConvexLexer(cs);
+			lexer.removeErrorListeners();
+			lexer.setTokenFactory(new CommonTokenFactory(true));
+			TokenStream tokens = new InteractiveTokenStream(lexer);
+			ConvexParser parser = new ConvexParser(tokens);
+			parser.removeErrorListeners();
+
+			ParseTree tree = parser.form();
+
+			r.unread(cs.LA(1));
+
+			CRListener visitor=new CRListener();
+			ParseTreeWalker.DEFAULT.walk(visitor, tree);
+
+			ArrayList<ACell> top=visitor.popList();
+			if (top.size()!=1) {
+				throw new ParseException("Bad parse output: "+top);
+			}
+
+			return top.get(0);
+
+		} catch (IOException e) {
+			throw Utils.sneakyThrow(e);
+		}
+
+	}
+
+
+
+}
+
+class InteractiveCharStream extends UnbufferedCharStream {
+
+	InteractiveCharStream(java.io.Reader input)
+	{
+		super(input);
+	}
+
+	@Override
+	public void consume() {
+
+		if (LA(1) == IntStream.EOF) {
+			throw new IllegalStateException("cannot consume EOF");
+		}
+
+		// buf always has at least data[p==0] in this method due to ctor
+		lastChar = data[p];   // track last char for LA(-1)
+		if (p == n-1 && numMarkers==0) {
+			n = 0;
+			p = -1; // p++ will leave this at 0
+			lastCharBufferStart = lastChar;
+		}
+
+		p++;
+		currentCharIndex++;
+		//sync(1);
+	}
+}
+
+class InteractiveTokenStream extends UnbufferedTokenStream<Token> {
+
+	InteractiveTokenStream(TokenSource tokenSource)
+	{
+		super(tokenSource);
+	}
+
+	@Override
+	public void consume() {
+		if (LA(1) == Token.EOF) {
+			throw new IllegalStateException("cannot consume EOF");
+		}
+
+		// buf always has at least tokens[p==0] in this method due to ctor
+		lastToken = tokens[p];   // track last token for LT(-1)
+
+		// if we're at last token and no markers, opportunity to flush buffer
+		if ( p == n-1 && numMarkers==0 ) {
+			n = 0;
+			p = -1; // p++ will leave this at 0
+			lastTokenBufferStart = lastToken;
+		}
+
+		p++;
+		currentTokenIndex++;
+		//sync(1);
+	}
 }

convex-core/src/test/java/convex/core/lang/reader/ANTLRTest.java

@@ -42,6 +42,11 @@ private <R extends ACell> R readAll(String s) {
 		return (R) AntlrReader.readAll(s);
 	}
 
+    @SuppressWarnings("unchecked")
+	private <R extends ACell> R readOne(String s) {
+		return (R) AntlrReader.readOne(new java.io.PushbackReader(new java.io.StringReader(s)));
+	}
+
 	@Test public void testNil() {
 		assertNull(read("nil"));
 	}
@@ -316,6 +321,15 @@ private void doDifferentPrintTests(String src, String dst) {
 		doRoundTripTest(dst); // final value should round trip normally
 	}
 
+    @Test public void testReadOne() {
+        assertEquals(Lists.of(1,2),readOne("(1 2) 3"));
+        assertEquals(Lists.of(1,2),readOne("(1 2)(3 4)"));
+        assertEquals(Lists.of(1,2),readOne("(1 2))))"));
+        assertThrows(ParseException.class,()->readOne(")"));
+        assertThrows(ParseException.class,()->readOne("1.0e0.1234"));
+        assertThrows(ParseException.class,()->readOne(":"));
+	}
+
 
 
 }