Merge pull request #242 from parsingdata/current-index

Resolves #236. Add support to allow access to value of current iteration inside iterable tokens (e.g., Rep, RepN and While).

core/src/main/java/io/parsingdata/metal/Shorthand.java

@@ -46,9 +46,9 @@
 import io.parsingdata.metal.expression.value.ConstantFactory;
 import io.parsingdata.metal.expression.value.Elvis;
 import io.parsingdata.metal.expression.value.Expand;
+import io.parsingdata.metal.expression.value.FoldCat;
 import io.parsingdata.metal.expression.value.FoldLeft;
 import io.parsingdata.metal.expression.value.FoldRight;
-import io.parsingdata.metal.expression.value.FoldCat;
 import io.parsingdata.metal.expression.value.Reverse;
 import io.parsingdata.metal.expression.value.UnaryValueExpression;
 import io.parsingdata.metal.expression.value.Value;
@@ -62,6 +62,7 @@
 import io.parsingdata.metal.expression.value.bitwise.ShiftLeft;
 import io.parsingdata.metal.expression.value.bitwise.ShiftRight;
 import io.parsingdata.metal.expression.value.reference.Count;
+import io.parsingdata.metal.expression.value.reference.CurrentIteration;
 import io.parsingdata.metal.expression.value.reference.CurrentOffset;
 import io.parsingdata.metal.expression.value.reference.First;
 import io.parsingdata.metal.expression.value.reference.Last;
@@ -89,6 +90,7 @@ public final class Shorthand {
     public static final Token EMPTY = def(EMPTY_NAME, 0L);
     public static final ValueExpression SELF = new Self();
     public static final ValueExpression CURRENT_OFFSET = new CurrentOffset();
+    public static final ValueExpression CURRENT_ITERATION = new CurrentIteration();
     public static final Expression TRUE = new True();
 
     private Shorthand() {}

core/src/main/java/io/parsingdata/metal/data/ImmutablePair.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2013-2016 Netherlands Forensic Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.parsingdata.metal.data;
+
+import static io.parsingdata.metal.Util.checkNotNull;
+
+import java.util.Objects;
+
+import io.parsingdata.metal.Util;
+
+public class ImmutablePair<L, R> {
+
+    public final L left;
+    public final R right;
+
+    public ImmutablePair(final L left, final R right) {
+        this.left = checkNotNull(left, "left");
+        this.right = checkNotNull(right, "right");
+    }
+
+    @Override
+    public String toString() {
+        return left + "=" + right;
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return Util.notNullAndSameClass(this, obj)
+            && Objects.equals(left, ((ImmutablePair)obj).left)
+            && Objects.equals(right, ((ImmutablePair)obj).right);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getClass(), left, right);
+    }
+
+}

core/src/main/java/io/parsingdata/metal/data/ParseState.java

@@ -16,6 +16,7 @@
 
 package io.parsingdata.metal.data;
 
+import static java.math.BigInteger.ONE;
 import static java.math.BigInteger.ZERO;
 
 import static io.parsingdata.metal.Util.checkNotNegative;
@@ -36,43 +37,52 @@ public class ParseState {
     public final ParseGraph order;
     public final BigInteger offset;
     public final Source source;
+    public final ImmutableList<ImmutablePair<Token, BigInteger>> iterations;
 
-    public ParseState(final ParseGraph order, final Source source, final BigInteger offset) {
+    public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList<ImmutablePair<Token, BigInteger>> iterations) {
         this.order = checkNotNull(order, "order");
         this.source = checkNotNull(source, "source");
         this.offset = checkNotNegative(offset, "offset");
+        this.iterations = checkNotNull(iterations, "iterations");
     }
 
     public static ParseState createFromByteStream(final ByteStream input, final BigInteger offset) {
-        return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset);
+        return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset, new ImmutableList<>());
     }
 
     public static ParseState createFromByteStream(final ByteStream input) {
         return createFromByteStream(input, ZERO);
     }
 
     public ParseState addBranch(final Token token) {
-        return new ParseState(order.addBranch(token), source, offset);
+        return new ParseState(order.addBranch(token), source, offset, token.isIterable() ? iterations.add(new ImmutablePair<>(token, ZERO)) : iterations);
     }
 
-    public ParseState closeBranch() {
-        return new ParseState(order.closeBranch(), source, offset);
+    public ParseState closeBranch(final Token token) {
+        if (token.isIterable() && !iterations.head.left.equals(token)) {
+            throw new IllegalStateException(String.format("Cannot close branch for iterable token %s. Current iteration state is for token %s.", token.name, iterations.head.left.name));
+        }
+        return new ParseState(order.closeBranch(), source, offset, token.isIterable() ? iterations.tail : iterations);
     }
 
     public ParseState add(final ParseValue parseValue) {
-        return new ParseState(order.add(parseValue), source, offset);
+        return new ParseState(order.add(parseValue), source, offset, iterations);
     }
 
     public ParseState add(final ParseReference parseReference) {
-        return new ParseState(order.add(parseReference), source, offset);
+        return new ParseState(order.add(parseReference), source, offset, iterations);
+    }
+
+    public ParseState iterate() {
+        return new ParseState(order, source, offset, iterations.tail.add(new ImmutablePair<>(iterations.head.left, iterations.head.right.add(ONE))));
     }
 
     public Optional<ParseState> seek(final BigInteger newOffset) {
-        return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset)) : Optional.empty();
+        return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset, iterations)) : Optional.empty();
     }
 
     public ParseState source(final ValueExpression dataExpression, final int index, final ParseState parseState, final Encoding encoding) {
-        return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO);
+        return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO, iterations);
     }
 
     public Optional<Slice> slice(final BigInteger length) {
@@ -81,20 +91,22 @@ public Optional<Slice> slice(final BigInteger length) {
 
     @Override
     public String toString() {
-        return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + ")";
+        final String iterationString = iterations.isEmpty() ? "" : ";iterations:" + iterations.toString();
+        return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + iterationString + ")";
     }
 
     @Override
     public boolean equals(final Object obj) {
         return Util.notNullAndSameClass(this, obj)
             && Objects.equals(order, ((ParseState)obj).order)
             && Objects.equals(offset, ((ParseState)obj).offset)
-            && Objects.equals(source, ((ParseState)obj).source);
+            && Objects.equals(source, ((ParseState)obj).source)
+            && Objects.equals(iterations, ((ParseState)obj).iterations);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(getClass(), order, offset, source);
+        return Objects.hash(getClass(), order, offset, source, iterations);
     }
 
 }

core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2013-2018 Netherlands Forensic Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.parsingdata.metal.expression.value.reference;
+
+import static io.parsingdata.metal.expression.value.ConstantFactory.createFromNumeric;
+
+import java.util.Optional;
+
+import io.parsingdata.metal.Util;
+import io.parsingdata.metal.data.ImmutableList;
+import io.parsingdata.metal.data.ParseState;
+import io.parsingdata.metal.encoding.Encoding;
+import io.parsingdata.metal.expression.value.Value;
+import io.parsingdata.metal.expression.value.ValueExpression;
+import io.parsingdata.metal.token.Rep;
+import io.parsingdata.metal.token.RepN;
+import io.parsingdata.metal.token.Token;
+import io.parsingdata.metal.token.While;
+
+/**
+ * A {@link ValueExpression} that represents the 0-based current iteration in an
+ * iterable {@link Token} (when {@link Token#isIterable()} returns true, e.g. when
+ * inside a {@link Rep}, {@link RepN}) or {@link While}).
+ */
+public class CurrentIteration implements ValueExpression {
+
+    @Override
+    public ImmutableList<Optional<Value>> eval(final ParseState parseState, final Encoding encoding) {
+        if (parseState.iterations.isEmpty()) {
+            return ImmutableList.create(Optional.empty());
+        }
+        return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head.right, new Encoding())));
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return Util.notNullAndSameClass(this, obj);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+
+}

core/src/main/java/io/parsingdata/metal/token/Cho.java

@@ -62,7 +62,7 @@ private Trampoline<Optional<ParseState>> iterate(final Environment environment,
         }
         return list.head
             .parse(environment)
-            .map(result -> complete(() -> success(result.closeBranch())))
+            .map(result -> complete(() -> success(result.closeBranch(this))))
             .orElseGet(() -> intermediate(() -> iterate(environment, list.tail)));
     }
 

core/src/main/java/io/parsingdata/metal/token/IterableToken.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013-2018 Netherlands Forensic Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.parsingdata.metal.token;
+
+import static io.parsingdata.metal.Trampoline.complete;
+import static io.parsingdata.metal.Trampoline.intermediate;
+import static io.parsingdata.metal.Util.checkNotNull;
+import static io.parsingdata.metal.Util.success;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import io.parsingdata.metal.Trampoline;
+import io.parsingdata.metal.data.Environment;
+import io.parsingdata.metal.data.ParseState;
+import io.parsingdata.metal.encoding.Encoding;
+
+public abstract class IterableToken extends Token {
+
+    public final Token token;
+
+    IterableToken(final String name, final Token token, final Encoding encoding) {
+        super(name, encoding);
+        this.token = checkNotNull(token, "token");
+    }
+
+    protected final Optional<ParseState> parse(final Environment environment, final Predicate<Environment> stopCondition, final Function<Environment, Optional<ParseState>> ifIterationFails) {
+        return iterate(environment.addBranch(this), stopCondition, ifIterationFails).computeResult();
+    }
+
+    /**
+     * Iteratively parse iterations of the token, given a stop condition and the logic how to handle a failed parse.
+     *
+     * @param environment the environment to apply the parse to
+     * @param stopCondition a function to determine when to stop the iteration
+     * @param ifIterationFails a function to determine how to handle a failed parse
+     * @return a trampolined {@code Optional<ParseState>}
+     */
+    private Trampoline<Optional<ParseState>> iterate(final Environment environment, final Predicate<Environment> stopCondition, final Function<Environment, Optional<ParseState>> ifIterationFails) {
+        if (stopCondition.test(environment)) {
+            return complete(() -> success(environment.parseState.closeBranch(this)));
+        }
+        return token
+                .parse(environment)
+                .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState.iterate()), stopCondition, ifIterationFails)))
+                .orElseGet(() -> complete(() -> ifIterationFails.apply(environment)));
+    }
+
+    @Override
+    public boolean isIterable() {
+        return true;
+    }
+
+    @Override
+    public boolean equals(final Object obj) {
+        return super.equals(obj)
+                && Objects.equals(token, ((IterableToken)obj).token);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), token);
+    }
+
+}

core/src/main/java/io/parsingdata/metal/token/Post.java

@@ -55,7 +55,7 @@ public Post(final String name, final Token token, final Expression predicate, fi
     protected Optional<ParseState> parseImpl(final Environment environment) {
         return token
             .parse(environment.addBranch(this))
-            .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch()) : failure())
+            .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch(this)) : failure())
             .orElseGet(Util::failure);
     }
 

core/src/main/java/io/parsingdata/metal/token/Pre.java

@@ -59,7 +59,7 @@ protected Optional<ParseState> parseImpl(final Environment environment) {
         }
         return token
             .parse(environment.addBranch(this))
-            .map(resultParseState -> success(resultParseState.closeBranch()))
+            .map(resultParseState -> success(resultParseState.closeBranch(this)))
             .orElseGet(Util::failure);
     }