diff --git a/.gitignore b/.gitignore index 22b58e75a..bd020be00 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -/bin -/wiki -/inputs **/playground /.settings /.idea @@ -9,11 +6,10 @@ target/ .cache-main .classpath .project -.jvmopts *.class *.log .vscode .metals .bloop project/project -project/metals.sbt \ No newline at end of file +project/metals.sbt diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 000000000..56aefe058 --- /dev/null +++ b/.jvmopts @@ -0,0 +1 @@ +-Xmx4G diff --git a/README.md b/README.md index 5094eb634..45924ba49 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,15 @@ Parsley is distributed on Maven Central, and can be added to your project via: ```scala // SBT -libraryDependencies += "com.github.j-mie6" %% "parsley" % "4.3.1" +libraryDependencies += "com.github.j-mie6" %% "parsley" % "4.4.0" // scala-cli ---dependency com.github.j-mie6::parsley:4.3.1 +--dependency com.github.j-mie6::parsley:4.4.0 // or in file -//> using dep com.github.j-mie6::parsley:4.3.1 +//> using dep com.github.j-mie6::parsley:4.4.0 // mill -ivy"com.github.j-mie6::parsley:4.3.1" +ivy"com.github.j-mie6::parsley:4.4.0" ``` Documentation can be found [**here**](https://javadoc.io/doc/com.github.j-mie6/parsley_2.13/latest/index.html) @@ -166,7 +166,8 @@ _An exception to this policy is made for any version `3.x.y`, which reaches EoL | `4.0.0` | 30th November 2022 | EoL reached (`4.0.4`) | | `4.1.0` | 18th January 2023 | EoL reached (`4.1.8`) | | `4.2.0` | 22nd January 2023 | EoL reached (`4.2.14`) | -| `4.3.0` | 8th July 2023 | Enjoying indefinite support | +| `4.3.0` | 8th July 2023 | EoL reached (`4.3.1`) | +| `4.4.0` | 6th October 2023 | Enjoying indefinite support | ## Bug Reports [![Percentage of issues still open](https://isitmaintained.com/badge/open/j-mie6/Parsley.svg)](https://isitmaintained.com/project/j-mie6/Parsley "Percentage of issues still open") [![Maintainability](https://img.shields.io/codeclimate/maintainability/j-mie6/parsley)](https://codeclimate.com/github/j-mie6/parsley) [![Test Coverage](https://img.shields.io/codeclimate/coverage-letter/j-mie6/parsley)](https://codeclimate.com/github/j-mie6/parsley) diff --git a/build.sbt b/build.sbt index 65cd5538f..4fc6e6576 100644 --- a/build.sbt +++ b/build.sbt @@ -16,7 +16,7 @@ val releaseFlags = Seq("-Xdisable-assertions", "-opt:l:method,inline", "-opt-inl val noReleaseFlagsScala3 = true // maybe some day this can be turned off... inThisBuild(List( - tlBaseVersion := "4.3", + tlBaseVersion := "4.4", organization := "com.github.j-mie6", organizationName := "Parsley Contributors ", startYear := Some(2020), // true start is 2018, but license is from 2020 @@ -40,6 +40,7 @@ inThisBuild(List( tlCiHeaderCheck := true, githubWorkflowJavaVersions := Seq(Java8, JavaLTS, JavaLatest), githubWorkflowAddedJobs += testCoverageJob(githubWorkflowGeneratedCacheSteps.value.toList), + //githubWorkflowConcurrency := None, // Website Configuration tlSitePublishBranch := Some(mainBranch), tlSiteApiUrl := Some(url("https://www.javadoc.io/doc/com.github.j-mie6/parsley_2.13/latest/")), @@ -59,7 +60,7 @@ lazy val parsley = crossProject(JSPlatform, JVMPlatform, NativePlatform) resolvers ++= Opts.resolver.sonatypeOssReleases, // Will speed up MiMA during fast back-to-back releases libraryDependencies ++= Seq( - "org.scalatest" %%% "scalatest" % "3.2.16" % Test, + "org.scalatest" %%% "scalatest" % "3.2.17" % Test, "org.scalacheck" %%% "scalacheck" % "1.17.0" % Test, "org.scalatestplus" %%% "scalacheck-1-17" % "3.2.15.0" % Test, ), diff --git a/parsley/shared/src/main/scala/parsley/Parsley.scala b/parsley/shared/src/main/scala/parsley/Parsley.scala index 498a4a806..416f26129 100644 --- a/parsley/shared/src/main/scala/parsley/Parsley.scala +++ b/parsley/shared/src/main/scala/parsley/Parsley.scala @@ -12,7 +12,7 @@ import parsley.expr.{chain, infix} import parsley.internal.deepembedding.{frontend, singletons} import parsley.internal.machine.Context -import Parsley.pure +import Parsley.{emptyErr, pure} import XCompat._ // substituteCo /** @@ -85,7 +85,7 @@ import XCompat._ // substituteCo * * @define attemptreason * The reason for this behaviour is that it prevents space leaks and improves error messages. If this behaviour - * is not desired, use `attempt(this)` to rollback any input consumed on failure. + * is not desired, use `atomic(this)` to rollback any input consumed on failure. * * @define or * parses `q` if this parser fails '''without''' consuming input. @@ -181,7 +181,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * * @example {{{ * scala> import parsley.character.string - * scala> (string("true").as(true)).parse("true") + * scala> string("true").as(true).parse("true") * val res0 = Success(true) * }}} * @@ -349,7 +349,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * * @example {{{ * scala> import parsley.Parsley, parsley.character.char - * scala> val sign: Parsley[Int => Int] = char('+') #> (identity[Int] _) <|> char('-') #> (x => -x) + * scala> val sign: Parsley[Int => Int] = char('+').as(identity[Int] _) <|> char('-').as(x => -x) * scala> val nat: Parsley[Int] = .. * scala> val int = sign <*> nat * scala> int.parse("-7") @@ -384,7 +384,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * * @example {{{ * // this has a common prefix "term" and requires backtracking - * val expr1 = attempt(lift2(Add, term <* char('+'), expr2)) <|> term + * val expr1 = atomic(lift2(Add, term <* char('+'), expr2)) <|> term * // common prefix factored out, and branches return a function to recombine * val expr2 = term <**> (char('+') *> expr2.map(y => Add(_, y)) (identity[Expr] _)) * }}} @@ -585,7 +585,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * @note $autoAmend * @group filter */ - def filter(pred: A => Boolean): Parsley[A] = new Parsley(new frontend.Filter(this.internal, pred)) + def filter(pred: A => Boolean): Parsley[A] = parsley.errors.combinator.filterWith(this)(pred, emptyErr) /** This combinator filters the result of this parser using a given predicate, succeeding only if the predicate returns `false`. * * First, parse this parser. If it succeeds then take its result `x` and apply it to the predicate `pred`. If `pred(x) is @@ -638,7 +638,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * @note $autoAmend * @group filter */ - def collect[B](pf: PartialFunction[A, B]): Parsley[B] = this.mapFilter(pf.lift) + def collect[B](pf: PartialFunction[A, B]): Parsley[B] = parsley.errors.combinator.collectWith(this)(pf, emptyErr) /** This combinator applies a function `f` to the result of this parser: if it returns a * `Some(y)`, `y` is returned, otherwise the parser fails. * @@ -665,7 +665,7 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front * @note $autoAmend * @group filter */ - def mapFilter[B](f: A => Option[B]): Parsley[B] = new Parsley(new frontend.MapFilter(this.internal, f)) + def mapFilter[B](f: A => Option[B]): Parsley[B] = parsley.errors.combinator.mapFilterWith(this)(f, emptyErr) // FOLDING COMBINATORS /** This combinator will parse this parser '''zero''' or more times combining the results with the function `f` and base value `k` from the right. @@ -901,6 +901,21 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front */ def flatten[B](implicit ev: A <:< Parsley[B]): Parsley[B] = this.flatMap[B](ev) + /** This combinator ignores the result of this parser and instead returns the input parsed by it. + * + * This combinator executes this parser: if it fails, this combinator fails. Otherwise, if this + * parser succeeded, its result is discarded and the input it consumed in the process of parsing + * is returned directly. This can be used to efficiently obtain the underlying string of some parser + * without having to reconstruct it. One potential application is to suppress complex results of + * things within the `Lexer` without having to re-implement its functionality. However, it should be + * used judiciously. + * + * @return a parser which returns the parsed input directly. + * @since 4.4.0 + * @group map + */ + def span: Parsley[String] = new Parsley(new frontend.Span(this.internal)) + // SPECIAL METHODS // $COVERAGE-OFF$ /** Forces the compilation of a parser as opposed to the regular lazy evaluation. @@ -916,15 +931,27 @@ final class Parsley[+A] private [parsley] (private [parsley] val internal: front */ def overflows(): Unit = internal.overflows() - /** Using this method signifies that the parser it is invoked on is impure and any optimisations which assume purity + /** This combinator signifies that the parser it is invoked on is impure and any optimisations which assume purity * are disabled. * + * @example Any parsers that make use of mutable state may need to use this combinator to control + * parsley's aggressive optimisations that remove results that are not needed: in this case, + * the optimiser cannot see that the result of a parser is mutating some value, and may remove it. * @group special */ - def unsafe(): Parsley[A] = { - internal.unsafe() - this - } + def impure: Parsley[A] = new Parsley(new frontend.Opaque(this.internal)) + // TODO: deprecate in 4.5, remove 5.0 + /** This combinator signifies that the parser it is invoked on is impure and any optimisations which assume purity + * are disabled. + * + * @example Any parsers that make use of mutable state may need to use this combinator to control + * parsley's aggressive optimisations that remove results that are not needed: in this case, + * the optimiser cannot see that the result of a parser is mutating some value, and may remove it. + * @note old alias for `impure` + * @group special + */ + def unsafe(): Parsley[A] = impure + // $COVERAGE-ON$ // $COVERAGE-OFF$ @@ -1019,9 +1046,9 @@ object Parsley { * * @example {{{ * // this works fine, even though all of `zipped`'s parsers are strict - * lazy val expr = (attempt(term) <* '+', ~expr).zipped(_ + _) <|> term + * lazy val expr = (atomic(term) <* '+', ~expr).zipped(_ + _) <|> term * // in this case, however, the following would fix the problem more elegantly: - * lazy val expr = (attempt(term), '+' *> expr).zipped(_ + _) <|> term + * lazy val expr = (atomic(term), '+' *> expr).zipped(_ + _) <|> term * }}} * * @return the parser `p`, but guaranteed to be lazy. @@ -1136,6 +1163,8 @@ object Parsley { * @see [[Parsley.flatten `flatten`]] for details and examples. */ def join[A](p: Parsley[Parsley[A]]): Parsley[A] = p.flatten + // $COVERAGE-OFF$ + // TODO: deprecate in 4.5.0 /** This combinator parses its argument `p`, but rolls back any consumed input on failure. * * If the parser `p` succeeds, then `attempt(p)` has no effect. However, if `p` failed, @@ -1155,15 +1184,41 @@ object Parsley { * * @param p the parser to execute, if it fails, it will not have consumed input. * @return a parser that tries `p`, but never consumes input if it fails. + * @note `atomic` should be used instead. + * @group prim + */ + def attempt[A](p: Parsley[A]): Parsley[A] = atomic(p) + // $COVERAGE-ON$ + /** This combinator parses its argument `p`, but rolls back any consumed input on failure. + * + * If the parser `p` succeeds, then `atomic(p)` has no effect. However, if `p` failed, + * then any input that it consumed is rolled back. This ensures that + * the parser `p` is all-or-nothing when consuming input. While there are many legimate + * uses for all-or-nothing behaviour, one notable, if discouraged, use is to allow the + * `<|>` combinator to backtrack -- recall it can only parse its alternative if the + * first failed ''without'' consuming input. This is discouraged, however, as it can + * affect the complexity of the parser and harm error messages. + * + * @example {{{ + * scala> import parsley.character.string, parsley.Parsley.atomic + * scala> (string("abc") <|> string("abd")).parse("abd") + * val res0 = Failure(..) // first parser consumed a, so no backtrack + * scala> (atomic(string("abc")) <|> string("abd")).parse("abd") + * val res1 = Success("abd") // first parser does not consume input on failure now + * }}} + * + * @param p the parser to execute, if it fails, it will not have consumed input. + * @return a parser that tries `p`, but never consumes input if it fails. + * @since 4.4.0 * @group prim */ - def attempt[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.Attempt(p.internal)) + def atomic[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.Attempt(p.internal)) /** This combinator parses its argument `p`, but does not consume input if it succeeds. * * If the parser `p` succeeds, then `lookAhead(p)` will roll back any input consumed * whilst parsing `p`. If `p` fails, however, then the whole combinator fails and * any input consumed '''remains consumed'''. If this behaviour is not desirable, - * consider pairing `lookAhead` with `attempt`. + * consider pairing `lookAhead` with `atomic`. * * @example {{{ * scala> import parsley.Parsley.lookAhead, parsley.character.string @@ -1191,7 +1246,7 @@ object Parsley { * {{{ * import parsley.character.{string, letterOrDigit} * import parsley.Parsley.notFollowedBy - * def keyword(kw: String): Parsley[Unit] = attempt { + * def keyword(kw: String): Parsley[Unit] = atomic { * string(kw) *> notFollowedBy(letterOrDigit) * } * }}} @@ -1201,6 +1256,18 @@ object Parsley { * @group prim */ def notFollowedBy(p: Parsley[_]): Parsley[Unit] = new Parsley(new frontend.NotFollowedBy(p.internal)) + /** This combinator fails immediately, with a caret of the given width and no other information. + * + * By producing basically no information, this combinator is principally for adjusting the + * caret-width of another error, rather than the value `empty`, which is used to fail with + * no effect on error content. + * + * @param caretWidth the width of the caret for the error produced by this combinator. + * @return a parser that fails. + * @since 4.4.0 + * @group basic + */ + def empty(caretWidth: Int): Parsley[Nothing] = new Parsley(singletons.Empty(caretWidth)) /** This parser fails immediately, with an unknown parse error. * * @example {{{ @@ -1210,12 +1277,13 @@ object Parsley { * }}} * * @return a parser that fails. + * @note equivalent to `empty(0)` * @group basic */ - val empty: Parsley[Nothing] = errors.combinator.empty(0) - /** This combinator produces `()` without having any other effect. + val empty: Parsley[Nothing] = empty(0) + /** This parser produces `()` without having any other effect. * - * When this combinator is ran, no input is required, nor consumed, and + * When this parser is ran, no input is required, nor consumed, and * the given value will always be successfully returned. It has no other * effect on the state of the parser. * @@ -1233,6 +1301,9 @@ object Parsley { * @group basic */ val unit: Parsley[Unit] = pure(()) + + private val emptyErr = new parsley.errors.VanillaGen[Any] + // $COVERAGE-OFF$ /** This parser returns the current line number of the input without having any other effect. * diff --git a/parsley/shared/src/main/scala/parsley/ap.scala b/parsley/shared/src/main/scala/parsley/ap.scala index 850b61134..eceaa5395 100644 --- a/parsley/shared/src/main/scala/parsley/ap.scala +++ b/parsley/shared/src/main/scala/parsley/ap.scala @@ -15,10 +15,10 @@ package parsley * scala> import parsley.character.char * scala> import parsley.ap.{ap2, ap3} * scala> case class Add(x: Int, y: Int) - * scala> val p = ap2(pure(Add), char('a') #> 4, char('b') #> 5) + * scala> val p = ap2(pure(Add), char('a').as(4), char('b').as(5)) * scala> p.parse("ab") * val res0 = Success(Add(4, 5)) - * scala> val q = ap3(pure((x: Int, y: Int, z: Int) => x * y + z), char('a') #> 3, char('b') #> 2, char('c') #> 5) + * scala> val q = ap3(pure((x: Int, y: Int, z: Int) => x * y + z), char('a').as(3), char('b').as(2), char('c').as(5)) * scala> q.parse("abc") * val res1 = Success(11) * scala> q.parse("ab") diff --git a/parsley/shared/src/main/scala/parsley/character.scala b/parsley/shared/src/main/scala/parsley/character.scala index fd7e64b57..54a6e2539 100644 --- a/parsley/shared/src/main/scala/parsley/character.scala +++ b/parsley/shared/src/main/scala/parsley/character.scala @@ -5,11 +5,10 @@ */ package parsley -import scala.annotation.switch import scala.collection.immutable.NumericRange -import parsley.Parsley.{attempt, empty, fresh, pure} -import parsley.combinator.{choice, skipMany} +import parsley.Parsley.{atomic, empty, fresh, pure} +import parsley.combinator.{choice, skipMany, skipSome} import parsley.errors.combinator.ErrorMethods import parsley.token.errors.{Label, LabelConfig, NotConfigured} @@ -59,9 +58,9 @@ import parsley.internal.deepembedding.singletons * for `java.lang.Character` to see which Unicode version is supported by your JVM. A table of the Unicode versions * up to JDK 17 can be found [[https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Character.html here]]. * - * These parsers are only able to parse unicode characters in the range `'\u0000'` to `'\uFFFF'`, known as + * These parsers are only able to parse unicode characters in the range `'\u0000'` to `'\uffff'`, known as * the ''Basic Multilingual Plane (BMP)''. Unicode characters wider than a single 16-bit character should be - * parsed using multi-character combinators such as `string`, or, alternatively, `satisfyUtf16` or `charUtf16`. + * parsed using multi-character combinators such as `string`, or, alternatively, combinators found in [[unicode `unicode`]]. * * @groupprio string 22 * @groupname string String Combinators @@ -98,13 +97,16 @@ object character { * * @param c the character to parse * @return a parser that tries to read a single `c`, or fails. - * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[string `string`]] or [[codePoint `codePoint`]]. + * @note this combinator can only handle 16-bit characters: for larger codepoints, + * consider using [[string `string`]] or [[unicode.char `unicode.char`]]. * @group core */ def char(c: Char): Parsley[Char] = char(c, NotConfigured) private def char(c: Char, label: String): Parsley[Char] = char(c, Label(label)) - private def char(c: Char, label: LabelConfig): Parsley[Char] = new Parsley(new singletons.CharTok(c, label)) + private def char(c: Char, label: LabelConfig): Parsley[Char] = new Parsley(new singletons.CharTok(c, c, label)) + //TODO: deprecate in 4.5 + // $COVERAGE-OFF$ /** This combinator tries to parse a single specific codepoint `c` from the input. * * Like [[char `char`]], except it may consume two characters from the input, @@ -114,11 +116,11 @@ object character { * * @example {{{ * scala> import parsley.character.codePoint - * scala> codePoint(0x1F643).parse("") + * scala> codePoint(0x1f642).parse("") * val res0 = Failure(..) - * scala> codePoint(0x1F643).parse("🙂") - * val res1 = Success(0x1F643) - * scala> codePoint(0x1F643).parse("b🙂") + * scala> codePoint(0x1f642).parse("🙂") + * val res1 = Success(0x1f642) + * scala> codePoint(0x1f642).parse("b🙂") * val res2 = Failure(..) * }}} * @@ -126,12 +128,8 @@ object character { * @return * @group core */ - def codePoint(c: Int): Parsley[Int] = { - if (Character.isBmpCodePoint(c)) char(c.toChar) #> c - else new Parsley(new singletons.SupplementaryCharTok(c, NotConfigured)) - } - @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") - private [parsley] def charUtf16(c: Int): Parsley[Int] = codePoint(c) + def codePoint(c: Int): Parsley[Int] = unicode.char(c) + // $COVERAGE-ON$ /** This combinator tries to parse a single character from the input that matches the given predicate. * @@ -152,18 +150,39 @@ object character { * * @param pred the predicate to test the next character against, should one exist. * @return a parser that tries to read a single character `c`, such that `pred(c)` is true, or fails. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.satisfy `unicode.satisfy`]]. * @group core */ def satisfy(pred: Char => Boolean): Parsley[Char] = satisfy(pred, NotConfigured) private def satisfy(pred: Char => Boolean, label: String): Parsley[Char] = satisfy(pred, Label(label)) private def satisfy(pred: Char => Boolean, label: LabelConfig) = new Parsley(new singletons.Satisfy(pred, label)) - // we want this down the line :) - //def satisfy[A](pred: PartialFunction[Char, A]): Parsley[A] = satisfy(pred.isDefinedAt(_)).map(pred) - - // TODO: document - private [parsley] def satisfyUtf16(pred: Int => Boolean): Parsley[Int] = new Parsley(new singletons.UniSatisfy(pred, NotConfigured)) + /** This combinator tries to parse and process a character from the input if it is defined for the given function. + * + * Attempts to read a character from the input and tests to see if it is in the domain of `f`. If a character + * `c` can be read and `f(c)` is defined, then `c` is consumed and `f(c)` is returned. Otherwise, no input is consumed + * and this combinator will fail. + * + * @example {{{ + * scala> import parsley.character.satisfyMap + * scala> val digit = satisfyMap { + * case c if c.isDigit => c.asDigit + * } + * scala> digit.parse("") + * val res0 = Failure(..) + * scala> digit.parse("7") + * val res1 = Success(7) + * scala> digit.parse("a5") + * val res2 = Failure(..) + * }}} + * + * @param f the function to test the next character against and transform it with, should one exist. + * @return a parser that tries to read a single character `c`, such that `f(c)` is defined, and returns `f(c)` if so, or fails. + * @since 4.4.0 + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.satisfyMap `unicode.satisfyMap`]]. + * @group core + */ + def satisfyMap[A](f: PartialFunction[Char, A]): Parsley[A] = satisfy(f.isDefinedAt(_)).map(f) /** This combinator attempts to parse a given string from the input, and fails otherwise. * @@ -191,10 +210,10 @@ object character { * @group string */ def string(s: String): Parsley[String] = string(s, NotConfigured) - private def string(s: String, label: String): Parsley[String] = string(s, Label(label)) - private def string(s: String, label: LabelConfig): Parsley[String] = { + private [parsley] def string(s: String, label: String): Parsley[String] = string(s, Label(label)) + private [parsley] def string(s: String, label: LabelConfig): Parsley[String] = { require(s.nonEmpty, "`string` may not be passed the empty string (`string(\"\")` is meaningless, perhaps you meant `pure(\"\")`?)") - new Parsley(new singletons.StringTok(s, label)) + new Parsley(new singletons.StringTok(s, s, label)) } /** $oneOf @@ -215,7 +234,7 @@ object character { * * @param cs the set of characters to check. * @return a parser that parses one of the member of the set `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.oneOf(cs:Set* `unicode.oneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -246,7 +265,7 @@ object character { * * @param cs the characters to check. * @return a parser that parses one of the elements of `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.oneOf(cs:Int* `unicode.oneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -272,7 +291,7 @@ object character { * * @param cs the range of characters to check. * @return a parser that parses a character within the range `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.oneOf(cs:Range* `unicode.oneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -305,7 +324,7 @@ object character { * * @param cs the set of characters to check. * @return a parser that parses one character that is not a member of the set `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.noneOf(cs:Set* `unicode.noneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -338,7 +357,7 @@ object character { * * @param cs the set of characters to check. * @return a parser that parses one character that is not an element of `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.noneOf(cs:Int* `unicode.noneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -366,7 +385,7 @@ object character { * * @param cs the range of characters to check. * @return a parser that parses a character outside the range `cs`. - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.noneOf(cs:Range* `unicode.noneOf`]]. * @see [[satisfy `satisfy`]] * @group class */ @@ -408,12 +427,34 @@ object character { expr.infix.secretLeft1(fresh(new StringBuilder), pc, pf).map(_.toString) } - // TODO: document - private [parsley] def stringOfManyUtf16(pc: Parsley[Int]): Parsley[String] = { - val pf = pure(addCodepoint(_, _)) - // Can't use the regular foldLeft here, because we need a fresh StringBuilder each time. - expr.infix.secretLeft1(fresh(new StringBuilder), pc, pf).map(_.toString) - } + // TODO: optimise, this can be _really_ tightly implemented with a substring on the input + /** This combinator parses characters matching the given predicate '''zero''' or more times, collecting + * the results into a string. + * + * Repeatly reads characters that satisfy the given predicate `pred`. When no more characters + * can be successfully read, the results are stitched together into a `String` and returned. + * This combinator can never fail, since `satisfy` can never fail having consumed input. + * + * @example {{{ + * scala> import parsley.character.{letter, stringOfMany} + * scala> import parsley.implicits.zipped.Zipped2 + * scala> val ident = (letter, stringOfMany(_.isLetterOrDigit)).zipped((c, s) => s"$c$s") + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc9d") + * scala> ident.parse("a") + * val res1 = Success("a") + * scala> ident.parse("9") + * val res2 = Failure(..) + * }}} + * + * @param pred the predicate to test characters against. + * @return a parser that returns the span of characters satisfying `pred` + * @note this acts exactly like `stringOfMany(satisfy(pred))`, but may be more efficient. + * @note analogous to the `megaparsec` `takeWhileP` combinator. + * @since 4.4.0 + * @group string + */ + def stringOfMany(pred: Char => Boolean): Parsley[String] = skipMany(satisfy(pred)).span /** This combinator parses `pc` '''one''' or more times, collecting its results into a string. * @@ -442,19 +483,39 @@ object character { expr.infix.secretLeft1(pc.map(new StringBuilder += _), pc, pf).map(_.toString) } - // TODO: document - private [parsley] def stringOfSomeUtf16(pc: Parsley[Int]): Parsley[String] = { - val pf = pure(addCodepoint(_, _)) - // Can't use the regular foldLeft1 here, because we need a fresh StringBuilder each time. - expr.infix.secretLeft1(pc.map(addCodepoint(new StringBuilder, _)), pc, pf).map(_.toString) - } + // TODO: optimise, this can be _really_ tightly implemented with a substring on the input + /** This combinator parses characters matching the given predicate '''one''' or more times, collecting + * the results into a string. + * + * Repeatly reads characters that satisfy the given predicate `pred`. When no more characters + * can be successfully read, the results are stitched together into a `String` and returned. + * This combinator can never fail having consumed input, since `satisfy` can never fail having + * consumed input. + * + * @example {{{ + * scala> import parsley.character.{stringOfSome} + * scala> val ident = stringOfSome(_.isLetter) + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc") + * scala> ident.parse("") + * val res1 = Failure(..) + * }}} + * + * @param pred the predicate to test characters against. + * @return a parser that returns the span of characters satisfying `pred` + * @note this acts exactly like `stringOfSome(satisfy(pred))`, but may be more efficient. + * @note analogous to the `megaparsec` `takeWhile1P` combinator. + * @since 4.4.0 + * @group string + */ + def stringOfSome(pred: Char => Boolean): Parsley[String] = skipSome(satisfy(pred)).span /** This combinator tries to parse each of the strings `strs` (and `str0`), until one of them succeeds. * - * Unlike `choice`, or more accurately `attemptChoice`, this combinator will not - * necessarily parse the strings in the order provided. It will favour strings that have another string + * Unlike `choice`, or more accurately `atomicChoice`, this combinator will not + * necessarily parse the strings in the order provided. It will avoid strings that have another string * as a prefix first, so that it has ''Longest Match'' semantics. It will try to minimise backtracking - * too, making it a much more efficient option than `attemptChoice`. + * too, making it a much more efficient option than `atomicChoice`. * * The longest succeeding string will be returned. If no strings match then the combinator fails. * @@ -484,10 +545,10 @@ object character { /** This combinator tries to parse each of the key-value pairs `kvs` (and `kv0`), until one of them succeeds. * * Each argument to this combinator is a pair of a string and a parser to perform if that string can be parsed. - * `strings(s0 -> p0, ...)` can be thought of as `attemptChoice(string(s0) *> p0, ...)`, however, the given + * `strings(s0 -> p0, ...)` can be thought of as `atomicChoice(string(s0) *> p0, ...)`, however, the given * ordering of key-value pairs does not dictate the order in which the parses are tried. In particular, it - * will favour keys that are the prefix of another key first, so that it has ''Longest Match'' semantics. - * it will try to minimise backtracking too, making it a much more efficient option than `attemptChoice`. + * will avoid keys that are the prefix of another key first, so that it has ''Longest Match'' semantics. + * It will try to minimise backtracking too, making it a much more efficient option than `atomicChoice`. * * @example {{{ * scala> import parsley.character.strings @@ -505,15 +566,15 @@ object character { * }}} * * @note the scope of any backtracking performed is isolated to the key itself, as it is assumed that once a - * key parses correctly, the branch has been committed to. Putting an `attempt` around the values will not affect + * key parses correctly, the branch has been committed to. Putting an `atomic` around the values will not affect * this behaviour. * * @param kv0 the first key-value pair to try to parse. * @param kvs the remaining key-value pairs to try to parse. * @return a parser that tries to parse all the given key-value pairs, returning the (possibly failing) result * of the value that corresponds to the longest matching key. - @since 4.0.0 - @group string + * @since 4.0.0 + * @group string */ def strings[A](kv0: (String, Parsley[A]), kvs: (String, Parsley[A])*): Parsley[A] = { // this isn't the best we could do: it's possible to eliminate backtracking with a Trie... @@ -523,18 +584,18 @@ object character { val ss = kv0 +: kvs choice(ss.groupBy(_._1.head).toList.sortBy(_._1).view.map(_._2).flatMap { s => val (sLast, pLast) :: rest = s.toList.sortBy(_._1.length): @unchecked - ((string(sLast) *> pLast) :: rest.map { case (s, p) => attempt(string(s)) *> p }).reverse + ((string(sLast) *> pLast) :: rest.map { case (s, p) => atomic(string(s)) *> p }).reverse }.toSeq: _*) } /** This parser will parse '''any''' single character from the input, failing if there is no input remaining. * - * @note this combinator can only handle 16-bit characters. + * @note this combinator can only handle 16-bit characters: for larger codepoints, consider using [[unicode.item `unicode.item`]]. * @group core */ val item: Parsley[Char] = satisfy(_ => true, "any character") - /** This parser tries to parse a space or tab character, and returns it if successful + /** This parser tries to parse a space or tab character, and returns it if successful. * * @see [[isSpace `isSpace`]] * @group spec @@ -561,7 +622,7 @@ object character { * @see [[isWhitespace `isWhitespace`]] * @group spec */ - val whitespace: Parsley[Char] = satisfy(isWhitespace(_), "whitespace") + val whitespace: Parsley[Char] = satisfy(_.isWhitespace, "whitespace") /** This parser skips zero or more space characters using [[whitespace `whitespace`]]. * @@ -581,11 +642,11 @@ object character { /** This parser tries to parse a `CRLF` newline character pair, returning `'\n'` if successful. * * A `CRLF` character is the pair of carriage return (`'\r'`) and line feed (`'\n'`). These - * two characters will be parsed together or not at all. The parser is made atomic using `attempt`. + * two characters will be parsed together or not at all. The parser is made atomic using `atomic`. * * @group spec */ - val crlf: Parsley[Char] = string("\r\n", "end of crlf") #> '\n' + val crlf: Parsley[Char] = atomic(string("\r\n", "end of crlf")).as('\n') /** This parser will parse either a line feed (`LF`) or a `CRLF` newline, returning `'\n'` if successful. * @@ -604,7 +665,7 @@ object character { /** This parser tries to parse an uppercase letter, and returns it if successful. * - * An uppercase letter is any character `c <= '\uFFFF'` whose Unicode ''Category Type'' is Uppercase Letter (`Lu`). + * An uppercase letter is any character `c <= '\uffff'` whose Unicode ''Category Type'' is Uppercase Letter (`Lu`). * Examples of characters within this category include: * - the Latin letters `'A'` through `'Z'` * - Latin special character such as `'Å'`, `'Ç'`, `'Õ'` @@ -620,7 +681,7 @@ object character { /** This parser tries to parse a lowercase letter, and returns it if successful. * - * A lowercase letter is any character `c <= '\uFFFF'` whose Unicode ''Category Type'' is Lowercase Letter (`Ll`). + * A lowercase letter is any character `c <= '\uffff'` whose Unicode ''Category Type'' is Lowercase Letter (`Ll`). * Examples of characters within this category include: * - the Latin letters `'a'` through `'z'` * - Latin special character such as `'é'`, `'ß'`, `'ð'` @@ -646,7 +707,7 @@ object character { /** This parser tries to parse a letter, and returns it if successful. * - * A letter is any character `c <= '\uFFFF'` whose Unicode ''Category Type'' is any of the following: + * A letter is any character `c <= '\uffff'` whose Unicode ''Category Type'' is any of the following: * 1. Uppercase Letter (`Lu`) * 1. Lowercase Letter (`Ll`) * 1. Titlecase Letter (`Lt`) @@ -661,13 +722,13 @@ object character { /** This parser tries to parse a digit, and returns it if successful. * - * A digit is any character `c <= '\uFFFF'` whose Unicode ''Category Type'' is Decimal Number (`Nd`). + * A digit is any character `c <= '\uffff'` whose Unicode ''Category Type'' is Decimal Number (`Nd`). * Examples of (inclusive) ranges within this category include: * - the Latin digits `'0'` through `'9'` * - the Arabic-Indic digits `'\u0660'` through `'\u0669'` - * - the Extended Arabic-Indic digits `'\u06F0'` through `'\u06F9'` - * - the Devangari digits `'\u0966'` through `'\u096F'` - * - the Fullwidth digits `'\uFF10'` through `'\uFF19'` + * - the Extended Arabic-Indic digits `'\u06f0'` through `'\u06f9'` + * - the Devangari digits `'\u0966'` through `'\u096f'` + * - the Fullwidth digits `'\uff10'` through `'\uff19'` * * $categories * @@ -696,15 +757,17 @@ object character { */ val octDigit: Parsley[Char] = satisfy(isOctDigit(_), "octal digit") - /** This parser tries to parse a bit and returns it if successful. + /** This parser tries to parse a binary digit (bit) and returns it if successful. * - * A bit (binary digit) is either `'0'` or `'1'`. + * A bit is either `'0'` or `'1'`. * * @group spec */ - val bit: Parsley[Char] = oneOf('0', '1').label("bit") + val bit: Parsley[Char] = satisfy(c => Character.digit(c, 2) != -1, "bit") // Functions + // TODO: deprecate in 4.5 + // $COVERAGE-OFF$ /** This function returns true if a character is a whitespace character. * * A whitespace character is one of: @@ -713,15 +776,13 @@ object character { * 1. a line feed (`'\n'`) * 1. a carriage return (`'\r'`) * 1. a form feed (`'\f'`) - * 1. a vertical tab (`'\u000B'`) + * 1. a vertical tab (`'\u000b'`) * * @see [[whitespace `whitespace`]] * @group pred */ - def isWhitespace(c: Char): Boolean = (c: @switch) match { - case ' ' | '\t' | '\n' | '\r' | '\f' | '\u000b' => true - case _ => false - } + def isWhitespace(c: Char): Boolean = c.isWhitespace + // $COVERAGE-ON$ /** This function returns true if a character is a hexadecimal digit. * @@ -729,16 +790,12 @@ object character { * 1. the digits `'0'` through `'9'` * 1. the letters `'a'` through `'f'` * 1. the letters `'A'` through `'Z'` + * 1. an equivalent from another charset * * @see [[hexDigit `hexDigit`]] * @group pred */ - def isHexDigit(c: Char): Boolean = (c: @switch) match { - case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' - | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' - | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' => true - case _ => false - } + def isHexDigit(c: Char): Boolean = Character.digit(c, 16) != -1 /** This function returns true if a character is an octal digit. * @@ -747,7 +804,7 @@ object character { * @group pred * @see [[octDigit `octDigit`]] */ - def isOctDigit(c: Char): Boolean = c <= '7' && c >= '0' + def isOctDigit(c: Char): Boolean = Character.digit(c, 8) != -1 /** This function returns true if a character is either a space or a tab character. * @@ -759,11 +816,16 @@ object character { // Sue me. private def renderChar(c: Char): String = parsley.errors.helpers.renderRawString(s"$c") - private [parsley] def addCodepoint(sb: StringBuilder, codepoint: Int): StringBuilder = { - if (Character.isSupplementaryCodePoint(codepoint)) { - sb += Character.highSurrogate(codepoint) - sb += Character.lowSurrogate(codepoint) - } - else sb += codepoint.toChar - } + // $COVERAGE-OFF$ + @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") + private [parsley] def charUtf16(c: Int): Parsley[Int] = unicode.char(c) + @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") + private [parsley] def satisfyUtf16(f: Int => Boolean): Parsley[Int] = unicode.satisfy(f) + @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") + private [parsley] def stringOfManyUtf16(pc: Parsley[Int]): Parsley[String] = unicode.stringOfMany(pc) + @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") + private [parsley] def stringOfSomeUtf16(pc: Parsley[Int]): Parsley[String] = unicode.stringOfSome(pc) + @deprecated("this is an old naming, which I believe was never exposed but to be safe it'll remain till 5.0.0", "4.3.0") + private [parsley] def addCodepoint(sb: StringBuilder, codepoint: Int): StringBuilder = unicode.addCodepoint(sb, codepoint) + // $COVERAGE-ON$ } diff --git a/parsley/shared/src/main/scala/parsley/combinator.scala b/parsley/shared/src/main/scala/parsley/combinator.scala index f3df843c2..1407e0f05 100644 --- a/parsley/shared/src/main/scala/parsley/combinator.scala +++ b/parsley/shared/src/main/scala/parsley/combinator.scala @@ -6,9 +6,11 @@ package parsley import scala.annotation.tailrec +import scala.collection.mutable -import parsley.Parsley.{attempt, empty, notFollowedBy, pure, select, unit} +import parsley.Parsley.{atomic, empty, fresh, notFollowedBy, pure, select, unit} import parsley.implicits.zipped.{Zipped2, Zipped3} +import parsley.registers.{RegisterMaker, RegisterMethods} import parsley.internal.deepembedding.{frontend, singletons} @@ -62,6 +64,12 @@ import parsley.internal.deepembedding.{frontend, singletons} * it is possible to use the [[parsley.Parsley.LazyParsley.unary_~ prefix `~`]] combinator to make any individual * arguments lazy as required, for example `skip(p, ~q, r)`. * + * @groupprio range 65 + * @groupname range Range Combinators + * @groupdesc range + * These combinators allow for the parsing of a specific parser either a specific number of times, or between a certain + * amount of times. + * * @groupprio cond 75 * @groupname cond Conditional Combinators * @groupdesc cond @@ -102,6 +110,8 @@ object combinator { */ def choice[A](ps: Parsley[A]*): Parsley[A] = ps.reduceRightOption(_ <|> _).getOrElse(empty) + // TODO: deprecate in 4.5.0 + // $COVERAGE-OFF$ /** This combinator tries to parse each of the parsers `ps` in order, until one of them succeeds. * * Finds the first parser in `ps` which succeeds, returning its result. If none of the parsers @@ -130,7 +140,40 @@ object combinator { * @see [[parsley.Parsley$.attempt `attempt`]] * @note this combinator is not particularly efficient, because it may unnecessarily backtrack for each alternative. */ - def attemptChoice[A](ps: Parsley[A]*): Parsley[A] = ps.reduceRightOption((p, q) => attempt(p) <|> q).getOrElse(empty) + def attemptChoice[A](ps: Parsley[A]*): Parsley[A] = atomicChoice(ps: _*) + // $COVERAGE-ON$ + + /** This combinator tries to parse each of the parsers `ps` in order, until one of them succeeds. + * + * Finds the first parser in `ps` which succeeds, returning its result. If none of the parsers + * succeed, then this combinator fails. This combinator will always try and parse each of the + * combinators until one succeeds, regardless of how they fail. The last argument will '''not''' + * be wrapped in `atomic`, as this is not necessary. + * + * @example {{{ + * scala> import parsley.combinator.atomicChoice + * scala> import parsley.character.string + * scala> val p = atomicChoice(string("abc"), string("ab"), string("bc"), string("d")) + * scala> p.parse("abc") + * val res0 = Success("abc") + * scala> p.parse("ab") + * val res1 = Success("ab") + * scala> p.parse("bc") + * val res2 = Success("bc") + * scala> p.parse("x") + * val res3 = Failure(..) + * }}} + * + * @param ps the parsers to try, in order. + * @return a parser that tries to parse one of `ps`. + * @group multi + * @see [[parsley.Parsley.<|> `<|>`]] + * @see [[parsley.Parsley$.atomic `atomic`]] + * @since 4.4.0 + * @note this combinator is not particularly efficient, because it may unnecessarily backtrack for each alternative: a more efficient alternative + * for `String` is [[character.strings(str0* `strings`]]. + */ + def atomicChoice[A](ps: Parsley[A]*): Parsley[A] = ps.reduceRightOption((p, q) => atomic(p) <|> q).getOrElse(empty) /** This combinator will parse each of `ps` in order, collecting the results. * @@ -185,9 +228,9 @@ object combinator { * @note $strict */ def traverse[A, B](f: A => Parsley[B], xs: A*): Parsley[List[B]] = sequence(xs.map(f): _*) - // this will be used in future! + // TODO: this will be used in future! private [parsley] def traverse5[A, B](xs: A*)(f: A => Parsley[B]): Parsley[List[B]] = traverse(f, xs: _*) - private [parsley] def traverse_[A](xs: A*)(f: A => Parsley[_]): Parsley[Unit] = skip(unit, xs.map(f): _*) + private [parsley] def traverse_[A](xs: A*)(f: A => Parsley[_]): Parsley[Unit] = skip(unit, xs.map(f): _*) // TODO: does traverse.void just work? /** This combinator will parse each of `ps` in order, discarding the results. * @@ -212,33 +255,7 @@ object combinator { * @see [[parsley.Parsley.*> `*>`]] * @note $strict */ - def skip(p: Parsley[_], ps: Parsley[_]*): Parsley[Unit] = ps.foldLeft(p.void)(_ <* _) - - /** This combinator parses exactly `n` occurrences of `p`, returning these `n` results in a list. - * - * Parses `p` repeatedly up to `n` times. If `p` fails before `n` is reached, then this combinator - * fails. It is not required for `p` to fail after the `n`^th^ parse. The results produced by - * `p`, `x,,1,,` through `x,,n,,`, are returned as `List(x,,1,,, .., x,,n,,)`. - * - * @example {{{ - * scala> import parsley.character.item - * scala> import parsley.combinator.exactly - * scala> val p = exactly(3, item) - * scala> p.parse("ab") - * val res0 = Failure(..) - * scala> p.parse("abc") - * val res1 = Success(List('a', 'b', 'c')) - * scala> p.parse("abcd") - * val res2 = Success(List('a', 'b', 'c')) - * }}} - * - * @param n the number of times to repeat `p`. - * @param p the parser to repeat. - * @return a parser that parses `p` exactly `n` times, returning a list of the results. - * @group misc - * @since 4.0.0 - */ - def exactly[A](n: Int, p: Parsley[A]): Parsley[List[A]] = traverse[Int, A](_ => p, (1 to n): _*) + def skip(p: Parsley[_], ps: Parsley[_]*): Parsley[Unit] = ps.foldLeft(p.void)(_ <* _) // TODO: does sequence.void just work? /** This combinator tries to parse `p`, wrapping its result in a `Some` if it succeeds, or returns `None` if it fails. * @@ -309,9 +326,7 @@ object combinator { * @return a parser that tries to parse `p`, returning `x` regardless of success or failure. * @group opt */ - def optionalAs[A](p: Parsley[_], x: A): Parsley[A] = { - (p #> x).getOrElse(x) - } + def optionalAs[A](p: Parsley[_], x: A): Parsley[A] = p.as(x).getOrElse(x) /** This combinator can eliminate an `Option` from the result of the parser `p`. * @@ -480,7 +495,7 @@ object combinator { * @since 2.2.0 * @group iter */ - def skipMany(p: Parsley[_]): Parsley[Unit] = new Parsley(new frontend.SkipMany(p.internal)) + def skipMany(p: Parsley[_]): Parsley[Unit] = many(p).void /** This combinator repeatedly parses a given parser '''one''' or more times, ignoring the results. * @@ -506,7 +521,7 @@ object combinator { * @return a parser that parses `p` until it fails, returning unit. * @group iter */ - def skipSome(p: Parsley[_]): Parsley[Unit] = skipManyN(1, p) + def skipSome(p: Parsley[_]): Parsley[Unit] = some(p).void /** This combinator repeatedly parses a given parser '''`n`''' or more times, ignoring the results. * @@ -541,6 +556,60 @@ object combinator { go(n) } + /** This combinator repeatedly parses a given parser '''zero''' or more times, returning how many times it succeeded. + * + * Parses a given parser, `p`, repeatedly until it fails. If `p` failed having consumed input, + * this combinator fails. Otherwise when `p` fails '''without consuming input''', this combinator + * will succeed. The number of times `p` succeeded is returned as the result. + * + * @example {{{ + * scala> import parsley.character.string + * scala> import parsley.combinator.count + * scala> val p = count(string("ab")) + * scala> p.parse("") + * val res0 = Success(0) + * scala> p.parse("ab") + * val res1 = Success(1) + * scala> p.parse("abababab") + * val res2 = Success(4) + * scala> p.parse("aba") + * val res3 = Failure(..) + * }}} + * + * @param p the parser to execute multiple times. + * @return the number of times `p` successfully parses + * @group iter + * @since 4.4.0 + */ + def count(p: Parsley[_]): Parsley[Int] = p.foldLeft(0)((n, _) => n + 1) + + /** This combinator repeatedly parses a given parser '''one''' or more times, returning how many times it succeeded. + * + * Parses a given parser, `p`, repeatedly until it fails. If `p` failed having consumed input, + * this combinator fails. Otherwise when `p` fails '''without consuming input''', this combinator + * will succeed. The parser `p` must succeed at least once. The number of times `p` succeeded is returned as the result. + * + * @example {{{ + * scala> import parsley.character.string + * scala> import parsley.combinator.count1 + * scala> val p = count1(string("ab")) + * scala> p.parse("") + * val res0 = Failure(..) + * scala> p.parse("ab") + * val res1 = Success(1) + * scala> p.parse("abababab") + * val res2 = Success(4) + * scala> p.parse("aba") + * val res3 = Failure(..) + * }}} + * + * @param p the parser to execute multiple times. + * @return the number of times `p` successfully parses + * @group iter + * @since 4.4.0 + */ + def count1(p: Parsley[_]): Parsley[Int] = p.foldLeft1(0)((n, _) => n + 1) + /** This combinator parses '''zero''' or more occurrences of `p`, separated by `sep`. * * Behaves just like `sepBy1`, except does not require an initial `p`, returning the empty list instead. @@ -751,12 +820,12 @@ object combinator { * @group iter */ def manyUntil[A](p: Parsley[A], end: Parsley[_]): Parsley[List[A]] = { - new Parsley(new frontend.ManyUntil((end #> ManyUntil.Stop <|> p: Parsley[Any]).internal)) + new Parsley(new frontend.ManyUntil((end.as(ManyUntil.Stop) <|> p: Parsley[Any]).internal)) } - // TODO: document and test before release + // TODO: find a way to make this redundant private [parsley] def skipManyUntil(p: Parsley[_], end: Parsley[_]): Parsley[Unit] = { - new Parsley(new frontend.SkipManyUntil((end #> ManyUntil.Stop <|> p: Parsley[Any]).internal)) + new Parsley(new frontend.SkipManyUntil((end.as(ManyUntil.Stop) <|> p.void: Parsley[Any]).internal)) } private [parsley] object ManyUntil { @@ -793,10 +862,10 @@ object combinator { notFollowedBy(end) *> (p <::> manyUntil(p, end)) } - // TODO: document and test before release - private [parsley] def skipSomeUntil(p: Parsley[_], end: Parsley[_]): Parsley[Unit] = { - notFollowedBy(end) *> (p *> skipManyUntil(p, end)) - } + // TODO: remove + // $COVERAGE-OFF$ + private [parsley] def skipSomeUntil(p: Parsley[_], end: Parsley[_]): Parsley[Unit] = notFollowedBy(end) *> (p *> skipManyUntil(p, end)) + // $COVERAGE-ON$ /** This combinator parses one of `thenP` or `elseP` depending on the result of parsing `condP`. * @@ -839,15 +908,29 @@ object combinator { * * @param condP the parser that yields the condition value. * @param thenP the parser to execute if the condition is `true`. - * @return a parser that conditionally parses `thenP` or `elseP` after `condP`. + * @return a parser that conditionally parses `thenP` after `condP`. * @group cond */ def when(condP: Parsley[Boolean], thenP: =>Parsley[Unit]): Parsley[Unit] = ifP(condP, thenP, unit) - // TODO: document and test before release - private [parsley] def ensure[A](condP: Parsley[Boolean], beforeP: =>Parsley[A]): Parsley[A] = - //ifP(condP, beforeP, empty) - condP.filter(identity) *> beforeP + /** This combinator verfies that the given parser returns `true`, or else fails. + * + * First, parse `p`; if it succeeds then, so long at returns `true`, this `guard(p)` succeeds. Otherwise, + * if `p` either fails, or returns `false`, `guard(p)` will fail. + * + * @example {{{ + * guard(pure(true)) == unit + * guard(pure(false)) == empty + * when(p.map(!_), empty) == guard(p) + * }}} + * + * @param p the parser that yields the condition value. + * @group cond + */ + def guard(p: Parsley[Boolean]): Parsley[Unit] = ifP(p, unit, empty) + + // TODO: remove + private [parsley] def ensure[A](condP: Parsley[Boolean], beforeP: =>Parsley[A]): Parsley[A] = guard(condP) *> beforeP /** This combinator repeatedly parses `p` so long as it returns `true`. * @@ -875,4 +958,131 @@ object combinator { lazy val whilePP: Parsley[Unit] = when(p, whilePP) whilePP } + + /** This combinator parses exactly `n` occurrences of `p`, returning these `n` results in a list. + * + * Parses `p` repeatedly up to `n` times. If `p` fails before `n` is reached, then this combinator + * fails. It is not required for `p` to fail after the `n`^th^ parse. The results produced by + * `p`, `x,,1,,` through `x,,n,,`, are returned as `List(x,,1,,, .., x,,n,,)`. + * + * @example {{{ + * scala> import parsley.character.item + * scala> import parsley.combinator.exactly + * scala> val p = exactly(3, item) + * scala> p.parse("ab") + * val res0 = Failure(..) + * scala> p.parse("abc") + * val res1 = Success(List('a', 'b', 'c')) + * scala> p.parse("abcd") + * val res2 = Success(List('a', 'b', 'c')) + * }}} + * + * @param n the number of times to repeat `p`. + * @param p the parser to repeat. + * @return a parser that parses `p` exactly `n` times, returning a list of the results. + * @group range + * @since 4.0.0 + */ + def exactly[A](n: Int, p: Parsley[A]): Parsley[List[A]] = traverse[Int, A](_ => p, (1 to n): _*) + private def skipExactly(n: Int, p: Parsley[_]): Parsley[Unit] = traverse_[Int](1 to n: _*)(_ => p) + + /** This combinator parses between `min` and `max` occurrences of `p`, returning these `n` results in a list. + * + * Parses `p` repeatedly a minimum of `min` times and up to `max` times both inclusive. If `p` fails before + * `min` is reached, then this combinator fails. It is not required for `p` to fail after the `max`^th^ parse. The results produced by + * `p`, `x,,min,,` through `x,,max,,`, are returned as `List(x,,min,,, .., x,,max,,)`. + * + * @example {{{ + * scala> import parsley.character.item + * scala> import parsley.combinator.range + * scala> val p = range(min=3, max=5)(item) + * scala> p.parse("ab") + * val res0 = Failure(..) + * scala> p.parse("abc") + * val res1 = Success(List('a', 'b', 'c')) + * scala> p.parse("abcd") + * val res2 = Success(List('a', 'b', 'c', 'd')) + * scala> p.parse("abcde") + * val res2 = Success(List('a', 'b', 'c', 'd', 'e')) + * scala> p.parse("abcdef") + * val res2 = Success(List('a', 'b', 'c', 'd', 'e')) + * }}} + * + * @param min the minimum number of times to repeat `p`, inclusive. + * @param max the maximum number of times to repeat `p`, inclusive. + * @param p the parser to repeat. + * @return the results of the successful parses of `p`. + * @group range + * @since 4.4.0 + */ + def range[A](min: Int, max: Int)(p: Parsley[A]): Parsley[List[A]] = fresh(mutable.ListBuffer.empty[A]).persist { xs => + count(min, max)((xs, p).zipped(_ += _).unsafe()) ~> + xs.map(_.toList) + } + + /** This combinator parses between `min` and `max` occurrences of `p` but ignoring the results. + * + * Parses `p` repeatedly a minimum of `min` times and up to `max` times both inclusive. If `p` fails before + * `min` is reached, then this combinator fails. It is not required for `p` to fail after the `max`^th^ parse. + * The results are discarded and `()` is returned instead. + * + * @example {{{ + * scala> import parsley.character.item + * scala> import parsley.combinator.range_ + * scala> val p = range_(min=3, max=5)(item) + * scala> p.parse("ab") + * val res0 = Failure(..) + * scala> p.parse("abc") + * val res1 = Success(()) + * scala> p.parse("abcd") + * val res2 = Success(()) + * scala> p.parse("abcde") + * val res2 = Success(()) + * scala> p.parse("abcdef") + * val res2 = Success(()) + * }}} + * + * @param min the minimum number of times to repeat `p`, inclusive. + * @param max the maximum number of times to repeat `p`, inclusive. + * @param p the parser to repeat. + * @group range + * @since 4.4.0 + */ + def range_(min: Int, max: Int)(p: Parsley[_]): Parsley[Unit] = count(min, max)(p).void + + /** This combinator parses between `min` and `max` occurrences of `p`, returning the number of successes. + * + * Parses `p` repeatedly a minimum of `min` times and up to `max` times both inclusive. If `p` fails before + * `min` is reached, then this combinator fails. It is not required for `p` to fail after the `max`^th^ parse. + * The results are discarded and the number of successful parses of `p`, `n`, is returned instead, such that + * `min <= n <= max`. + * + * @example {{{ + * scala> import parsley.character.item + * scala> import parsley.combinator.count + * scala> val p = count(min=3, max=5)(item) + * scala> p.parse("ab") + * val res0 = Failure(..) + * scala> p.parse("abc") + * val res1 = Success(3) + * scala> p.parse("abcd") + * val res2 = Success(4) + * scala> p.parse("abcde") + * val res2 = Success(5) + * scala> p.parse("abcdef") + * val res2 = Success(5) + * }}} + * + * @param min the minimum number of times to repeat `p`, inclusive. + * @param max the maximum number of times to repeat `p`, inclusive. + * @param p the parser to repeat. + * @return the number of times `p` parsed successfully. + * @group range + * @since 4.4.0 + */ + def count(min: Int, max: Int)(p: Parsley[_]): Parsley[Int] = min.makeReg { i => + skipExactly(min, p) ~> + skipMany(ensure(i.gets(_ < max), p) ~> i.modify(_ + 1)) ~> + i.get + } } diff --git a/parsley/shared/src/main/scala/parsley/debug.scala b/parsley/shared/src/main/scala/parsley/debug.scala index 467475778..8aed91351 100644 --- a/parsley/shared/src/main/scala/parsley/debug.scala +++ b/parsley/shared/src/main/scala/parsley/debug.scala @@ -5,7 +5,11 @@ */ package parsley +import scala.annotation.tailrec +import scala.collection.mutable + import parsley.errors.ErrorBuilder +import parsley.registers.Reg import parsley.internal.deepembedding.frontend @@ -28,8 +32,8 @@ import parsley.internal.deepembedding.frontend * @groupdesc ctrl * These methods can control how the debug mechanism functions in a general way. */ -// $COVERAGE-OFF$ object debug { + // $COVERAGE-OFF$ /** Base trait for breakpoints. * * @group break @@ -117,9 +121,14 @@ object debug { * @param name The name to be assigned to this parser * @param break The breakpoint properties of this parser, defaults to NoBreak * @param coloured Whether to render with colour (default true: render colours) + * @param watchedRegs Which registers to also track the values of and their names, if any */ - def debug(name: String, break: Breakpoint, coloured: Boolean): Parsley[A] = { - new Parsley(new frontend.Debug[A](con(p).internal, name, !coloured, break)) + def debug(name: String, break: Breakpoint, coloured: Boolean, watchedRegs: (Reg[_], String)*): Parsley[A] = { + new Parsley(new frontend.Debug[A](con(p).internal, name, !coloured, break, watchedRegs)) + } + + private [parsley] def debug(name: String, break: Breakpoint, coloured: Boolean): Parsley[A] = { + debug(name, break, coloured, Seq.empty[(Reg[_], String)]: _*) } /** $debug @@ -153,8 +162,11 @@ object debug { * * @param name The name to be assigned to this parser * @param break The breakpoint properties of this parser, defaults to NoBreak + * @param watchedRegs Which registers to also track the values of and their names, if any */ - def debug(name: String, break: Breakpoint): Parsley[A] = debug(name, break, coloured = true) + def debug(name: String, break: Breakpoint, watchedRegs: (Reg[_], String)*): Parsley[A] = debug(name, break, coloured = true, watchedRegs: _*) + + private [parsley] def debug(name: String, break: Breakpoint): Parsley[A] = debug(name, break, Seq.empty[(Reg[_], String)]: _*) /** $debug * @@ -187,8 +199,11 @@ object debug { * * @param name The name to be assigned to this parser * @param coloured Whether to render with colour + * @param watchedRegs Which registers to also track the values of and their names, if any */ - def debug(name: String, coloured: Boolean): Parsley[A] = debug(name, break = NoBreak, coloured) + def debug(name: String, coloured: Boolean, watchedRegs: (Reg[_], String)*): Parsley[A] = debug(name, break = NoBreak, coloured, watchedRegs: _*) + + private [parsley] def debug(name: String, coloured: Boolean): Parsley[A] = debug(name, coloured, Seq.empty[(Reg[_], String)]: _*) /** $debug * @@ -220,14 +235,216 @@ object debug { * Renders in colour with no break-point. * * @param name The name to be assigned to this parser + * @param watchedRegs Which registers to also track the values of and their names, if any */ - def debug(name: String): Parsley[A] = debug(name, break = NoBreak, coloured = true) + def debug(name: String, watchedRegs: (Reg[_], String)*): Parsley[A] = debug(name, break = NoBreak, coloured = true, watchedRegs: _*) + + private [parsley] def debug(name: String): Parsley[A] = debug(name, Seq.empty[(Reg[_], String)]: _*) + /** Display information about the error messages generated by this parser. + * + * This is an experimental debugger that provides internal information about error messages. + * This provides more detail than one might normally see inside a regular error message, but + * may help isolate the root cause of an error message not being as expected: this can form the + * bulk of a specific question on the discussion board. + * + * @param name The name to be assigned to this parser + * @param coloured Whether the output should be colourful + * @param errBuilder The error builder used for formatting messages in the "real parser", + * which is used to help format information in the debugger. + * @since 4.0.0 + */ def debugError(name: String, coloured: Boolean)(implicit errBuilder: ErrorBuilder[_]): Parsley[A] = { new Parsley(new frontend.DebugError[A](con(p).internal, name, !coloured, errBuilder)) } + /** Display information about the error messages generated by this parser. + * + * This is an experimental debugger that provides internal information about error messages. + * This provides more detail than one might normally see inside a regular error message, but + * may help isolate the root cause of an error message not being as expected: this can form the + * bulk of a specific question on the discussion board. + * + * @param name The name to be assigned to this parser + * @param coloured Whether the output should be colourful + * @param errBuilder The error builder used for formatting messages in the "real parser", + * which is used to help format information in the debugger. + * @since 4.0.0 + */ def debugError(name: String)(implicit errBuilder: ErrorBuilder[_]): Parsley[A] = debugError(name, coloured = true) + + /** This combinator allows for the runtime of this parser to be measured. + * + * When this parser executes, its start and end times will be logged using `System.nanoTime()`, + * which has a resolution of 100ns. These will be logged into the given `Profiler` object. + * + * @param name the ''unique'' name of this parser, which will represent it in the table + * @param profiler the profiling object that will collect and process the data + * @note usual disclaimers about profiling apply: results are just data; use your judgement + * @see [[Profiler `Profiler`]] + * @since 4.4.0 + */ + def profile(name: String)(implicit profiler: Profiler): Parsley[A] = new Parsley(new frontend.Profile[A](con(p).internal, name, profiler)) + } + // $COVERAGE-ON$ + + /** This class is used to store the profile data for a specific group of sub-parsers. + * + * It records the start and end timestamps of the parsers that interact with it. It is possible + * to use multiple different profilers if you want to establish the cumulative time for a sub-parser + * instead of the self-time. + * + * This class is mutable, so care must be taken to call `reset()` between runs, unless you want to + * accumulate the data. + * + * @since 4.4.0 + */ + class Profiler { + private val entries = mutable.Map.empty[String, mutable.Buffer[Long]] + private val exits = mutable.Map.empty[String, mutable.Buffer[Long]] + private var lastTime: Long = 0 + private var lastTimeCount: Long = 0 + + // $COVERAGE-OFF$ + /** Prints a summary of the data sampled by this profiler. + * + * After the run(s) of the parser are complete, this method can be used to + * generate the summary of the sampled data. It will print a table where the + * total "self-time", number of invocations and average "self-time" are displayed + * for each profiled sub-parser. + * + * * '''self-time''': this is the amount of time spend in a specific parser, removing + * the times from within the child parsers. + * + * @note to measure cumulative time of a parser, consider using a separate `Profiler` + * object for it instead. + * @since 4.4.0 + */ + def summary(): Unit = { + val (selfTotals, invocations) = process + render(selfTotals, invocations) + } + // $COVERAGE-ON$ + + /** Clears the data within this profiler. + * @since 4.4.0 + */ + def reset(): Unit = { + // can't clear the maps, because the instructions may have already captured the lists + for ((_, timings) <- entries) timings.clear() + for ((_, timings) <- exits) timings.clear() + lastTime = 0 + lastTimeCount = 0 + } + + private [parsley] def entriesFor(name: String): mutable.Buffer[Long] = entries.getOrElseUpdate(name, mutable.ListBuffer.empty) + private [parsley] def exitsFor(name: String): mutable.Buffer[Long] = exits.getOrElseUpdate(name, mutable.ListBuffer.empty) + private [parsley] def monotone(n: Long) = { + if (n == lastTime) { + lastTimeCount += 1 + n + lastTimeCount + } + else { + lastTime = n + lastTimeCount = 0 + n + } + } + + private [parsley] def process: (Map[String, Long], Map[String, Int]) = { + val allEntries = collapse(entries).sortBy(_._2) + val allExits = collapse(exits).sortBy(_._2) + + require((allEntries ::: allExits).toSet.size == (allExits.length + allExits.length), + "recorded times must all be monotonically increasing") + + val selfTotals = mutable.Map.empty[String, Long] + val invocations = mutable.Map.empty[String, Int] + + @tailrec + def go(entries: List[(String, Long)], exits: List[(String, Long)], stack: List[((String, Long), Long)], cum: Long): Unit = { + (entries, exits, stack) match { + case (Nil, Nil, Nil) => + // final unwinding or stuff to clear on the stack (cum here is for the children) + case (ens, (n2, t2)::exs, ((n1, t1), oldCum)::stack) if ens.headOption.forall(t2 < _._2) => + assert(n1 == n2, "unwinding should result in matching values") + add(invocations, n1)(1) + add(selfTotals, n1)(t2 - t1 - cum) + go(ens, exs, stack, oldCum + t2 - t1) + // in this case, the scope closes quickly (cum here is for your siblings) + case ((n1, t1)::ens, (n2, t2)::exs, stack) if ens.headOption.forall(t2 < _._2) && n1 == n2 => + assert(ens.nonEmpty || n1 == n2, "unwinding should result in matching values") + add(invocations, n1)(1) + add(selfTotals, n1)(t2 - t1) + go(ens, exs, stack, cum + t2 - t1) + // the next one opens first, or the entry and exit don't match + // in either case, this isn't our exit, push ourselves onto the stack (cum here is for your siblings) + case (nt::ens, exs@(_ :: _), stack) => go(ens, exs, (nt, cum)::stack, 0) + // $COVERAGE-OFF$ + case (Nil, Nil, _::_) + | (Nil, _::_, Nil) + | (_ ::_, Nil, _) => assert(false, "something has gone very wrong") + case (Nil, _::_, _::_) => ??? // deadcode from case 2 + // $COVERAGE-ON$ + } + } + + //println(allEntries.map { case (name, t) => (name, t - allEntries.head._2) }) + //println(allExits.map { case (name, t) => (name, t - allEntries.head._2) }) + go(allEntries, allExits, Nil, 0) + (selfTotals.toMap, invocations.toMap) + } + + private def collapse(timings: Iterable[(String, Iterable[Long])]): List[(String, Long)] = timings.flatMap { + case (name, times) => times.map(t => (name, t)) + }.toList + + private def add[A: Numeric](m: mutable.Map[String, A], name: String)(n: A): Unit = m.get(name) match { + case Some(x) => m(name) = implicitly[Numeric[A]].plus(x, n) + case None => m(name) = n + } + + // $COVERAGE-OFF$ + private def render(selfTimes: Map[String, Long], invocations: Map[String, Int]): Unit = { + val combined = selfTimes.map { + case (name, selfTime) => + val invokes = invocations(name) + (name, (f"${selfTime/1000.0}%.1fμs", invocations(name), f"${selfTime/invokes/1000.0}%.3fμs")) + } + val head1 = "name" + val head2 = "self time" + val head3 = "num calls" + val head4 = "average self time" + + val (names, data) = combined.unzip + val (selfs, invokes, avs) = data.unzip3 + + val col1Width = (head1.length :: names.map(_.length).toList).max + val col2Width = (head2.length :: selfs.map(_.length).toList).max + val col3Width = (head3.length :: invokes.map(digits(_)).toList).max + val col4Width = (head4.length :: avs.map(_.length).toList).max + + val header = List(pad(head1, col1Width), tab(col1Width), + pad(head2, col2Width), tab(col2Width), + pad(head3, col3Width), tab(col3Width), + pad(head4, col4Width)).mkString + val hline = header.map(_ => '-') + + println(header) + println(hline) + for ((name, (selfTime, invokes, avSelfTime)) <- combined) { + println(List(pad(name, col1Width), tab(col1Width), + prePad(selfTime, col2Width), tab(col2Width), + prePad(invokes.toString, col3Width), tab(col3Width), + prePad(avSelfTime, col4Width)).mkString) + } + println(hline) + } + + private def pad(str: String, n: Int) = str + " " * (n - str.length) + private def prePad(str: String, n: Int) = " " * (n - str.length) + str + private def digits[A: Numeric](n: A): Int = Math.log10(implicitly[Numeric[A]].toDouble(n)).toInt + 1 + private def tab(n: Int) = " " * (4 - n % 4) + // $COVERAGE-ON$ } } -// $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/errors/ErrorGen.scala b/parsley/shared/src/main/scala/parsley/errors/ErrorGen.scala new file mode 100644 index 000000000..1a99f4a7e --- /dev/null +++ b/parsley/shared/src/main/scala/parsley/errors/ErrorGen.scala @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Parsley Contributors + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package parsley.errors + +import parsley.Parsley + +import parsley.internal.deepembedding.frontend.LazyParsley +import parsley.internal.deepembedding.singletons +import parsley.internal.errors.{RigidCaret, UnexpectDesc} +import parsley.internal.machine.errors.{DefuncError, EmptyError, ExpectedError, UnexpectedError} + +import org.typelevel.scalaccompat.annotation.unused + +/** This class can be used to generate hand-tuned error messages without using `flatMap`. + * + * This class, and its subclasses, describe special primitives that can use the results of + * a previous parser to form an error message and then raise it. This is not something that + * is normally possible with raw combinators, without using `flatMap`, which is expensive. + * + * Primarily, these are designed to be used with `filterWith`/`verifiedWith`/`preventWith` + * but can be used in other parsers as well. See the methods of the class for information. + * + * @since 4.4.0 + */ +sealed abstract class ErrorGen[-A] { + /** This combinator takes a given parser and raises an error based on its returned results. + * + * The given parser produces a value and a width, which are used to synthesise and raise + * an error message derived from the value with the given width. This is a safe way of using + * `parser`, since it ensures that the result of the given parser `p` is not optimised out. + * `errGen(p)` is similar to `withWidth(p).flatMap { case (x, w) => failCombinator(...) }`, + * in that it generates errors in a context-sensitive way. However, this is much more efficient + * than using the expensive `flatMap`, so it is provided as a primitive operation. + * + * @since 4.4.0 + */ + final def apply(p: Parsley[(A, Int)]): Parsley[Nothing] = (p <**> parser).impure + + /** This parser can be applied (postfix) to a parser returning a value and a width to generate an + * error message tailored to them. + * + * This is '''not''' a generally safe operation to be performing, and should only be used + * within a combinator that is guaranteed to use its results. The optimiser is not aware that + * the results of the parser this will be applied to will actually be needed, and so may optimise + * them out. Using this parser inside an arm of `select` or `branch`, say, would be safe, because + * these combinators force the result of their condition to be generated, but `p <**> this.parser` + * is not generally safe without a use of `impure` to guard it. This is what `apply` accomplishes + * more safely. + * + * @since 4.4.0 + */ + final def parser: Parsley[((A, Int)) => Nothing] = new Parsley(internal) + private [errors] def internal: LazyParsley[((A, Int)) => Nothing] + + /** This method can be overridden to control how wide an error is based on the value and width + * that produces it. + * + * The width provides to this error generator likely comes directly from the span of the + * parser used to produce the required result. However, this may not be entirely accurate + * for how the user might want the error to be sized (perhaps there was whitespace, or the + * parser consumed more input than was necessary to pin-point the problem). In these cases, + * this method allows for custom logic to derive the actual width of the error message. By + * default, just returns the given `width`. + * + * @since 4.4.0 + */ + def adjustWidth(@unused x: A, width: Int): Int = width +} + +/** An error generator for ''Vanilla'' errors, which can tune the unexpected message and a + * generated reason. + * + * @since 4.4.0 + */ +class VanillaGen[-A] extends ErrorGen[A] { + /** What should the unexpected component of the error message be based on the result the + * offending parser produced? + * + * @since 4.4.0 + */ + def unexpected(@unused x: A): VanillaGen.UnexpectedItem = VanillaGen.EmptyItem + + /** What should the reason component of the error message be (if any) based on the result the + * offending parser produced? + * + * @since 4.4.0 + */ + def reason(@unused x: A): Option[String] = None + + private [errors] override def internal: LazyParsley[((A, Int)) => Nothing] = new singletons.VanillaGen(this) +} + +/** This object contain the `UnexpectedItem` classes, needed to define the `unexpected` component of `VanillaGen`. + * + * @since 4.4.0 + */ +object VanillaGen { + /** Base class for describing how to form the unexpected component of a vanilla error message from `VanillaGen`. + * + * @since 4.4.0 + */ + sealed abstract class UnexpectedItem { + private [parsley] def makeError(offset: Int, line: Int, col: Int, caretWidth: Int): DefuncError + } + + /** Signifies that the error generated should use whatever input was consumed by the offending parser + * verbatim. + * + * @since 4.4.0 + */ + case object RawItem extends UnexpectedItem { + private[parsley] def makeError(offset: Int, line: Int, col: Int, caretWidth: Int): DefuncError = + new ExpectedError(offset, line, col, None, caretWidth) + } + + /** Signifies that the error generated should not have an unexpected component at all (as in `filter`). + * + * @since 4.4.0 + */ + case object EmptyItem extends UnexpectedItem { + private[parsley] def makeError(offset: Int, line: Int, col: Int, caretWidth: Int): DefuncError = + new EmptyError(offset, line, col, caretWidth) + } + + /** Signifies that the error generated should use the given name as the unexpected component. + * + * @since 4.4.0 + */ + final case class NamedItem(name: String) extends UnexpectedItem { + private[parsley] def makeError(offset: Int, line: Int, col: Int, caretWidth: Int): DefuncError = + new UnexpectedError(offset, line, col, None, new UnexpectDesc(name, new RigidCaret(caretWidth))) + } +} + +/** An error generate for ''Specialised'' errors, which can tune the freeform messages of the error. + * + * @since 4.4.0 + */ +abstract class SpecialisedGen[-A] extends ErrorGen[A] { + /** What should the messages of the error message be based on the result the + * offending parser produced? + * + * @since 4.4.0 + */ + def messages(x: A): Seq[String] + + private [errors] override def internal: LazyParsley[((A, Int)) => Nothing] = new singletons.SpecialisedGen(this) +} diff --git a/parsley/shared/src/main/scala/parsley/errors/Token.scala b/parsley/shared/src/main/scala/parsley/errors/Token.scala index c923de86e..a93fd2bca 100644 --- a/parsley/shared/src/main/scala/parsley/errors/Token.scala +++ b/parsley/shared/src/main/scala/parsley/errors/Token.scala @@ -5,6 +5,7 @@ */ package parsley.errors +// TODO: move into ErrorBuilder object in 5.5.0? /** This class represents an extracted token returned by `unexpectedToken` in `ErrorBuilder`. * * There is deliberately no analogue for `EndOfInput` because we guarantee that non-empty diff --git a/parsley/shared/src/main/scala/parsley/errors/TokenSpan.scala b/parsley/shared/src/main/scala/parsley/errors/TokenSpan.scala index 3128adacf..840fae001 100644 --- a/parsley/shared/src/main/scala/parsley/errors/TokenSpan.scala +++ b/parsley/shared/src/main/scala/parsley/errors/TokenSpan.scala @@ -36,6 +36,7 @@ object TokenSpan { override private [parsley] def toCaretLength(col: Int, lengthLine: Int, lengthAfters: =>List[Int]): Int = w } + // $COVERAGE-OFF$ /** This span is designed to be used by token extractors that try and parse the * remaining input: it indicates the number of lines and columns that were * parsed in the process of extracting the token. @@ -66,4 +67,5 @@ object TokenSpan { } } } + // $COVERAGE-ON$ } diff --git a/parsley/shared/src/main/scala/parsley/errors/combinator.scala b/parsley/shared/src/main/scala/parsley/errors/combinator.scala index fae9f45fc..b166cb61b 100644 --- a/parsley/shared/src/main/scala/parsley/errors/combinator.scala +++ b/parsley/shared/src/main/scala/parsley/errors/combinator.scala @@ -40,7 +40,10 @@ import parsley.internal.errors.{CaretWidth, FlexibleCaret, RigidCaret} * parsers that interact with the error system in some way. */ object combinator { - private [parsley] def empty(caretWidth: Int): Parsley[Nothing] = new Parsley(singletons.Empty(caretWidth)) + // TODO: remove in 5.0, for MiMA's sake + // $COVERAGE-OFF$ + private [parsley] def empty(caretWidth: Int): Parsley[Nothing] = Parsley.empty(caretWidth) + // $COVERAGE-ON$ /** This combinator consumes no input and fails immediately with the given error messages. * @@ -114,6 +117,22 @@ object combinator { * on the output of a parser that may render it invalid, but the error should point to the * beginning of the structure. This combinators effect can be cancelled with [[entrench `entrench`]]. * + * @example {{{ + * scala> val greeting = string("hello world") <* char('!') + * scala> greeting.label("greeting").parse("hello world.") + * val res0 = Failure((line 1, column 12): + * unexpected "." + * expected "!" + * >hello world. + * ^) + * scala> amend(greeting).label("greeting").parse("hello world.") + * val res1 = Failure((line 1, column 1): + * unexpected "h" + * expected greeting + * >hello world. + * ^) + * }}} + * * @param p a parser whose error messages should be adjusted. * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. * @since 3.1.0 @@ -121,6 +140,45 @@ object combinator { */ def amend[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorAmend(p.internal, partial = false)) + /** This combinator adjusts any error messages generated by the given parser so that they + * occur at the position recorded on entry to this combinator, but retains the original offset. + * + * Similar to [[amend `amend`]], but retains the original offset the error occurred at. This is known + * as its ''underlying offset'' as opposed to the visual ''presentation offset''. To the reader, the + * error messages appears as if no input was consumed, but for the purposes of error message merging + * the error is still deeper. A key thing to note is that two errors can only merge if they are at + * the same presentation ''and'' underlying offsets: if they are not the deeper of the two ''dominates''. + * + * The ability for an error to still dominate others after partial amendment can be useful for allowing + * it to avoid being lost when merging with errors that are deeper than the presentation offset but + * shallower than the underlying. + * + * @example {{{ + * scala> val greeting = string("hello world") <* char('!') + * scala> val shortGreeting = string("h") <* (char('i') | string("ey")) <* char('!') + * // here, the shortGreeting, despite not getting as far into the input is dominating the amended long greeting + * scala> (amend(atomic(greeting)).label("hello world!") | shortGreeting).parse("hello world.") + * val res0 = Failure((line 1, column 2): + * unexpected "el" + * expected "ey" or "i" + * >hello world. + * ^^) + * // here it appears to start at the `h` point, but notably dominates the short greeting + * scala> (amend(atomic(greeting)).label("hello world!") | shortGreeting).parse("hello world.") + * val res1= Failure((line 1, column 1): + * unexpected "h" + * expected hello world! + * >hello world. + * ^) + * }}} + * + * @param p a parser whose error messages should be adjusted. + * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. + * @since 4.4.0 + * @group adj + */ + def partialAmend[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorAmend(p.internal, partial = true)) + /** This combinator prevents the action of any enclosing `amend` on the errors generated by the given * parser. * @@ -141,6 +199,8 @@ object combinator { * } * }}} * + * '''In reality though, `filterOut` has an `amend` and `entrench` built into it.''' + * * @param p a parser whose error messages should not be adjusted by any surrounding [[amend `amend`]]. * @return a parser that parses `p` but ensures any error messages are generated normally. * @since 3.1.0 @@ -148,11 +208,11 @@ object combinator { */ def entrench[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorEntrench(p.internal)) - /** This combinator undoes the action of the `entrench` combinator on the given parser. + /** This combinator undoes the action of any `entrench` combinators on the given parser. * * Entrenchment is important for preventing the incorrect amendment of certain parts of sub-errors * for a parser, but it may be then undesireable to block further amendments from elsewhere in the - * parser. This combinator can be used to cancel and entrenchment after the critical section has + * parser. This combinator can be used to cancel all entrenchment after the critical section has * passed. * * @param p a parser that should no longer be under the affect of an `entrench` combinator @@ -161,7 +221,20 @@ object combinator { * @group adj */ def dislodge[A](p: Parsley[A]): Parsley[A] = dislodge(Int.MaxValue)(p) - private def dislodge[A](by: Int)(p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorDislodge(by, p.internal)) + /** This combinator undoes the action of `by` many `entrench` combinators on the given parser. + * + * Entrenchment is important for preventing the incorrect amendment of certain parts of sub-errors + * for a parser, but it may be then undesireable to block further amendments from elsewhere in the + * parser. This combinator can be used to cancel several, but potentially not all entrenchments after the + * critical section has passed. + * + * @param by the number of entrenchments to undo + * @param p a parser that should no longer be under the affect of an `entrench` combinator + * @return a parser that parses `p` and ''may'' allows its error messages to be amended if all entrenchments are undone + * @since 4.4.0 + * @group adj + */ + def dislodge[A](by: Int)(p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorDislodge(by, p.internal)) /** This combinator first tries to amend the position of any error generated by the given parser, * and if the error was entrenched will dislodge it instead. @@ -169,14 +242,44 @@ object combinator { * @param p a parser whose error messages should be amended unless its been entrenched. * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. * @since 4.2.0 - * @see [[amend `amend`]] and `[[dislodge `dislodge`]] + * @see [[amend `amend`]] and [[dislodge[A](p:parsley\.Parsley[A])* `dislodge`]] + * @group adj + */ + def amendThenDislodge[A](p: Parsley[A]): Parsley[A] = amendThenDislodge(Int.MaxValue)(p) + /** This combinator first tries to amend the position of any error generated by the given parser, + * and if the error was entrenched will dislodge it `by` many times instead. + * + * @param p a parser whose error messages should be amended unless its been entrenched. + * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. + * @since 4.4.0 + * @see [[amend `amend`]] and [[dislodge[A](by:Int)* `dislodge`]] * @group adj */ - def amendThenDislodge[A](p: Parsley[A]): Parsley[A] = dislodge(amend(p)) + def amendThenDislodge[A](by: Int)(p: Parsley[A]): Parsley[A] = dislodge(by)(amend(p)) - // TODO: test, document, and expose :) - private [parsley] def partialAmend[A](p: Parsley[A]): Parsley[A] = new Parsley(new frontend.ErrorAmend(p.internal, partial = true)) - private [parsley] def partialAmendThenDislodge[A](p: Parsley[A]): Parsley[A] = dislodge(partialAmend(p)) + // These don't need coverage really, they are basically the same as the ones above + // $COVERAGE-OFF$ + /** This combinator first tries to partially amend the position of any error generated by the given parser, + * and if the error was entrenched will dislodge it instead. + * + * @param p a parser whose error messages should be amended unless its been entrenched. + * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. + * @since 4.4.0 + * @see [[partialAmend `partialAmend`]] and [[dislodge[A](p:parsley\.Parsley[A])* `dislodge`]] + * @group adj + */ + def partialAmendThenDislodge[A](p: Parsley[A]): Parsley[A] = partialAmendThenDislodge(Int.MaxValue)(p) + /** This combinator first tries to partially amend the position of any error generated by the given parser, + * and if the error was entrenched will dislodge it `by` many times instead. + * + * @param p a parser whose error messages should be amended unless its been entrenched. + * @return a parser that parses `p` but ensures any errors generated occur as if no input were consumed. + * @since 4.4.0 + * @see [[partialAmend `partialAmend`]] and [[dislodge[A](by:Int)* `dislodge`]] + * @group adj + */ + def partialAmendThenDislodge[A](by: Int)(p: Parsley[A]): Parsley[A] = dislodge(by)(partialAmend(p)) + // $COVERAGE-ON$ /** This combinator marks any errors within the given parser as being ''lexical errors''. * @@ -225,6 +328,14 @@ object combinator { * defined, this means that it is invalid and the filtering will fail using the message obtained from the * succesful partial function invocation. * + * @groupprio genFilter 15 + * @groupname genFilter Generic Filtering Combinators + * @groupdesc genFilter + * This combinators generalise the combinators from above, which are all special cases of them. Each of these + * takes the characteristic predicate or function of the regular variants, but takes an `errGen` object that + * can be used to fine-tune the error messages. These offer some flexiblity not offered by the specialised + * filtering combinators, but are a little more verbose to use. + * * @groupprio fail 20 * * @define observably @@ -276,10 +387,15 @@ object combinator { * @return a parser that returns the result of this parser if it fails the predicate. * @see [[parsley.Parsley.filterNot `filterNot`]], which is a basic version of this same combinator with no customised reason. * @see [[guardAgainst `guardAgainst`]], which is similar to `filterOut`, except it generates a ''specialised'' error as opposed to just a reason. + * @note implemented in terms of [[filterWith `filterWith`]]. * @note $autoAmend * @group filter */ - def filterOut(pred: PartialFunction[A, String]): Parsley[A] = new Parsley(new frontend.FilterOut(con(p).internal, pred)) + def filterOut(pred: PartialFunction[A, String]): Parsley[A] = { + this.filterWith(new VanillaGen[A] { + override def reason(x: A) = Some(pred(x)) + })(!pred.isDefinedAt(_)) + } /** This combinator filters the result of this parser using the given partial-predicate, succeeding only when the predicate is undefined. * @@ -321,9 +437,14 @@ object combinator { * @see [[filterOut `filterOut`]], which is similar to `guardAgainst`, except it generates a reason for failure and not a ''specialised'' error. * @see [[[collectMsg[B](msggen:A=>Seq[String])* `collectMsg`]]], which is similar to `guardAgainst`, but can also transform the data on success. * @note $autoAmend + * @note implemented in terms of [[filterWith `filterWith`]]. * @group filter */ - def guardAgainst(pred: PartialFunction[A, Seq[String]]): Parsley[A] = new Parsley(new frontend.GuardAgainst(con(p).internal, pred)) + def guardAgainst(pred: PartialFunction[A, Seq[String]]): Parsley[A] = { + this.filterWith(new SpecialisedGen[A] { + override def messages(x: A) = pred(x) + })(!pred.isDefinedAt(_)) + } /** This combinator applies a partial function `pf` to the result of this parser if its result is defined for `pf`, failing if it is not. * @@ -350,6 +471,7 @@ object combinator { * @see [[parsley.Parsley.collect `collect`]], which is a basic version of this same combinator with no customised error message. * @see [[guardAgainst `guardAgainst`]], which is similar to `collectMsg`, except it does not transform the data. * @note $autoAmend + * @note implemented in terms of [[collectWith `collectWith`]]. * @group filter */ def collectMsg[B](msg0: String, msgs: String*)(pf: PartialFunction[A, B]): Parsley[B] = this.collectMsg(_ => msg0 +: msgs)(pf) @@ -378,14 +500,13 @@ object combinator { * @see [[parsley.Parsley.collect `collect`]], which is a basic version of this same combinator with no customised error message. * @see [[guardAgainst `guardAgainst`]], which is similar to `collectMsg`, except it does not transform the data. * @note $autoAmend + * @note implemented in terms of [[collectWith `collectWith`]]. * @group filter */ def collectMsg[B](msggen: A => Seq[String])(pf: PartialFunction[A, B]): Parsley[B] = { - this.guardAgainst{case x if !pf.isDefinedAt(x) => msggen(x)}.map(pf) - } - - private def _unexpectedWhen(pred: PartialFunction[A, (String, Option[String])]): Parsley[A] = { - new Parsley(new frontend.UnexpectedWhen(con(p).internal, pred)) + this.collectWith(new SpecialisedGen[A] { + override def messages(x: A) = msggen(x) + })(pf) } /** This combinator filters the result of this parser using the given partial-predicate, succeeding only when the predicate is undefined. @@ -418,10 +539,13 @@ object combinator { * @see [[guardAgainst `guardAgainst`]], which is similar to `unexpectedWhen`, except it generates a ''specialised'' error instead. * @see [[unexpectedWithReasonWhen `unexpectedWithReasonWhen`]], which is similar, but also has a reason associated. * @note $autoAmend + * @note implemented in terms of [[filterWith `filterWith`]]. * @group filter */ - def unexpectedWhen(pred: PartialFunction[A, String]): Parsley[A] = this._unexpectedWhen { - case x if pred.isDefinedAt(x) => (pred(x), None) + def unexpectedWhen(pred: PartialFunction[A, String]): Parsley[A] = { + this.filterWith(new VanillaGen[A] { + override def unexpected(x: A) = VanillaGen.NamedItem(pred(x)) + })(!pred.isDefinedAt(_)) } /** This combinator filters the result of this parser using the given partial-predicate, succeeding only when the predicate is undefined. @@ -453,12 +577,14 @@ object combinator { * @see [[guardAgainst `guardAgainst`]], which is similar to `unexpectedWhen`, except it generates a ''specialised'' error instead. * @see [[unexpectedWhen `unexpectedWhen`]], which is similar, but with no associated reason. * @since 4.2.0 + * @note implemented in terms of [[filterWith `filterWith`]]. * @group filter */ - def unexpectedWithReasonWhen(pred: PartialFunction[A, (String, String)]): Parsley[A] = this._unexpectedWhen { - case x if pred.isDefinedAt(x) => - val (unex, reason) = pred(x) - (unex, Some(reason)) + def unexpectedWithReasonWhen(pred: PartialFunction[A, (String, String)]): Parsley[A] = { + this.filterWith(new VanillaGen[A] { + override def unexpected(x: A) = VanillaGen.NamedItem(pred(x)._1) + override def reason(x: A) = Some(pred(x)._2) + })(!pred.isDefinedAt(_)) } /** This combinator changes the expected component of any errors generated by this parser. @@ -468,11 +594,16 @@ object combinator { * * $observably * @param item the name to give to the expected component of any qualifying errors. + * @param items any further labels to assign to this parser. * @return a parser that expects `item` on failure. * @since 3.0.0 * @group rich */ - def label(item: String): Parsley[A] = if (item.isEmpty) this.hide else this.labels(item)//new Parsley(new frontend.ErrorLabel(con(p).internal, item)) + def label(item: String, items: String*): Parsley[A] = { + if (item.isEmpty && items.isEmpty) this.hide + else this.labels(item +: items: _*) + } + private [combinator] def label(item: String): Parsley[A] = if (item.isEmpty) this.hide else this.labels(item) /** This combinator changes the expected component of any errors generated by this parser. * @@ -486,6 +617,7 @@ object combinator { */ def ?(item: String): Parsley[A] = this.label(item) + // needs to be private up to parsley to support the ConfigImplUntyped private [parsley] def labels(items: String*): Parsley[A] = { require(items.forall(_.nonEmpty), "Labels cannot be empty strings") new Parsley(new frontend.ErrorLabel(con(p).internal, items)) @@ -521,11 +653,13 @@ object combinator { * $observably * @since 3.0.0 * @return a parser that does not produce an expected component on failure. - * @see [[label `label`]] * @group rich */ def hide: Parsley[A] = this.labels() + // TODO: this will become the new hide in 5.0.0 + private [parsley] def newHide: Parsley[A] = new Parsley(new frontend.ErrorHide(con(p).internal)) + // $COVERAGE-OFF$ /** This combinator parses this parser and then fails, using the result of this parser to customise the error message. * @@ -542,8 +676,8 @@ object combinator { */ @deprecated("This combinator will be removed in 5.0.0, without direct replacement", "4.2.0") def !(msggen: A => String): Parsley[Nothing] = partialAmendThenDislodge { - parsley.position.internalOffsetSpan(entrench(con(p))).flatMap { case (os, x, oe) => - combinator.fail(oe - os, msggen(x)) + parsley.position.withWidth(entrench(con(p))).flatMap { case (x, width) => + combinator.fail(width, msggen(x)) } } @@ -564,10 +698,76 @@ object combinator { */ @deprecated("This combinator will be removed in 5.0.0, without direct replacement", "4.2.0") def unexpected(msggen: A => String): Parsley[Nothing] = partialAmendThenDislodge { - parsley.position.internalOffsetSpan(entrench(con(p))).flatMap { case (os, x, oe) => - combinator.unexpected(oe - os, msggen(x)) + parsley.position.withWidth(entrench(con(p))).flatMap { case (x, width) => + combinator.unexpected(width, msggen(x)) } } // $COVERAGE-ON$ + + // $COVERAGE-OFF$ + // TODO: remove in 5.0.0 + @deprecated("better argument currying now, don't use", "4.4.0-RC2") + private [parsley] def filterWith(pred: A => Boolean, errGen: ErrorGen[A]): Parsley[A] = combinator.filterWith(con(p))(pred, errGen) + // $COVERAGE-ON$ + /** This combinator filters the result of this parser with the given predicate, generating an error with the + * given error generator if the function returned `false`. + * + * Like [[Parsley.filter `filter`]], except allows for the error message generated to be fine-tuned with + * respect to the parsers result and width of input consumed using an [[ErrorGen `ErrorGen`]] object. + * + * @param errGen how to generate error messages based on the result of this parser. + * @param pred the predicate that is tested against the parser result. + * @since 4.4.0 + * @group genFilter + */ + def filterWith(errGen: ErrorGen[A])(pred: A => Boolean): Parsley[A] = combinator.filterWith(con(p))(pred, errGen) + + // $COVERAGE-OFF$ + // TODO: remove in 5.0.0 + @deprecated("better argument currying now, don't use", "4.4.0-RC2") + private [parsley] def collectWith[B](pf: PartialFunction[A, B], errGen: ErrorGen[A]): Parsley[B] = combinator.collectWith(con(p))(pf, errGen) + // $COVERAGE-ON$ + /** This combinator conditionally transforms the result of this parser with a given partial function, generating an error with the + * given error generator if the function is not defined on the result of this parser. + * + * Like [[Parsley.collect `collect`]], except allows for the error message generated to be fine-tuned with + * respect to the parsers result and width of input consumed using an [[ErrorGen `ErrorGen`]] object. + * + * @param pf the partial function used to both filter the result of this parser and transform it. + * @param errGen how to generate error messages based on the result of this parser. + * @since 4.4.0 + * @group genFilter + */ + def collectWith[B](errGen: ErrorGen[A])(pf: PartialFunction[A, B]): Parsley[B] = combinator.collectWith(con(p))(pf, errGen) + + // $COVERAGE-OFF$ + // TODO: remove in 5.0.0 + @deprecated("better argument currying now, don't use", "4.4.0-RC2") + private [parsley] def mapFilterWith[B](f: A => Option[B], errGen: ErrorGen[A]): Parsley[B] = combinator.mapFilterWith(con(p))(f, errGen) + // $COVERAGE-ON$ + /** This combinator conditionally transforms the result of this parser with a given function, generating an error with the + * given error generator if the function returns `None` given the result of this parser. + * + * Like [[Parsley.mapFilter `mapFilter`]], except allows for the error message generated to be fine-tuned with + * respect to the parsers result and width of input consumed using an [[ErrorGen `ErrorGen`]] object. + * + * @param f the function used to both filter the result of this parser and transform it. + * @param errGen how to generate error messages based on the result of this parser. + * @since 4.4.0 + * @group genFilter + */ + def mapFilterWith[B](errGen: ErrorGen[A])(f: A => Option[B]): Parsley[B] = combinator.mapFilterWith(con(p))(f, errGen) + } + + @inline private [parsley] def filterWith[A](p: Parsley[A])(f: A => Boolean, err: ErrorGen[A]): Parsley[A] = { + new Parsley(new frontend.Filter(p.internal, f, err.internal)) + } + + @inline private [parsley] def collectWith[A, B](p: Parsley[A])(f: PartialFunction[A, B], err: ErrorGen[A]): Parsley[B] = { + mapFilterWith(p)(f.lift, err) + } + + @inline private [parsley] def mapFilterWith[A, B](p: Parsley[A])(f: A => Option[B], err: ErrorGen[A]): Parsley[B] = { + new Parsley(new frontend.MapFilter(p.internal, f, err.internal)) } } diff --git a/parsley/shared/src/main/scala/parsley/errors/patterns.scala b/parsley/shared/src/main/scala/parsley/errors/patterns.scala index 6d0929909..74d0d1ed6 100644 --- a/parsley/shared/src/main/scala/parsley/errors/patterns.scala +++ b/parsley/shared/src/main/scala/parsley/errors/patterns.scala @@ -5,9 +5,9 @@ */ package parsley.errors -import parsley.Parsley - -import parsley.internal.deepembedding.frontend +import parsley.Parsley, Parsley.{atomic, select, unit} +import parsley.errors.combinator.{amend, ErrorMethods} +import parsley.position.withWidth /** This module contains combinators that help facilitate the error message generational patterns ''Verified Errors'' and ''Preventative Errors''. * @@ -33,11 +33,9 @@ object patterns { * parse (as if [[parsley.errors.combinator$.amend `amend`]] had been used) and the caret will span the entire * successful parse of this parser. * - * @define attemptNonTerminal - * when this parser is not to be considered as a terminal error, use `attempt` around the ''entire'' combinator to + * @define atomicNonTerminal + * when this parser is not to be considered as a terminal error, use `atomic` around the ''entire'' combinator to * allow for backtracking if this parser succeeds (and therefore fails). - * - * @define Ensures this parser does not succeed, failing with a */ implicit final class VerifiedErrors[P, A](p: P)(implicit con: P => Parsley[A]) { /** Ensures this parser does not succeed, failing with a specialised error based on this parsers result if it does. @@ -49,9 +47,13 @@ object patterns { * @param msggen the function that generates the error messages from the parsed value. * @since 4.2.0 * @note $autoAmend - * @note $attemptNonTerminal + * @note $atomicNonTerminal */ - def verifiedFail(msggen: A => Seq[String]): Parsley[Nothing] = verified(Left(msggen)) + def verifiedFail(msggen: A => Seq[String]): Parsley[Nothing] = this.verifiedWith { + new SpecialisedGen[A] { + override def messages(x: A) = msggen(x) + } + } /** Ensures this parser does not succeed, failing with a specialised error if it does. * @@ -63,7 +65,7 @@ object patterns { * @param msgs the remaining messages that will make up the error message. * @since 4.2.0 * @note $autoAmend - * @note $attemptNonTerminal + * @note $atomicNonTerminal */ def verifiedFail(msg0: String, msgs: String*): Parsley[Nothing] = this.verifiedFail(_ => msg0 +: msgs) @@ -75,9 +77,9 @@ object patterns { * * @since 4.2.0 * @note $autoAmend - * @note $attemptNonTerminal + * @note $atomicNonTerminal */ - def verifiedUnexpected: Parsley[Nothing] = this.verifiedUnexpected(None) + def verifiedUnexpected: Parsley[Nothing] = this.verifiedWithVanillaRaw(_ => None) /** Ensures this parser does not succeed, failing with a vanilla error with an unexpected message and caret spanning the parse and a given reason. * @@ -88,9 +90,9 @@ object patterns { * @param reason the reason that this parser is illegal. * @since 4.2.0 * @note $autoAmend - * @note $attemptNonTerminal + * @note $atomicNonTerminal */ - def verifiedUnexpected(reason: String): Parsley[Nothing] = this.verifiedUnexpected(_ => reason) + def verifiedUnexpected(reason: String): Parsley[Nothing] = this.verifiedWithVanillaRaw(_ => Some(reason)) /** Ensures this parser does not succeed, failing with a vanilla error with an unexpected message and caret spanning the parse and a reason generated * from this parser's result. @@ -102,11 +104,142 @@ object patterns { * @param reason a function that produces a reason for the error given the parsed result. * @since 4.2.0 * @note $autoAmend - * @note $attemptNonTerminal + * @note $atomicNonTerminal + */ + def verifiedUnexpected(reason: A => String): Parsley[Nothing] = this.verifiedWithVanillaRaw(x => Some(reason(x))) + + /** Ensures this parser does not succeed, failing with an error as described by the given `ErrorGen` object. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an error message using + * the given `errGen` with width the same as the parsed data. However, if this parser fails, no input is consumed + * and an empty error is generated. This parser will produce no labels if it fails. + * + * @param err the generator that produces the error message. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal + */ + def verifiedWith(err: ErrorGen[A]): Parsley[Nothing] = amend(err(withWidth(atomic(con(p)).newHide))) + + @inline private def verifiedWithVanilla(unexGen: A => VanillaGen.UnexpectedItem, reasonGen: A => Option[String]) = verifiedWith { + new VanillaGen[A] { + override def unexpected(x: A) = unexGen(x) + override def reason(x: A) = reasonGen(x) + } + } + + @inline private def verifiedWithVanillaRaw(reasonGen: A => Option[String]) = verifiedWithVanilla(_ => VanillaGen.RawItem, reasonGen) + } + + /** This class exposes combinators related to the ''Preventative Errors'' parser design pattern. + * + * This extension class operates on values that are convertible to parsers. The combinators it enables + * allow for the parsing of known illegal values, providing richer error messages in case they succeed. + * + * @constructor This constructor should not be called manually, it is designed to be used via Scala's implicit resolution. + * @param p the value that this class is enabling methods on. + * @param con a conversion that allows values convertible to parsers to be used. + * @tparam P the type of base value that this class is used on (the conversion to `Parsley`) is summoned automatically. + * @since 4.4.0 + * + * @define autoAmend + * when this combinator fails (and not this parser itself), it will generate errors rooted at the start of the + * parse (as if [[parsley.errors.combinator$.amend `amend`]] had been used) and the caret will span the entire + * successful parse of this parser. + * + * @define atomicNonTerminal + * when this parser is not to be considered as a terminal error, use `atomic` around the ''entire'' combinator to + * allow for backtracking if this parser succeeds (and therefore fails). + */ + implicit final class PreventativeErrors[P, A](p: P)(implicit con: P => Parsley[A]) { + /** Ensures this parser does not succeed, failing with a specialised error based on this parsers result if it does. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an error message + * based on the parsed result. However, if this parser fails, no input is consumed and this combinator succeeds. + * This parser will produce no evidence of running if it succeeds. + * + * @param msggen the function that generates the error messages from the parsed value. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal + */ + def preventativeFail(msggen: A => Seq[String]): Parsley[Unit] = this.preventWith(new SpecialisedGen[A] { + override def messages(x: A) = msggen(x) + }) + + /** Ensures this parser does not succeed, failing with a fixed specialised error if it does. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an error message with the + * given messages. However, if this parser fails, no input is consumed and this combinator succeeds. + * This parser will produce no evidence of running if it succeeds. + * + * @param msg0 the first message in the error message. + * @param msgs the remaining messages that will make up the error message. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal + */ + def preventativeFail(msg0: String, msgs: String*): Parsley[Unit] = this.preventativeFail(_ => msg0 +: msgs) + + /** Ensures this parser does not succeed, failing with a vanilla error with an unexpected message and caret spanning the parse and a reason generated + * from this parser's result. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an unexpected message the same width as + * the parse along with a reason generated from the successful parse along with the given labels. However, if this parser fails, no input is + * consumed and this combinator succeeds. This parser will produce no evidence of running if it succeeds. + * + * @param reason a function that produces a reason for the error given the parsed result. + * @param labels the labels that should be expected if this parser hadn't succeeded. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal + */ + def preventativeExplain(reason: A => String, labels: String*): Parsley[Unit] = this.preventWithVanillaRaw(x => Some(reason(x)), labels: _*) + + /** Ensures this parser does not succeed, failing with a vanilla error with an unexpected message and caret spanning the parse and a given reason. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an unexpected message the same width as + * the parse along with the given reason and given labels. However, if this parser fails, no input is consumed and this combinator succeeds. + * This parser will produce no evidence of running if it succeeds. + * + * @param reason the reason that this parser is illegal. + * @param labels the labels that should be expected if this parser hadn't succeeded. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal + */ + def preventativeExplain(reason: String, labels: String*): Parsley[Unit] = this.preventativeExplain(_ => reason, labels: _*) + + /** Ensures this parser does not succeed, failing with an error as described by the given `ErrorGen` object. + * + * If this parser succeeds, input is consumed and this combinator will fail, producing an error message using + * the given `errGen` with width the same as the parsed data along with the given labels. However, if this parser + * fails, no input is consumed and this combinator succeeds. This parser will produce no evidence of running if it succeeds. + * + * @param err the generator that produces the error message. + * @param labels the labels that should be expected if this parser hadn't succeeded. + * @since 4.4.0 + * @note $autoAmend + * @note $atomicNonTerminal */ - def verifiedUnexpected(reason: A => String): Parsley[Nothing] = this.verifiedUnexpected(Some(reason)) + def preventWith(err: ErrorGen[A], labels: String*): Parsley[Unit] = { + val inner: Parsley[Either[(A, Int), Unit]] = withWidth(atomic(con(p)).newHide) <+> unit + val labelledErr = labels match { + case l1 +: ls => err.parser.label(l1, ls: _*) + case _ => err.parser + } + amend(select(inner, labelledErr)) + } + + @inline private def preventWithVanilla(unexGen: A => VanillaGen.UnexpectedItem, reasonGen: A => Option[String], labels: String*) = { + this.preventWith(new VanillaGen[A] { + override def unexpected(x: A) = unexGen(x) + override def reason(x: A) = reasonGen(x) + }, labels: _*) + } - private def verified(msggen: Either[A => Seq[String], Option[A => String]]) = new Parsley(new frontend.VerifiedError(con(p).internal, msggen)) - private def verifiedUnexpected(reason: Option[A => String]) = verified(Right(reason)) + @inline private def preventWithVanillaRaw(reasonGen: A => Option[String], labels: String*) = { + this.preventWithVanilla(_ => VanillaGen.RawItem, reasonGen, labels: _*) + } } } diff --git a/parsley/shared/src/main/scala/parsley/errors/tokenextractors/LexToken.scala b/parsley/shared/src/main/scala/parsley/errors/tokenextractors/LexToken.scala index aac57bf37..1475a2e85 100644 --- a/parsley/shared/src/main/scala/parsley/errors/tokenextractors/LexToken.scala +++ b/parsley/shared/src/main/scala/parsley/errors/tokenextractors/LexToken.scala @@ -7,7 +7,7 @@ package parsley.errors.tokenextractors import scala.collection.immutable.WrappedString -import parsley.Parsley, Parsley.{attempt, lookAhead, notFollowedBy} +import parsley.Parsley, Parsley.{atomic, lookAhead, notFollowedBy} import parsley.Success import parsley.XAssert.assert import parsley.character.{item, stringOfSome} @@ -59,7 +59,7 @@ trait LexToken { this: ErrorBuilder[_] => // this parser cannot and must not fail private lazy val makeParser: Parsley[Either[::[(String, (Int, Int))], String]] = { - val toks = traverse5(tokens: _*)(p => option(lookAhead(attempt(p) <~> position.pos))).map(_.flatten).collect { case toks@(_::_) => toks } + val toks = traverse5(tokens: _*)(p => option(lookAhead(atomic(p) <~> position.pos))).map(_.flatten).collect { case toks@(_::_) => toks } // this can only fail if either there is no input (which there must be), or there is a token at the front, in which case `rawTok` is not parsed anyway val rawTok = stringOfSome(traverse_(tokens: _*)(notFollowedBy) *> item) toks <+> rawTok @@ -123,6 +123,6 @@ object LexToken { * @since 4.0.0 */ def constantSymbols(ps: (Parsley[_], String)*): Seq[Parsley[String]] = ps.map { - case (p, n) => p #> n + case (p, n) => p.as(n) } } diff --git a/parsley/shared/src/main/scala/parsley/errors/tokenextractors/TillNextWhitespace.scala b/parsley/shared/src/main/scala/parsley/errors/tokenextractors/TillNextWhitespace.scala index d950b48f1..e1e7c2ede 100644 --- a/parsley/shared/src/main/scala/parsley/errors/tokenextractors/TillNextWhitespace.scala +++ b/parsley/shared/src/main/scala/parsley/errors/tokenextractors/TillNextWhitespace.scala @@ -27,10 +27,18 @@ trait TillNextWhitespace { this: ErrorBuilder[_] => */ def trimToParserDemand: Boolean + /** Describes what characters are considered whitespace. + * + * Defaults to `_.isWhitespace`. + * + * @since 4.4.0 + */ + def isWhitespace(c: Char): Boolean = c.isWhitespace + /** @see [[parsley.errors.ErrorBuilder.unexpectedToken `ErrorBuilder.unexpectedToken`]] */ override final def unexpectedToken(cs: Iterable[Char], amountOfInputParserWanted: Int, @unused lexicalError: Boolean): Token = { - if (trimToParserDemand) TillNextWhitespace.unexpectedToken(cs, amountOfInputParserWanted) - else TillNextWhitespace.unexpectedToken(cs) + if (trimToParserDemand) TillNextWhitespace.unexpectedToken(cs, amountOfInputParserWanted, isWhitespace(_)) + else TillNextWhitespace.unexpectedToken(cs, isWhitespace(_)) } } @@ -45,10 +53,18 @@ object TillNextWhitespace { * * @since 4.0.0 */ - def unexpectedToken(cs: Iterable[Char]): Token = cs match { + def unexpectedToken(cs: Iterable[Char]): Token = unexpectedToken(cs, _.isWhitespace) + + /** The implementation of `unexpectedToken` as done by `TillNextWhitespace`, with redundant arguments removed. + * + * This function will not trim the token to parser demand + * + * @since 4.4.0 + */ + def unexpectedToken(cs: Iterable[Char], isWhitespace: Char => Boolean): Token = cs match { case helpers.WhitespaceOrUnprintable(name) => Token.Named(name, TokenSpan.Width(1)) // these cases automatically handle the utf-16 surrogate pairs - case cs => Token.Raw(extractTillNextWhitespace(cs)) + case cs => Token.Raw(extractTillNextWhitespace(cs, isWhitespace)) } /** The implementation of `unexpectedToken` as done by `TillNextWhitespace`, with redundant arguments removed. @@ -57,22 +73,30 @@ object TillNextWhitespace { * * @since 4.0.0 */ - def unexpectedToken(cs: Iterable[Char], amountOfInputParserWanted: Int): Token = cs match { + def unexpectedToken(cs: Iterable[Char], amountOfInputParserWanted: Int): Token = unexpectedToken(cs, amountOfInputParserWanted, _.isWhitespace) + + /** The implementation of `unexpectedToken` as done by `TillNextWhitespace`, with redundant arguments removed. + * + * This function will not trim the token to parser demand + * + * @since 4.4.0 + */ + def unexpectedToken(cs: Iterable[Char], amountOfInputParserWanted: Int, isWhitespace: Char => Boolean): Token = cs match { case helpers.WhitespaceOrUnprintable(name) => Token.Named(name, TokenSpan.Width(1)) // these cases automatically handle the utf-16 surrogate pairs - case cs => Token.Raw(helpers.takeCodePoints(extractTillNextWhitespace(cs), amountOfInputParserWanted)) + case cs => Token.Raw(helpers.takeCodePoints(extractTillNextWhitespace(cs, isWhitespace), amountOfInputParserWanted)) } // TODO: we should take to minimum of parser demand and next whitespace, this would potentially be much much cheaper // Assumption: there are no non-BMP whitespace characters - private def extractTillNextWhitespace(cs: Iterable[Char]): String = cs match { + private def extractTillNextWhitespace(cs: Iterable[Char], isWhitespace: Char => Boolean): String = cs match { case cs: WrappedString => // These do not require allocation on the string val idx = { - val idx = cs.indexWhere(_.isWhitespace) + val idx = cs.indexWhere(isWhitespace) if (idx != -1) idx else cs.length } cs.slice(0, idx).toString - case cs => cs.takeWhile(!_.isWhitespace).mkString + case cs => cs.takeWhile(!isWhitespace(_)).mkString } } diff --git a/parsley/shared/src/main/scala/parsley/expr/chain.scala b/parsley/shared/src/main/scala/parsley/expr/chain.scala index 955af1c89..1868915ee 100644 --- a/parsley/shared/src/main/scala/parsley/expr/chain.scala +++ b/parsley/shared/src/main/scala/parsley/expr/chain.scala @@ -38,7 +38,7 @@ object chain { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.right1(digit.map(d => Num(d.asDigit)), char('+') #> Add) + * scala> val expr = chain.right1(digit.map(d => Num(d.asDigit)), char('+').as(Add)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Num(1), Add(Num(2), Add(Num(3), Num(4))))) * scala> expr.parse("") @@ -66,7 +66,7 @@ object chain { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.left1(digit.map(d => Num(d.asDigit)), char('+') #> Add) + * scala> val expr = chain.left1(digit.map(d => Num(d.asDigit)), char('+').as(Add)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Add(Add(Num(1), Num(2)), Num(3)), Num(4))) * scala> expr.parse("") @@ -95,7 +95,7 @@ object chain { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.right(digit.map(d => Num(d.asDigit)), char('+') #> Add, Num(0)) + * scala> val expr = chain.right(digit.map(d => Num(d.asDigit)), char('+').as(Add), Num(0)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Num(1), Add(Num(2), Add(Num(3), Num(4))))) * scala> expr.parse("") @@ -126,7 +126,7 @@ object chain { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.left(digit.map(d => Num(d.asDigit)), char('+') #> Add, Num(0)) + * scala> val expr = chain.left(digit.map(d => Num(d.asDigit)), char('+').as(Add), Num(0)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Add(Add(Num(1), Num(2)), Num(3)), Num(4))) * scala> expr.parse("") @@ -158,7 +158,7 @@ object chain { * scala> case class Negate(x: Expr) extends Expr * scala> case class Id(x: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.prefix(char('-') #> Negate <|> char('+') #> Id, digit.map(d => Num(d.asDigit))) + * scala> val expr = chain.prefix(char('-').as(Negate) <|> char('+').as(Id), digit.map(d => Num(d.asDigit))) * scala> expr.parse("--+1") * val res0 = Success(Negate(Negate(Id(Num(1))))) * scala> expr.parse("1") @@ -189,7 +189,7 @@ object chain { * scala> case class Inc(x: Expr) extends Expr * scala> case class Dec(x: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.postfix(digit.map(d => Num(d.asDigit)), string("++") #> Inc <|> string("--") #> Dec) + * scala> val expr = chain.postfix(digit.map(d => Num(d.asDigit)), string("++").as(Inc) <|> string("--").as(Dec)) * scala> expr.parse("1++----") * val res0 = Success(Dec(Dec(Inc(Num(1))))) * scala> expr.parse("1") @@ -220,7 +220,7 @@ object chain { * scala> case class Negate(x: Expr) extends Expr * scala> case class Id(x: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.prefix1(char('-') #> Negate <|> char('+') #> Id, digit.map(d => Num(d.asDigit))) + * scala> val expr = chain.prefix1(char('-').as(Negate) <|> char('+').as(Id), digit.map(d => Num(d.asDigit))) * scala> expr.parse("--+1") * val res0 = Success(Negate(Negate(Id(Num(1))))) * scala> expr.parse("1") @@ -251,7 +251,7 @@ object chain { * scala> case class Inc(x: Expr) extends Expr * scala> case class Dec(x: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = chain.postfix1(digit.map(d => Num(d.asDigit)), string("++") #> Inc <|> string("--") #> Dec) + * scala> val expr = chain.postfix1(digit.map(d => Num(d.asDigit)), string("++").as(Inc) <|> string("--").as(Dec)) * scala> expr.parse("1++----") * val res0 = Success(Dec(Dec(Inc(Num(1))))) * scala> expr.parse("1") diff --git a/parsley/shared/src/main/scala/parsley/expr/infix.scala b/parsley/shared/src/main/scala/parsley/expr/infix.scala index 7ec702476..8dbb6ea81 100644 --- a/parsley/shared/src/main/scala/parsley/expr/infix.scala +++ b/parsley/shared/src/main/scala/parsley/expr/infix.scala @@ -38,7 +38,7 @@ object infix { * scala> sealed trait Expr * scala> case class Add(x: Num, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = infix.right1[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+') #> Add)) + * scala> val expr = infix.right1[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+').as(Add))) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Num(1), Add(Num(2), Add(Num(3), Num(4))))) * scala> expr.parse("") @@ -76,7 +76,7 @@ object infix { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Num) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = infix.left1[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+') #> Add) + * scala> val expr = infix.left1[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+').as(Add)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Add(Add(Num(1), Num(2)), Num(3)), Num(4))) * scala> expr.parse("") @@ -120,7 +120,7 @@ object infix { * scala> sealed trait Expr * scala> case class Add(x: Num, y: Expr) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = infix.right[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+') #> Add, Num(0)) + * scala> val expr = infix.right[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+').as(Add), Num(0)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Num(1), Add(Num(2), Add(Num(3), Num(4))))) * scala> expr.parse("") @@ -159,7 +159,7 @@ object infix { * scala> sealed trait Expr * scala> case class Add(x: Expr, y: Num) extends Expr * scala> case class Num(x: Int) extends Expr - * scala> val expr = infix.left[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+') #> Add, Num(0)) + * scala> val expr = infix.left[Num, Add, Expr](digit.map(d => Num(d.asDigit)), char('+').as(Add), Num(0)) * scala> expr.parse("1+2+3+4") * val res0 = Success(Add(Add(Add(Num(1), Num(2)), Num(3)), Num(4))) * scala> expr.parse("") diff --git a/parsley/shared/src/main/scala/parsley/expr/mixed.scala b/parsley/shared/src/main/scala/parsley/expr/mixed.scala index e0e5c32de..0ae32443d 100644 --- a/parsley/shared/src/main/scala/parsley/expr/mixed.scala +++ b/parsley/shared/src/main/scala/parsley/expr/mixed.scala @@ -35,7 +35,7 @@ object mixed { * scala> case class B(l: C, r: Expr) extends Expr * scala> case class U(c: Expr) extends Expr * scala> case class C(x: Char) extends Expr - * scala> val p = mixed.right1(digit.map(C), char('-') #> U, char('+') #> B) + * scala> val p = mixed.right1(digit.map(C), char('-').as(U), char('+').as(B)) * scala> p.parse("-1+--2+3") * val res0 = Success(U(B(C('1'), U(U(B(C('2'), C('3'))))))) * }}} @@ -76,7 +76,7 @@ object mixed { * scala> case class B(l: Expr, r: C) extends Expr * scala> case class U(c: Expr) extends Expr * scala> case class C(x: Char) extends Expr - * scala> val p = mixed.left1(digit.map(Constant), char('?') #> U, char('+') #> B) + * scala> val p = mixed.left1(digit.map(Constant), char('?').as(U), char('+').as(B)) * scala> p.parse("1?+2+3??") * val res0 = Success(U(U(B(B(U(C('1')), C('2')), C('3'))))) * }}} diff --git a/parsley/shared/src/main/scala/parsley/expr/precedence.scala b/parsley/shared/src/main/scala/parsley/expr/precedence.scala index 213f54b2d..15ce35d12 100644 --- a/parsley/shared/src/main/scala/parsley/expr/precedence.scala +++ b/parsley/shared/src/main/scala/parsley/expr/precedence.scala @@ -29,8 +29,8 @@ object precedence { * scala> import parsley.Parsley, parsley.character.{char, digit} * scala> import parsley.expr.{Ops, InfixL, precedence} * scala> val expr = precedence(digit.map(_.asDigit)) - * (Ops(InfixL)(char('*') #> (_ * _)), - * Ops(InfixL)(char('+') #> (_ + _), char('-') #> (_ - _))) + * (Ops(InfixL)(char('*').as(_ * _)), + * Ops(InfixL)(char('+').as(_ + _), char('-').as(_ - _))) * scala> expr.parse("1+8*7+4") * val res0 = Success(61) * }}} @@ -57,8 +57,8 @@ object precedence { * @example {{{ * scala> import parsley.Parsley, parsley.character.{char, digit} * scala> import parsley.expr.{Ops, InfixL, precedence} - * scala> val expr = precedence[Int](Ops(InfixL)(char('+') #> (_ + _), char('-') #> (_ - _))), - * Ops(InfixL)(char('*') #> (_ * _)) + * scala> val expr = precedence[Int](Ops(InfixL)(char('+').as(_ + _), char('-').as(_ - _))), + * Ops(InfixL)(char('*').as(_ * _)) * (digit.map(_.asDigit))) * scala> expr.parse("1+8*7+4") * val res0 = Success(61) @@ -94,8 +94,8 @@ object precedence { * scala> import parsley.Parsley, parsley.character.{char, digit} * scala> import parsley.expr.{Atoms, Ops, InfixL, precedence} * scala> val expr = precedence(Atoms(digit.map(_.asDigit)) :+ - * Ops[Int](InfixL)(char('*') #> (_ * _)) :+ - * Ops[Int](InfixL)(char('+') #> (_ + _), char('-') #> (_ - _))) + * Ops[Int](InfixL)(char('*').as(_ * _)) :+ + * Ops[Int](InfixL)(char('+').as(_ + _), char('-').as(_ - _))) * scala> expr.parse("1+8*7+4") * val res0 = Success(61) * }}} diff --git a/parsley/shared/src/main/scala/parsley/implicits/lift.scala b/parsley/shared/src/main/scala/parsley/implicits/lift.scala index c57d26c5a..ff768fd7a 100644 --- a/parsley/shared/src/main/scala/parsley/implicits/lift.scala +++ b/parsley/shared/src/main/scala/parsley/implicits/lift.scala @@ -16,11 +16,11 @@ import parsley.lift._ * scala> import parsley.character.char * scala> import parsley.implicits.lift.{Lift2, Lift3} * scala> case class Add(x: Int, y: Int) - * scala> val p = Add.lift(char('a') #> 4, char('b') #> 5) + * scala> val p = Add.lift(char('a').as(4), char('b').as(5)) * scala> p.parse("ab") * val res0 = Success(Add(4, 5)) * scala> val f = (x: Int, y: Int, z: Int) => x * y + z - * scala> val q = f.lift(char('a') #> 3, char('b') #> 2, char('c') #> 5) + * scala> val q = f.lift(char('a').as(3), char('b').as(2), char('c').as(5)) * scala> q.parse("abc") * val res1 = Success(11) * scala> q.parse("ab") diff --git a/parsley/shared/src/main/scala/parsley/implicits/zipped.scala b/parsley/shared/src/main/scala/parsley/implicits/zipped.scala index 120759937..a7393bd01 100644 --- a/parsley/shared/src/main/scala/parsley/implicits/zipped.scala +++ b/parsley/shared/src/main/scala/parsley/implicits/zipped.scala @@ -19,10 +19,10 @@ import parsley.lift._ * scala> import parsley.character.char * scala> import parsley.implicits.zipped.{Zipped2, Zipped3} * scala> case class Add(x: Int, y: Int) - * scala> val p = (char('a') #> 4, char('b') #> 5).zipped(Add) + * scala> val p = (char('a').as(4), char('b').as(5)).zipped(Add) * scala> p.parse("ab") * val res0 = Success(Add(4, 5)) - * scala> val q = (char('a') #> 3, char('b') #> 2, char('c') #> 5).zipped((x, y, z) => x * y + z) + * scala> val q = (char('a').as(3), char('b').as(2), char('c').as(5)).zipped((x, y, z) => x * y + z) * scala> q.parse("abc") * val res1 = Success(11) * scala> q.parse("ab") diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala index 840242290..96b377b77 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala @@ -13,7 +13,7 @@ import parsley.XAssert._ import parsley.internal.collection.mutable.SinglyLinkedList, SinglyLinkedList.LinkedListIterator import parsley.internal.deepembedding.ContOps, ContOps.{result, suspend, ContAdapter} import parsley.internal.deepembedding.singletons._ -import parsley.internal.errors.ExpectItem +import parsley.internal.errors.{ExpectDesc, ExpectItem} import parsley.internal.machine.instructions // scalastyle:off underscore.import @@ -21,6 +21,8 @@ import Choice._ import StrictParsley.InstrBuffer // scalastyle:on underscore.import +// TODO: can we tabilify across a Let? +// FIXME: It's annoying this doesn't work if the first thing is not tablable: let's make it more fine-grained to create groupings? private [deepembedding] final class Choice[A](private [backend] val alt1: StrictParsley[A], private [backend] var alt2: StrictParsley[A], private [backend] var alts: SinglyLinkedList[StrictParsley[A]]) extends StrictParsley[A] { @@ -59,11 +61,11 @@ private [deepembedding] final class Choice[A](private [backend] val alt1: Strict case _ => this } - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { this.tablify match { // If the tablified list is single element (or the next is None), that implies that this should be generated as normal! - case (_ :: Nil) | (_ :: (_, None) :: Nil) => codeGenChain(alt1, alt2, alts.iterator) - case tablified => codeGenJumpTable(tablified) + case (_ :: Nil) | (_ :: (_, None) :: Nil) => codeGenChain(alt1, alt2, alts.iterator, producesResults) + case tablified => codeGenJumpTable(tablified, producesResults) } } @@ -99,12 +101,13 @@ private [backend] object Choice { private def unapply[A](self: Choice[A]): Some[(StrictParsley[A], StrictParsley[A], SinglyLinkedList[StrictParsley[A]])] = Some((self.alt1, self.alt2, self.alts)) - private def scopedState[A, M[_, +_]: ContOps, R](p: StrictParsley[A])(generateHandler: =>M[R, Unit]) + private def scopedState[A, M[_, +_]: ContOps, R](p: StrictParsley[A], producesResults: Boolean)(generateHandler: =>M[R, Unit]) (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = state.freshLabel() val skip = state.freshLabel() - instrs += new instructions.PushHandlerAndState(handler, saveHints = true, hideHints = false) - suspend(p.codeGen[M, R]) >> { + // FIXME: check this, this is the only one that uses this instruction, and I think it was a mistake + instrs += new instructions.PushHandlerAndStateAndClearHints(handler) + suspend(p.codeGen[M, R](producesResults)) >> { instrs += new instructions.JumpAndPopState(skip) instrs += new instructions.Label(handler) generateHandler |> { @@ -113,12 +116,12 @@ private [backend] object Choice { } } - private def scopedCheck[A, M[_, +_]: ContOps, R](p: StrictParsley[A])(generateHandler: =>M[R, Unit]) + private def scopedCheck[A, M[_, +_]: ContOps, R](p: StrictParsley[A], producesResults: Boolean)(generateHandler: =>M[R, Unit]) (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = state.freshLabel() val skip = state.freshLabel() - instrs += new instructions.PushHandlerAndCheck(handler, saveHints = true) - suspend(p.codeGen[M, R]) >> { + instrs += new instructions.PushHandlerAndClearHints(handler) + suspend(p.codeGen[M, R](producesResults)) >> { instrs += new instructions.JumpAndPopCheck(skip) instrs += new instructions.Label(handler) generateHandler |> { @@ -127,39 +130,42 @@ private [backend] object Choice { } } - private def codeGenChain[A, M[_, +_]: ContOps, R](alt1: StrictParsley[A], alt2: StrictParsley[A], alts: Iterator[StrictParsley[A]]) - (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + private def codeGenChain[A, M[_, +_]: ContOps, R] + (alt1: StrictParsley[A], alt2: StrictParsley[A], alts: Iterator[StrictParsley[A]], producesResults: Boolean) + (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { if (alts.hasNext) { val alt3 = alts.next() - codeGenAlt(alt1, suspend(codeGenChain[A, M, R](alt2, alt3, alts))) + codeGenAlt(alt1, suspend(codeGenChain[A, M, R](alt2, alt3, alts, producesResults)), producesResults) } else alt2 match { case Pure(x) => alt1 match { - case Attempt(u) => scopedState(u) { + case Attempt(u) => scopedState(u, producesResults) { instrs += new instructions.AlwaysRecoverWith[A](x) + if (!producesResults) instrs += instructions.Pop result(()) } - case u => scopedCheck(u) { + case u => scopedCheck(u, producesResults) { instrs += new instructions.RecoverWith[A](x) + if (!producesResults) instrs += instructions.Pop result(()) } } - case v => codeGenAlt(alt1, suspend(v.codeGen[M, R])) + case v => codeGenAlt(alt1, suspend(v.codeGen[M, R](producesResults)), producesResults) } } // Why is rest lazy? because Cont could be Id, and Id forces the argument immediately! - private def codeGenAlt[A, M[_, +_]: ContOps, R](p: StrictParsley[A], rest: =>M[R, Unit]) + private def codeGenAlt[A, M[_, +_]: ContOps, R](p: StrictParsley[A], rest: =>M[R, Unit], producesResults: Boolean) (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val merge = state.getLabel(instructions.MergeErrorsAndFail) p match { - case Attempt(u) => scopedState(u) { + case Attempt(u) => scopedState(u, producesResults) { instrs += new instructions.RestoreAndPushHandler(merge) rest |> { instrs += instructions.ErrorToHints } } - case u => scopedCheck(u) { + case u => scopedCheck(u, producesResults) { instrs += new instructions.Catch(merge) rest |> { instrs += instructions.ErrorToHints @@ -175,22 +181,21 @@ private [backend] object Choice { case Nil => corrected } - private def codeGenRoots[M[_, +_]: ContOps, R](roots: List[List[StrictParsley[_]]], ls: List[Int], end: Int) + private def codeGenRoots[M[_, +_]: ContOps, R](roots: List[List[StrictParsley[_]]], ls: List[Int], end: Int, producesResults: Boolean) (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = roots match { case root::roots_ => instrs += new instructions.Label(ls.head) - codeGenAlternatives(root) >> { + codeGenAlternatives(root, producesResults) >> { instrs += instructions.ErrorToHints instrs += new instructions.JumpAndPopCheck(end) - suspend(codeGenRoots[M, R](roots_, ls.tail, end)) + suspend(codeGenRoots[M, R](roots_, ls.tail, end, producesResults)) } case Nil => result(()) } - private def codeGenAlternatives[M[_, +_]: ContOps, R] - (alts: List[StrictParsley[_]]) - (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = (alts: @unchecked) match { - case alt::Nil => alt.codeGen - case alt::alts_ => codeGenAlt(alt, suspend(codeGenAlternatives[M, R](alts_))) + private def codeGenAlternatives[M[_, +_]: ContOps, R](alts: List[StrictParsley[_]], producesResults: Boolean) + (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = (alts: @unchecked) match { + case alt::Nil => alt.codeGen(producesResults) + case alt::alts_ => codeGenAlt(alt, suspend(codeGenAlternatives[M, R](alts_, producesResults)), producesResults) } // TODO: Refactor @tailrec private def foldTablified(tablified: List[(StrictParsley[_], (Char, Iterable[ExpectItem], Int, Boolean))], // scalastyle:ignore parameter.number @@ -222,17 +227,25 @@ private [backend] object Choice { expecteds, expectedss.zip(leads.toList.reverseIterator.map(backtracking(_)).toList)) } - @tailrec private def tablable(p: StrictParsley[_], backtracks: Boolean): Option[(Char, Iterable[ExpectItem], Int, Boolean)] = p match { + private def tablable(p: StrictParsley[_], backtracks: Boolean): Option[(Char, Iterable[ExpectItem], Int, Boolean)] = p match { // CODO: Numeric parsers by leading digit (This one would require changing the foldTablified function a bit) - case ct@CharTok(c) => Some((c, ct.expected.asExpectItems(c), 1, backtracks)) - case ct@SupplementaryCharTok(c) => Some((Character.highSurrogate(c), ct.expected.asExpectItems(Character.toChars(c).mkString), 1, backtracks)) - case st@StringTok(s) => Some((s.head, st.expected.asExpectItems(s), s.codePointCount(0, s.length), backtracks)) + case ct@CharTok(c, _) => Some((c, ct.expected.asExpectItems(c), 1, backtracks)) + case ct@SupplementaryCharTok(c, _) => Some((Character.highSurrogate(c), ct.expected.asExpectItems(Character.toChars(c).mkString), 1, backtracks)) + case st@StringTok(s, _) => Some((s.head, st.expected.asExpectItems(s), s.codePointCount(0, s.length), backtracks)) //case op@MaxOp(o) => Some((o.head, Some(Desc(o)), o.size, backtracks)) //case _: StringLiteral | RawStringLiteral => Some(('"', Some(Desc("string")), 1, backtracks)) // TODO: This can be done for case insensitive things too, but with duplicated branching case t@token.SoftKeyword(s) if t.caseSensitive => Some((s.head, t.expected.asExpectDescs(s), s.codePointCount(0, s.length), backtracks)) case t@token.SoftOperator(s) => Some((s.head, t.expected.asExpectDescs(s), s.codePointCount(0, s.length), backtracks)) case Attempt(t) => tablable(t, backtracks = true) + case ErrorLabel(t, labels) => tablable(t, backtracks).map { + case (c, _, width, backtracks) => (c, labels.map(new ExpectDesc(_)), width, backtracks) + } + case ErrorHide(t) => tablable(t, backtracks).map { + case (c, _, _, backtracks) => (c, None, 0, backtracks) + } + case Profile(t) => tablable(t, backtracks) + case TablableErrors(t) => tablable(t, backtracks) case (_: Pure[_]) <*> t => tablable(t, backtracks) case Lift2(_, t, _) => tablable(t, backtracks) case Lift3(_, t, _, _) => tablable(t, backtracks) @@ -241,8 +254,9 @@ private [backend] object Choice { case _ => None } - private def codeGenJumpTable[M[_, +_]: ContOps, R, A](tablified: List[(StrictParsley[_], Option[(Char, Iterable[ExpectItem], Int, Boolean)])]) - (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + private def codeGenJumpTable[M[_, +_]: ContOps, R, A](tablified: List[(StrictParsley[_], Option[(Char, Iterable[ExpectItem], Int, Boolean)])], + producesResults: Boolean) + (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val needsDefault = tablified.last._2.nonEmpty val end = state.freshLabel() val default = state.freshLabel() @@ -253,7 +267,7 @@ private [backend] object Choice { val (roots, leads, ls, size, expecteds, expectedss) = foldTablified(tablified_, state, mutable.Map.empty, mutable.Map.empty, mutable.ListBuffer.empty, mutable.ListBuffer.empty, 0, Nil, Nil) instrs += new instructions.JumpTable(leads, ls, default, merge, size, expecteds, propagateExpecteds(expectedss, expecteds, Nil)) - codeGenRoots(roots, ls, end) >> { + codeGenRoots(roots, ls, end, producesResults) >> { instrs += new instructions.Catch(merge) //This instruction is reachable as default - 1 instrs += new instructions.Label(default) if (needsDefault) { @@ -261,7 +275,7 @@ private [backend] object Choice { result(instrs += new instructions.Label(end)) } else { - tablified.last._1.codeGen |> { + tablified.last._1.codeGen(producesResults) |> { instrs += instructions.ErrorToHints instrs += new instructions.Label(end) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/ErrorEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/ErrorEmbedding.scala index 5add9e35c..f886d24cb 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/ErrorEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/ErrorEmbedding.scala @@ -10,19 +10,18 @@ import parsley.token.errors.Label import parsley.internal.deepembedding.singletons._ import parsley.internal.machine.instructions -private [deepembedding] final class ErrorLabel[A](val p: StrictParsley[A], private [ErrorLabel] val labels: scala.Seq[String]) extends ScopedUnary[A, A] { - // This needs to save the hints because label should relabel only the hints generated _within_ its context, then merge with the originals after - override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndCheck(label, saveHints = true) +private [deepembedding] final class ErrorLabel[A](val p: StrictParsley[A], private val labels: scala.Seq[String]) extends ScopedUnary[A, A] { + override def setup(label: Int): instructions.Instr = new instructions.PushHandler(label) // was AndClearHints override def instr: instructions.Instr = new instructions.RelabelHints(labels) override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabelForRelabelError(labels) // don't need to be limited to not hidden when the thing can never internally generate hints final override def optimise: StrictParsley[A] = p match { - case CharTok(c) /*if ct.expected ne Hidden */ => new CharTok(c, Label(labels: _*)).asInstanceOf[StrictParsley[A]] - case SupplementaryCharTok(c) /*if ct.expected ne Hidden */ => new SupplementaryCharTok(c, Label(labels: _*)).asInstanceOf[StrictParsley[A]] - case StringTok(s) /*if st.expected ne Hidden */ => new StringTok(s, Label(labels: _*)).asInstanceOf[StrictParsley[A]] - case Satisfy(f) /*if sat.expected ne Hidden */ => new Satisfy(f, Label(labels: _*)).asInstanceOf[StrictParsley[A]] - case UniSatisfy(f) /*if sat.expected ne Hidden */ => new UniSatisfy(f, Label(labels: _*)).asInstanceOf[StrictParsley[A]] + case CharTok(c, x) => new CharTok(c, x, Label(labels: _*)).asInstanceOf[StrictParsley[A]] + case SupplementaryCharTok(c, x) => new SupplementaryCharTok(c, x, Label(labels: _*)).asInstanceOf[StrictParsley[A]] + case StringTok(s, x) => new StringTok(s, x, Label(labels: _*)).asInstanceOf[StrictParsley[A]] + case Satisfy(f) => new Satisfy(f, Label(labels: _*)).asInstanceOf[StrictParsley[A]] + case UniSatisfy(f) => new UniSatisfy(f, Label(labels: _*)).asInstanceOf[StrictParsley[A]] case ErrorLabel(p, label2) if label2.nonEmpty => ErrorLabel(p, labels) case _ => this } @@ -31,9 +30,19 @@ private [deepembedding] final class ErrorLabel[A](val p: StrictParsley[A], priva final override def pretty(p: String): String = s"$p.label($labels)" // $COVERAGE-ON$ } +private [deepembedding] final class ErrorHide[A](val p: StrictParsley[A]) extends ScopedUnary[A, A] { + override def setup(label: Int): instructions.Instr = new instructions.PushHandler(label) + override def instr: instructions.Instr = instructions.HideHints + override def instrNeedsLabel: Boolean = false + override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.HideErrorAndFail) + + // $COVERAGE-OFF$ + final override def pretty(p: String): String = s"$p.hide" + // $COVERAGE-ON$ +} private [deepembedding] final class ErrorExplain[A](val p: StrictParsley[A], reason: String) extends ScopedUnary[A, A] { - override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndCheck(label, saveHints = false) - override def instr: instructions.Instr = instructions.PopHandlerAndCheck + override def setup(label: Int): instructions.Instr = new instructions.PushHandler(label) + override def instr: instructions.Instr = instructions.PopHandler override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabelForApplyReason(reason) // $COVERAGE-OFF$ @@ -41,7 +50,7 @@ private [deepembedding] final class ErrorExplain[A](val p: StrictParsley[A], rea // $COVERAGE-ON$ } -private [deepembedding] final class ErrorAmend[A](val p: StrictParsley[A], partial: Boolean) extends ScopedUnaryWithState[A, A](false) { +private [deepembedding] final class ErrorAmend[A](val p: StrictParsley[A], partial: Boolean) extends ScopedUnaryWithState[A, A] { override val instr: instructions.Instr = instructions.PopHandlerAndState override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.AmendAndFail(partial)) @@ -71,8 +80,8 @@ private [deepembedding] final class ErrorDislodge[A](n: Int, val p: StrictParsle private [deepembedding] final class ErrorLexical[A](val p: StrictParsley[A]) extends ScopedUnary[A, A] { // This needs to save the hints because error label will relabel the first hint, which because the list is ordered would be the hints that came _before_ // entering labels context. Instead label should relabel the first hint generated _within_ its context, then merge with the originals after - override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndCheck(label, saveHints = false) - override def instr: instructions.Instr = instructions.PopHandlerAndCheck + override def setup(label: Int): instructions.Instr = new instructions.PushHandler(label) + override def instr: instructions.Instr = instructions.PopHandler override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.SetLexicalAndFail) @@ -81,22 +90,21 @@ private [deepembedding] final class ErrorLexical[A](val p: StrictParsley[A]) ext // $COVERAGE-ON$ } -private [deepembedding] final class VerifiedError[A](val p: StrictParsley[A], msggen: Either[A => scala.Seq[String], Option[A => String]]) - extends ScopedUnary[A, Nothing] { - override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndState(label, saveHints = true, hideHints = true) - override def instr: instructions.Instr = instructions.MakeVerifiedError(msggen) - override def instrNeedsLabel: Boolean = false - override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.NoVerifiedError) - - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"verifiedError($p)" - // $COVERAGE-ON$ -} - private [backend] object ErrorLabel { def apply[A](p: StrictParsley[A], labels: scala.Seq[String]): ErrorLabel[A] = new ErrorLabel(p, labels) def unapply[A](self: ErrorLabel[A]): Some[(StrictParsley[A], scala.Seq[String])] = Some((self.p, self.labels)) } +private [backend] object ErrorHide { + def unapply[A](self: ErrorHide[A]): Some[StrictParsley[A]] = Some(self.p) +} private [backend] object ErrorExplain { def apply[A](p: StrictParsley[A], reason: String): ErrorExplain[A] = new ErrorExplain(p, reason) } + +private [backend] object TablableErrors { + def unapply[A](self: StrictParsley[A]): Option[StrictParsley[A]] = self match { + case self: ErrorAmend[_] => Some(self.p) + case self: ErrorLexical[_] => Some(self.p) // is this correct? + case _ => None + } +} diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/GeneralisedEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/GeneralisedEmbedding.scala index 3e2c02dcd..f779a4f05 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/GeneralisedEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/GeneralisedEmbedding.scala @@ -25,16 +25,16 @@ private [backend] abstract class ScopedUnary[A, B] extends Unary[A, B] { def setup(label: Int): instructions.Instr def handlerLabel(state: CodeGenState): Int def instrNeedsLabel: Boolean - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + final override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = handlerLabel(state) instrs += setup(handler) - suspend[M, R, Unit](p.codeGen) |> { + suspend[M, R, Unit](p.codeGen(producesResults)) |> { if (instrNeedsLabel) instrs += new instructions.Label(handler) instrs += instr } } } -private [backend] abstract class ScopedUnaryWithState[A, B](doesNotProduceHints: Boolean) extends ScopedUnary[A, B] { - override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndState(label, doesNotProduceHints, doesNotProduceHints) +private [backend] abstract class ScopedUnaryWithState[A, B] extends ScopedUnary[A, B] { + override def setup(label: Int): instructions.Instr = new instructions.PushHandlerAndState(label) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IntrinsicEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IntrinsicEmbedding.scala index 66a5d1531..5114f8c14 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IntrinsicEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IntrinsicEmbedding.scala @@ -13,26 +13,28 @@ import parsley.internal.machine.instructions import StrictParsley.InstrBuffer // TODO: Perform applicative fusion optimisations -private [deepembedding] final class Lift2[A, B, C](private [Lift2] val f: (A, B) => C, val left: StrictParsley[A], val right: StrictParsley[B]) +private [deepembedding] final class Lift2[A, B, C](private val f: (A, B) => C, val left: StrictParsley[A], val right: StrictParsley[B]) extends StrictParsley[C] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - suspend(left.codeGen[M, R]) >> - suspend(right.codeGen[M, R]) |> - (instrs += instructions.Lift2(f)) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + suspend(left.codeGen[M, R](producesResults)) >> + suspend(right.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Lift2(f) + } } // $COVERAGE-OFF$ final override def pretty: String = s"lift2(?, ${left.pretty}, ${right.pretty})" // $COVERAGE-ON$ } -private [deepembedding] final class Lift3[A, B, C, D](val f: (A, B, C) => D, val p: StrictParsley[A], val q: StrictParsley[B], val r: StrictParsley[C]) +private [deepembedding] final class Lift3[A, B, C, D](private val f: (A, B, C) => D, val p: StrictParsley[A], val q: StrictParsley[B], val r: StrictParsley[C]) extends StrictParsley[D] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - suspend(p.codeGen[M, R]) >> - suspend(q.codeGen[M, R]) >> - suspend(r.codeGen[M, R]) |> - (instrs += instructions.Lift3(f)) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + suspend(p.codeGen[M, R](producesResults)) >> + suspend(q.codeGen[M, R](producesResults)) >> + suspend(r.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Lift3(f) + } } // $COVERAGE-OFF$ final override def pretty: String = s"lift3(?, ${p.pretty}, ${q.pretty}, ${r.pretty})" @@ -41,12 +43,12 @@ private [deepembedding] final class Lift3[A, B, C, D](val f: (A, B, C) => D, val private [deepembedding] final class Local[S, A](reg: Reg[S], left: StrictParsley[S], right: StrictParsley[A]) extends StrictParsley[A] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - suspend(left.codeGen[M, R]) >> { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + suspend(left.codeGen[M, R](producesResults = true)) >> { instrs += new instructions.Get(reg.addr) instrs += new instructions.SwapAndPut(reg.addr) - suspend(right.codeGen[M, R])|> { - instrs += new instructions.SwapAndPut(reg.addr) + suspend(right.codeGen[M, R](producesResults)) |> { + instrs += (if (producesResults) new instructions.SwapAndPut(reg.addr) else new instructions.Put(reg.addr)) } } } @@ -56,10 +58,10 @@ private [deepembedding] final class Local[S, A](reg: Reg[S], left: StrictParsley } private [backend] object Lift2 { - def unapply[A, B, C](self: Lift2[A, B, C]): Option[((A, B) => C, StrictParsley[A], StrictParsley[B])] = Some((self.f, self.left, self.right)) + def unapply[A, B, C](self: Lift2[A, B, C]): Some[((A, B) => C, StrictParsley[A], StrictParsley[B])] = Some((self.f, self.left, self.right)) } private [backend] object Lift3 { - def unapply[A, B, C, D](self: Lift3[A, B, C, D]): Option[((A, B, C) => D, StrictParsley[A], StrictParsley[B], StrictParsley[C])] = { + def unapply[A, B, C, D](self: Lift3[A, B, C, D]): Some[((A, B, C) => D, StrictParsley[A], StrictParsley[B], StrictParsley[C])] = { Some((self.f, self.p, self.q, self.r)) } } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IterativeEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IterativeEmbedding.scala index 6fa716b28..ab4206778 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IterativeEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/IterativeEmbedding.scala @@ -15,40 +15,29 @@ import parsley.internal.machine.instructions import StrictParsley.InstrBuffer -private [backend] sealed abstract class ManyLike[A, B](name: String, unit: B) extends Unary[A, B] { - def instr(label: Int): instructions.Instr - def preamble(instrs: InstrBuffer): Unit - final override def optimise: StrictParsley[B] = p match { - case _: Pure[_] => throw new NonProductiveIterationException(name) // scalastyle:ignore throw - case _: MZero => new Pure(unit) +private [deepembedding] final class Many[A](val p: StrictParsley[A]) extends Unary[A, List[A]] { + final override def optimise: StrictParsley[List[A]] = p match { + case _: Pure[_] => throw new NonProductiveIterationException("many") // scalastyle:ignore throw + case _: MZero => new Pure(Nil) case _ => this } - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + final override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val body = state.freshLabel() val handler = state.freshLabel() - preamble(instrs) - instrs += new instructions.PushHandlerIterative(handler) + if (producesResults) instrs += new instructions.Fresh(mutable.ListBuffer.empty[Any]) + instrs += new instructions.PushHandler(handler) instrs += new instructions.Label(body) - suspend(p.codeGen[M, R]) |> { + suspend(p.codeGen[M, R](producesResults)) |> { instrs += new instructions.Label(handler) - instrs += instr(body) + instrs += (if (producesResults) new instructions.Many(body) else new instructions.SkipMany(body)) } } -} -private [deepembedding] final class Many[A](val p: StrictParsley[A]) extends ManyLike[A, List[A]]("many", Nil) { - override def instr(label: Int): instructions.Instr = new instructions.Many(label) - override def preamble(instrs: InstrBuffer): Unit = instrs += new instructions.Fresh(mutable.ListBuffer.empty[Any]) + // $COVERAGE-OFF$ final override def pretty(p: String): String = s"many($p)" // $COVERAGE-ON$ } -private [deepembedding] final class SkipMany[A](val p: StrictParsley[A]) extends ManyLike[A, Unit]("skipMany", ()) { - override def instr(label: Int): instructions.Instr = new instructions.SkipMany(label) - override def preamble(instrs: InstrBuffer): Unit = () - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"skipMany($p)" - // $COVERAGE-ON$ -} + private [backend] sealed abstract class ChainLike[A](p: StrictParsley[A], op: StrictParsley[A => A]) extends StrictParsley[A] { def inlinable: Boolean = false override def optimise: StrictParsley[A] = op match { @@ -61,16 +50,17 @@ private [backend] sealed abstract class ChainLike[A](p: StrictParsley[A], op: St protected def pretty(p: String, op: String): String // $COVERAGE-ON$ } + private [deepembedding] final class ChainPost[A](p: StrictParsley[A], op: StrictParsley[A => A]) extends ChainLike[A](p, op) { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val body = state.freshLabel() val handler = state.freshLabel() - suspend(p.codeGen[M, R]) >> { - instrs += new instructions.PushHandlerIterative(handler) + suspend(p.codeGen[M, R](producesResults)) >> { + instrs += new instructions.PushHandler(handler) instrs += new instructions.Label(body) - suspend(op.codeGen[M, R]) |> { + suspend(op.codeGen[M, R](producesResults)) |> { instrs += new instructions.Label(handler) - instrs += new instructions.ChainPost(body) + instrs += (if (producesResults) new instructions.ChainPost(body) else new instructions.SkipMany(body)) } } } @@ -78,36 +68,39 @@ private [deepembedding] final class ChainPost[A](p: StrictParsley[A], op: Strict final override def pretty(p: String, op: String): String = s"chainPost($p, $op)" // $COVERAGE-ON$ } + private [deepembedding] final class ChainPre[A](p: StrictParsley[A], op: StrictParsley[A => A]) extends ChainLike[A](p, op) { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val body = state.freshLabel() val handler = state.freshLabel() - instrs += new instructions.Push(identity[Any] _) - instrs += new instructions.PushHandlerIterative(handler) + if (producesResults) instrs += new instructions.Push(identity[Any] _) + instrs += new instructions.PushHandler(handler) instrs += new instructions.Label(body) - suspend(op.codeGen[M, R]) >> { + suspend(op.codeGen[M, R](producesResults)) >> { instrs += new instructions.Label(handler) - instrs += new instructions.ChainPre(body) - suspend(p.codeGen[M, R]) |> - (instrs += instructions.Apply) + instrs += (if (producesResults) new instructions.ChainPre(body) else new instructions.SkipMany(body)) + suspend(p.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Apply + } } } // $COVERAGE-OFF$ final override def pretty(p: String, op: String): String = s"chainPre($op, $p)" // $COVERAGE-ON$ } + private [deepembedding] final class Chainl[A, B](init: StrictParsley[B], p: StrictParsley[A], op: StrictParsley[(B, A) => B]) extends StrictParsley[B] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val body = state.freshLabel() val handler = state.freshLabel() - suspend(init.codeGen[M, R]) >> { - instrs += new instructions.PushHandlerIterative(handler) + suspend(init.codeGen[M, R](producesResults)) >> { + instrs += new instructions.PushHandler(handler) instrs += new instructions.Label(body) - op.codeGen[M, R] >> - suspend(p.codeGen[M, R]) |> { + suspend(op.codeGen[M, R](producesResults)) >> + suspend(p.codeGen[M, R](producesResults)) |> { instrs += new instructions.Label(handler) - instrs += new instructions.Chainl(body) + instrs += (if (producesResults) new instructions.Chainl(body) else new instructions.SkipMany(body)) } } } @@ -115,46 +108,57 @@ private [deepembedding] final class Chainl[A, B](init: StrictParsley[B], p: Stri final override def pretty: String = s"chainl1(${init.pretty}, ${p.pretty}, ${op.pretty})" // $COVERAGE-ON$ } + private [deepembedding] final class Chainr[A, B](p: StrictParsley[A], op: StrictParsley[(A, B) => B], private [Chainr] val wrap: A => B) extends StrictParsley[B] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit]= { - val body = state.freshLabel() - val handler1 = state.getLabel(instructions.ChainrWholeHandler) - val handler2 = state.freshLabel() - instrs += new instructions.Push(identity[Any] _) - instrs += new instructions.PushHandlerIterative(handler1) - instrs += new instructions.Label(body) - suspend(p.codeGen[M, R]) >> { - instrs += new instructions.PushHandlerIterative(handler2) - suspend(op.codeGen[M, R]) |> { - instrs += new instructions.ChainrJump(body) - instrs += new instructions.Label(handler2) - instrs += instructions.ChainrOpHandler(wrap) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit]= { + if (producesResults) { + val body = state.freshLabel() + // handler1 is where the check offset is kept + val handler1 = state.getLabel(instructions.Refail) + val handler2 = state.freshLabel() + instrs += new instructions.Push(identity[Any] _) + instrs += new instructions.PushHandler(handler1) + instrs += new instructions.Label(body) + suspend(p.codeGen[M, R](producesResults=true)) >> { + instrs += new instructions.PushHandler(handler2) + suspend(op.codeGen[M, R](producesResults=true)) |> { + instrs += new instructions.ChainrJump(body) + instrs += new instructions.Label(handler2) + instrs += instructions.ChainrOpHandler(wrap) + if (!producesResults) instrs += instructions.Pop + } } } + // if we don't care about the results, there is no difference between chainl1 and chainr1, and chainl1 is more efficient + else new Chainl[Nothing, Nothing](p.asInstanceOf[StrictParsley[Nothing]], + p.asInstanceOf[StrictParsley[Nothing]], + op.asInstanceOf[StrictParsley[Nothing]]).codeGen[M, R](producesResults = false) } // $COVERAGE-OFF$ final override def pretty: String = s"chainr1(${p.pretty}, ${op.pretty})" // $COVERAGE-ON$ } -private [deepembedding] final class SepEndBy1[A, B](p: StrictParsley[A], sep: StrictParsley[B]) extends StrictParsley[List[A]] { + +private [deepembedding] final class SepEndBy1[A](p: StrictParsley[A], sep: StrictParsley[_]) extends StrictParsley[List[A]] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val body = state.freshLabel() val handler1 = state.freshLabel() val handler2 = state.freshLabel() instrs += new instructions.Fresh(mutable.ListBuffer.empty[Any]) - instrs += new instructions.PushHandlerIterative(handler1) + instrs += new instructions.PushHandler(handler1) instrs += new instructions.Label(body) - suspend(p.codeGen[M, R]) >> { - instrs += new instructions.PushHandlerIterative(handler2) - suspend(sep.codeGen[M, R]) |> { + suspend(p.codeGen[M, R](producesResults = true)) >> { + instrs += new instructions.PushHandler(handler2) + suspend(sep.codeGen[M, R](producesResults = false)) |> { instrs += new instructions.SepEndBy1Jump(body) instrs += new instructions.Label(handler2) instrs += instructions.SepEndBy1SepHandler instrs += new instructions.Label(handler1) instrs += instructions.SepEndBy1WholeHandler + if (!producesResults) instrs += instructions.Pop } } } @@ -162,31 +166,31 @@ private [deepembedding] final class SepEndBy1[A, B](p: StrictParsley[A], sep: St final override def pretty: String = s"sepEndBy1(${p.pretty}, ${sep.pretty})" // $COVERAGE-ON$ } + +// TODO: unify :/ private [deepembedding] final class ManyUntil[A](val p: StrictParsley[Any]) extends Unary[Any, List[A]] { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val start = state.freshLabel() - val loop = state.freshLabel() instrs += new instructions.Fresh(mutable.ListBuffer.empty[Any]) - instrs += new instructions.PushHandler(loop) instrs += new instructions.Label(start) - suspend(p.codeGen[M, R]) |> { - instrs += new instructions.Label(loop) + suspend(p.codeGen[M, R](producesResults = true)) |> { instrs += new instructions.ManyUntil(start) + if (!producesResults) instrs += instructions.Pop } } // $COVERAGE-OFF$ final override def pretty(p: String): String = s"manyUntil($p)" // $COVERAGE-ON$ } + private [deepembedding] final class SkipManyUntil(val p: StrictParsley[Any]) extends Unary[Any, Unit] { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val start = state.freshLabel() - val loop = state.freshLabel() - instrs += new instructions.PushHandler(loop) instrs += new instructions.Label(start) - suspend(p.codeGen[M, R]) |> { - instrs += new instructions.Label(loop) + // requires the control flow through for the end token + suspend(p.codeGen[M, R](producesResults = true)) |> { instrs += new instructions.SkipManyUntil(start) + if (producesResults) instrs += instructions.Push.Unit } } // $COVERAGE-OFF$ diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/PrimitiveEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/PrimitiveEmbedding.scala index 4c4154ec9..6e41755ec 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/PrimitiveEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/PrimitiveEmbedding.scala @@ -5,7 +5,7 @@ */ package parsley.internal.deepembedding.backend -import parsley.debug.{Breakpoint, EntryBreak, ExitBreak, FullBreak} +import parsley.debug.{Breakpoint, EntryBreak, ExitBreak, FullBreak, Profiler} import parsley.errors.ErrorBuilder import parsley.registers.Reg @@ -14,21 +14,21 @@ import parsley.internal.deepembedding.singletons._ import parsley.internal.machine.instructions import StrictParsley.InstrBuffer -private [deepembedding] final class Attempt[A](val p: StrictParsley[A]) extends ScopedUnaryWithState[A, A](false) { +private [deepembedding] final class Attempt[A](val p: StrictParsley[A]) extends ScopedUnaryWithState[A, A] { override val instr: instructions.Instr = instructions.PopHandlerAndState override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.RestoreAndFail) override def optimise: StrictParsley[A] = p match { - case p: CharTok => p + case p: CharTok[_] => p case p: Attempt[_] => p - case StringTok(s) if s.size == 1 => p + //case StringTok(s, _) if s.size == 1 => p case _ => this } // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"attempt($p)" + final override def pretty(p: String): String = s"atomic($p)" // $COVERAGE-ON$ } -private [deepembedding] final class Look[A](val p: StrictParsley[A]) extends ScopedUnaryWithState[A, A](true) { +private [deepembedding] final class Look[A](val p: StrictParsley[A]) extends ScopedUnaryWithState[A, A] { override val instr: instructions.Instr = instructions.RestoreHintsAndState override def instrNeedsLabel: Boolean = false override def handlerLabel(state: CodeGenState): Int = state.getLabel(instructions.PopStateRestoreHintsAndFail) @@ -37,17 +37,14 @@ private [deepembedding] final class Look[A](val p: StrictParsley[A]) extends Sco // $COVERAGE-ON$ } private [deepembedding] final class NotFollowedBy[A](val p: StrictParsley[A]) extends Unary[A, Unit] { - /*override def optimise: StrictParsley[Unit] = p match { - case _: MZero => new Pure(()) - case _ => this - }*/ - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + final override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = state.freshLabel() - instrs += new instructions.PushHandlerAndState(handler, saveHints = true, hideHints = true) - suspend[M, R, Unit](p.codeGen) |> { + instrs += new instructions.PushHandlerAndState(handler) + suspend[M, R, Unit](p.codeGen(producesResults = false)) |> { instrs += instructions.NegLookFail instrs += new instructions.Label(handler) instrs += instructions.NegLookGood + if (producesResults) instrs += instructions.Push.Unit } } // $COVERAGE-OFF$ @@ -55,32 +52,35 @@ private [deepembedding] final class NotFollowedBy[A](val p: StrictParsley[A]) ex // $COVERAGE-ON$ } -private [deepembedding] final class Rec[A](val call: instructions.Call) extends StrictParsley[A] { +private [deepembedding] final class Let[A] extends StrictParsley[A] { def inlinable: Boolean = true - // Must be a def, since call.label can change! - def label: Int = call.label - - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = result(instrs += call) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = result { + instrs += new instructions.Call(state.getLabel(this, producesResults)) + } // $COVERAGE-OFF$ def pretty: String = this.toString // $COVERAGE-ON$ } -private [deepembedding] final class Let[A](val p: StrictParsley[A]) extends StrictParsley[A] { - def inlinable: Boolean = true - def label(implicit state: CodeGenState): Int = state.getLabel(this) - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - result(instrs += new instructions.Call(label)) +private [deepembedding] final class Opaque[A](p: StrictParsley[A]) extends StrictParsley[A] { + def inlinable: Boolean = p.inlinable + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R,Unit] = { + // this blocks result suppression, because the ErrorGen combinators have non-inspectible control flow + p.codeGen(producesResults = true) |> { + if (!producesResults) instrs += instructions.Pop + } } - // $COVERAGE-OFF$ - def pretty: String = this.toString + def pretty: String = p.pretty // $COVERAGE-ON$ } + private [deepembedding] final class Put[S](reg: Reg[S], val p: StrictParsley[S]) extends Unary[S, Unit] { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - suspend(p.codeGen[M, R]) |> - (instrs += new instructions.Put(reg.addr)) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + suspend(p.codeGen[M, R](producesResults = true)) |> { + instrs += new instructions.Put(reg.addr) + if (producesResults) instrs += instructions.Push.Unit + } } // $COVERAGE-OFF$ final override def pretty(p: String): String = s"put(r${reg.addr}, $p)" @@ -89,14 +89,14 @@ private [deepembedding] final class Put[S](reg: Reg[S], val p: StrictParsley[S]) private [deepembedding] final class NewReg[S, A](reg: Reg[S], init: StrictParsley[S], body: StrictParsley[A]) extends StrictParsley[A] { def inlinable: Boolean = false - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = state.getLabelForPutAndFail(reg) - suspend(init.codeGen[M, R]) >> { + suspend(init.codeGen[M, R](producesResults = true)) >> { instrs += new instructions.Get(reg.addr) instrs += new instructions.SwapAndPut(reg.addr) instrs += new instructions.PushHandler(handler) - suspend(body.codeGen[M, R]) |> { - instrs += new instructions.SwapAndPut(reg.addr) + suspend(body.codeGen[M, R](producesResults)) |> { + instrs += (if (producesResults) new instructions.SwapAndPut(reg.addr) else new instructions.Put(reg.addr)) instrs += instructions.PopHandler } } @@ -106,30 +106,67 @@ private [deepembedding] final class NewReg[S, A](reg: Reg[S], init: StrictParsle // $COVERAGE-ON$ } +private [deepembedding] final class Span(p: StrictParsley[_]) extends StrictParsley[String] { + def inlinable: Boolean = false + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + if (producesResults) { + val handler = state.getLabel(instructions.PopStateAndFail) + instrs += new instructions.PushHandlerAndState(handler) + suspend[M, R, Unit](p.codeGen(producesResults = false)) |> { + instrs += instructions.Span + } + } + else p.codeGen(producesResults = false) + } + // $COVERAGE-OFF$ + final override def pretty: String = s"${p.pretty}.span" + // $COVERAGE-ON$ +} + // $COVERAGE-OFF$ -private [deepembedding] final class Debug[A](val p: StrictParsley[A], name: String, ascii: Boolean, break: Breakpoint) extends Unary[A, A] { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { +private [deepembedding] final class Debug[A](val p: StrictParsley[A], name: String, ascii: Boolean, break: Breakpoint, watchedRegs: scala.Seq[(Reg[_], String)]) + extends Unary[A, A] { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + val watchedAddrs = watchedRegs.map { + case (r, name) => (r.addr, name) + } val handler = state.freshLabel() - instrs += new instructions.LogBegin(handler, name, ascii, (break eq EntryBreak) || (break eq FullBreak)) - suspend(p.codeGen[M, R]) |> { + instrs += new instructions.LogBegin(handler, name, ascii, (break eq EntryBreak) || (break eq FullBreak), watchedAddrs) + suspend(p.codeGen[M, R](producesResults)) |> { instrs += new instructions.Label(handler) - instrs += new instructions.LogEnd(name, ascii, (break eq ExitBreak) || (break eq FullBreak)) + instrs += new instructions.LogEnd(name, ascii, (break eq ExitBreak) || (break eq FullBreak), watchedAddrs) } } final override def pretty(p: String): String = p } + private [deepembedding] final class DebugError[A](val p: StrictParsley[A], name: String, ascii: Boolean, errBuilder: ErrorBuilder[_]) extends Unary[A, A] { - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val handler = state.freshLabel() instrs += new instructions.LogErrBegin(handler, name, ascii)(errBuilder) - suspend(p.codeGen[M, R]) |> { - instrs += instructions.Swap + suspend(p.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Swap instrs += new instructions.Label(handler) instrs += new instructions.LogErrEnd(name, ascii)(errBuilder) } } final override def pretty(p: String): String = p } + +private [deepembedding] final class Profile[A](val p: StrictParsley[A], name: String, profiler: Profiler) extends Unary[A, A] { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R,Unit] = { + val handler = state.freshLabel() + instrs += new instructions.ProfileEnter(handler, name, profiler) + suspend(p.codeGen[M, R](producesResults)) |> { + instrs += new instructions.Label(handler) + instrs += new instructions.ProfileExit(name, profiler) + } + } + final override def pretty(p: String): String = p +} +private [backend] object Profile { + def unapply[A](p: Profile[A]): Some[StrictParsley[A]] = Some(p.p) +} // $COVERAGE-ON$ private [backend] object Attempt { diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SelectiveEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SelectiveEmbedding.scala index 0bea967a7..224096e21 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SelectiveEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SelectiveEmbedding.scala @@ -19,17 +19,23 @@ private [backend] sealed abstract class BranchLike[A, B, C, D](finaliser: Option def instr(label: Int): instructions.Instr def inlinable: Boolean = false - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + final override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { val toP = state.freshLabel() val end = state.freshLabel() - suspend(b.codeGen[M, R]) >> { + suspend(b.codeGen[M, R](producesResults = true)) >> { instrs += instr(toP) - suspend(q.codeGen[M, R]) >> { - for (instr <- finaliser) instrs += instr + suspend(q.codeGen[M, R](producesResults)) >> { + for (instr <- finaliser) { + if (producesResults) instrs += instr + else instrs += instructions.Pop + } instrs += new instructions.Jump(end) instrs += new instructions.Label(toP) - suspend(p.codeGen[M, R]) |> { - for (instr <- finaliser) instrs += instr + suspend(p.codeGen[M, R](producesResults)) |> { + for (instr <- finaliser) { + if (producesResults) instrs += instr + else instrs += instructions.Pop + } instrs += new instructions.Label(end) } } @@ -66,58 +72,42 @@ private [deepembedding] final class If[A](val b: StrictParsley[Boolean], val p: // $COVERAGE-ON$ } -private [backend] sealed abstract class FilterLike[A](instr: instructions.Instr) extends Unary[A, A] { - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - val handler = state.getLabel(instructions.PopStateAndFail) - instrs += new instructions.PushHandlerAndState(handler, saveHints = false, hideHints = false) - suspend(p.codeGen[M, R]) |> { - instrs += instr - } - } -} -private [deepembedding] final class Filter[A](val p: StrictParsley[A], pred: A => Boolean) extends FilterLike[A](new instructions.Filter(pred)) { - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"$p.filter(?)" - // $COVERAGE-ON$ -} -private [deepembedding] final class MapFilter[A, B](val p: StrictParsley[A], f: A => Option[B]) extends Unary[A, B] { - final override def optimise: StrictParsley[B] = p match { - case Pure(x) => f(x).map(new Pure(_)).getOrElse(Empty.Zero) - case z: MZero => z - case _ => this - } +private [backend] sealed abstract class FilterLike[A, B] extends StrictParsley[B] { + protected val p: StrictParsley[A] + protected val err: StrictParsley[((A, Int)) => Nothing] + def inlinable: Boolean = false + protected def instr(handler: Int, jumpLabel: Int): instructions.Instr - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - val handler = state.getLabel(instructions.PopStateAndFail) - instrs += new instructions.PushHandlerAndState(handler, saveHints = false, hideHints = false) - suspend(p.codeGen[M, R]) |> { - instrs += new instructions.MapFilter(f) + final override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + val handler1 = state.getLabel(instructions.PopStateAndFail) + val handler2 = state.getLabel(instructions.AmendAndFail(false)) + val jumpLabel = state.freshLabel() + instrs += new instructions.PushHandlerAndState(handler1) + suspend(p.codeGen[M, R](producesResults = true)) >> { + instrs += instr(handler2, jumpLabel) + suspend(err.codeGen[M, R](producesResults = true)) |> { + instrs += new instructions.Label(jumpLabel) + if (!producesResults) instrs += instructions.Pop + } } } // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"$p.mapFilter(?)" + final override def pretty: String = pretty(p.pretty, err.pretty) + protected def pretty(p: String, err: String): String // $COVERAGE-ON$ } -private [deepembedding] final class FilterOut[A](val p: StrictParsley[A], pred: PartialFunction[A, String]) - extends FilterLike[A](new instructions.FilterOut(pred)) { - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"$p.filterOut(?)" - // $COVERAGE-ON$ -} -private [deepembedding] final class GuardAgainst[A](val p: StrictParsley[A], pred: PartialFunction[A, scala.Seq[String]]) - extends FilterLike[A](instructions.GuardAgainst(pred)) { - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"$p.guardAgainst(?)" - // $COVERAGE-ON$ +private [deepembedding] final class Filter[A](val p: StrictParsley[A], pred: A => Boolean, val err: StrictParsley[((A, Int)) => Nothing]) + extends FilterLike[A, A] { + final override def instr(handler: Int, jumpLabel: Int): instructions.Instr = new instructions.Filter(pred, jumpLabel, handler) + final override def pretty(p: String, err: String): String = s"filterWith($p, ???, $err)" } -private [deepembedding] final class UnexpectedWhen[A](val p: StrictParsley[A], pred: PartialFunction[A, (String, Option[String])]) - extends FilterLike[A](instructions.UnexpectedWhen(pred)) { - // $COVERAGE-OFF$ - final override def pretty(p: String): String = s"$p.unexpectedWhen(?)" - // $COVERAGE-ON$ +private [deepembedding] final class MapFilter[A, B](val p: StrictParsley[A], pred: A => Option[B], val err: StrictParsley[((A, Int)) => Nothing]) + extends FilterLike[A, B] { + final override def instr(handler: Int, jumpLabel: Int): instructions.Instr = new instructions.MapFilter(pred, jumpLabel, handler) + final override def pretty(p: String, err: String): String = s"mapFilterWith($p, ???, $err)" } private [backend] object Branch { diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SequenceEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SequenceEmbedding.scala index dcc2bbe93..4cda3f864 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SequenceEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/SequenceEmbedding.scala @@ -19,10 +19,9 @@ import StrictParsley.InstrBuffer private [deepembedding] final class <*>[A, B](var left: StrictParsley[A => B], var right: StrictParsley[A]) extends StrictParsley[B] { def inlinable: Boolean = false // TODO: Refactor - // FIXME: Needs more interation with .safe override def optimise: StrictParsley[B] = (left, right) match { // Fusion laws - case (uf, ux@Pure(x)) if (uf.isInstanceOf[Pure[_]] || uf.isInstanceOf[_ <*> _]) && uf.safe && ux.safe => uf match { + case (uf, Pure(x)) if (uf.isInstanceOf[Pure[_]] || uf.isInstanceOf[_ <*> _]) => uf match { // first position fusion case Pure(f) => new Pure(f(x)) // second position fusion @@ -48,6 +47,10 @@ private [deepembedding] final class <*>[A, B](var left: StrictParsley[A => B], v left = new Pure(f.compose(g)).asInstanceOf[StrictParsley[A => B]] right = u.asInstanceOf[StrictParsley[A]] this + // Fusion for special combinators + case (Pure(f), ct@CharTok(c, x)) => new CharTok(c, f(x), ct.expected) + case (Pure(f), ct@SupplementaryCharTok(c, x)) => new SupplementaryCharTok(c, f(x), ct.expected) + case (Pure(f), ct@StringTok(c, x)) => new StringTok(c, f(x), ct.expected) // TODO: functor law with lift2! // right absorption law: mzero <*> p = mzero case (z: MZero, _) => z @@ -64,27 +67,19 @@ private [deepembedding] final class <*>[A, B](var left: StrictParsley[A => B], v } // consequence of left zero law and monadic definition of <*>, preserving error properties of left case (u, z: MZero) => *>(u, z) - // interchange law: u <*> pure y == pure ($y) <*> u == ($y) <$> u (single instruction, so we benefit at code-gen) - case (uf, Pure(x)) => - left = new Pure((f: A => B) => f(x)).asInstanceOf[StrictParsley[A => B]] - right = uf.asInstanceOf[StrictParsley[A]] - this case _ => this } - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = left match { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = left match { // pure f <*> p = f <$> p - case Pure(f) => right match { - case ct@CharTok(c) => result(instrs += instructions.CharTokFastPerform[Char, B](c, f.asInstanceOf[Char => B], ct.expected)) - case ct@SupplementaryCharTok(c) => result(instrs += instructions.SupplementaryCharTokFastPerform[Int, B](c, f.asInstanceOf[Int => B], ct.expected)) - case st@StringTok(s) => result(instrs += instructions.StringTokFastPerform(s, f.asInstanceOf[String => B], st.expected)) - case _ => - suspend(right.codeGen[M, R]) |> - (instrs += instructions.Lift1(f)) - } + case Pure(f) => + suspend(right.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Lift1(f) + } case _ => - suspend(left.codeGen[M, R]) >> - suspend(right.codeGen[M, R]) |> - (instrs += instructions.Apply) + suspend(left.codeGen[M, R](producesResults)) >> + suspend(right.codeGen[M, R](producesResults)) |> { + if (producesResults) instrs += instructions.Apply + } } // $COVERAGE-OFF$ final override def pretty: String = s"(${left.pretty} <*> ${right.pretty})" @@ -93,30 +88,19 @@ private [deepembedding] final class <*>[A, B](var left: StrictParsley[A => B], v private [deepembedding] final class >>=[A, B](val p: StrictParsley[A], private [>>=] val f: A => frontend.LazyParsley[B]) extends Unary[A, B] { override def optimise: StrictParsley[B] = p match { - // monad law 1: pure x >>= f = f x: unsafe because it might expose recursion - //case Pure(x) if safe => new Rec(() => f(x)) - // char/string x = char/string x *> pure x and monad law 1 - //case p@CharTok(c) => *>(p, new Rec(() => f(c.asInstanceOf[A]), expected)) - //case p@StringTok(s) => *>(p, new Rec(() => f(s.asInstanceOf[A]), expected)) - // (q *> p) >>= f = q *> (p >>= f) - //case u *> v => *>(u, >>=(v, f).optimise) - // monad law 3: (m >>= g) >>= f = m >>= (\x -> g x >>= f) Note: this *could* help if g x ended with a pure, since this would be optimised out! - //case (m: Parsley[T] @unchecked) >>= (g: (T => A) @unchecked) => - // p = m.asInstanceOf[Parsley[A]] - // f = (x: T) => >>=(g(x), f, expected).optimise - // this - // monadplus law (left zero) case z: MZero => z case _ => this } - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { - suspend(p.codeGen[M, R]) |> { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + suspend(p.codeGen[M, R](producesResults = true)) |> { instrs += instructions.DynCall[A] { x => val p = f(x) + // FIXME: suppress results within p, then can remove pop p.demandCalleeSave(state.numRegs) if (implicitly[ContOps[M]].isStackSafe) p.overflows() p.instrs } + if (!producesResults) instrs += instructions.Pop } } // $COVERAGE-OFF$ @@ -151,6 +135,8 @@ private [deepembedding] final class Seq[A](private [backend] var before: DoublyL case _ => after } + // TODO: Get behaves much like pure except for shifting positions + // TODO: can this be optimised to reduce repeated matching? override def optimise: StrictParsley[A] = this match { // Assume that this is eliminated first, so not other before or afters case (_: Pure[_]) **> u => u @@ -198,7 +184,7 @@ private [deepembedding] final class Seq[A](private [backend] var before: DoublyL case _ => this } - override def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = res match { + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = res match { case Pure(x) => // peephole here involves CharTokFastPerform, StringTokFastPerform, and Exchange assume(after.isEmpty, "The pure in question is normalised to the end: if result is pure, after is empty.") @@ -206,19 +192,18 @@ private [deepembedding] final class Seq[A](private [backend] var before: DoublyL val last = before.last before.initInPlace() suspend(Seq.codeGenMany[M, R](before.iterator)) >> { + before.addOne(last) // undo the initInPlace, you'll thank me later last match { - case ct@CharTok(c) => result(instrs += instructions.CharTokFastPerform[Char, A](c, _ => x, ct.expected)) - case st@StringTok(s) => result(instrs += instructions.StringTokFastPerform(s, _ => x, st.expected)) - case st@Satisfy(f) => result(instrs += new instructions.SatisfyExchange(f, x, st.expected)) + case st@Satisfy(f) if producesResults => result(instrs += new instructions.SatisfyExchange(f, x, st.expected)) case _ => - suspend(last.codeGen[M, R]) |> { - instrs += new instructions.Exchange(x) + suspend(last.codeGen[M, R](producesResults = false)) |> { + if (producesResults) instrs += new instructions.Push(x) } } } case _ => suspend(Seq.codeGenMany[M, R](before.iterator)) >> { - suspend(res.codeGen[M, R]) >> { + suspend(res.codeGen[M, R](producesResults)) >> { suspend(Seq.codeGenMany(after.iterator)) } } @@ -234,12 +219,9 @@ private [backend] object Seq { } private [Seq] def codeGenMany[M[_, +_]: ContOps, R](it: Iterator[StrictParsley[_]]) - (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { + (implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] = { if (it.hasNext) { - suspend(it.next().codeGen[M, R]) >> { - instrs += instructions.Pop - suspend(codeGenMany(it)) - } + suspend(it.next().codeGen[M, R](producesResults = false)) >> suspend(codeGenMany(it)) } else result(()) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/StrictParsley.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/StrictParsley.scala index f454d6190..9a252d6cf 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/StrictParsley.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/StrictParsley.scala @@ -44,17 +44,16 @@ private [deepembedding] trait StrictParsley[+A] { * @return the final array of instructions for this parser */ final private [deepembedding] def generateInstructions[M[_, +_]: ContOps](numRegsUsedByParent: Int, usedRegs: Set[Reg[_]], - recs: Iterable[(Rec[_], M[Unit, StrictParsley[_]])]) + bodyMap: Map[Let[_], StrictParsley[_]]) (implicit state: CodeGenState): Array[Instr] = { implicit val instrs: InstrBuffer = newInstrBuffer perform { - generateCalleeSave[M, Array[Instr]](numRegsUsedByParent, this.codeGen, usedRegs) |> { + generateCalleeSave[M, Array[Instr]](numRegsUsedByParent, this.codeGen(producesResults = true), usedRegs) |> { // When `numRegsUsedByParent` is -1 this is top level, otherwise it is a flatMap instrs += (if (numRegsUsedByParent >= 0) instructions.Return else instructions.Halt) - val recRets = finaliseRecs(recs) - val letRets = finaliseLets() + val letRets = finaliseLets(bodyMap) generateHandlers(state.handlers) - finaliseInstrs(instrs, state.nlabels, recRets ::: letRets) + finaliseInstrs(instrs, state.nlabels, letRets) } } } @@ -64,10 +63,11 @@ private [deepembedding] trait StrictParsley[+A] { * It is fine for this method to perform peephole optimisation on the combinators and generate * more optimal sequences of instructions in specific circumstances. * + * @param producesResults is this parser expected to push its result onto the stack? * @param instrs the current buffer of instructions to generate into * @param state code generator state, for the generation of labels */ - protected [backend] def codeGen[M[_, +_]: ContOps, R](implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] + protected [backend] def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: InstrBuffer, state: CodeGenState): M[R, Unit] /** This method is directly called by the "frontend" and is used to perform domain-specific * optimisations on this parser (usually following the laws of the parser combinators). @@ -82,9 +82,6 @@ private [deepembedding] trait StrictParsley[+A] { */ private [deepembedding] def inlinable: Boolean - /** Is this combinator known to be pure? */ - final private [deepembedding] var safe = true - // $COVERAGE-OFF$ /** Pretty-prints a combinator tree, for internal debugging purposes only. */ private [deepembedding] def pretty: String @@ -174,45 +171,23 @@ private [deepembedding] object StrictParsley { else bodyGen } - /** Generates each of the given recursive parsers in turn into the instruction buffer. - * Each has already been assigned a jump-label, which is stored in the `Rec` node itself. - * The layout of generation is the jump-label, followed by the body, followed by a `Return`. - * - * @param recs a supply of `Rec` nodes along with the generator for their strict parsers - * @param instrs the instruction buffer to generate into - * @param state the code generation state, for label generation - * @return the list of return labels for each of the parsers (for TCO) - */ - private def finaliseRecs[M[_, +_]: ContOps](recs: Iterable[(Rec[_], M[Unit, StrictParsley[_]])]) - (implicit instrs: InstrBuffer, state: CodeGenState): List[RetLoc] = { - val retLocs = mutable.ListBuffer.empty[RetLoc] - for ((rec, p) <- recs) { - instrs += new instructions.Label(rec.label) - perform(p.flatMap(_.codeGen)) - val retLoc = state.freshLabel() - instrs += new instructions.Label(retLoc) - instrs += instructions.Return - retLocs += retLoc - } - retLocs.toList - } - /** Generates each of the shared, non-recursive, parsers that have been ''used'' by * the parser. These are stored within the code generation state. This is done because * some of the identified shared parsers may have been inlined and so do not need to * be generated. The state tracks which of these parsers were actually demanded, so * dead-code is automatically eliminated. * + * @param bodyMap the map of lets to bodies * @param instrs the instruction buffer to generate into * @param state the code generation state, which contains the shared parsers * @return the list of return labels for each of the parsers (for TCO) */ - private def finaliseLets[M[_, +_]: ContOps]()(implicit instrs: InstrBuffer, state: CodeGenState): List[RetLoc] = { + private def finaliseLets[M[_, +_]: ContOps](bodyMap: Map[Let[_], StrictParsley[_]])(implicit instrs: InstrBuffer, state: CodeGenState): List[RetLoc] = { val retLocs = mutable.ListBuffer.empty[RetLoc] while (state.more) { - val let = state.nextLet() - instrs += new instructions.Label(let.label) - perform[M, Unit](let.p.codeGen) + val (let, producesResults, label) = state.nextLet() + instrs += new instructions.Label(label) + perform[M, Unit](bodyMap(let).codeGen(producesResults)) val retLoc = state.freshLabel() instrs += new instructions.Label(retLoc) instrs += instructions.Return @@ -313,9 +288,9 @@ private [deepembedding] class CodeGenState(val numRegs: Int) { /** The next jump-label identifier. */ private var current = 0 /** The shared-parsers that have been referenced at some point in the generation so far. */ - private val queue = mutable.ListBuffer.empty[Let[_]] + private val queue = mutable.ListBuffer.empty[(Let[_], Boolean, Int)] /** The mapping between a shared-parser and its generated jump-label. */ - private val map = mutable.Map.empty[Let[_], Int] + private val map = mutable.Map.empty[(Let[_], Boolean), Int] /** Generates a unique jump-label. */ def freshLabel(): Int = { @@ -332,13 +307,14 @@ private [deepembedding] class CodeGenState(val numRegs: Int) { * @param sub the shared parser to collect a label for * @return the label assigned the given parser */ - def getLabel(sub: Let[_]): Int = map.getOrElseUpdate(sub, { - sub +=: queue - freshLabel() + def getLabel(sub: Let[_], producesResults: Boolean): Int = map.getOrElseUpdate((sub, producesResults), { + val label = freshLabel() + (sub, producesResults, label) +=: queue + label }) /** Returns the next shared-parser that has been refered during code generation */ - def nextLet(): Let[_] = queue.remove(0) + def nextLet(): (Let[_], Boolean, Int) = queue.remove(0) /** Are there any more shared-parsers left on the processing queue? */ def more: Boolean = queue.nonEmpty diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/AlternativeEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/AlternativeEmbedding.scala index 4e3ecdb74..21a347fda 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/AlternativeEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/AlternativeEmbedding.scala @@ -12,7 +12,7 @@ private [parsley] final class <|>[A](p: LazyParsley[A], q: LazyParsley[A]) exten final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R,Unit] = { suspend(p.findLets[M, R](seen)) >> suspend(q.findLets(seen)) } - final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = + final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] = for { p <- suspend(p.optimised[M, R, A]) q <- suspend(q.optimised[M, R, A]) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/ErrorEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/ErrorEmbedding.scala index eb94054a7..66ba48b1d 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/ErrorEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/ErrorEmbedding.scala @@ -7,11 +7,16 @@ package parsley.internal.deepembedding.frontend import parsley.internal.deepembedding.backend, backend.StrictParsley -private [parsley] final class ErrorLabel[A](p: LazyParsley[A], private [ErrorLabel] val labels: Seq[String]) extends Unary[A, A](p) { +private [parsley] final class ErrorLabel[A](p: LazyParsley[A], labels: Seq[String]) extends Unary[A, A](p) { override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.ErrorLabel(p, labels) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, labels) } +private [parsley] final class ErrorHide[A](p: LazyParsley[A]) extends Unary[A, A](p) { + override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.ErrorHide(p) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p) +} private [parsley] final class ErrorExplain[A](p: LazyParsley[A], reason: String) extends Unary[A, A](p) { override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.ErrorExplain(p, reason) @@ -39,9 +44,3 @@ private [parsley] final class ErrorLexical[A](p: LazyParsley[A]) extends Unary[A override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p) } - -private [parsley] final class VerifiedError[A](p: LazyParsley[A], msggen: Either[A => Seq[String], Option[A => String]]) extends Unary[A, Nothing](p) { - override def make(p: StrictParsley[A]): StrictParsley[Nothing] = new backend.VerifiedError(p, msggen) - - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Nothing] = visitor.visit(this, context)(p, msggen) -} diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/GeneralisedEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/GeneralisedEmbedding.scala index 11eb1dbd5..a7eb4c152 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/GeneralisedEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/GeneralisedEmbedding.scala @@ -15,7 +15,7 @@ private [frontend] abstract class Unary[A, B](p: LazyParsley[A]) extends LazyPar final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R,Unit] = suspend(p.findLets(seen)) - override def preprocess[M[_, +_]: ContOps, R, B_ >: B](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[B_]] = + override def preprocess[M[_, +_]: ContOps, R, B_ >: B](implicit lets: LetMap): M[R, StrictParsley[B_]] = for (p <- suspend(p.optimised[M, R, A])) yield make(p) } @@ -27,7 +27,7 @@ private [frontend] abstract class Binary[A, B, C](left: LazyParsley[A], _right: final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R,Unit] = { suspend(left.findLets[M, R](seen)) >> suspend(right.findLets(seen)) } - final override def preprocess[M[_, +_]: ContOps, R, C_ >: C](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[C_]] = + final override def preprocess[M[_, +_]: ContOps, R, C_ >: C](implicit lets: LetMap): M[R, StrictParsley[C_]] = for { left <- suspend(left.optimised[M, R, A]) right <- suspend(right.optimised[M, R, B]) @@ -43,7 +43,7 @@ private [frontend] abstract class Ternary[A, B, C, D](first: LazyParsley[A], _se final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R, Unit] = { suspend(first.findLets[M, R](seen)) >> suspend(second.findLets(seen)) >> suspend(third.findLets(seen)) } - final override def preprocess[M[_, +_]: ContOps, R, D_ >: D](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[D_]] = + final override def preprocess[M[_, +_]: ContOps, R, D_ >: D](implicit lets: LetMap): M[R, StrictParsley[D_]] = for { first <- suspend(first.optimised[M, R, A]) second <- suspend(second.optimised[M, R, B]) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IntrinsicEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IntrinsicEmbedding.scala index a13ceea07..5563824bb 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IntrinsicEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IntrinsicEmbedding.scala @@ -9,12 +9,12 @@ import parsley.registers.Reg import parsley.internal.deepembedding.backend, backend.StrictParsley -private [parsley] final class Lift2[A, B, C](private [Lift2] val f: (A, B) => C, p: LazyParsley[A], q: =>LazyParsley[B]) extends Binary[A, B, C](p, q) { +private [parsley] final class Lift2[A, B, C](private val f: (A, B) => C, p: LazyParsley[A], q: =>LazyParsley[B]) extends Binary[A, B, C](p, q) { override def make(p: StrictParsley[A], q: StrictParsley[B]): StrictParsley[C] = new backend.Lift2(f, p, q) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[C] = visitor.visit(this, context)(f, p, q) } -private [parsley] final class Lift3[A, B, C, D](private [Lift3] val f: (A, B, C) => D, p: LazyParsley[A], q: =>LazyParsley[B], r: =>LazyParsley[C]) +private [parsley] final class Lift3[A, B, C, D](private val f: (A, B, C) => D, p: LazyParsley[A], q: =>LazyParsley[B], r: =>LazyParsley[C]) extends Ternary[A, B, C, D](p, q, r) { override def make(p: StrictParsley[A], q: StrictParsley[B], r: StrictParsley[C]): StrictParsley[D] = new backend.Lift3(f, p, q, r) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IterativeEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IterativeEmbedding.scala index 1d1f7aaec..e3beab3f9 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IterativeEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/IterativeEmbedding.scala @@ -13,11 +13,6 @@ private [parsley] final class Many[A](p: LazyParsley[A]) extends Unary[A, List[A override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[List[A]] = visitor.visit(this, context)(p) } -private [parsley] final class SkipMany[A](p: LazyParsley[A]) extends Unary[A, Unit](p) { - override def make(p: StrictParsley[A]): StrictParsley[Unit] = new backend.SkipMany(p) - - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context)(p) -} private [parsley] final class ChainPost[A](p: LazyParsley[A], _op: =>LazyParsley[A => A]) extends Binary[A, A => A, A](p, _op) { override def make(p: StrictParsley[A], op: StrictParsley[A => A]): StrictParsley[A] = new backend.ChainPost(p, op) @@ -27,7 +22,7 @@ private [parsley] final class ChainPre[A](p: LazyParsley[A], op: LazyParsley[A = final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R, Unit] = { suspend(p.findLets[M, R](seen)) >> suspend(op.findLets(seen)) } - final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = + final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] = for { p <- suspend(p.optimised[M, R, A]) op <- suspend(op.optimised[M, R, A => A]) @@ -47,8 +42,8 @@ private [parsley] final class Chainr[A, B](p: LazyParsley[A], op: =>LazyParsley[ override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[B] = visitor.visit(this, context)(p, op, wrap) } -private [parsley] final class SepEndBy1[A, B](p: LazyParsley[A], sep: =>LazyParsley[B]) extends Binary[A, B, List[A]](p, sep) { - override def make(p: StrictParsley[A], sep: StrictParsley[B]): StrictParsley[List[A]] = new backend.SepEndBy1(p, sep) +private [parsley] final class SepEndBy1[A](p: LazyParsley[A], sep: =>LazyParsley[_]) extends Binary[A, Any, List[A]](p, sep) { + override def make(p: StrictParsley[A], sep: StrictParsley[Any]): StrictParsley[List[A]] = new backend.SepEndBy1(p, sep) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[List[A]] = visitor.visit(this, context)(p, sep) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/LazyParsley.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/LazyParsley.scala index 9ae97ea6c..a48a58275 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/LazyParsley.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/LazyParsley.scala @@ -27,8 +27,9 @@ import parsley.internal.machine.instructions, instructions.Instr private [parsley] abstract class LazyParsley[+A] private [deepembedding] { // Public API // $COVERAGE-OFF$ + // TODO: remove in 5.0 /** Denotes this parser is unsafe, which will disable certain law-based optimisations that assume purity. */ - private [parsley] final def unsafe(): Unit = sSafe = false + private [parsley] final def unsafe(): Unit = () /** Force the parser, which eagerly computes its instructions immediately */ private [parsley] final def force(): Unit = instrs: @nowarn /** Denote that this parser is large enough that it might stack-overflow during @@ -73,17 +74,13 @@ private [parsley] abstract class LazyParsley[+A] private [deepembedding] { */ protected def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R, Unit] - /** Describes how to recursively convert this combinator into a `StrictParsley` by - * `optimise`ing its sub-trees. + /** Describes how to recursively convert this combinator into a `StrictParsley` by `optimise`ing its sub-trees. * - * @param lets the known non-recursive shared parsers mapped to their corresponding join-point nodes - * @param recs the known recursive parsers mapped to their corresponding join-point nodes + * @param lets the known shared parsers mapped to their corresponding join-point nodes * @return the strict, finite, version of this tree, with all shared parsers factored out into join-points */ - protected def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] + protected def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] - /** should the underlying strict tree be considered safe? */ - final private var sSafe = true /** should the `Id` instance be skipped? */ final private var cps = false /** how many registers are used by the ''parent'' of this combinator (this combinator is part of a `flatMap` when this is not -1) */ @@ -117,11 +114,11 @@ private [parsley] abstract class LazyParsley[+A] private [deepembedding] { (perform[M, Array[Instr]] { findLets(Set.empty) >> { val usedRegs: Set[Reg[_]] = letFinderState.usedRegs - implicit val state: backend.CodeGenState = new backend.CodeGenState(letFinderState.numRegs) - implicit val recMap: RecMap = RecMap(letFinderState.recs) - implicit val letMap: LetMap = LetMap(letFinderState.lets) - val recs_ = recMap.map { case (p, rec) => (rec, p.unsafeOptimised[M, Unit, Any]) } - for { sp <- this.optimised } yield sp.generateInstructions(numRegsUsedByParent, usedRegs, recs_) + implicit val letMap: LetMap = LetMap(letFinderState.lets, letFinderState.recs) + for { sp <- this.optimised } yield { + implicit val state: backend.CodeGenState = new backend.CodeGenState(letFinderState.numRegs) + sp.generateInstructions(numRegsUsedByParent, usedRegs, letMap.bodies) + } } }, letFinderState.numRegs) } @@ -164,27 +161,21 @@ private [parsley] abstract class LazyParsley[+A] private [deepembedding] { /** Performs the factoring out of shared parsers and then converts this parser into its strict form performing * optimisations on that translated form. * - * @param lets the known non-recursive shared parsers mapped to their corresponding join-point nodes - * @param recs the known recursive parsers mapped to their corresponding join-point nodes + * @param lets the known shared parsers mapped to their corresponding join-point nodes * @return the strict, finite, version of this tree, with all shared parsers factored out into join-points */ - final protected [frontend] def optimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = { - if (recs.contains(this)) result(recs(this)) - else if (lets.contains(this)) result(lets(this)) + final protected [frontend] def optimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] = { + if (lets.contains(this)) result(lets(this)) else this.unsafeOptimised } /** Similar to `optimised` but should be '''only''' used on things known to be let-bindings (to avoid infinite expansion!). */ - final private [frontend] def knownLetTopOptimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = { + final private [frontend] def knownLetTopOptimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] = { assume(lets.contains(this), "the let check can only be skipped for known let-bindings") - assume(!recs.contains(this), "rec membership can be skipped for known let-binding bodies") this.unsafeOptimised } - /** Similar to `optimised` but does not check for inclusion in the `lets` or `recs` sets. */ - private def unsafeOptimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = { - for {p <- this.preprocess} yield { - p.safe = this.sSafe - p.optimise - } + /** Similar to `optimised` but does not check for inclusion in the `lets` set. */ + private def unsafeOptimised[M[_, +_]: ContOps, R, A_ >: A](implicit lets: LetMap): M[R, StrictParsley[A_]] = { + for { p <- this.preprocess } yield p.optimise } // Processing with visitors. @@ -197,12 +188,11 @@ private [parsley] abstract class LazyParsley[+A] private [deepembedding] { implicit val ops = Id.ops implicit val letFinderState: LetFinderState = new LetFinderState findLets(Set.empty) + implicit val letMap: LetMap = LetMap(letFinderState.lets, letFinderState.recs) implicit val state: backend.CodeGenState = new backend.CodeGenState(0) - implicit val recMap: RecMap = RecMap(letFinderState.recs) - implicit val letMap: LetMap = LetMap(letFinderState.lets) val mrecs = for { - (p, rec) <- recMap - } yield s"${rec.label}: ${p.unsafeOptimised[Id.Impl, String, Any].pretty}" + (let, p) <- letMap.bodies + } yield s"${state.getLabel(let, producesResults = true)}: ${p.pretty}" s"main body: ${this.optimised.pretty}\n${mrecs.mkString("\n")}" } // $COVERAGE-ON$ @@ -240,10 +230,11 @@ private [deepembedding] class LetFinderState { /** Has the given parser never been analysed before? */ private [frontend] def notProcessedBefore(p: LazyParsley[_]): Boolean = _preds(p) == 1 + // TODO: I think this can just drop the !_recs(p) constraint and it'll just work? (check whether recs get a pred bump) /** Returns all the non-recursive parsers which are referenced two or more times across the tree. */ private [frontend] def lets: Iterable[LazyParsley[_]] = _preds.toSeq.view.collect { case (p, refs) if refs >= 2 && !_recs(p) => p - } + } ++ recs /** Returns all the recursive parsers in the tree */ private [frontend] lazy val recs: Set[LazyParsley[_]] = _recs.toSet /** Returns all the registers used by the parser */ @@ -253,9 +244,10 @@ private [deepembedding] class LetFinderState { } /** Represents a map of let-bound lazy parsers to their strict equivalents. */ -private [deepembedding] final class LetMap private (letGen: Map[LazyParsley[_], LetMap => StrictParsley[_]]) { +private [deepembedding] final class LetMap private (letGen: Map[LazyParsley[_], LetMap => StrictParsley[_]], recs: Set[LazyParsley[_]]) { // This might not necessarily contain Let nodes: if they were inlined then they will not be present here - private val mutMap = mutable.Map.empty[LazyParsley[_], StrictParsley[_]] + private val letMap = mutable.Map.empty[LazyParsley[_], StrictParsley[_]] + private val bodyMap = mutable.Map.empty[backend.Let[_], StrictParsley[_]] /** Is the given parser a let-binding? */ def contains(p: LazyParsley[_]): Boolean = letGen.contains(p) @@ -264,14 +256,28 @@ private [deepembedding] final class LetMap private (letGen: Map[LazyParsley[_], * * @note this does not necessary return a `Let` node, as the underlying parser may be inlined. */ - def apply[A](p: LazyParsley[A]): StrictParsley[A] = mutMap.getOrElseUpdate(p, { - assume(contains(p), "only let-bound parsers can be mapped to a strict version in the let-map") - val sp = letGen(p)(this) - if (sp.inlinable) sp else new backend.Let(sp) - }).asInstanceOf[StrictParsley[A]] + def apply[A](p: LazyParsley[A]): StrictParsley[A] = letMap.get(p) match { + case Some(let) => let.asInstanceOf[StrictParsley[A]] + case None => + assume(contains(p), "only let-bound parsers can be mapped to a strict version in the let-map") + val ourLet = new backend.Let[A] + // MUST be added before processing the body + letMap(p) = ourLet + val sp = letGen(p)(this) + if (sp.inlinable && !recs.contains(p)) { + letMap(p) = sp + sp.asInstanceOf[StrictParsley[A]] + } + else { + bodyMap(ourLet) = sp + ourLet + } + } + + def bodies: Map[backend.Let[_], StrictParsley[_]] = bodyMap.toMap // $COVERAGE-OFF$ - override def toString: String = mutMap.toString + override def toString: String = letMap.toString // $COVERAGE-ON$ } private [frontend] object LetMap { @@ -280,37 +286,10 @@ private [frontend] object LetMap { * @param lets the identified shared non-recursive parsers to include * @param recs the identified recursive parsers that may be required in the translation */ - def apply[M[_, +_]: ContOps](lets: Iterable[LazyParsley[_]])(implicit recs: RecMap): LetMap = { + def apply[M[_, +_]: ContOps](lets: Iterable[LazyParsley[_]], recs: Set[LazyParsley[_]]): LetMap = { new LetMap(lets.map(p => p -> ((_self: LetMap) => { implicit val self: LetMap = _self perform[M, StrictParsley[_]](p.knownLetTopOptimised) - })).toMap) - } -} - -/** Represents the map of lazy recursive parsers to their strict `Rec` node join-points. */ -private [deepembedding] final class RecMap private (map: Map[LazyParsley[_], backend.Rec[_]]) extends Iterable[(LazyParsley[_], backend.Rec[_])] { - /** Is the given parser recursive? */ - def contains(p: LazyParsley[_]): Boolean = map.contains(p) - - /** Returns the `Rec` node that corresponds to a given recursive parser. */ - def apply[A](p: LazyParsley[A]): backend.Rec[A] = map(p).asInstanceOf[backend.Rec[A]] - - /** An iterator over all the key-value pairs in this map */ - override def iterator: Iterator[(LazyParsley[_], backend.Rec[_])] = map.iterator - - // $COVERAGE-OFF$ - override def toString: String = map.toString - // $COVERAGE-ON$ -} -private [frontend] object RecMap { - /** Creates a `RecMap` given all the recursive parsers identified by let-finding. - * This will map each parser to a `Rec` node in the strict combinator tree. - * - * @param recs all of the recursive parsers to fill the map with - * @param state code-generation state, required to generate labels for the `Call` instructions. - */ - def apply(recs: Iterable[LazyParsley[_]])(implicit state: backend.CodeGenState): RecMap = { - new RecMap(recs.map(p => p -> new backend.Rec(new instructions.Call(state.freshLabel()))).toMap) + })).toMap, recs) } } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/PrimitiveEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/PrimitiveEmbedding.scala index 5abff9ba2..989831d3e 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/PrimitiveEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/PrimitiveEmbedding.scala @@ -5,7 +5,7 @@ */ package parsley.internal.deepembedding.frontend -import parsley.debug.Breakpoint +import parsley.debug.{Breakpoint, Profiler} import parsley.errors.ErrorBuilder import parsley.registers.Reg @@ -37,15 +37,34 @@ private [parsley] final class NewReg[S, A](val reg: Reg[S], init: LazyParsley[S] override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(reg, init, body) } +private [parsley] final class Span(p: LazyParsley[_]) extends Unary[Any, String](p) { + override def make(p: StrictParsley[Any]): StrictParsley[String] = new backend.Span(p) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[String] = visitor.visit(this, context)(p) +} + // $COVERAGE-OFF$ -private [parsley] final class Debug[A](p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint) extends Unary[A, A](p) { - override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.Debug(p, name, ascii, break) +private [parsley] final class Debug[A](p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint, watchedRegs: Seq[(Reg[_], String)]) + extends Unary[A, A](p) { + override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.Debug(p, name, ascii, break, watchedRegs) - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, name, ascii, break) + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, name, ascii, break, watchedRegs) } private [parsley] final class DebugError[A](p: LazyParsley[A], name: String, ascii: Boolean, errBuilder: ErrorBuilder[_]) extends Unary[A, A](p) { override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.DebugError(p, name, ascii, errBuilder) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, name, ascii, errBuilder) } + +private [parsley] final class Profile[A](p: LazyParsley[A], name: String, profiler: Profiler) extends Unary[A, A](p) { + override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.Profile(p, name, profiler) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, name, profiler) +} + +private [parsley] final class Opaque[A](p: LazyParsley[A]) extends Unary[A, A](p) { + override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.Opaque(p) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = p.visit(visitor, context) +} // $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SelectiveEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SelectiveEmbedding.scala index 1d222c1d8..ea0d15437 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SelectiveEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SelectiveEmbedding.scala @@ -20,28 +20,16 @@ private [parsley] final class If[A](b: LazyParsley[Boolean], p: =>LazyParsley[A] override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(b, p, q) } -private [parsley] final class Filter[A](p: LazyParsley[A], pred: A => Boolean) extends Unary[A, A](p) { - override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.Filter(p, pred) +private [parsley] final class Filter[A](p: LazyParsley[A], pred: A => Boolean, err: =>LazyParsley[((A, Int)) => Nothing]) + extends Binary[A, ((A, Int)) => Nothing, A](p, err) { + override def make(p: StrictParsley[A], err: StrictParsley[((A, Int)) => Nothing]): StrictParsley[A] = new backend.Filter(p, pred, err) - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, pred) + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, pred, err) } -private [parsley] final class MapFilter[A, B](p: LazyParsley[A], f: A => Option[B]) extends Unary[A, B](p) { - override def make(p: StrictParsley[A]): StrictParsley[B] = new backend.MapFilter(p, f) - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[B] = visitor.visit(this, context)(p, f) -} -private [parsley] final class FilterOut[A](p: LazyParsley[A], pred: PartialFunction[A, String]) extends Unary[A, A](p) { - override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.FilterOut(p, pred) - - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, pred) -} -private [parsley] final class GuardAgainst[A](p: LazyParsley[A], pred: PartialFunction[A, Seq[String]]) extends Unary[A, A](p) { - override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.GuardAgainst(p, pred) - - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, pred) -} -private [parsley] final class UnexpectedWhen[A](p: LazyParsley[A], pred: PartialFunction[A, (String, Option[String])]) extends Unary[A, A](p) { - override def make(p: StrictParsley[A]): StrictParsley[A] = new backend.UnexpectedWhen(p, pred) +private [parsley] final class MapFilter[A, B](p: LazyParsley[A], pred: A => Option[B], err: =>LazyParsley[((A, Int)) => Nothing]) + extends Binary[A, ((A, Int)) => Nothing, B](p, err) { + override def make(p: StrictParsley[A], err: StrictParsley[((A, Int)) => Nothing]): StrictParsley[B] = new backend.MapFilter(p, pred, err) - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(p, pred) + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[B] = visitor.visit(this, context)(p, pred, err) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SequenceEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SequenceEmbedding.scala index 98384347b..92c6e841e 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SequenceEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/SequenceEmbedding.scala @@ -13,7 +13,7 @@ private [parsley] final class <*>[A, B](pf: LazyParsley[A => B], px: =>LazyParsl override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[B] = visitor.visit(this, context)(pf, px) } -private [parsley] final class >>=[A, B](p: LazyParsley[A], private [>>=] val f: A => LazyParsley[B]) extends Unary[A, B](p) { +private [parsley] final class >>=[A, B](p: LazyParsley[A], private val f: A => LazyParsley[B]) extends Unary[A, B](p) { override def make(p: StrictParsley[A]): StrictParsley[B] = new backend.>>=(p, f) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[B] = visitor.visit(this, context)(p, f) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/Visitors.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/Visitors.scala index 6587397d5..007a0ca0e 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/Visitors.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/frontend/Visitors.scala @@ -5,7 +5,7 @@ */ package parsley.internal.deepembedding.frontend -import parsley.debug.Breakpoint +import parsley.debug.{Breakpoint, Profiler} import parsley.errors.ErrorBuilder import parsley.registers.Reg import parsley.token.descriptions.SpaceDesc @@ -32,7 +32,7 @@ import parsley.internal.errors.CaretWidth private [parsley] abstract class LazyParsleyIVisitor[-T, +U[+_]] { // scalastyle:ignore number.of.methods // Singleton parser visitors. def visit[A](self: Pure[A], context: T)(x: A): U[A] - def visit[A](self: Fresh[A], context: T)(x: => A): U[A] + def visit[A](self: Fresh[A], context: T)(x: =>A): U[A] def visit(self: Satisfy, context: T)(f: Char => Boolean, expected: LabelConfig): U[Char] def visit(self: Line.type, context: T): U[Int] def visit(self: Col.type, context: T): U[Int] @@ -47,15 +47,17 @@ private [parsley] abstract class LazyParsleyIVisitor[-T, +U[+_]] { // scalastyle start: Char => Boolean, letter: Char => Boolean, illegal: String => Boolean): U[String] - def visit(self: CharTok, context: T)(c: Char, exp: LabelConfig): U[Char] - def visit(self: SupplementaryCharTok, context: T)(codepoint: Int, exp: LabelConfig): U[Int] - def visit(self: StringTok, context: T)(s: String, exp: LabelConfig): U[String] + def visit[A](self: CharTok[A], context: T)(c: Char, x: A, exp: LabelConfig): U[A] + def visit[A](self: SupplementaryCharTok[A], context: T)(codepoint: Int, x: A, exp: LabelConfig): U[A] + def visit[A](self: StringTok[A], context: T)(s: String, x: A, exp: LabelConfig): U[A] def visit(self: Eof.type, context: T): U[Unit] def visit(self: UniSatisfy, context: T)(f: Int => Boolean, exp: LabelConfig): U[Int] def visit[S](self: Modify[S], context: T)(reg: Reg[S], f: S => S): U[Unit] def visit(self: Empty, context: T)(width: Int): U[Nothing] def visit(self: Fail, context: T)(width: CaretWidth, msgs: Seq[String]): U[Nothing] def visit(self: Unexpected, context: T)(msg: String, width: CaretWidth): U[Nothing] + def visit[A](self: VanillaGen[A], context: T)(gen: parsley.errors.VanillaGen[A]): U[((A, Int)) => Nothing] + def visit[A](self: SpecialisedGen[A], context: T)(gen: parsley.errors.SpecialisedGen[A]): U[((A, Int)) => Nothing] def visit(self: EscapeMapped, context: T)(escTrie: Trie[Int], escs: Set[String]): U[Int] def visit(self: EscapeAtMost, context: T)(n: Int, radix: Int): U[BigInt] def visit(self: EscapeOneOfExactly, context: T)(radix: Int, ns: List[Int], ie: SpecialisedFilterConfig[Int]): U[BigInt] @@ -71,52 +73,50 @@ private [parsley] abstract class LazyParsleyIVisitor[-T, +U[+_]] { // scalastyle def visit[A](self: Look[A], context: T)(p: LazyParsley[A]): U[A] def visit[A](self: NotFollowedBy[A], context: T)(p: LazyParsley[A]): U[Unit] def visit[S](self: Put[S], context: T)(reg: Reg[S], p: LazyParsley[S]): U[Unit] - def visit[S, A](self: NewReg[S, A], context: T)(reg: Reg[S], init: LazyParsley[S], body: => LazyParsley[A]): U[A] - def visit[A](self: Debug[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint): U[A] + def visit[S, A](self: NewReg[S, A], context: T)(reg: Reg[S], init: LazyParsley[S], body: =>LazyParsley[A]): U[A] + def visit(self: Span, context: T)(p: LazyParsley[_]): U[String] + def visit[A](self: Debug[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint, watchedRegs: Seq[(Reg[_], String)]): U[A] def visit[A](self: DebugError[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, errBuilder: ErrorBuilder[_]): U[A] + def visit[A](self: Profile[A], context: T)(p: LazyParsley[A], name: String, profiler: Profiler): U[A] // Selective parser visitors. - def visit[A, B, C](self: Branch[A, B, C], context: T)(b: LazyParsley[Either[A, B]], p: => LazyParsley[A => C], q: => LazyParsley[B => C]): U[C] - def visit[A](self: If[A], context: T)(b: LazyParsley[Boolean], p: => LazyParsley[A], q: => LazyParsley[A]): U[A] - def visit[A](self: Filter[A], context: T)(p: LazyParsley[A], pred: A => Boolean): U[A] - def visit[A, B](self: MapFilter[A, B], context: T)(p: LazyParsley[A], f: A => Option[B]): U[B] - def visit[A](self: FilterOut[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, String]): U[A] - def visit[A](self: GuardAgainst[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, Seq[String]]): U[A] - def visit[A](self: UnexpectedWhen[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, (String, Option[String])]): U[A] + def visit[A, B, C](self: Branch[A, B, C], context: T)(b: LazyParsley[Either[A, B]], p: =>LazyParsley[A => C], q: =>LazyParsley[B => C]): U[C] + def visit[A](self: If[A], context: T)(b: LazyParsley[Boolean], p: =>LazyParsley[A], q: =>LazyParsley[A]): U[A] + def visit[A](self: Filter[A], context: T)(p: LazyParsley[A], pred: A => Boolean, err: =>LazyParsley[((A, Int)) => Nothing]): U[A] + def visit[A, B](self: MapFilter[A, B], context: T)(p: LazyParsley[A], pred: A => Option[B], err: =>LazyParsley[((A, Int)) => Nothing]): U[B] // Alternative parser visitors. def visit[A](self: <|>[A])(context: T, p: LazyParsley[A], q: LazyParsley[A]): U[A] // Intrinsic parser visitors. - def visit[A, B, C](self: Lift2[A, B, C], context: T)(f: (A, B) => C, p: LazyParsley[A], q: => LazyParsley[B]): U[C] - def visit[A, B, C, D](self: Lift3[A, B, C, D], context: T)(f: (A, B, C) => D, p: LazyParsley[A], q: => LazyParsley[B], r: => LazyParsley[C]): U[D] - def visit[S, A](self: Local[S, A], context: T)(reg: Reg[S], p: LazyParsley[S], q: => LazyParsley[A]): U[A] + def visit[A, B, C](self: Lift2[A, B, C], context: T)(f: (A, B) => C, p: LazyParsley[A], q: =>LazyParsley[B]): U[C] + def visit[A, B, C, D](self: Lift3[A, B, C, D], context: T)(f: (A, B, C) => D, p: LazyParsley[A], q: =>LazyParsley[B], r: =>LazyParsley[C]): U[D] + def visit[S, A](self: Local[S, A], context: T)(reg: Reg[S], p: LazyParsley[S], q: =>LazyParsley[A]): U[A] // Sequence parser visitors. - def visit[A, B](self: A <*> B, context: T)(pf: LazyParsley[A => B], px: => LazyParsley[A]): U[B] + def visit[A, B](self: A <*> B, context: T)(pf: LazyParsley[A => B], px: =>LazyParsley[A]): U[B] def visit[A, B](self: A >>= B, context: T)(p: LazyParsley[A], f: A => LazyParsley[B]): U[B] - def visit[A](self: *>[A], context: T)(p: LazyParsley[_], _q: => LazyParsley[A]): U[A] - def visit[A](self: <*[A], context: T)(p: LazyParsley[A], _q: => LazyParsley[_]): U[A] + def visit[A](self: *>[A], context: T)(p: LazyParsley[_], _q: =>LazyParsley[A]): U[A] + def visit[A](self: <*[A], context: T)(p: LazyParsley[A], _q: =>LazyParsley[_]): U[A] // Iterative parser visitors. def visit[A](self: Many[A], context: T)(p: LazyParsley[A]): U[List[A]] - def visit[A](self: SkipMany[A], context: T)(p: LazyParsley[A]): U[Unit] - def visit[A](self: ChainPost[A], context: T)(p: LazyParsley[A], _op: => LazyParsley[A => A]): U[A] - def visit[A](self: ChainPre[A], context: T)(p: LazyParsley[A], op: => LazyParsley[A => A]): U[A] - def visit[A, B](self: Chainl[A, B], context: T)(init: LazyParsley[B], p: => LazyParsley[A], op: => LazyParsley[(B, A) => B]): U[B] - def visit[A, B](self: Chainr[A, B], context: T)(p: LazyParsley[A], op: => LazyParsley[(A, B) => B], wrap: A => B): U[B] - def visit[A, B](self: SepEndBy1[A, B], context: T)(p: LazyParsley[A], sep: => LazyParsley[B]): U[List[A]] + def visit[A](self: ChainPost[A], context: T)(p: LazyParsley[A], _op: =>LazyParsley[A => A]): U[A] + def visit[A](self: ChainPre[A], context: T)(p: LazyParsley[A], op: =>LazyParsley[A => A]): U[A] + def visit[A, B](self: Chainl[A, B], context: T)(init: LazyParsley[B], p: =>LazyParsley[A], op: =>LazyParsley[(B, A) => B]): U[B] + def visit[A, B](self: Chainr[A, B], context: T)(p: LazyParsley[A], op: =>LazyParsley[(A, B) => B], wrap: A => B): U[B] + def visit[A](self: SepEndBy1[A], context: T)(p: LazyParsley[A], sep: =>LazyParsley[_]): U[List[A]] def visit[A](self: ManyUntil[A], context: T)(body: LazyParsley[Any]): U[List[A]] def visit(self: SkipManyUntil, context: T)(body: LazyParsley[Any]): U[Unit] // Error parser visitors. def visit[A](self: ErrorLabel[A], context: T)(p: LazyParsley[A], labels: Seq[String]): U[A] + def visit[A](self: ErrorHide[A], context: T)(p: LazyParsley[A]): U[A] def visit[A](self: ErrorExplain[A], context: T)(p: LazyParsley[A], reason: String): U[A] def visit[A](self: ErrorAmend[A], context: T)(p: LazyParsley[A], partial: Boolean): U[A] def visit[A](self: ErrorEntrench[A], context: T)(p: LazyParsley[A]): U[A] def visit[A](self: ErrorDislodge[A], context: T)(n: Int, p: LazyParsley[A]): U[A] def visit[A](self: ErrorLexical[A], context: T)(p: LazyParsley[A]): U[A] - def visit[A](self: VerifiedError[A], context: T)(p: LazyParsley[A], msggen: Either[A => Seq[String], Option[A => String]]): U[Nothing] } /** Generalised version of [[LazyParsleyIVisitor]] that allows you to define default implementations @@ -133,159 +133,124 @@ private [frontend] abstract class GenericLazyParsleyIVisitor[-T, +U[+_]] extends // XXX: These names are different as otherwise some visit methods recurse in an unwanted manner. def visitSingleton[A](self: Singleton[A], context: T): U[A] def visitUnary[A, B](self: Unary[A, B], context: T)(p: LazyParsley[A]): U[B] - def visitBinary[A, B, C](self: Binary[A, B, C], context: T)(l: LazyParsley[A], r: => LazyParsley[B]): U[C] - def visitTernary[A, B, C, D](self: Ternary[A, B, C, D], context: T)(f: LazyParsley[A], s: => LazyParsley[B], t: => LazyParsley[C]): U[D] + def visitBinary[A, B, C](self: Binary[A, B, C], context: T)(l: LazyParsley[A], r: =>LazyParsley[B]): U[C] + def visitTernary[A, B, C, D](self: Ternary[A, B, C, D], context: T)(f: LazyParsley[A], s: =>LazyParsley[B], t: =>LazyParsley[C]): U[D] // Singleton overrides. - override def visit[A](self: Pure[A], context: T)(x: A): U[A] = - visitSingleton(self, context) - override def visit[A](self: Fresh[A], context: T)(x: => A): U[A] = - visitSingleton(self, context) - override def visit(self: Satisfy, context: T)(f: Char => Boolean, expected: LabelConfig): U[Char] = - visitSingleton(self, context) - override def visit(self: Line.type, context: T): U[Int] = - visitSingleton(self, context) - override def visit(self: Col.type, context: T): U[Int] = - visitSingleton(self, context) - override def visit(self: Offset.type, context: T): U[Int] = - visitSingleton(self, context) - override def visit[S](self: Get[S], context: T)(reg: Reg[S]): U[S] = - visitSingleton(self, context) - override def visit(self: WhiteSpace, context: T)(ws: Char => Boolean, desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = - visitSingleton(self, context) - override def visit(self: SkipComments, context: T)(desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = - visitSingleton(self, context) - override def visit(self: Comment, context: T)(desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = - visitSingleton(self, context) - override def visit[A](self: Sign[A], context: T)(ty: SignType, signPresence: PlusSignPresence): U[A => A] = - visitSingleton(self, context) + override def visit[A](self: Pure[A], context: T)(x: A): U[A] = visitSingleton(self, context) + override def visit[A](self: Fresh[A], context: T)(x: =>A): U[A] = visitSingleton(self, context) + override def visit(self: Satisfy, context: T)(f: Char => Boolean, expected: LabelConfig): U[Char] = visitSingleton(self, context) + override def visit(self: Line.type, context: T): U[Int] = visitSingleton(self, context) + override def visit(self: Col.type, context: T): U[Int] = visitSingleton(self, context) + override def visit(self: Offset.type, context: T): U[Int] = visitSingleton(self, context) + override def visit[S](self: Get[S], context: T)(reg: Reg[S]): U[S] = visitSingleton(self, context) + override def visit(self: WhiteSpace, context: T)(ws: Char => Boolean, desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = visitSingleton(self, context) + override def visit(self: SkipComments, context: T)(desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = visitSingleton(self, context) + override def visit(self: Comment, context: T)(desc: SpaceDesc, errorConfig: ErrorConfig): U[Unit] = visitSingleton(self, context) + override def visit[A](self: Sign[A], context: T)(ty: SignType, signPresence: PlusSignPresence): U[A => A] = visitSingleton(self, context) override def visit(self: NonSpecific, context: T)(name: String, ue: String => String, start: Char => Boolean, letter: Char => Boolean, - illegal: String => Boolean): U[String] = - visitSingleton(self, context) - override def visit(self: CharTok, context: T)(c: Char, exp: LabelConfig): U[Char] = - visitSingleton(self, context) - override def visit(self: SupplementaryCharTok, context: T)(codepoint: Int, exp: LabelConfig): U[Int] = - visitSingleton(self, context) - override def visit(self: StringTok, context: T)(s: String, exp: LabelConfig): U[String] = - visitSingleton(self, context) - override def visit(self: Eof.type, context: T): U[Unit] = - visitSingleton(self, context) - override def visit(self: UniSatisfy, context: T)(f: Int => Boolean, exp: LabelConfig): U[Int] = - visitSingleton(self, context) - override def visit[S](self: Modify[S], context: T)(reg: Reg[S], f: S => S): U[Unit] = - visitSingleton(self, context) - override def visit(self: Empty, context: T)(width: Int): U[Nothing] = - visitSingleton(self, context) - override def visit(self: Fail, context: T)(width: CaretWidth, msgs: Seq[String]): U[Nothing] = - visitSingleton(self, context) - override def visit(self: Unexpected, context: T)(msg: String, width: CaretWidth): U[Nothing] = - visitSingleton(self, context) - override def visit(self: EscapeMapped, context: T)(escTrie: Trie[Int], escs: Set[String]): U[Int] = - visitSingleton(self, context) - override def visit(self: EscapeAtMost, context: T)(n: Int, radix: Int): U[BigInt] = + illegal: String => Boolean): U[String] = visitSingleton(self, context) + override def visit[A](self: CharTok[A], context: T)(c: Char, x: A, exp: LabelConfig): U[A] = visitSingleton(self, context) + override def visit[A](self: SupplementaryCharTok[A], context: T)(codepoint: Int, x: A, exp: LabelConfig): U[A] = visitSingleton(self, context) + override def visit[A](self: StringTok[A], context: T)(s: String, x: A, exp: LabelConfig): U[A] = visitSingleton(self, context) + override def visit(self: Eof.type, context: T): U[Unit] = visitSingleton(self, context) + override def visit(self: UniSatisfy, context: T)(f: Int => Boolean, exp: LabelConfig): U[Int] = visitSingleton(self, context) + override def visit[S](self: Modify[S], context: T)(reg: Reg[S], f: S => S): U[Unit] = visitSingleton(self, context) + override def visit(self: Empty, context: T)(width: Int): U[Nothing] = visitSingleton(self, context) + override def visit(self: Fail, context: T)(width: CaretWidth, msgs: Seq[String]): U[Nothing] = visitSingleton(self, context) + override def visit(self: Unexpected, context: T)(msg: String, width: CaretWidth): U[Nothing] = visitSingleton(self, context) + override def visit[A](self: VanillaGen[A], context: T)(gen: parsley.errors.VanillaGen[A]): U[((A, Int)) => Nothing] = visitSingleton(self, context) + override def visit[A](self: SpecialisedGen[A], context: T)(gen: parsley.errors.SpecialisedGen[A]): U[((A, Int)) => Nothing] = { visitSingleton(self, context) - override def visit(self: EscapeOneOfExactly, context: T)(radix: Int, ns: List[Int], ie: SpecialisedFilterConfig[Int]): U[BigInt] = + } + override def visit(self: EscapeMapped, context: T)(escTrie: Trie[Int], escs: Set[String]): U[Int] = visitSingleton(self, context) + override def visit(self: EscapeAtMost, context: T)(n: Int, radix: Int): U[BigInt] = visitSingleton(self, context) + override def visit(self: EscapeOneOfExactly, context: T)(radix: Int, ns: List[Int], ie: SpecialisedFilterConfig[Int]): U[BigInt] = { visitSingleton(self, context) + } override def visit(self: SoftKeyword, context: T)(specific: String, letter: CharPredicate, caseSensitive: Boolean, expected: LabelConfig, - expectedEnd: String): U[Unit] = - visitSingleton(self, context) + expectedEnd: String): U[Unit] = visitSingleton(self, context) override def visit(self: SoftOperator, context: T)(specific: String, letter: CharPredicate, ops: Trie[Unit], expected: LabelConfig, - expectedEnd: String): U[Unit] = - visitSingleton(self, context) + expectedEnd: String): U[Unit] = visitSingleton(self, context) // Primitive overrides. - override def visit[A](self: Attempt[A], context: T)(p: LazyParsley[A]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: Look[A], context: T)(p: LazyParsley[A]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: NotFollowedBy[A], context: T)(p: LazyParsley[A]): U[Unit] = - visitUnary(self, context)(p) - override def visit[S](self: Put[S], context: T)(reg: Reg[S], p: LazyParsley[S]): U[Unit] = - visitUnary(self, context)(p) - override def visit[S, A](self: NewReg[S, A], context: T)(reg: Reg[S], init: LazyParsley[S], body: => LazyParsley[A]): U[A] = + override def visit[A](self: Attempt[A], context: T)(p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: Look[A], context: T)(p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: NotFollowedBy[A], context: T)(p: LazyParsley[A]): U[Unit] = visitUnary(self, context)(p) + override def visit[S](self: Put[S], context: T)(reg: Reg[S], p: LazyParsley[S]): U[Unit] = visitUnary(self, context)(p) + override def visit[S, A](self: NewReg[S, A], context: T)(reg: Reg[S], init: LazyParsley[S], body: =>LazyParsley[A]): U[A] = { visitBinary(self, context)(init, body) - override def visit[A](self: Debug[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint): U[A] = + } + override def visit(self: Span, context: T)(p: LazyParsley[_]): U[String] = visitUnary[Any, String](self, context)(p) + override def visit[A](self: Debug[A], context: T) + (p: LazyParsley[A], name: String, ascii: Boolean, break: Breakpoint, watchedRegs: Seq[(Reg[_], String)]): U[A] = { visitUnary(self, context)(p) - override def visit[A](self: DebugError[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, errBuilder: ErrorBuilder[_]): U[A] = + } + override def visit[A](self: DebugError[A], context: T)(p: LazyParsley[A], name: String, ascii: Boolean, errBuilder: ErrorBuilder[_]): U[A] = { visitUnary(self, context)(p) + } + override def visit[A](self: Profile[A], context: T)(p: LazyParsley[A], name: String, profiler: Profiler): U[A] = visitUnary(self, context)(p) // Selective overrides. - override def visit[A, B, C](self: Branch[A, B, C], context: T)(b: LazyParsley[Either[A, B]], p: => LazyParsley[A => C], q: => LazyParsley[B => C]): U[C] = + override def visit[A, B, C](self: Branch[A, B, C], context: T)(b: LazyParsley[Either[A, B]], p: =>LazyParsley[A => C], q: =>LazyParsley[B => C]): U[C] = { visitTernary(self, context)(b, p, q) - override def visit[A](self: If[A], context: T)(b: LazyParsley[Boolean], p: => LazyParsley[A], q: => LazyParsley[A]): U[A] = + } + override def visit[A](self: If[A], context: T)(b: LazyParsley[Boolean], p: =>LazyParsley[A], q: =>LazyParsley[A]): U[A] = { visitTernary(self, context)(b, p, q) - override def visit[A](self: Filter[A], context: T)(p: LazyParsley[A], pred: A => Boolean): U[A] = - visitUnary(self, context)(p) - override def visit[A, B](self: MapFilter[A, B], context: T)(p: LazyParsley[A], f: A => Option[B]): U[B] = - visitUnary(self, context)(p) - override def visit[A](self: FilterOut[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, String]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: GuardAgainst[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, Seq[String]]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: UnexpectedWhen[A], context: T)(p: LazyParsley[A], pred: PartialFunction[A, (String, Option[String])]): U[A] = - visitUnary(self, context)(p) + } + override def visit[A](self: Filter[A], context: T)(p: LazyParsley[A], pred: A => Boolean, err: =>LazyParsley[((A, Int)) => Nothing]): U[A] = { + visitBinary(self, context)(p, err) + } + override def visit[A, B](self: MapFilter[A, B], context: T)(p: LazyParsley[A], pred: A => Option[B], err: =>LazyParsley[((A, Int)) => Nothing]): U[B] = { + visitBinary(self, context)(p, err) + } // Intrinsic overrides. - override def visit[A, B, C](self: Lift2[A, B, C], context: T)(f: (A, B) => C, p: LazyParsley[A], q: => LazyParsley[B]): U[C] = + override def visit[A, B, C](self: Lift2[A, B, C], context: T)(f: (A, B) => C, p: LazyParsley[A], q: =>LazyParsley[B]): U[C] = { visitBinary(self, context)(p, q) + } override def visit[A, B, C, D](self: Lift3[A, B, C, D], context: T)(f: (A, B, C) => D, p: LazyParsley[A], - q: => LazyParsley[B], - r: => LazyParsley[C]): U[D] = - visitTernary(self, context)(p, q, r) - override def visit[S, A](self: Local[S, A], context: T)(reg: Reg[S], p: LazyParsley[S], q: => LazyParsley[A]): U[A] = - visitBinary(self, context)(p, q) + q: =>LazyParsley[B], + r: =>LazyParsley[C]): U[D] = visitTernary(self, context)(p, q, r) + override def visit[S, A](self: Local[S, A], context: T)(reg: Reg[S], p: LazyParsley[S], q: =>LazyParsley[A]): U[A] = visitBinary(self, context)(p, q) // Sequence overrides. - override def visit[A, B](self: A <*> B, context: T)(pf: LazyParsley[A => B], px: => LazyParsley[A]): U[B] = - visitBinary(self, context)(pf, px) - override def visit[A, B](self: A >>= B, context: T)(p: LazyParsley[A], f: A => LazyParsley[B]): U[B] = - visitUnary(self, context)(p) - override def visit[A](self: *>[A], context: T)(p: LazyParsley[_], _q: => LazyParsley[A]): U[A] = - visitBinary[Any, A, A](self, context)(p, _q) - override def visit[A](self: <*[A], context: T)(p: LazyParsley[A], _q: => LazyParsley[_]): U[A] = - visitBinary[A, Any, A](self, context)(p, _q) + override def visit[A, B](self: A <*> B, context: T)(pf: LazyParsley[A => B], px: =>LazyParsley[A]): U[B] = visitBinary(self, context)(pf, px) + override def visit[A, B](self: A >>= B, context: T)(p: LazyParsley[A], f: A => LazyParsley[B]): U[B] = visitUnary(self, context)(p) + override def visit[A](self: *>[A], context: T)(p: LazyParsley[_], _q: =>LazyParsley[A]): U[A] = visitBinary[Any, A, A](self, context)(p, _q) + override def visit[A](self: <*[A], context: T)(p: LazyParsley[A], _q: =>LazyParsley[_]): U[A] = visitBinary[A, Any, A](self, context)(p, _q) // Iterative overrides. - override def visit[A](self: Many[A], context: T)(p: LazyParsley[A]): U[List[A]] = - visitUnary(self, context)(p) - override def visit[A](self: SkipMany[A], context: T)(p: LazyParsley[A]): U[Unit] = - visitUnary(self, context)(p) - override def visit[A](self: ChainPost[A], context: T)(p: LazyParsley[A], _op: => LazyParsley[A => A]): U[A] = - visitBinary(self, context)(p, _op) - override def visit[A, B](self: Chainl[A, B], context: T)(init: LazyParsley[B], p: => LazyParsley[A], op: => LazyParsley[(B, A) => B]): U[B] = + override def visit[A](self: Many[A], context: T)(p: LazyParsley[A]): U[List[A]] = visitUnary(self, context)(p) + override def visit[A](self: ChainPost[A], context: T)(p: LazyParsley[A], _op: =>LazyParsley[A => A]): U[A] = visitBinary(self, context)(p, _op) + override def visit[A, B](self: Chainl[A, B], context: T)(init: LazyParsley[B], p: =>LazyParsley[A], op: =>LazyParsley[(B, A) => B]): U[B] = { visitTernary(self, context)(init, p, op) - override def visit[A, B](self: Chainr[A, B], context: T)(p: LazyParsley[A], op: => LazyParsley[(A, B) => B], wrap: A => B): U[B] = + } + override def visit[A, B](self: Chainr[A, B], context: T)(p: LazyParsley[A], op: =>LazyParsley[(A, B) => B], wrap: A => B): U[B] = { visitBinary(self, context)(p, op) - override def visit[A, B](self: SepEndBy1[A, B], context: T)(p: LazyParsley[A], sep: => LazyParsley[B]): U[List[A]] = - visitBinary(self, context)(p, sep) - override def visit[A](self: ManyUntil[A], context: T)(body: LazyParsley[Any]): U[List[A]] = - visitUnary[Any, List[A]](self, context)(body) - override def visit(self: SkipManyUntil, context: T)(body: LazyParsley[Any]): U[Unit] = - visitUnary[Any, Unit](self, context)(body) + } + override def visit[A](self: SepEndBy1[A], context: T)(p: LazyParsley[A], sep: =>LazyParsley[_]): U[List[A]] = { + visitBinary[A, Any, List[A]](self, context)(p, sep) + } + override def visit[A](self: ManyUntil[A], context: T)(body: LazyParsley[Any]): U[List[A]] = visitUnary[Any, List[A]](self, context)(body) + override def visit(self: SkipManyUntil, context: T)(body: LazyParsley[Any]): U[Unit] = visitUnary[Any, Unit](self, context)(body) // Error overrides. - override def visit[A](self: ErrorLabel[A], context: T)(p: LazyParsley[A], labels: Seq[String]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: ErrorExplain[A], context: T)(p: LazyParsley[A], reason: String): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: ErrorAmend[A], context: T)(p: LazyParsley[A], partial: Boolean): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: ErrorEntrench[A], context: T)(p: LazyParsley[A]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: ErrorDislodge[A], context: T)(n: Int, p: LazyParsley[A]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: ErrorLexical[A], context: T)(p: LazyParsley[A]): U[A] = - visitUnary(self, context)(p) - override def visit[A](self: VerifiedError[A], context: T)(p: LazyParsley[A], msggen: Either[A => Seq[String], Option[A => String]]): U[Nothing] = - visitUnary(self, context)(p) + override def visit[A](self: ErrorLabel[A], context: T)(p: LazyParsley[A], labels: Seq[String]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorHide[A], context: T)(p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorExplain[A], context: T)(p: LazyParsley[A], reason: String): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorAmend[A], context: T)(p: LazyParsley[A], partial: Boolean): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorEntrench[A], context: T)(p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorDislodge[A], context: T)(n: Int, p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) + override def visit[A](self: ErrorLexical[A], context: T)(p: LazyParsley[A]): U[A] = visitUnary(self, context)(p) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/ErrorEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/ErrorEmbedding.scala index ffbf8b380..8af0e9dc7 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/ErrorEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/ErrorEmbedding.scala @@ -6,6 +6,7 @@ package parsley.internal.deepembedding.singletons import parsley.internal.deepembedding.backend.MZero +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.errors.CaretWidth import parsley.internal.machine.instructions @@ -15,7 +16,7 @@ private [parsley] final class Empty private (val width: Int) extends Singleton[N // $COVERAGE-OFF$ override val pretty: String = "empty" // $COVERAGE-ON$ - override val instr: instructions.Instr = new instructions.Empty(width) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = instrs += new instructions.Empty(width) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Nothing] = visitor.visit(this, context)(width) } @@ -24,7 +25,7 @@ private [parsley] final class Fail(width: CaretWidth, msgs: String*) extends Sin // $COVERAGE-OFF$ override def pretty: String = s"fail(${msgs.mkString(", ")})" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Fail(width, msgs: _*) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = instrs += new instructions.Fail(width, msgs: _*) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Nothing] = visitor.visit(this, context)(width, msgs) } @@ -33,11 +34,32 @@ private [parsley] final class Unexpected(msg: String, width: CaretWidth) extends // $COVERAGE-OFF$ override def pretty: String = s"unexpected($msg)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Unexpected(msg, width) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = instrs += new instructions.Unexpected(msg, width) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Nothing] = visitor.visit(this, context)(msg, width) } +// From the thesis +private [parsley] final class VanillaGen[A](gen: parsley.errors.VanillaGen[A]) extends Singleton[((A, Int)) => Nothing] { + // $COVERAGE-OFF$ + override def pretty: String = "VanillaGen" + // $COVERAGE-ON$ + + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = instrs += new instructions.VanillaGen(gen) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[((A, Int)) => Nothing] = visitor.visit(this, context)(gen) +} + +private [parsley] final class SpecialisedGen[A](gen: parsley.errors.SpecialisedGen[A]) extends Singleton[((A, Int)) => Nothing] { + // $COVERAGE-OFF$ + override def pretty: String = "SpecialisedGen" + // $COVERAGE-ON$ + + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = instrs += new instructions.SpecialisedGen(gen) + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[((A, Int)) => Nothing] = visitor.visit(this, context)(gen) +} + private [parsley] object Empty { val Zero = new Empty(0) def apply(width: Int): Empty = if (width == 0) Zero else new Empty(width) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/IntrinsicEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/IntrinsicEmbedding.scala index 0a2b3cf22..479ce6b39 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/IntrinsicEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/IntrinsicEmbedding.scala @@ -8,41 +8,56 @@ package parsley.internal.deepembedding.singletons import parsley.registers.Reg import parsley.token.errors.LabelConfig +import parsley.internal.deepembedding.backend.StrictParsley, StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.{LazyParsleyIVisitor, UsesRegister} import parsley.internal.machine.instructions -private [parsley] final class CharTok(private [CharTok] val c: Char, val expected: LabelConfig) extends Singleton[Char] { +private [parsley] final class CharTok[A](private val c: Char, private val x: A, val expected: LabelConfig) extends Singleton[A] { // $COVERAGE-OFF$ - override def pretty: String = s"char($c)" + override def pretty: String = s"char($c).as($x)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.CharTok(c, expected) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.CharTok(c, expected) + if (producesResults) instrs += new instructions.Push(x) + } - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Char] = visitor.visit(this, context)(c, expected) + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(c, x, expected) } -private [parsley] final class SupplementaryCharTok(private [SupplementaryCharTok] val codepoint: Int, val expected: LabelConfig) extends Singleton[Int] { +private [parsley] final class SupplementaryCharTok[A](private val codepoint: Int, private val x: A, val expected: LabelConfig) extends Singleton[A] { // $COVERAGE-OFF$ - override def pretty: String = s"char(${Character.toChars(codepoint).mkString})" + override def pretty: String = s"char(${Character.toChars(codepoint).mkString}).as($x)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.SupplementaryCharTok(codepoint, expected) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.SupplementaryCharTok(codepoint, expected) + if (producesResults) instrs += new instructions.Push(x) + } - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context)(codepoint, expected) + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(codepoint, x, expected) } -private [parsley] final class StringTok(private [StringTok] val s: String, val expected: LabelConfig) extends Singleton[String] { +private [parsley] final class StringTok[A](private val s: String, private val x: A, val expected: LabelConfig) extends Singleton[A] { // $COVERAGE-OFF$ - override def pretty: String = s"string($s)" + override def pretty: String = s"string($s).as($x)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.StringTok(s, expected) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.StringTok(s, expected) + if (producesResults) instrs += new instructions.Push(x) + } - override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[String] = visitor.visit(this, context)(s, expected) + override protected[deepembedding] def optimise: StrictParsley[A] = if (s.length == 1) new CharTok(s.head, x, expected) else this + + override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(s, x, expected) } private [parsley] object Eof extends Singleton[Unit] { // $COVERAGE-OFF$ - override val pretty: String = "eof" + override def pretty: String = "eof" // $COVERAGE-ON$ - override val instr: instructions.Instr = instructions.Eof + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += instructions.Eof + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context) } @@ -51,7 +66,10 @@ private [parsley] final class UniSatisfy(private [UniSatisfy] val f: Int => Bool // $COVERAGE-OFF$ override def pretty: String = "satisfyUnicode(?)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.UniSat(f, expected) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.UniSat(f, expected) + if (!producesResults) instrs += instructions.Pop + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context)(f, expected) } @@ -60,20 +78,23 @@ private [parsley] final class Modify[S](val reg: Reg[S], f: S => S) extends Sing // $COVERAGE-OFF$ override def pretty: String = s"modify($reg, ?)" // $COVERAGE-ON$ - override def instr: instructions.Instr = instructions.Modify(reg.addr, f) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += instructions.Modify(reg.addr, f) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context)(reg, f) } private [deepembedding] object CharTok { - def unapply(self: CharTok): Option[Char] = Some(self.c) + def unapply[A](self: CharTok[A]): Some[(Char, A)] = Some((self.c, self.x)) } private [deepembedding] object SupplementaryCharTok { - def unapply(self: SupplementaryCharTok): Option[Int] = Some(self.codepoint) + def unapply[A](self: SupplementaryCharTok[A]): Some[(Int, A)] = Some((self.codepoint, self.x)) } private [deepembedding] object StringTok { - def unapply(self: StringTok): Option[String] = Some(self.s) + def unapply[A](self: StringTok[A]): Some[(String, A)] = Some((self.s, self.x)) } private [deepembedding] object UniSatisfy { - def unapply(self: UniSatisfy): Option[Int => Boolean] = Some(self.f) + def unapply(self: UniSatisfy): Some[Int => Boolean] = Some(self.f) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/PrimitiveEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/PrimitiveEmbedding.scala index 49f37a0ae..6ef45d219 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/PrimitiveEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/PrimitiveEmbedding.scala @@ -8,14 +8,18 @@ package parsley.internal.deepembedding.singletons import parsley.registers.Reg import parsley.token.errors.LabelConfig +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.machine.instructions -private [parsley] final class Satisfy(private [Satisfy] val f: Char => Boolean, val expected: LabelConfig) extends Singleton[Char] { +private [parsley] final class Satisfy(private val f: Char => Boolean, val expected: LabelConfig) extends Singleton[Char] { // $COVERAGE-OFF$ override val pretty: String = "satisfy(f)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Satisfies(f, expected) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.Satisfies(f, expected) + if (!producesResults) instrs += instructions.Pop + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Char] = visitor.visit(this, context)(f, expected) } @@ -24,7 +28,7 @@ private [parsley] object Line extends Singleton[Int] { // $COVERAGE-OFF$ override val pretty: String = "line" // $COVERAGE-ON$ - override val instr: instructions.Instr = instructions.Line + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += instructions.Line override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context) } @@ -32,7 +36,7 @@ private [parsley] object Col extends Singleton[Int] { // $COVERAGE-OFF$ override val pretty: String = "col" // $COVERAGE-ON$ - override val instr: instructions.Instr = instructions.Col + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += instructions.Col override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context) } @@ -40,7 +44,7 @@ private [parsley] object Offset extends Singleton[Int] { // $COVERAGE-OFF$ override val pretty: String = "offset" // $COVERAGE-ON$ - override val instr: instructions.Instr = instructions.Offset + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += instructions.Offset override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context) } @@ -51,7 +55,7 @@ private [parsley] final class Get[S](reg: Reg[S]) extends Singleton[S] { // $COVERAGE-OFF$ override def pretty: String = s"get($reg)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Get(reg.addr) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += new instructions.Get(reg.addr) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[S] = visitor.visit(this, context)(reg) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/SequenceEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/SequenceEmbedding.scala index 07e90cfac..dd1c72625 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/SequenceEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/SequenceEmbedding.scala @@ -5,15 +5,16 @@ */ package parsley.internal.deepembedding.singletons +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.machine.instructions // Core Embedding -private [parsley] final class Pure[A](private [Pure] val x: A) extends Singleton[A] { +private [parsley] final class Pure[A](private val x: A) extends Singleton[A] { // $COVERAGE-OFF$ override def pretty: String = s"pure($x)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Push(x) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += new instructions.Push(x) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(x) } @@ -22,7 +23,7 @@ private [parsley] final class Fresh[A](x: =>A) extends Singleton[A] { // $COVERAGE-OFF$ override def pretty: String = s"fresh($x)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.Fresh(x) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = if (producesResults) instrs += new instructions.Fresh(x) override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A] = visitor.visit(this, context)(x) } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/Singletons.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/Singletons.scala index d40cc2a17..a17f00e52 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/Singletons.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/Singletons.scala @@ -8,7 +8,6 @@ package parsley.internal.deepembedding.singletons import parsley.internal.deepembedding.ContOps, ContOps.result import parsley.internal.deepembedding.backend, backend.StrictParsley import parsley.internal.deepembedding.frontend, frontend.LazyParsley -import parsley.internal.machine.instructions /** Singletons are special instances of combinators which do not take other parsers as arguments. * As such, they cannot have recursive parsers inside them and do not need to distinguish between @@ -21,16 +20,15 @@ import parsley.internal.machine.instructions * @note due to the fact these appear in the frontend, they must not be mutable, for the same * reasons as detailed in `LazyParsley` */ -private [deepembedding] abstract class Singleton[A] extends LazyParsley[A] with StrictParsley[A] { +private [deepembedding] abstract class Singleton[+A] extends LazyParsley[A] with StrictParsley[A] { /** The instruction that should be generated during the code generation for this combinator */ - def instr: instructions.Instr + def genInstrs(producesResults: Boolean)(implicit instrs: StrictParsley.InstrBuffer): Unit final override def inlinable: Boolean = true final override def findLetsAux[M[_, +_]: ContOps, R](seen: Set[LazyParsley[_]])(implicit state: frontend.LetFinderState): M[R, Unit] = result(()) - final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: frontend.LetMap, recs: frontend.RecMap): M[R, StrictParsley[A_]] = { - result(this) - } - final override def codeGen[M[_, +_]: ContOps, R](implicit instrs: StrictParsley.InstrBuffer, state: backend.CodeGenState): M[R, Unit] = { - result(instrs += instr) + final override def preprocess[M[_, +_]: ContOps, R, A_ >: A](implicit lets: frontend.LetMap): M[R, StrictParsley[A_]] = result(this) + override def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean) + (implicit instrs: StrictParsley.InstrBuffer, state: backend.CodeGenState): M[R, Unit] = { + result(genInstrs(producesResults)) } } diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala index a367ce324..b73ad7e70 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala @@ -10,6 +10,7 @@ import parsley.token.descriptions.numeric.PlusSignPresence import parsley.token.errors.ErrorConfig import parsley.internal.deepembedding.Sign.SignType +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.machine.instructions @@ -17,7 +18,10 @@ private [parsley] final class WhiteSpace(ws: Char => Boolean, desc: SpaceDesc, e extends Singleton[Unit] { // $COVERAGE-OFF$ override val pretty: String = "whiteSpace" - override def instr: instructions.Instr = new instructions.TokenWhiteSpace(ws, desc, errConfig) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.TokenWhiteSpace(ws, desc, errConfig) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context)(ws, desc, errConfig) } @@ -25,7 +29,10 @@ private [parsley] final class WhiteSpace(ws: Char => Boolean, desc: SpaceDesc, e private [parsley] final class SkipComments(desc: SpaceDesc, errConfig: ErrorConfig) extends Singleton[Unit] { // $COVERAGE-OFF$ override val pretty: String = "skipComments" - override def instr: instructions.Instr = new instructions.TokenSkipComments(desc, errConfig) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.TokenSkipComments(desc, errConfig) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context)(desc, errConfig) } @@ -33,7 +40,10 @@ private [parsley] final class SkipComments(desc: SpaceDesc, errConfig: ErrorConf private [parsley] final class Comment(desc: SpaceDesc, errConfig: ErrorConfig) extends Singleton[Unit] { // $COVERAGE-OFF$ override val pretty: String = "comment" - override def instr: instructions.Instr = new instructions.TokenComment(desc, errConfig) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.TokenComment(desc, errConfig) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = visitor.visit(this, context)(desc, errConfig) } @@ -41,17 +51,23 @@ private [parsley] final class Comment(desc: SpaceDesc, errConfig: ErrorConfig) e private [parsley] final class Sign[A](ty: SignType, signPresence: PlusSignPresence) extends Singleton[A => A] { // $COVERAGE-OFF$ override val pretty: String = "sign" - override def instr: instructions.Instr = new instructions.TokenSign(ty, signPresence) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.TokenSign(ty, signPresence) + if (!producesResults) instrs += instructions.Pop + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[A => A] = visitor.visit(this, context)(ty, signPresence) } -private [parsley] class NonSpecific(name: String, unexpectedIllegal: String => String, - start: Char => Boolean, letter: Char => Boolean, illegal: String => Boolean) extends Singleton[String] { +private [parsley] final class NonSpecific(name: String, unexpectedIllegal: String => String, + start: Char => Boolean, letter: Char => Boolean, illegal: String => Boolean) extends Singleton[String] { // $COVERAGE-OFF$ override def pretty: String = "nonspecificName" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.TokenNonSpecific(name, unexpectedIllegal)(start, letter, illegal) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.TokenNonSpecific(name, unexpectedIllegal)(start, letter, illegal) + if (!producesResults) instrs += instructions.Pop + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[String] = { visitor.visit(this, context)(name, unexpectedIllegal, start, letter, illegal) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala index bfde85279..3f06be5bc 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala @@ -9,6 +9,7 @@ import parsley.token.errors.LabelConfig import parsley.token.predicate.CharPredicate import parsley.internal.collection.immutable.Trie +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.deepembedding.singletons.Singleton import parsley.internal.machine.instructions @@ -18,7 +19,10 @@ private [parsley] final class SoftKeyword(private [SoftKeyword] val specific: St // $COVERAGE-OFF$ override def pretty: String = s"softKeyword($specific)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.token.SoftKeyword(specific, letter, caseSensitive, expected, expectedEnd) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.token.SoftKeyword(specific, letter, caseSensitive, expected, expectedEnd) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = { visitor.visit(this, context)(specific, letter, caseSensitive, expected, expectedEnd) @@ -30,7 +34,10 @@ private [parsley] final class SoftOperator(private [SoftOperator] val specific: // $COVERAGE-OFF$ override def pretty: String = s"softOperator($specific)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.token.SoftOperator(specific, letter, ops, expected, expectedEnd) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.token.SoftOperator(specific, letter, ops, expected, expectedEnd) + if (producesResults) instrs += instructions.Push.Unit + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Unit] = { visitor.visit(this, context)(specific, letter, ops, expected, expectedEnd) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/TextEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/TextEmbedding.scala index c345c69a2..18e0e3a4f 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/TextEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/TextEmbedding.scala @@ -8,6 +8,7 @@ package parsley.internal.deepembedding.singletons.token import parsley.token.errors.SpecialisedFilterConfig import parsley.internal.collection.immutable.Trie +import parsley.internal.deepembedding.backend.StrictParsley.InstrBuffer import parsley.internal.deepembedding.frontend.LazyParsleyIVisitor import parsley.internal.deepembedding.singletons.Singleton import parsley.internal.machine.instructions @@ -16,13 +17,19 @@ private [parsley] final class EscapeMapped(escTrie: Trie[Int], escs: Set[String] // $COVERAGE-OFF$ override def pretty: String = "escapeMapped" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.token.EscapeMapped(escTrie, escs) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.token.EscapeMapped(escTrie, escs) + if (!producesResults) instrs += instructions.Pop + } override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Int] = visitor.visit(this, context)(escTrie, escs) } private [parsley] final class EscapeAtMost(n: Int, radix: Int) extends Singleton[BigInt] { - override def instr: instructions.Instr = new instructions.token.EscapeAtMost(n, radix) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.token.EscapeAtMost(n, radix) + if (!producesResults) instrs += instructions.Pop + } // $COVERAGE-OFF$ override def pretty: String = "escapeAtMost" // $COVERAGE-ON$ @@ -30,7 +37,10 @@ private [parsley] final class EscapeAtMost(n: Int, radix: Int) extends Singleton } private [parsley] final class EscapeOneOfExactly(radix: Int, ns: List[Int], inexactErr: SpecialisedFilterConfig[Int]) extends Singleton[BigInt] { - override def instr: instructions.Instr = new instructions.token.EscapeOneOfExactly(radix, ns, inexactErr) + override def genInstrs(producesResults: Boolean)(implicit instrs: InstrBuffer): Unit = { + instrs += new instructions.token.EscapeOneOfExactly(radix, ns, inexactErr) + if (!producesResults) instrs += instructions.Pop + } // $COVERAGE-OFF$ override def pretty: String = "escapeOneOfExactly" // $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/Context.scala b/parsley/shared/src/main/scala/parsley/internal/machine/Context.scala index b93f4e790..1a726a6a7 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/Context.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/Context.scala @@ -11,13 +11,14 @@ import scala.annotation.tailrec import parsley.Failure import parsley.Result import parsley.Success +import parsley.XAssert._ import parsley.errors.ErrorBuilder import parsley.internal.errors.{CaretWidth, ExpectItem, LineBuilder, UnexpectDesc} import parsley.internal.machine.errors.{ClassicFancyError, DefuncError, DefuncHints, EmptyHints, ErrorItemBuilder, ExpectedError, UnexpectedError} import instructions.Instr -import stacks.{ArrayStack, CallStack, CheckStack, ErrorStack, HandlerStack, HintStack, Stack, StateStack}, Stack.StackExt +import stacks.{ArrayStack, CallStack, ErrorStack, HandlerStack, Stack, StateStack}, Stack.StackExt private [parsley] final class Context(private [machine] var instrs: Array[Instr], private [machine] val input: String, @@ -34,7 +35,6 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] /** State stack consisting of offsets and positions that can be rolled back */ private [machine] var states: StateStack = Stack.empty /** Stack consisting of offsets at previous checkpoints, which may query to test for consumed input */ - private [machine] var checkStack: CheckStack = Stack.empty /** Current operational status of the machine */ private [machine] var good: Boolean = true private [machine] var running: Boolean = true @@ -52,23 +52,14 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] private [machine] var debuglvl: Int = 0 // NEW ERROR MECHANISMS - private var hints: DefuncHints = EmptyHints + private [machine] var hints: DefuncHints = EmptyHints private var hintsValidOffset = 0 - private var hintStack = Stack.empty[HintStack] private [machine] var errs: ErrorStack = Stack.empty - private [machine] def saveHints(shadow: Boolean): Unit = { - hintStack = new HintStack(hints, hintsValidOffset, hintStack) - if (!shadow) hints = EmptyHints - } private [machine] def restoreHints(): Unit = { - val hintFrame = this.hintStack - this.hintsValidOffset = hintFrame.validOffset + val hintFrame = this.handlers + this.hintsValidOffset = hintFrame.hintOffset this.hints = hintFrame.hints - this.commitHints() - } - private [machine] def commitHints(): Unit = { - this.hintStack = this.hintStack.tail } /* Error Debugging Info */ @@ -78,9 +69,8 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] /* ERROR RELABELLING BEGIN */ private [machine] def mergeHints(): Unit = { - val hintFrame = this.hintStack - if (hintFrame.validOffset == offset) this.hints = hintFrame.hints.merge(this.hints) - commitHints() + val hintFrame = this.handlers + if (hintFrame.hintOffset == offset) this.hints = hintFrame.hints.merge(this.hints) } private [machine] def replaceHint(labels: Iterable[String]): Unit = hints = hints.rename(labels) private [machine] def popHints(): Unit = hints = hints.pop @@ -108,12 +98,11 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] private [machine] def addHints(expecteds: Set[ExpectItem], unexpectedWidth: Int) = { assume(expecteds.nonEmpty, "hints must always be non-empty") invalidateHints() - // TODO: this can be optimised further - hints = hints.addError(new ExpectedError(this.offset, this.line, this.col, expecteds, unexpectedWidth)) + hints = hints.addError(new ExpectedError(this.offset, this.line, this.col, expecteds, unexpectedWidth)) // TODO: this can be optimised further } private [machine] def updateCheckOffset() = { - this.checkStack.offset = this.offset + this.handlers.check = this.offset } // $COVERAGE-OFF$ @@ -128,10 +117,8 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] | rets = ${calls.mkString(", ")} | handlers = ${handlers.mkString(", ")} | recstates = ${states.mkString(", ")} - | checks = ${checkStack.mkString(", ")} | registers = ${regs.zipWithIndex.map{case (r, i) => s"r$i = $r"}.toList.mkString("\n ")} | errors = ${errs.mkString(", ")} - | hints = ($hintsValidOffset, ${hints.toSet}):${hintStack.mkString(", ")} |]""".stripMargin } // $COVERAGE-ON$ @@ -143,21 +130,17 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] run[Err, A]() } else if (good) { - assert(stack.size == 1, "stack must end a parse with exactly one item") + assert(stack.size == 1, s"stack must end a parse with exactly one item, it has ${stack.size}") assert(calls.isEmpty, "there must be no more calls to unwind on end of parser") assert(handlers.isEmpty, "there must be no more handlers on end of parse") - assert(checkStack.isEmpty, "there must be no residual check remaining on end of parse") assert(states.isEmpty, "there must be no residual states left at end of parse") assert(errs.isEmpty, "there should be no parse errors remaining at end of parse") - assert(hintStack.isEmpty, "there should be no hints remaining at end of parse") Success(stack.peek[A]) } else { assert(!errs.isEmpty && errs.tail.isEmpty, "there should be exactly 1 parse error remaining at end of parse") assert(handlers.isEmpty, "there must be no more handlers on end of parse") - assert(checkStack.isEmpty, "there must be no residual check remaining on end of parse") assert(states.isEmpty, "there must be no residual states left at end of parse") - assert(hintStack.isEmpty, "there should be no hints remaining at end of parse") Failure(errs.error.asParseError.format(sourceFile)) } } @@ -179,14 +162,16 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] calls = calls.tail } - private [machine] def catchNoConsumed(handler: =>Unit): Unit = { + private [machine] def catchNoConsumed(check: Int)(handler: =>Unit): Unit = { assert(!good, "catching can only be performed in a handler") - if (offset != checkStack.offset) fail() + if (offset != check) { + handlers = handlers.tail + fail() + } else { good = true handler } - checkStack = checkStack.tail } private [machine] def pushError(err: DefuncError): Unit = this.errs = new ErrorStack(this.useHints(err), this.errs) @@ -219,7 +204,6 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] if (handlers.isEmpty) running = false else { val handler = handlers - handlers = handlers.tail instrs = handler.instrs calls = handler.calls pc = handler.pc @@ -266,8 +250,9 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr] offset += n col += n } - private [machine] def pushHandler(label: Int): Unit = handlers = new HandlerStack(calls, instrs, label, stack.usize, handlers) - private [machine] def pushCheck(): Unit = checkStack = new CheckStack(offset, checkStack) + private [machine] def pushHandler(label: Int): Unit = { + handlers = new HandlerStack(calls, instrs, label, stack.usize, offset, hints, hintsValidOffset, handlers) + } private [machine] def saveState(): Unit = states = new StateStack(offset, line, col, states) private [machine] def restoreState(): Unit = { val state = states diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncError.scala b/parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncError.scala index c4d9eafce..78099c429 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncError.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncError.scala @@ -302,7 +302,7 @@ private [errors] sealed abstract class BaseError extends TrivialDefuncError { } } -private [machine] final class ExpectedError(val presentationOffset: Int, val line: Int, val col: Int, +private [parsley] final class ExpectedError(val presentationOffset: Int, val line: Int, val col: Int, val expected: Iterable[ExpectItem], val unexpectedWidth: Int) extends BaseError { override final val flags = if (expected.isEmpty) (DefuncError.ExpectedEmptyMask | DefuncError.TrivialErrorMask) else DefuncError.TrivialErrorMask } @@ -344,6 +344,7 @@ private [parsley] final class EmptyError(val presentationOffset: Int, val line: builder.updateEmptyUnexpected(unexpectedWidth) } } +// TODO: remove private [parsley] final class EmptyErrorWithReason(val presentationOffset: Int, val line: Int, val col: Int, val reason: String, val unexpectedWidth: Int) extends BaseError { override final val flags = DefuncError.ExpectedEmptyMask | DefuncError.TrivialErrorMask diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/CoreInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/CoreInstrs.scala index da176aaf2..028f2f3f8 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/CoreInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/CoreInstrs.scala @@ -5,9 +5,11 @@ */ package parsley.internal.machine.instructions +import parsley.XAssert._ + import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ -import parsley.internal.machine.errors.EmptyError +import parsley.internal.machine.errors.{EmptyError, EmptyHints} // Stack Manipulators private [internal] final class Push[A](x: A) extends Instr { @@ -19,6 +21,9 @@ private [internal] final class Push[A](x: A) extends Instr { override def toString: String = s"Push($x)" // $COVERAGE-ON$ } +private [internal] object Push { + val Unit = new Push(()) +} private [internal] final class Fresh[A](x: =>A) extends Instr { override def apply(ctx: Context): Unit = { @@ -155,54 +160,52 @@ private [internal] object PopHandler extends Instr { // $COVERAGE-ON$ } -private [internal] final class PushHandlerAndState(var label: Int, saveHints: Boolean, hideHints: Boolean) extends InstrWithLabel { +private [internal] final class PushHandlerAndClearHints(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) ctx.pushHandler(label) - ctx.saveState() - // FIXME: shadow = !hide so why is it like this? - if (saveHints) ctx.saveHints(shadow = hideHints) + ctx.hints = EmptyHints ctx.inc() } // $COVERAGE-OFF$ - override def toString: String = s"PushHandlerAndState($label)" + override def toString: String = s"PushHandlerAndClearHints($label)" // $COVERAGE-ON$ } -private [internal] object PopHandlerAndState extends Instr { +private [internal] final class PushHandlerAndStateAndClearHints(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - ctx.states = ctx.states.tail - ctx.handlers = ctx.handlers.tail + ctx.pushHandler(label) + ctx.saveState() + ctx.hints = EmptyHints ctx.inc() } // $COVERAGE-OFF$ - override def toString: String = "PopHandlerAndState" + override def toString: String = s"PushHandlerAndStateAmdClearHints($label)" // $COVERAGE-ON$ } -private [internal] final class PushHandlerAndCheck(var label: Int, saveHints: Boolean) extends InstrWithLabel { +private [internal] final class PushHandlerAndState(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - ctx.pushCheck() ctx.pushHandler(label) - if (saveHints) ctx.saveHints(false) + ctx.saveState() ctx.inc() } // $COVERAGE-OFF$ - override def toString: String = s"PushHandlerAndCheck($label)" + override def toString: String = s"PushHandlerAndState($label)" // $COVERAGE-ON$ } -private [internal] object PopHandlerAndCheck extends Instr { +private [internal] object PopHandlerAndState extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - ctx.checkStack = ctx.checkStack.tail + ctx.states = ctx.states.tail ctx.handlers = ctx.handlers.tail ctx.inc() } // $COVERAGE-OFF$ - override def toString: String = "PopHandlerAndCheck" + override def toString: String = "PopHandlerAndState" // $COVERAGE-ON$ } @@ -219,9 +222,8 @@ private [internal] final class Jump(var label: Int) extends InstrWithLabel { private [internal] final class JumpAndPopCheck(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) + // TODO: should this be mergeHints? ctx.handlers = ctx.handlers.tail - ctx.checkStack = ctx.checkStack.tail - ctx.commitHints() // TODO: should this be mergeHints? ctx.pc = label } // $COVERAGE-OFF$ @@ -234,7 +236,6 @@ private [internal] final class JumpAndPopState(var label: Int) extends InstrWith ensureRegularInstruction(ctx) ctx.handlers = ctx.handlers.tail ctx.states = ctx.states.tail - ctx.commitHints() ctx.pc = label } // $COVERAGE-OFF$ @@ -246,8 +247,12 @@ private [internal] final class Catch(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) ctx.restoreHints() - ctx.catchNoConsumed { - ctx.pushHandler(label) + val handler = ctx.handlers + ctx.catchNoConsumed(handler.check) { + assume(handler.stacksz == ctx.stack.usize && handler.check == ctx.offset + && handler.hints == ctx.hints && handler.hintOffset == ctx.currentHintsValidOffset, + "the handler can be re-used") + handler.pc = label ctx.inc() } } @@ -262,10 +267,26 @@ private [internal] final class RestoreAndPushHandler(var label: Int) extends Ins ctx.restoreState() ctx.restoreHints() ctx.good = true - ctx.pushHandler(label) + val handler = ctx.handlers + assume(handler.stacksz == ctx.stack.usize && handler.check == ctx.offset + && handler.hints == ctx.hints && handler.hintOffset == ctx.currentHintsValidOffset, + "the handler can be re-used") + handler.pc = label ctx.inc() } // $COVERAGE-OFF$ override def toString: String = s"RestoreAndPushHandler($label)" // $COVERAGE-ON$ } + +private [internal] object Refail extends Instr { + override def apply(ctx: Context): Unit = { + ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail + ctx.fail() + } + + // $COVERAGE-OFF$ + override def toString: String = "Refail" + // $COVERAGE-ON$ +} diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/DebugInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/DebugInstrs.scala index ffb92b2de..703dacaff 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/DebugInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/DebugInstrs.scala @@ -7,6 +7,7 @@ package parsley.internal.machine.instructions import parsley.XAssert._ +import parsley.debug.Profiler import parsley.errors.ErrorBuilder import parsley.internal.errors.{ExpectItem, FancyError, ParseError, TrivialError} @@ -79,11 +80,16 @@ private [instructions] object InputSlicer { } private [instructions] trait Logger extends PrettyPortal with InputSlicer with Colours { - final protected def preludeString(dir: Direction, ctx: Context, ends: String) = { + final protected def preludeString(dir: Direction, ctx: Context, ends: String, watchedRegs: Seq[(Int, String)]) = { val input = this.slice(ctx) val prelude = s"${portal(dir, ctx)} (${ctx.line}, ${ctx.col}): " val caret = (" " * prelude.length) + this.caret(ctx) - indentAndUnlines(ctx, s"$prelude$input$ends", caret) + val regSummary = if (watchedRegs.isEmpty) Seq.empty else { + "watched registers:" +: watchedRegs.map { + case (addr, name) => s" $name = ${ctx.regs(addr)}" + } :+ "" + } + indentAndUnlines(ctx, s"$prelude$input$ends" +: caret +: regSummary: _*) } final protected def doBreak(ctx: Context): Unit = { print(indentAndUnlines(ctx, @@ -94,11 +100,11 @@ private [instructions] trait Logger extends PrettyPortal with InputSlicer with C } } -private [internal] final class LogBegin(var label: Int, override val name: String, override val ascii: Boolean, break: Boolean) +private [internal] final class LogBegin(var label: Int, override val name: String, override val ascii: Boolean, break: Boolean, watchedRegs: Seq[(Int, String)]) extends InstrWithLabel with Logger { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - println(preludeString(Enter, ctx, "")) + println(preludeString(Enter, ctx, "", watchedRegs)) if (break) doBreak(ctx) ctx.debuglvl += 1 ctx.pushHandler(label) @@ -107,13 +113,13 @@ private [internal] final class LogBegin(var label: Int, override val name: Strin override def toString: String = s"LogBegin($label, $name)" } -private [internal] final class LogEnd(val name: String, override val ascii: Boolean, break: Boolean) extends Instr with Logger { +private [internal] final class LogEnd(val name: String, val ascii: Boolean, break: Boolean, watchedRegs: Seq[(Int, String)]) extends Instr with Logger { override def apply(ctx: Context): Unit = { assert(ctx.running, "cannot wrap a Halt with a debug") ctx.debuglvl -= 1 + ctx.handlers = ctx.handlers.tail val end = " " + { if (ctx.good) { - ctx.handlers = ctx.handlers.tail ctx.inc() green("Good") } @@ -122,7 +128,7 @@ private [internal] final class LogEnd(val name: String, override val ascii: Bool red("Fail") } } - println(preludeString(Exit, ctx, end)) + println(preludeString(Exit, ctx, end, watchedRegs)) if (break) doBreak(ctx) } override def toString: String = s"LogEnd($name)" @@ -173,6 +179,7 @@ private [internal] final class LogErrEnd(override val name: String, override val override def apply(ctx: Context): Unit = { assert(ctx.running, "cannot wrap a Halt with a debug") ctx.debuglvl -= 1 + ctx.handlers = ctx.handlers.tail @unused val currentHintsValidOffset = ctx.currentHintsValidOffset if (ctx.good) { // In this case, the currently in-flight hints should be reported @@ -180,7 +187,6 @@ private [internal] final class LogErrEnd(override val name: String, override val val inFlightHints = ctx.inFlightHints.toSet val formattedInFlight = inFlightHints.map(_.formatExpect) val msgInit = s": ${green("Good")}, current hints are $formattedInFlight with" - ctx.handlers = ctx.handlers.tail if (!oldData.stillValid(ctx.currentHintsValidOffset)) { println(preludeString(Exit, ctx, s"$msgInit old hints discarded (valid at offset ${ctx.currentHintsValidOffset})")) } @@ -230,4 +236,29 @@ private [instructions] object LogErrEnd { "}" } } + +private [internal] final class ProfileEnter(var label: Int, name: String, profiler: Profiler) extends InstrWithLabel { + private [this] val entries = profiler.entriesFor(name) + override def apply(ctx: Context): Unit = { + ensureRegularInstruction(ctx) + ctx.pushHandler(label) + ctx.inc() + entries += profiler.monotone(System.nanoTime()) + } + + override def toString: String = s"ProfileEnter($label, $name)" +} + +private [internal] final class ProfileExit(name: String, profiler: Profiler) extends Instr { + private [this] val exits = profiler.exitsFor(name) + override def apply(ctx: Context): Unit = { + exits += profiler.monotone(System.nanoTime()) + ctx.handlers = ctx.handlers.tail + if (ctx.good) ctx.inc() + else ctx.fail() + } + + override def toString: String = s"ProfileExit($name)" +} + // $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/ErrorInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/ErrorInstrs.scala index ef412682d..66ce96b12 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/ErrorInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/ErrorInstrs.scala @@ -8,7 +8,7 @@ package parsley.internal.machine.instructions import parsley.internal.errors.{CaretWidth, RigidCaret, UnexpectDesc} import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ -import parsley.internal.machine.errors.{ClassicFancyError, EmptyError, ExpectedError, ExpectedErrorWithReason} +import parsley.internal.machine.errors.EmptyError private [internal] final class RelabelHints(labels: Iterable[String]) extends Instr { private [this] val isHide: Boolean = labels.isEmpty @@ -20,12 +20,11 @@ private [internal] final class RelabelHints(labels: Iterable[String]) extends In if (isHide) ctx.popHints() // EOK // replace the head of the hints with the singleton for our label - else if (ctx.offset == ctx.checkStack.offset) ctx.replaceHint(labels) + else if (ctx.offset == ctx.handlers.check) ctx.replaceHint(labels) // COK // do nothing ctx.mergeHints() ctx.handlers = ctx.handlers.tail - ctx.checkStack = ctx.checkStack.tail ctx.inc() } // $COVERAGE-OFF$ @@ -40,9 +39,9 @@ private [internal] final class RelabelErrorAndFail(labels: Iterable[String]) ext ctx.errs.error = ctx.useHints { // only use the label if the error message is generated at the same offset // as the check stack saved for the start of the `label` combinator. - ctx.errs.error.label(labels, ctx.checkStack.offset) + ctx.errs.error.label(labels, ctx.handlers.check) } - ctx.checkStack = ctx.checkStack.tail + ctx.handlers = ctx.handlers.tail ctx.fail() } // $COVERAGE-OFF$ @@ -50,6 +49,32 @@ private [internal] final class RelabelErrorAndFail(labels: Iterable[String]) ext // $COVERAGE-ON$ } +private [internal] object HideHints extends Instr { + override def apply(ctx: Context): Unit = { + ensureRegularInstruction(ctx) + ctx.popHints() + ctx.mergeHints() + ctx.handlers = ctx.handlers.tail + ctx.inc() + } + // $COVERAGE-OFF$ + override def toString: String = "HideHints" + // $COVERAGE-ON$ +} + +private [internal] object HideErrorAndFail extends Instr { + override def apply(ctx: Context): Unit = { + ensureHandlerInstruction(ctx) + ctx.restoreHints() + ctx.errs.error = new EmptyError(ctx.offset, ctx.line, ctx.col, unexpectedWidth = 0) + ctx.handlers = ctx.handlers.tail + ctx.fail() + } + // $COVERAGE-OFF$ + override def toString: String = "HideErrorAndFail" + // $COVERAGE-ON$ +} + private [internal] object ErrorToHints extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) @@ -66,6 +91,7 @@ private [internal] object ErrorToHints extends Instr { private [internal] object MergeErrorsAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail val err2 = ctx.errs.error ctx.errs = ctx.errs.tail ctx.errs.error = ctx.errs.error.merge(err2) @@ -80,8 +106,8 @@ private [internal] object MergeErrorsAndFail extends Instr { private [internal] class ApplyReasonAndFail(reason: String) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) - ctx.errs.error = ctx.errs.error.withReason(reason, ctx.checkStack.offset) - ctx.checkStack = ctx.checkStack.tail + ctx.errs.error = ctx.errs.error.withReason(reason, ctx.handlers.check) + ctx.handlers = ctx.handlers.tail ctx.fail() } @@ -93,6 +119,7 @@ private [internal] class ApplyReasonAndFail(reason: String) extends Instr { private [internal] class AmendAndFail private (partial: Boolean) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail ctx.errs.error = ctx.errs.error.amend(partial, ctx.states.offset, ctx.states.line, ctx.states.col) ctx.states = ctx.states.tail ctx.fail() @@ -111,6 +138,7 @@ private [internal] object AmendAndFail { private [internal] object EntrenchAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail ctx.errs.error = ctx.errs.error.entrench ctx.fail() } @@ -123,6 +151,7 @@ private [internal] object EntrenchAndFail extends Instr { private [internal] class DislodgeAndFail(n: Int) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail ctx.errs.error = ctx.errs.error.dislodge(n) ctx.fail() } @@ -135,8 +164,8 @@ private [internal] class DislodgeAndFail(n: Int) extends Instr { private [internal] object SetLexicalAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) - ctx.errs.error = ctx.errs.error.markAsLexical(ctx.checkStack.offset) - ctx.checkStack = ctx.checkStack.tail + ctx.errs.error = ctx.errs.error.markAsLexical(ctx.handlers.check) + ctx.handlers = ctx.handlers.tail ctx.fail() } @@ -166,43 +195,33 @@ private [internal] final class Unexpected(msg: String, width: CaretWidth) extend // $COVERAGE-ON$ } -private [internal] class MakeVerifiedError private (msggen: Either[Any => Seq[String], Option[Any => String]]) extends Instr { +private [internal] final class VanillaGen[A](gen: parsley.errors.VanillaGen[A]) extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - val state = ctx.states - ctx.states = ctx.states.tail - ctx.restoreHints() - // A previous success is a failure - ctx.handlers = ctx.handlers.tail - val caretWidth = ctx.offset - state.offset - val x = ctx.stack.upeek - val err = msggen match { - case Left(f) => new ClassicFancyError(state.offset, state.line, state.col, new RigidCaret(caretWidth), f(x): _*) - case Right(Some(f)) => new ExpectedErrorWithReason(state.offset, state.line, state.col, None, f(x), caretWidth) - case Right(None) => new ExpectedError(state.offset, state.line, state.col, None, caretWidth) - } - ctx.fail(err) + // stack will have an (A, Int) pair on it + val (x, caretWidth) = ctx.stack.pop[(A, Int)]() + val unex = gen.unexpected(x) + val reason = gen.reason(x) + val err = unex.makeError(ctx.offset, ctx.line, ctx.col, gen.adjustWidth(x, caretWidth)) + // Sorry, it's faster :( + if (reason.isDefined) ctx.fail(err.withReason(reason.get)) + else ctx.fail(err) } + // $COVERAGE-OFF$ - override def toString: String = "MakeVerifiedError" + override def toString: String = "VanillaGen" // $COVERAGE-ON$ } -private [internal] object MakeVerifiedError { - def apply[A](msggen: Either[A => Seq[String], Option[A => String]]): MakeVerifiedError = { - new MakeVerifiedError(msggen.asInstanceOf[Either[Any => Seq[String], Option[Any => String]]]) - } -} -private [internal] object NoVerifiedError extends Instr { +private [internal] final class SpecialisedGen[A](gen: parsley.errors.SpecialisedGen[A]) extends Instr { override def apply(ctx: Context): Unit = { - ensureHandlerInstruction(ctx) - // If a verified error goes wrong, then it should appear like nothing happened - ctx.restoreState() - ctx.restoreHints() - ctx.errs.error = new EmptyError(ctx.offset, ctx.line, ctx.col, unexpectedWidth = 0) - ctx.fail() + ensureRegularInstruction(ctx) + // stack will have an (A, Int) pair on it + val (x, caretWidth) = ctx.stack.pop[(A, Int)]() + ctx.failWithMessage(new RigidCaret(gen.adjustWidth(x, caretWidth)), gen.messages(x): _*) } + // $COVERAGE-OFF$ - override def toString: String = "VerifiedErrorHandler" + override def toString: String = "SpecialisedGen" // $COVERAGE-ON$ } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IntrinsicInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IntrinsicInstrs.scala index 12337dac2..003786774 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IntrinsicInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IntrinsicInstrs.scala @@ -7,12 +7,12 @@ package parsley.internal.machine.instructions import scala.annotation.tailrec +import parsley.XAssert._ import parsley.token.errors.LabelConfig -import parsley.internal.errors.{EndOfInput, ExpectDesc, ExpectItem, RigidCaret, UnexpectDesc} +import parsley.internal.errors.{EndOfInput, ExpectDesc, ExpectItem} import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ -import parsley.internal.machine.errors.{ClassicFancyError, DefuncError, EmptyError, EmptyErrorWithReason, UnexpectedError} private [internal] final class Lift2(f: (Any, Any) => Any) extends Instr { override def apply(ctx: Context): Unit = { @@ -43,25 +43,23 @@ private [internal] object Lift3 { def apply[A, B, C, D](f: (A, B, C) => D): Lift3 = new Lift3(f.asInstanceOf[(Any, Any, Any) => Any]) } -private [internal] class CharTok(c: Char, x: Any, errorItem: Iterable[ExpectItem]) extends Instr { - def this(c: Char, x: Any, expected: LabelConfig) = this(c, x, expected.asExpectItems(s"$c")) - def this(c: Char, expected: LabelConfig) = this(c, c, expected) +private [internal] class CharTok private (c: Char, errorItem: Iterable[ExpectItem]) extends Instr { + def this(c: Char, expected: LabelConfig) = this(c, expected.asExpectItems(s"$c")) override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) if (ctx.moreInput && ctx.peekChar == c) { ctx.consumeChar() - ctx.pushAndContinue(x) + ctx.inc() } else ctx.expectedFail(errorItem, unexpectedWidth = 1) } // $COVERAGE-OFF$ - override def toString: String = if (x == c) s"Chr($c)" else s"ChrExchange($c, $x)" + override def toString: String = s"Chr($c)" // $COVERAGE-ON$ } -private [internal] class SupplementaryCharTok(codepoint: Int, x: Any, errorItem: Iterable[ExpectItem]) extends Instr { - def this(codepoint: Int, x: Any, expected: LabelConfig) = this(codepoint, x, expected.asExpectItems(Character.toChars(codepoint).mkString)) - def this(codepoint: Int, expected: LabelConfig) = this(codepoint, codepoint, expected) +private [internal] class SupplementaryCharTok private (codepoint: Int, errorItem: Iterable[ExpectItem]) extends Instr { + def this(codepoint: Int, expected: LabelConfig) = this(codepoint, expected.asExpectItems(Character.toChars(codepoint).mkString)) assert(Character.isSupplementaryCodePoint(codepoint), "SupplementaryCharTok should only be used for supplementary code points") val h = Character.highSurrogate(codepoint) @@ -70,20 +68,17 @@ private [internal] class SupplementaryCharTok(codepoint: Int, x: Any, errorItem: ensureRegularInstruction(ctx) if (ctx.moreInput(2) && ctx.peekChar(0) == h && ctx.peekChar(1) == l) { ctx.fastConsumeSupplementaryChar() - ctx.pushAndContinue(x) + ctx.inc() } else ctx.expectedFail(errorItem, unexpectedWidth = 1) } // $COVERAGE-OFF$ - override def toString: String = - if (x == codepoint) s"SupplementaryChr($h$l)" - else s"SupplementaryChrExchange($h$l, $x)" + override def toString: String = s"SupplementaryChr($h$l)" // $COVERAGE-ON$ } -private [internal] final class StringTok(s: String, x: Any, errorItem: Iterable[ExpectItem]) extends Instr { - def this(s: String, x: Any, expected: LabelConfig) = this(s, x, expected.asExpectItems(s)) - def this(s: String, expected: LabelConfig) = this(s, s, expected) +private [internal] final class StringTok private (s: String, errorItem: Iterable[ExpectItem]) extends Instr { + def this(s: String, expected: LabelConfig) = this(s, expected.asExpectItems(s)) private [this] val sz = s.length private [this] val codePointLength = s.codePointCount(0, sz) @@ -124,7 +119,7 @@ private [internal] final class StringTok(s: String, x: Any, errorItem: Iterable[ ctx.col = colAdjust(ctx.col) ctx.line = lineAdjust(ctx.line) ctx.offset = i - ctx.pushAndContinue(x) + ctx.inc() } } @@ -133,7 +128,7 @@ private [internal] final class StringTok(s: String, x: Any, errorItem: Iterable[ go(ctx, ctx.offset, 0) } // $COVERAGE-OFF$ - override def toString: String = if (x.isInstanceOf[String] && (s eq x.asInstanceOf[String])) s"Str($s)" else s"StrPerform($s, $x)" + override def toString: String = s"Str($s)" // $COVERAGE-ON$ } @@ -187,107 +182,6 @@ private [internal] final class Case(var label: Int) extends InstrWithLabel { // $COVERAGE-ON$ } -private [internal] final class Filter[A](_pred: A => Boolean) extends Instr { - private [this] val pred = _pred.asInstanceOf[Any => Boolean] - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - ctx.handlers = ctx.handlers.tail - if (pred(ctx.stack.upeek)) ctx.inc() - else { - val state = ctx.states - val caretWidth = ctx.offset - state.offset - ctx.fail(new EmptyError(state.offset, state.line, state.col, caretWidth)) - } - ctx.states = ctx.states.tail - } - // $COVERAGE-OFF$ - override def toString: String = "Filter(?)" - // $COVERAGE-ON$ -} - -private [internal] final class MapFilter[A, B](_f: A => Option[B]) extends Instr { - private [this] val f = _f.asInstanceOf[Any => Option[Any]] - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - ctx.handlers = ctx.handlers.tail - f(ctx.stack.upeek) match { - case Some(x) => ctx.exchangeAndContinue(x) - case None => - val state = ctx.states - val caretWidth = ctx.offset - state.offset - ctx.fail(new EmptyError(state.offset, state.line, state.col, caretWidth)) - } - ctx.states = ctx.states.tail - } - // $COVERAGE-OFF$ - override def toString: String = "MapFilter(?)" - // $COVERAGE-ON$ -} - -private [internal] final class FilterOut[A](_pred: PartialFunction[A, String]) extends Instr { - private [this] val pred = _pred.asInstanceOf[PartialFunction[Any, String]] - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - ctx.handlers = ctx.handlers.tail - if (pred.isDefinedAt(ctx.stack.upeek)) { - val reason = pred(ctx.stack.upop()) - val state = ctx.states - val caretWidth = ctx.offset - state.offset - ctx.fail(new EmptyErrorWithReason(state.offset, state.line, state.col, reason, caretWidth)) - } - else ctx.inc() - ctx.states = ctx.states.tail - } - // $COVERAGE-OFF$ - override def toString: String = "FilterOut(?)" - // $COVERAGE-ON$ -} - -private [internal] final class GuardAgainst(pred: PartialFunction[Any, Seq[String]]) extends Instr { - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - ctx.handlers = ctx.handlers.tail - if (pred.isDefinedAt(ctx.stack.upeek)) { - val state = ctx.states - val caretWidth = ctx.offset - state.offset - ctx.fail(new ClassicFancyError(state.offset, state.line, state.col, new RigidCaret(caretWidth), pred(ctx.stack.upop()): _*)) - } - else ctx.inc() - ctx.states = ctx.states.tail - } - // $COVERAGE-OFF$ - override def toString: String = "GuardAgainst(?)" - // $COVERAGE-ON$ -} -private [internal] object GuardAgainst { - def apply[A](pred: PartialFunction[A, Seq[String]]): GuardAgainst = new GuardAgainst(pred.asInstanceOf[PartialFunction[Any, Seq[String]]]) -} - -private [internal] final class UnexpectedWhen(pred: PartialFunction[Any, (String, Option[String])]) extends Instr { - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - ctx.handlers = ctx.handlers.tail - if (pred.isDefinedAt(ctx.stack.upeek)) { - val state = ctx.states - val caretWidth = ctx.offset - state.offset - val (unex, reason) = pred(ctx.stack.upop()) - val err = new UnexpectedError(state.offset, state.line, state.col, Nil, new UnexpectDesc(unex, new RigidCaret(caretWidth))) - ctx.fail(reason.fold[DefuncError](err)(err.withReason(_))) - } - else ctx.inc() - ctx.states = ctx.states.tail - } - - // $COVERAGE-OFF$ - override def toString: String = "UnexpectedWhen(?)" - // $COVERAGE-ON$ -} -private [internal] object UnexpectedWhen { - def apply[A](pred: PartialFunction[A, (String, Option[String])]): UnexpectedWhen = { - new UnexpectedWhen(pred.asInstanceOf[PartialFunction[Any, (String, Option[String])]]) - } -} - private [internal] object NegLookFail extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) @@ -310,10 +204,11 @@ private [internal] object NegLookGood extends Instr { // Recover the previous state; notFollowedBy NEVER consumes input ctx.restoreState() ctx.restoreHints() + ctx.handlers = ctx.handlers.tail // A failure is what we wanted ctx.good = true ctx.errs = ctx.errs.tail - ctx.pushAndContinue(()) + ctx.inc() } // $COVERAGE-OFF$ override def toString: String = "NegLookGood" @@ -324,7 +219,7 @@ private [internal] object Eof extends Instr { private [this] final val expected = Some(EndOfInput) override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - if (ctx.offset == ctx.inputsz) ctx.pushAndContinue(()) + if (ctx.offset == ctx.inputsz) ctx.inc() else ctx.expectedFail(expected, unexpectedWidth = 1) } // $COVERAGE-OFF$ @@ -336,7 +231,7 @@ private [internal] final class Modify(reg: Int, f: Any => Any) extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) ctx.writeReg(reg, f(ctx.regs(reg))) - ctx.pushAndContinue(()) + ctx.inc() } // $COVERAGE-OFF$ override def toString: String = s"Modify($reg, f)" @@ -357,6 +252,63 @@ private [internal] final class SwapAndPut(reg: Int) extends Instr { // $COVERAGE-ON$ } +private [instructions] abstract class FilterLike extends Instr { + var good: Int + var bad: Int + + final override def relabel(labels: Array[Int]): this.type = { + good = labels(good) + bad = labels(bad) + this + } + + final def carryOn(ctx: Context): Unit = { + ctx.states = ctx.states.tail + ctx.handlers = ctx.handlers.tail + ctx.pc = good + } + + final def fail(ctx: Context, x: Any): Unit = { + ctx.handlers.pc = bad + ctx.exchangeAndContinue((x, ctx.offset - ctx.states.offset)) + } +} + +private [internal] final class Filter[A](_pred: A => Boolean, var good: Int, var bad: Int) extends FilterLike { + private [this] val pred = _pred.asInstanceOf[Any => Boolean] + + override def apply(ctx: Context): Unit = { + ensureRegularInstruction(ctx) + val x = ctx.stack.upeek + if (pred(x)) carryOn(ctx) + else fail(ctx, x) + } + + // $COVERAGE-OFF$ + override def toString: String = s"Filter(???, good = $good)" + // $COVERAGE-ON$ +} + +private [internal] final class MapFilter[A, B](_pred: A => Option[B], var good: Int, var bad: Int) extends FilterLike { + private [this] val pred = _pred.asInstanceOf[Any => Option[B]] + + override def apply(ctx: Context): Unit = { + ensureRegularInstruction(ctx) + val x = ctx.stack.upeek + val opt = pred(x) + // Sorry, it's faster :( + if (opt.isDefined) { + ctx.stack.exchange(opt.get) + carryOn(ctx) + } + else fail(ctx, x) + } + + // $COVERAGE-OFF$ + override def toString: String = s"MapFilter(???, good = $good)" + // $COVERAGE-ON$ +} + // Companion Objects private [internal] object StringTok { private [StringTok] abstract class Adjust { @@ -399,15 +351,3 @@ private [internal] object StringTok { } } } - -private [internal] object CharTokFastPerform { - def apply[A >: Char, B](c: Char, f: A => B, expected: LabelConfig): CharTok = new CharTok(c, f(c), expected) -} - -private [internal] object SupplementaryCharTokFastPerform { - def apply[A >: Int, B](c: Int, f: A => B, expected: LabelConfig): SupplementaryCharTok = new SupplementaryCharTok(c, f(c), expected) -} - -private [internal] object StringTokFastPerform { - def apply(s: String, f: String => Any, expected: LabelConfig): StringTok = new StringTok(s, f(s), expected) -} diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IterativeInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IterativeInstrs.scala index 95b1f552b..abec17ca9 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IterativeInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/IterativeInstrs.scala @@ -7,24 +7,11 @@ package parsley.internal.machine.instructions import scala.collection.mutable +import parsley.XAssert._ + import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ -// TODO: Now PushHandlerAndCheck(label, false), so could be removed again! -private [internal] final class PushHandlerIterative(var label: Int) extends InstrWithLabel { - override def apply(ctx: Context): Unit = { - ensureRegularInstruction(ctx) - // This is used for iterative parsers, which must ensure that invalidated hints are invalided _now_ - //ctx.invalidateHints() // FIXME: This has been removed because hint setting in updateCheckOffsetAndHints has been disabled, pending deep thought - ctx.pushCheck() - ctx.pushHandler(label) - ctx.inc() - } - // $COVERAGE-OFF$ - override def toString: String = s"PushHandlerIterative($label)" - // $COVERAGE-ON$ -} - private [internal] final class Many(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { if (ctx.good) { @@ -34,7 +21,8 @@ private [internal] final class Many(var label: Int) extends InstrWithLabel { ctx.pc = label } // If the head of input stack is not the same size as the head of check stack, we fail to next handler - else ctx.catchNoConsumed { + else ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.exchangeAndContinue(ctx.stack.peek[mutable.ListBuffer[Any]].toList) } @@ -44,17 +32,18 @@ private [internal] final class Many(var label: Int) extends InstrWithLabel { // $COVERAGE-ON$ } +// TODO: Factor these handlers out! private [internal] final class SkipMany(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { if (ctx.good) { - ctx.stack.pop_() ctx.updateCheckOffset() ctx.pc = label } // If the head of input stack is not the same size as the head of check stack, we fail to next handler - else ctx.catchNoConsumed { + else ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() - ctx.pushAndContinue(()) + ctx.inc() } } // $COVERAGE-OFF$ @@ -71,7 +60,8 @@ private [internal] final class ChainPost(var label: Int) extends InstrWithLabel ctx.pc = label } // If the head of input stack is not the same size as the head of check stack, we fail to next handler - else ctx.catchNoConsumed { + else ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.inc() } @@ -90,7 +80,8 @@ private [internal] final class ChainPre(var label: Int) extends InstrWithLabel { ctx.pc = label } // If the head of input stack is not the same size as the head of check stack, we fail to next handler - else ctx.catchNoConsumed { + else ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.inc() } @@ -109,7 +100,8 @@ private [internal] final class Chainl(var label: Int) extends InstrWithLabel { ctx.pc = label } // If the head of input stack is not the same size as the head of check stack, we fail to next handler - else ctx.catchNoConsumed { + else ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.inc() } @@ -122,7 +114,6 @@ private [internal] final class Chainl(var label: Int) extends InstrWithLabel { private [instructions] object DualHandler { def popSecondHandlerAndJump(ctx: Context, label: Int): Unit = { ctx.handlers = ctx.handlers.tail - ctx.checkStack = ctx.checkStack.tail ctx.updateCheckOffset() ctx.pc = label } @@ -147,13 +138,14 @@ private [internal] final class ChainrJump(var label: Int) extends InstrWithLabel private [internal] final class ChainrOpHandler(wrap: Any => Any) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + val check = ctx.handlers.check ctx.handlers = ctx.handlers.tail - ctx.catchNoConsumed { + ctx.catchNoConsumed(check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() val y = ctx.stack.upop() ctx.exchangeAndContinue(ctx.stack.peek[Any => Any](wrap(y))) } - ctx.checkStack = ctx.checkStack.tail } // $COVERAGE-OFF$ @@ -164,22 +156,9 @@ private [internal] object ChainrOpHandler { def apply[A, B](wrap: A => B): ChainrOpHandler = new ChainrOpHandler(wrap.asInstanceOf[Any => Any]) } -private [internal] object ChainrWholeHandler extends Instr { - override def apply(ctx: Context): Unit = { - ensureHandlerInstruction(ctx) - ctx.checkStack = ctx.checkStack.tail - ctx.fail() - } - - // $COVERAGE-OFF$ - override def toString: String = "ChainrWholeHandler" - // $COVERAGE-ON$ -} - private [internal] final class SepEndBy1Jump(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - ctx.stack.pop_() val x = ctx.stack.upop() ctx.stack.peek[mutable.ListBuffer[Any]] += x DualHandler.popSecondHandlerAndJump(ctx, label) @@ -191,8 +170,8 @@ private [internal] final class SepEndBy1Jump(var label: Int) extends InstrWithLa } private [instructions] object SepEndBy1Handlers { - def pushAccWhenCheckValidAndContinue(ctx: Context, acc: mutable.ListBuffer[Any]): Unit = { - if (ctx.offset != ctx.checkStack.offset || acc.isEmpty) ctx.fail() + def pushAccWhenCheckValidAndContinue(ctx: Context, check: Int, acc: mutable.ListBuffer[Any]): Unit = { + if (ctx.offset != check || acc.isEmpty) ctx.fail() else { ctx.addErrorToHintsAndPop() ctx.good = true @@ -204,6 +183,8 @@ private [instructions] object SepEndBy1Handlers { private [internal] object SepEndBy1SepHandler extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + val check = ctx.handlers.check + ctx.handlers = ctx.handlers.tail // p succeeded and sep didn't, so push p and fall-through to the whole handler val x = ctx.stack.upop() val acc = ctx.stack.peek[mutable.ListBuffer[Any]] @@ -213,8 +194,7 @@ private [internal] object SepEndBy1SepHandler extends Instr { assert(ctx.handlers.pc == ctx.pc + 1, "the top-most handler must be the whole handler in the sep handler") ctx.handlers = ctx.handlers.tail ctx.inc() - SepEndBy1Handlers.pushAccWhenCheckValidAndContinue(ctx, acc) - ctx.checkStack = ctx.checkStack.tail.tail + SepEndBy1Handlers.pushAccWhenCheckValidAndContinue(ctx, check, acc) } // $COVERAGE-OFF$ @@ -225,8 +205,9 @@ private [internal] object SepEndBy1SepHandler extends Instr { private [internal] object SepEndBy1WholeHandler extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) - SepEndBy1Handlers.pushAccWhenCheckValidAndContinue(ctx, ctx.stack.peek[mutable.ListBuffer[Any]]) - ctx.checkStack = ctx.checkStack.tail + val check = ctx.handlers.check + ctx.handlers = ctx.handlers.tail + SepEndBy1Handlers.pushAccWhenCheckValidAndContinue(ctx, check, ctx.stack.peek[mutable.ListBuffer[Any]]) } // $COVERAGE-OFF$ @@ -236,18 +217,13 @@ private [internal] object SepEndBy1WholeHandler extends Instr { private [internal] final class ManyUntil(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { - if (ctx.good) { - ctx.stack.upop() match { - case parsley.combinator.ManyUntil.Stop => - ctx.exchangeAndContinue(ctx.stack.peek[mutable.ListBuffer[Any]].toList) - ctx.handlers = ctx.handlers.tail - case x => - ctx.stack.peek[mutable.ListBuffer[Any]] += x - ctx.pc = label - } + ensureRegularInstruction(ctx) + ctx.stack.upop() match { + case parsley.combinator.ManyUntil.Stop => ctx.exchangeAndContinue(ctx.stack.peek[mutable.ListBuffer[Any]].toList) + case x => + ctx.stack.peek[mutable.ListBuffer[Any]] += x + ctx.pc = label } - // ManyUntil is a fallthrough handler, it must be visited during failure, but does nothing to the external state - else ctx.fail() } // $COVERAGE-OFF$ override def toString: String = s"ManyUntil($label)" @@ -256,20 +232,13 @@ private [internal] final class ManyUntil(var label: Int) extends InstrWithLabel private [internal] final class SkipManyUntil(var label: Int) extends InstrWithLabel { override def apply(ctx: Context): Unit = { - if (ctx.good) { - ctx.stack.upeek match { - case parsley.combinator.ManyUntil.Stop => - ctx.handlers = ctx.handlers.tail - ctx.exchangeAndContinue(()) - case _ => - ctx.pc = label - ctx.stack.pop_() - } + ensureRegularInstruction(ctx) + ctx.stack.upop() match { + case parsley.combinator.ManyUntil.Stop => ctx.inc() + case _ => ctx.pc = label } - // ManyUntil is a fallthrough handler, it must be visited during failure, but does nothing to the external state - else ctx.fail() } // $COVERAGE-OFF$ - override def toString: String = s"ManyUntil($label)" + override def toString: String = s"SkipManyUntil($label)" // $COVERAGE-ON$ } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/OptInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/OptInstrs.scala index 79c29ae6e..c2774ef64 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/OptInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/OptInstrs.scala @@ -13,7 +13,7 @@ import parsley.token.errors.LabelConfig import parsley.internal.errors.ExpectItem import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ -import parsley.internal.machine.errors.ExpectedError +import parsley.internal.machine.errors.{EmptyHints, ExpectedError} import parsley.internal.machine.stacks.ErrorStack private [internal] final class Lift1(f: Any => Any) extends Instr { @@ -58,7 +58,8 @@ private [internal] final class RecoverWith[A](x: A) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) ctx.restoreHints() // This must be before adding the error to hints - ctx.catchNoConsumed { + ctx.catchNoConsumed(ctx.handlers.check) { + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.pushAndContinue(x) } @@ -73,6 +74,7 @@ private [internal] final class AlwaysRecoverWith[A](x: A) extends Instr { ensureHandlerInstruction(ctx) ctx.restoreState() ctx.restoreHints() // This must be before adding the error to hints + ctx.handlers = ctx.handlers.tail ctx.addErrorToHintsAndPop() ctx.good = true ctx.pushAndContinue(x) @@ -99,9 +101,8 @@ private [internal] final class JumpTable(jumpTable: mutable.LongMap[(Int, Iterab val (dest, errorItems) = jumpTable.getOrElse(ctx.peekChar.toLong, (default, allErrorItems)) ctx.pc = dest if (dest != default) { - ctx.pushCheck() ctx.pushHandler(defaultPreamble) - ctx.saveHints(shadow = false) + ctx.hints = EmptyHints } addErrors(ctx, errorItems) // adds a handler } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/PrimitiveInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/PrimitiveInstrs.scala index 4028d7a38..ee0a47b55 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/PrimitiveInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/PrimitiveInstrs.scala @@ -27,6 +27,7 @@ private [internal] final class Satisfies(f: Char => Boolean, expected: Iterable[ private [internal] object RestoreAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail // Pop input off head then fail to next handler ctx.restoreState() ctx.fail() @@ -52,6 +53,7 @@ private [internal] object RestoreHintsAndState extends Instr { private [internal] object PopStateAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail ctx.states = ctx.states.tail ctx.fail() } @@ -64,6 +66,7 @@ private [internal] object PopStateRestoreHintsAndFail extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) ctx.restoreHints() + ctx.handlers = ctx.handlers.tail ctx.states = ctx.states.tail ctx.fail() } @@ -117,7 +120,7 @@ private [internal] final class Get(reg: Int) extends Instr { private [internal] final class Put(reg: Int) extends Instr { override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) - ctx.writeReg(reg, ctx.stack.peekAndExchange(())) + ctx.writeReg(reg, ctx.stack.upop()) ctx.inc() } // $COVERAGE-OFF$ @@ -128,6 +131,7 @@ private [internal] final class Put(reg: Int) extends Instr { private [internal] final class PutAndFail(reg: Int) extends Instr { override def apply(ctx: Context): Unit = { ensureHandlerInstruction(ctx) + ctx.handlers = ctx.handlers.tail ctx.writeReg(reg, ctx.stack.upeek) ctx.fail() } @@ -136,6 +140,20 @@ private [internal] final class PutAndFail(reg: Int) extends Instr { // $COVERAGE-ON$ } +private [internal] object Span extends Instr { + override def apply(ctx: Context): Unit = { + // this uses the state stack because post #132 we will need a save point to obtain the start of the input + ensureRegularInstruction(ctx) + val startOffset = ctx.states.offset + ctx.states = ctx.states.tail + ctx.handlers = ctx.handlers.tail + ctx.pushAndContinue(ctx.input.substring(startOffset, ctx.offset)) + } + // $COVERAGE-OFF$ + override def toString: String = "Span" + // $COVERAGE-ON$ +} + // This instruction holds mutate state, but it is safe to do so, because it's always the first instruction of a DynCall. private [parsley] final class CalleeSave(var label: Int, localRegs: Set[Reg[_]], reqSize: Int, slots: List[(Int, Int)], saveArray: Array[AnyRef]) extends InstrWithLabel { @@ -174,10 +192,8 @@ private [parsley] final class CalleeSave(var label: Int, localRegs: Set[Reg[_]], } private def continue(ctx: Context): Unit = { - if (ctx.good) { - ctx.handlers = ctx.handlers.tail - ctx.pc = label - } + ctx.handlers = ctx.handlers.tail + if (ctx.good) ctx.pc = label else ctx.fail() } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala index eeb70fec1..9c613ddc0 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala @@ -89,10 +89,10 @@ private [internal] final class TokenComment private ( // If neither comment is available we fail if (!ctx.moreInput || (!lineAllowed || !ctx.input.startsWith(line, ctx.offset)) && !startsMulti) ctx.expectedFail(expected = None, openingSize) // One of the comments must be available - else if (startsMulti && multiLineComment(ctx)) ctx.pushAndContinue(()) + else if (startsMulti && multiLineComment(ctx)) ctx.inc() else if (startsMulti) ctx.expectedFail(expected = endOfMultiComment, unexpectedWidth = 1) // It clearly wasn't the multi-line comment, so we are left with single line - else if (singleLineComment(ctx)) ctx.pushAndContinue(()) + else if (singleLineComment(ctx)) ctx.inc() else ctx.expectedFail(expected = endOfSingleComment, unexpectedWidth = 1) } @@ -111,9 +111,9 @@ private [instructions] abstract class WhiteSpaceLike extends CommentLexer { val startsSingle = ctx.input.startsWith(line, ctx.offset) if (startsSingle && singleLineComment(ctx)) singlesOnly(ctx) else if (startsSingle) ctx.expectedFail(expected = endOfSingleComment, unexpectedWidth = 1) - else ctx.pushAndContinue(()) + else ctx.inc() } - else ctx.pushAndContinue(()) + else ctx.inc() } @tailrec private final def multisOnly(ctx: Context): Unit = { @@ -121,7 +121,7 @@ private [instructions] abstract class WhiteSpaceLike extends CommentLexer { val startsMulti = ctx.moreInput && ctx.input.startsWith(start, ctx.offset) if (startsMulti && multiLineComment(ctx)) multisOnly(ctx) else if (startsMulti) ctx.expectedFail(expected = endOfMultiComment, numCodePointsEnd) - else ctx.pushAndContinue(()) + else ctx.inc() } private [this] final val sharedPrefix = line.view.zip(start).takeWhile(Function.tupled(_ == _)).map(_._1).mkString @@ -138,15 +138,15 @@ private [instructions] abstract class WhiteSpaceLike extends CommentLexer { val startsLine = ctx.input.startsWith(factoredLine, ctx.offset + sharedPrefix.length) if (startsLine && singleLineComment(ctx)) singlesAndMultis(ctx) else if (startsLine) ctx.expectedFail(expected = endOfSingleComment, unexpectedWidth = 1) - else ctx.pushAndContinue(()) + else ctx.inc() } } - else ctx.pushAndContinue(()) + else ctx.inc() } final def spacesAndContinue(ctx: Context): Unit = { spaces(ctx) - ctx.pushAndContinue(()) + ctx.inc() } private [this] final val impl = { diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala index da2e66d96..5f4865c34 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala @@ -80,7 +80,7 @@ private [internal] final class SoftKeyword(protected val specific: String, lette } else { ctx.states = ctx.states.tail - ctx.pushAndContinue(()) + ctx.inc() } } @@ -119,7 +119,7 @@ private [internal] final class SoftOperator(protected val specific: String, lett } else { ctx.states = ctx.states.tail - ctx.pushAndContinue(()) + ctx.inc() } } } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/TextInstructions.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/TextInstructions.scala index 01d040f32..9f83ab6a5 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/TextInstructions.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/TextInstructions.scala @@ -101,7 +101,7 @@ private [internal] final class EscapeAtMost(n: Int, radix: Int) extends EscapeSo } // $COVERAGE-OFF$ - override def toString: String = "EscapeAtMost" + override def toString: String = s"EscapeAtMost(n = $n, radix = $radix)" // $COVERAGE-ON$ } @@ -153,6 +153,6 @@ private [internal] final class EscapeOneOfExactly(radix: Int, ns: List[Int], ine } // $COVERAGE-OFF$ - override def toString: String = "EscapeOneOfExactly" + override def toString: String = s"EscapeOneOfExactly(ns = $ns, radix = $radix)" // $COVERAGE-ON$ } diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/CheckStack.scala b/parsley/shared/src/main/scala/parsley/internal/machine/stacks/CheckStack.scala deleted file mode 100644 index 80de35d83..000000000 --- a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/CheckStack.scala +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 Parsley Contributors - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package parsley.internal.machine.stacks - -private [machine] final class CheckStack(var offset: Int, val tail: CheckStack) -private [machine] object CheckStack extends Stack[CheckStack] { - implicit val inst: Stack[CheckStack] = this - type ElemTy = Int - // $COVERAGE-OFF$ - override protected def show(x: ElemTy): String = x.toString - override protected def head(xs: CheckStack): ElemTy = xs.offset - override protected def tail(xs: CheckStack): CheckStack = xs.tail - // $COVERAGE-ON$ -} diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HandlerStack.scala b/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HandlerStack.scala index 4e65070e4..ca6255e7e 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HandlerStack.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HandlerStack.scala @@ -5,18 +5,23 @@ */ package parsley.internal.machine.stacks +import parsley.internal.machine.errors.DefuncHints import parsley.internal.machine.instructions.Instr private [machine] final class HandlerStack( val calls: CallStack, val instrs: Array[Instr], - val pc: Int, + var pc: Int, val stacksz: Int, + var check: Int, + val hints: DefuncHints, + val hintOffset: Int, val tail: HandlerStack) private [machine] object HandlerStack extends Stack[HandlerStack] { implicit val inst: Stack[HandlerStack] = this type ElemTy = (Int, Int) // $COVERAGE-OFF$ + // TODO: needs to change override protected def show(x: ElemTy): String = { val (pc, stacksz) = x s"Handler:$pc(-${stacksz + 1})" diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HintStack.scala b/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HintStack.scala deleted file mode 100644 index e6e012e8c..000000000 --- a/parsley/shared/src/main/scala/parsley/internal/machine/stacks/HintStack.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2020 Parsley Contributors - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package parsley.internal.machine.stacks - -import parsley.internal.machine.errors.DefuncHints - -private [machine] final class HintStack(val hints: DefuncHints, val validOffset: Int, val tail: HintStack) -private [machine] object HintStack extends Stack[HintStack] { - implicit val inst: Stack[HintStack] = this - type ElemTy = (DefuncHints, Int) - // $COVERAGE-OFF$ - override protected def show(x: ElemTy): String = { - val (hints, validOffset) = x - s"($validOffset, $hints)" - } - override protected def head(xs: HintStack): ElemTy = (xs.hints, xs.validOffset) - override protected def tail(xs: HintStack): HintStack = xs.tail - // $COVERAGE-ON$ -} diff --git a/parsley/shared/src/main/scala/parsley/lift.scala b/parsley/shared/src/main/scala/parsley/lift.scala index 9fc03ca08..895f9bccc 100644 --- a/parsley/shared/src/main/scala/parsley/lift.scala +++ b/parsley/shared/src/main/scala/parsley/lift.scala @@ -20,15 +20,15 @@ import parsley.internal.deepembedding.frontend * scala> import parsley.character.char * scala> import parsley.lift.{lift2, lift3} * scala> case class Add(x: Int, y: Int) - * scala> val p = lift2(Add, char('a') #> 4, char('b') #> 5) + * scala> val p = lift2(Add, char('a').as(4), char('b').as(5)) * scala> p.parse("ab") * val res0 = Success(Add(4, 5)) - * scala> val q = lift3((x: Int, y: Int, z: Int) => x * y + z, char('a') #> 3, char('b') #> 2, char('c') #> 5) + * scala> val q = lift3((x: Int, y: Int, z: Int) => x * y + z, char('a').as(3), char('b').as(2), char('c').as(5)) * scala> q.parse("abc") * val res1 = Success(11) * scala> q.parse("ab") * val res2 = Failure(..) - * scala> val q2 = lift3[Int, Int, Int, Int](_ * _ + _, char('a') #> 3, char('b') #> 2, char('c') #> 5) + * scala> val q2 = lift3[Int, Int, Int, Int](_ * _ + _, char('a').as(3), char('b').as(2), char('c').as(5)) * }}} * @since 2.2.0 * diff --git a/parsley/shared/src/main/scala/parsley/position.scala b/parsley/shared/src/main/scala/parsley/position.scala index 3c993c3a1..c16a5e37e 100644 --- a/parsley/shared/src/main/scala/parsley/position.scala +++ b/parsley/shared/src/main/scala/parsley/position.scala @@ -106,8 +106,24 @@ object position { */ val offset: Parsley[Int] = internalOffset - // These are useless at 5.0.0 I think - private [parsley] def spanWith[A, S](end: Parsley[S])(p: Parsley[A]): Parsley[(S, A, S)] = (end, p, end).zipped - // this is subject to change at the slightest notice, do NOT expose - private [parsley] def internalOffsetSpan[A](p: Parsley[A]): Parsley[(Int, A, Int)] = spanWith(internalOffset)(p) + private [parsley] def withSpan[A, S](end: Parsley[S])(p: Parsley[A]): Parsley[(S, A, S)] = (end, p, end).zipped + + /** This combinator returns the result of a given parser and the number of characters it consumed. + * + * First records the initial `offset` on entry to given parser `p`, then executes `p`. If `p` succeeds, + * then the `offset` is taken again, and the two values are subtracted to give width `w`. The result of + * `p`, `x` is returned along with `w` as `(x, w)`. If `p` fails, this combinator will also fail. + * + * @example {{{ + * scala> import parsley.position.withWidth, parsley.character.string + * scala> withWidth(string("abc")).parse("abc") + * val res0 = Success(("abc", 3)) + * }}} + * + * @param p the parser to compute the width for + * @return a parser that pairs the result of the parser `p` with the number of characters it consumed + * @note the value returned is the number of 16-bit ''characters'' consumed, not unicode codepoints. + * @since 4.4.0 + */ + def withWidth[A](p: Parsley[A]): Parsley[(A, Int)] = (offset, p, offset).zipped((s, x, e) => (x, e-s)) } diff --git a/parsley/shared/src/main/scala/parsley/registers.scala b/parsley/shared/src/main/scala/parsley/registers.scala index 54f4199c9..b66d8ea00 100644 --- a/parsley/shared/src/main/scala/parsley/registers.scala +++ b/parsley/shared/src/main/scala/parsley/registers.scala @@ -8,6 +8,7 @@ package parsley import scala.collection.mutable import parsley.Parsley.{empty, fresh, pure} +import parsley.XAssert._ import parsley.combinator.{when, whileP} import parsley.exceptions.UnfilledRegisterException import parsley.implicits.zipped.Zipped2 @@ -126,7 +127,7 @@ object registers { * Without any other effect, the value `x` will be placed into this register. * * @example Put-Get Law: {{{ - * r.put(x) *> r.get == r.put(x) #> x + * r.put(x) *> r.get == r.put(x).as(x) * }}} * * @example Put-Put Law: {{{ @@ -315,6 +316,7 @@ object registers { * @param con a conversion that allows values convertible to parsers to be used. * @group ext */ + // TODO: rename? implicit final class RegisterMethods[P, A](p: P)(implicit con: P => Parsley[A]) { /** This combinator fills a fresh register with the result of this parser, this * register is provided to the given function, which continues the parse. @@ -362,6 +364,7 @@ object registers { * @param x the value to initialise a register with. * @group ext */ + // TODO: make AnyVal implicit final class RegisterMaker[A](x: A) { /** This combinator fills a fresh register with the this value. * @@ -425,8 +428,8 @@ object registers { * * r.put(0) *> * many('a' *> r.modify(_+1)) *> - * forYieldP_[Int](r.get, pure(_ != 0), pure(_ - 1)){_ => 'b'} *> - * forYieldP_[Int](r.get, pure(_ != 0), pure(_ - 1)){_ => 'c'} + * forYieldP_[Int, Char](r.get, pure(_ != 0), pure(_ - 1)){_ => 'b'} *> + * forYieldP_[Int, Char](r.get, pure(_ != 0), pure(_ - 1)){_ => 'c'} * }}} * * This will return a list `n` `'c'` characters. @@ -440,10 +443,10 @@ object registers { * @group comb */ def forYieldP_[A, B](init: Parsley[A], cond: =>Parsley[A => Boolean], step: =>Parsley[A => A])(body: Parsley[A] => Parsley[B]): Parsley[List[B]] = { - fresh(mutable.ListBuffer.empty[B]).fillReg { acc => + fresh(mutable.ListBuffer.empty[B]).persist { acc => forP_(init, cond, step) { x => - acc.put((acc.get, body(x)).zipped(_ += _)) - } *> acc.gets(_.toList) + (acc, body(x)).zipped(_ += _).unsafe() // we don't want this optimised out, it's a mutable operation in a resultless context + } ~> acc.map(_.toList) } } @@ -493,8 +496,8 @@ object registers { * * r.put(0) *> * many('a' *> r.modify(_+1)) *> - * forYieldP[Int](r.get, pure(_ != 0), pure(_ - 1)){'b'} *> - * forYieldP[Int](r.get, pure(_ != 0), pure(_ - 1)){'c'} + * forYieldP[Int, Char](r.get, pure(_ != 0), pure(_ - 1)){'b'} *> + * forYieldP[Int, Char](r.get, pure(_ != 0), pure(_ - 1)){'c'} * }}} * * This will return a list `n` `'c'` characters. @@ -509,10 +512,10 @@ object registers { * @group comb */ def forYieldP[A, B](init: Parsley[A], cond: =>Parsley[A => Boolean], step: =>Parsley[A => A])(body: =>Parsley[B]): Parsley[List[B]] = { - fresh(mutable.ListBuffer.empty[B]).fillReg { acc => + fresh(mutable.ListBuffer.empty[B]).persist { acc => forP(init, cond, step) { - acc.put((acc.get, body).zipped(_ += _)) - } *> acc.gets(_.toList) + (acc, body).zipped(_ += _).unsafe() + } ~> acc.map(_.toList) } } } diff --git a/parsley/shared/src/main/scala/parsley/token/Lexer.scala b/parsley/shared/src/main/scala/parsley/token/Lexer.scala index b224c1802..6271996f3 100644 --- a/parsley/shared/src/main/scala/parsley/token/Lexer.scala +++ b/parsley/shared/src/main/scala/parsley/token/Lexer.scala @@ -6,7 +6,6 @@ package parsley.token import parsley.Parsley, Parsley.unit -import parsley.character.satisfyUtf16 import parsley.combinator.{between, eof, sepBy, sepBy1, skipMany} import parsley.errors.combinator.{markAsToken, ErrorMethods} import parsley.registers.Reg @@ -17,6 +16,7 @@ import parsley.token.numeric.{LexemeCombined, LexemeInteger, LexemeReal, import parsley.token.predicate.{Basic, CharPredicate, NotRequired, Unicode} import parsley.token.symbol.{ConcreteSymbol, LexemeSymbol} import parsley.token.text.{ConcreteCharacter, ConcreteString, EscapableCharacter, Escape, LexemeCharacter, LexemeString, RawCharacter} +import parsley.unicode.satisfy import parsley.internal.deepembedding.singletons @@ -855,8 +855,8 @@ class Lexer(desc: descriptions.LexicalDesc, errConfig: errors.ErrorConfig) { case Basic(ws) => new Parsley(new singletons.WhiteSpace(ws, desc.spaceDesc, errConfig)) // satisfyUtf16 is effectively hidden, and so is Comment case Unicode(ws) if desc.spaceDesc.supportsComments => - skipMany(new Parsley(new singletons.Comment(desc.spaceDesc, errConfig)) <|> satisfyUtf16(ws))//.hide - case Unicode(ws) => skipMany(satisfyUtf16(ws))//.hide + skipMany(new Parsley(new singletons.Comment(desc.spaceDesc, errConfig)) <|> satisfy(ws)) + case Unicode(ws) => skipMany(satisfy(ws)) } } } diff --git a/parsley/shared/src/main/scala/parsley/token/errors/ConfigImplUntyped.scala b/parsley/shared/src/main/scala/parsley/token/errors/ConfigImplUntyped.scala index 18fd4a016..293be2d19 100644 --- a/parsley/shared/src/main/scala/parsley/token/errors/ConfigImplUntyped.scala +++ b/parsley/shared/src/main/scala/parsley/token/errors/ConfigImplUntyped.scala @@ -65,13 +65,15 @@ private [errors] final class Label private[errors] (val labels: Seq[String]) ext } private [parsley] final override def orElse(config: LabelConfig) = this } -/** This object has a factory for configurations producing labels: if the empty string is provided, this equivalent to [[Hidden `Hidden`]]. +/** This object has a factory for configurations producing labels: labels may not be empty. * @since 4.1.0 * @group labels */ object Label { def apply(label: String): LabelConfig = if (label.isEmpty) Hidden else new Label(Seq(label)) - private [parsley] def apply(labels: String*) = if (labels.isEmpty) Hidden else new Label(labels) + def apply(label1: String, label2: String, labels: String*): LabelConfig = new Label(label1 +: label2 +: labels) + // this is required internally, will go in parsley 5 + private [parsley] def apply(labels: String*) = new Label(labels) } /** This object configures labels by stating that it must be hidden. diff --git a/parsley/shared/src/main/scala/parsley/token/errors/VerifiedAndPreventativeErrors.scala b/parsley/shared/src/main/scala/parsley/token/errors/VerifiedAndPreventativeErrors.scala index 331e9e2ee..01442f0f9 100644 --- a/parsley/shared/src/main/scala/parsley/token/errors/VerifiedAndPreventativeErrors.scala +++ b/parsley/shared/src/main/scala/parsley/token/errors/VerifiedAndPreventativeErrors.scala @@ -6,8 +6,8 @@ package parsley.token.errors import parsley.Parsley, Parsley.empty -import parsley.character.satisfyUtf16 import parsley.errors.{combinator, patterns}, combinator.ErrorMethods, patterns.VerifiedErrors +import parsley.unicode.satisfy /** This class is used to configure what error is generated when `.` is parsed as a real number. * @since 4.1.0 @@ -70,7 +70,7 @@ sealed abstract class VerifiedBadChars { private [token] def checkBadChar: Parsley[Nothing] } private final class BadCharsFail private (cs: Map[Int, Seq[String]]) extends VerifiedBadChars { - private [token] def checkBadChar: Parsley[Nothing] = satisfyUtf16(cs.contains).verifiedFail(cs.apply(_)) + private [token] def checkBadChar: Parsley[Nothing] = satisfy(cs.contains).verifiedFail(cs.apply(_)) } /** This object makes "bad literal chars" generate a bunch of given messages in a ''specialised'' error. Requires a map from bad characters to their messages. * @since 4.1.0 @@ -81,7 +81,7 @@ object BadCharsFail { } private final class BadCharsReason private (cs: Map[Int, String]) extends VerifiedBadChars { - private [token] def checkBadChar: Parsley[Nothing] = satisfyUtf16(cs.contains).verifiedUnexpected(cs.apply(_)) + private [token] def checkBadChar: Parsley[Nothing] = satisfy(cs.contains).verifiedUnexpected(cs.apply(_)) } /** This object makes "bad literal chars" generate a reason in a ''vanilla'' error. Requires a map from bad characters to their reasons. * @since 4.1.0 diff --git a/parsley/shared/src/main/scala/parsley/token/names/ConcreteNames.scala b/parsley/shared/src/main/scala/parsley/token/names/ConcreteNames.scala index d40613efe..7b4c71a18 100644 --- a/parsley/shared/src/main/scala/parsley/token/names/ConcreteNames.scala +++ b/parsley/shared/src/main/scala/parsley/token/names/ConcreteNames.scala @@ -5,13 +5,14 @@ */ package parsley.token.names -import parsley.Parsley, Parsley.{attempt, empty, pure} -import parsley.character.{satisfy, satisfyUtf16, stringOfMany, stringOfManyUtf16} +import parsley.Parsley, Parsley.{atomic, empty, pure} +import parsley.character.{satisfy, stringOfMany} import parsley.errors.combinator.ErrorMethods import parsley.implicits.zipped.Zipped2 import parsley.token.descriptions.{NameDesc, SymbolDesc} import parsley.token.errors.ErrorConfig import parsley.token.predicate.{Basic, CharPredicate, NotRequired, Unicode} +import parsley.unicode.{satisfy => satisfyUtf16, stringOfMany => stringOfManyUtf16} import parsley.internal.deepembedding.singletons @@ -26,7 +27,7 @@ private [token] class ConcreteNames(nameDesc: NameDesc, symbolDesc: SymbolDesc, name: String, unexpectedIllegal: String => String) = { (startImpl, letterImpl) match { case (Basic(start), Basic(letter)) => new Parsley(new singletons.NonSpecific(name, unexpectedIllegal, start, letter, illegal)) - case _ => attempt { + case _ => atomic { complete(startImpl, letterImpl).unexpectedWhen { case x if illegal(x) => unexpectedIllegal(x) } @@ -34,8 +35,8 @@ private [token] class ConcreteNames(nameDesc: NameDesc, symbolDesc: SymbolDesc, } } private def trailer(impl: CharPredicate) = impl match { - case Basic(letter) => stringOfMany(satisfy(letter)) - case Unicode(letter) => stringOfManyUtf16(satisfyUtf16(letter)) + case Basic(letter) => stringOfMany(letter) + case Unicode(letter) => stringOfManyUtf16(letter) case NotRequired => pure("") } private def complete(start: CharPredicate, letter: CharPredicate) = start match { @@ -49,14 +50,14 @@ private [token] class ConcreteNames(nameDesc: NameDesc, symbolDesc: SymbolDesc, override lazy val identifier: Parsley[String] = keyOrOp(nameDesc.identifierStart, nameDesc.identifierLetter, symbolDesc.isReservedName, err.labelNameIdentifier, err.unexpectedNameIllegalIdentifier) - override def identifier(startChar: CharPredicate): Parsley[String] = attempt { + override def identifier(startChar: CharPredicate): Parsley[String] = atomic { err.filterNameIllFormedIdentifier.filter(identifier)(startChar.startsWith) } override lazy val userDefinedOperator: Parsley[String] = keyOrOp(nameDesc.operatorStart, nameDesc.operatorLetter, symbolDesc.isReservedOp, err.labelNameOperator, err.unexpectedNameIllegalOperator) - def userDefinedOperator(startChar: CharPredicate, endChar: CharPredicate): Parsley[String] = attempt { + def userDefinedOperator(startChar: CharPredicate, endChar: CharPredicate): Parsley[String] = atomic { err.filterNameIllFormedOperator.filter(userDefinedOperator)(x => startChar.startsWith(x) && endChar.endsWith(x)) } } diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/Generic.scala b/parsley/shared/src/main/scala/parsley/token/numeric/Generic.scala index 34b94d028..8e5ad1ee7 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/Generic.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/Generic.scala @@ -38,7 +38,7 @@ private [token] class Generic(err: ErrorConfig) { private def nonZeroOctDigit = satisfy(c => isOctDigit(c) && c != '0').label("octal digit") private def nonZeroBit = '1'.label("bit") // why secret? so that the above digits can be marked as digits without "non-zero or zero digit" nonsense - private def secretZero = '0'.hide #> BigInt(0) + private def secretZero = '0'.newHide.as(BigInt(0)) private def digit = character.digit private def hexDigit = character.hexDigit diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/SignedCombined.scala b/parsley/shared/src/main/scala/parsley/token/numeric/SignedCombined.scala index 72dcf4ca7..f56837ba1 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/SignedCombined.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/SignedCombined.scala @@ -5,7 +5,7 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.attempt +import parsley.Parsley, Parsley.atomic import parsley.token.descriptions.numeric.NumericDesc import parsley.token.errors.ErrorConfig @@ -15,11 +15,11 @@ import parsley.internal.deepembedding.singletons private [token] final class SignedCombined(desc: NumericDesc, unsigned: Combined, err: ErrorConfig) extends Combined(err) { private val sign = new Parsley(new singletons.Sign[CombinedType.resultType](CombinedType, desc.positiveSign)) - override def decimal: Parsley[Either[BigInt,BigDecimal]] = attempt(sign <*> unsigned.decimal) - override def hexadecimal: Parsley[Either[BigInt,BigDecimal]] = attempt(sign <*> unsigned.hexadecimal) - override def octal: Parsley[Either[BigInt,BigDecimal]] = attempt(sign <*> unsigned.octal) - override def binary: Parsley[Either[BigInt,BigDecimal]] = attempt(sign <*> unsigned.binary) - override def number: Parsley[Either[BigInt,BigDecimal]] = attempt(sign <*> unsigned.number) + override def decimal: Parsley[Either[BigInt,BigDecimal]] = atomic(sign <*> unsigned.decimal) + override def hexadecimal: Parsley[Either[BigInt,BigDecimal]] = atomic(sign <*> unsigned.hexadecimal) + override def octal: Parsley[Either[BigInt,BigDecimal]] = atomic(sign <*> unsigned.octal) + override def binary: Parsley[Either[BigInt,BigDecimal]] = atomic(sign <*> unsigned.binary) + override def number: Parsley[Either[BigInt,BigDecimal]] = atomic(sign <*> unsigned.number) override protected[numeric] def bounded[T](number: Parsley[Either[BigInt,BigDecimal]], bits: Bits, radix: Int) (implicit ev: CanHold[bits.self,T]): Parsley[Either[T,BigDecimal]] = { diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/SignedInteger.scala b/parsley/shared/src/main/scala/parsley/token/numeric/SignedInteger.scala index f53c79edc..c8777caf2 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/SignedInteger.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/SignedInteger.scala @@ -5,7 +5,7 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.attempt +import parsley.Parsley, Parsley.atomic import parsley.token.descriptions.numeric.NumericDesc import parsley.token.errors.{ErrorConfig, LabelWithExplainConfig} @@ -15,11 +15,11 @@ import parsley.internal.deepembedding.singletons private [token] final class SignedInteger(desc: NumericDesc, unsigned: UnsignedInteger, err: ErrorConfig) extends Integer(desc) { private val sign = new Parsley(new singletons.Sign[IntType.resultType](IntType, desc.positiveSign)) - override lazy val _decimal: Parsley[BigInt] = attempt(sign <*> err.labelIntegerDecimalEnd(unsigned._decimal)) - override lazy val _hexadecimal: Parsley[BigInt] = attempt(sign <*> err.labelIntegerHexadecimalEnd(unsigned._hexadecimal)) - override lazy val _octal: Parsley[BigInt] = attempt(sign <*> err.labelIntegerOctalEnd(unsigned._octal)) - override lazy val _binary: Parsley[BigInt] = attempt(sign <*> err.labelIntegerBinaryEnd(unsigned._binary)) - override lazy val _number: Parsley[BigInt] = attempt(sign <*> err.labelIntegerNumberEnd(unsigned._number)) + override lazy val _decimal: Parsley[BigInt] = atomic(sign <*> err.labelIntegerDecimalEnd(unsigned._decimal)) + override lazy val _hexadecimal: Parsley[BigInt] = atomic(sign <*> err.labelIntegerHexadecimalEnd(unsigned._hexadecimal)) + override lazy val _octal: Parsley[BigInt] = atomic(sign <*> err.labelIntegerOctalEnd(unsigned._octal)) + override lazy val _binary: Parsley[BigInt] = atomic(sign <*> err.labelIntegerBinaryEnd(unsigned._binary)) + override lazy val _number: Parsley[BigInt] = atomic(sign <*> err.labelIntegerNumberEnd(unsigned._number)) override def decimal: Parsley[BigInt] = err.labelIntegerSignedDecimal.apply(_decimal) override def hexadecimal: Parsley[BigInt] = err.labelIntegerSignedHexadecimal.apply(_hexadecimal) diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/SignedReal.scala b/parsley/shared/src/main/scala/parsley/token/numeric/SignedReal.scala index 8d7d33c59..733890746 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/SignedReal.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/SignedReal.scala @@ -5,7 +5,7 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.attempt +import parsley.Parsley, Parsley.atomic import parsley.token.descriptions.numeric.NumericDesc import parsley.token.errors.ErrorConfig @@ -15,11 +15,11 @@ import parsley.internal.deepembedding.singletons private [token] final class SignedReal(desc: NumericDesc, unsigned: Real, err: ErrorConfig) extends Real(err) { private val sign = new Parsley(new singletons.Sign[DoubleType.resultType](DoubleType, desc.positiveSign)) - override lazy val _decimal: Parsley[BigDecimal] = attempt(sign <*> err.labelRealDecimalEnd(unsigned._decimal)) - override lazy val _hexadecimal: Parsley[BigDecimal] = attempt(sign <*> err.labelRealHexadecimalEnd(unsigned._hexadecimal)) - override lazy val _octal: Parsley[BigDecimal] = attempt(sign <*> err.labelRealOctalEnd(unsigned._octal)) - override lazy val _binary: Parsley[BigDecimal] = attempt(sign <*> err.labelRealBinaryEnd(unsigned._binary)) - override lazy val _number: Parsley[BigDecimal] = attempt(sign <*> err.labelRealNumberEnd(unsigned._number)) + override lazy val _decimal: Parsley[BigDecimal] = atomic(sign <*> err.labelRealDecimalEnd(unsigned._decimal)) + override lazy val _hexadecimal: Parsley[BigDecimal] = atomic(sign <*> err.labelRealHexadecimalEnd(unsigned._hexadecimal)) + override lazy val _octal: Parsley[BigDecimal] = atomic(sign <*> err.labelRealOctalEnd(unsigned._octal)) + override lazy val _binary: Parsley[BigDecimal] = atomic(sign <*> err.labelRealBinaryEnd(unsigned._binary)) + override lazy val _number: Parsley[BigDecimal] = atomic(sign <*> err.labelRealNumberEnd(unsigned._number)) override def decimal: Parsley[BigDecimal] = err.labelRealDecimal(_decimal) override def hexadecimal: Parsley[BigDecimal] = err.labelRealHexadecimal(_hexadecimal) diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedCombined.scala b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedCombined.scala index 218871440..92dd2f220 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedCombined.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedCombined.scala @@ -5,18 +5,18 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.attempt +import parsley.Parsley, Parsley.atomic import parsley.token.descriptions.numeric.NumericDesc import parsley.token.errors.ErrorConfig import org.typelevel.scalaccompat.annotation.unused private [token] final class UnsignedCombined(@unused desc: NumericDesc, integer: Integer, rational: Real, err: ErrorConfig) extends Combined(err) { - override lazy val decimal: Parsley[Either[BigInt, BigDecimal]] = (attempt(rational.decimal) <+> integer.decimal).map(_.swap) - override lazy val hexadecimal: Parsley[Either[BigInt, BigDecimal]] = (attempt(rational.hexadecimal) <+> integer.hexadecimal).map(_.swap) - override lazy val octal: Parsley[Either[BigInt, BigDecimal]] = (attempt(rational.octal) <+> integer.octal).map(_.swap) - override lazy val binary: Parsley[Either[BigInt, BigDecimal]] = (attempt(rational.binary) <+> integer.binary).map(_.swap) - override lazy val number: Parsley[Either[BigInt, BigDecimal]] = (attempt(rational.number) <+> integer.number).map(_.swap) + override lazy val decimal: Parsley[Either[BigInt, BigDecimal]] = (atomic(rational.decimal) <+> integer.decimal).map(_.swap) + override lazy val hexadecimal: Parsley[Either[BigInt, BigDecimal]] = (atomic(rational.hexadecimal) <+> integer.hexadecimal).map(_.swap) + override lazy val octal: Parsley[Either[BigInt, BigDecimal]] = (atomic(rational.octal) <+> integer.octal).map(_.swap) + override lazy val binary: Parsley[Either[BigInt, BigDecimal]] = (atomic(rational.binary) <+> integer.binary).map(_.swap) + override lazy val number: Parsley[Either[BigInt, BigDecimal]] = (atomic(rational.number) <+> integer.number).map(_.swap) // FIXME: gross :( we should restructure this to be more reusable friendly // FIXME: This doesn't respect the number requirements on hex oct bin for both integer and rational @@ -28,9 +28,9 @@ private [token] final class UnsignedCombined(@unused desc: NumericDesc, integer: val decExponent = oneOf('e', 'E') *> integer.decimal32 val zeroPoint = (decFractional, decExponent <|> pure(0)).zipped[Either[BigInt, BigDecimal]] { case (f, e) => Right(f / 10 * BigDecimal(10).pow(e)) - } <|> decExponent #> Right(BigDecimal(0)) + } <|> decExponent.as(Right(BigDecimal(0))) val zeroLead = '0' *> (noZeroHexadecimal <|> noZeroOctal <|> noZeroBinary <|> zeroPoint <|> decimal <|> pure(Left(BigInt(0)))) - attempt(zeroLead <|> decimal) + atomic(zeroLead <|> decimal) }*/ // TODO: Using choice here will generate a jump table, which will be nicer for `number` (this requires enhancements to the jumptable optimisation) diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedInteger.scala b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedInteger.scala index a18c0679f..a0244b348 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedInteger.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedInteger.scala @@ -5,7 +5,7 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.{attempt, pure, unit} +import parsley.Parsley, Parsley.{atomic, pure, unit} import parsley.character.oneOf import parsley.combinator.optional import parsley.errors.combinator.ErrorMethods @@ -17,9 +17,9 @@ private [token] final class UnsignedInteger(desc: NumericDesc, err: ErrorConfig, // labelless versions protected [numeric] override lazy val _decimal: Parsley[BigInt] = generic.plainDecimal(desc, err.labelIntegerDecimalEnd) - protected [numeric] override lazy val _hexadecimal: Parsley[BigInt] = attempt('0' *> noZeroHexadecimal) - protected [numeric] override lazy val _octal: Parsley[BigInt] = attempt('0' *> noZeroOctal) - protected [numeric] override lazy val _binary: Parsley[BigInt] = attempt('0' *> noZeroBinary) + protected [numeric] override lazy val _hexadecimal: Parsley[BigInt] = atomic('0' *> noZeroHexadecimal) + protected [numeric] override lazy val _octal: Parsley[BigInt] = atomic('0' *> noZeroOctal) + protected [numeric] override lazy val _binary: Parsley[BigInt] = atomic('0' *> noZeroBinary) protected [numeric] override lazy val _number: Parsley[BigInt] = { if (desc.decimalIntegersOnly) decimal else { @@ -36,7 +36,7 @@ private [token] final class UnsignedInteger(desc: NumericDesc, err: ErrorConfig, else p } val zeroLead = '0'.label("digit") *> (addHex(addOct(addBin(decimal <|> pure(BigInt(0)))))) - attempt(zeroLead <|> decimal) + atomic(zeroLead <|> decimal) } } diff --git a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedReal.scala b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedReal.scala index 02195d0c0..d57f897af 100644 --- a/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedReal.scala +++ b/parsley/shared/src/main/scala/parsley/token/numeric/UnsignedReal.scala @@ -5,7 +5,7 @@ */ package parsley.token.numeric -import parsley.Parsley, Parsley.{attempt, empty, pure, unit} +import parsley.Parsley, Parsley.{atomic, empty, pure, unit} import parsley.character.{bit, digit, hexDigit, octDigit, oneOf} import parsley.combinator, combinator.optional import parsley.errors.combinator.{amendThenDislodge, entrench} @@ -16,10 +16,10 @@ import parsley.token.descriptions.numeric.{BreakCharDesc, ExponentDesc, NumericD import parsley.token.errors.{ErrorConfig, LabelConfig} private [token] final class UnsignedReal(desc: NumericDesc, natural: UnsignedInteger, err: ErrorConfig, generic: Generic) extends Real(err) { - override lazy val _decimal: Parsley[BigDecimal] = attempt(ofRadix(10, digit, err.labelRealDecimalEnd)) - override lazy val _hexadecimal: Parsley[BigDecimal] = attempt('0' *> noZeroHexadecimal) - override lazy val _octal: Parsley[BigDecimal] = attempt('0' *> noZeroOctal) - override lazy val _binary: Parsley[BigDecimal] = attempt('0' *> noZeroBinary) + override lazy val _decimal: Parsley[BigDecimal] = atomic(ofRadix(10, digit, err.labelRealDecimalEnd)) + override lazy val _hexadecimal: Parsley[BigDecimal] = atomic('0' *> noZeroHexadecimal) + override lazy val _octal: Parsley[BigDecimal] = atomic('0' *> noZeroOctal) + override lazy val _binary: Parsley[BigDecimal] = atomic('0' *> noZeroBinary) override lazy val _number: Parsley[BigDecimal] = { if (desc.decimalRealsOnly) decimal else { @@ -39,7 +39,7 @@ private [token] final class UnsignedReal(desc: NumericDesc, natural: UnsignedInt val leadingDotAllowedDecimal = if (desc.leadingDotAllowed) decimal else ofRadix(10, digit, leadingDotAllowed = true, err.labelRealNumberEnd) // not even accounting for the leading and trailing dot being allowed! val zeroLead = '0' *> (addHex(addOct(addBin(leadingDotAllowedDecimal <|> pure(BigDecimal(0)))))) - attempt(zeroLead <|> decimal) + atomic(zeroLead <|> decimal) } } diff --git a/parsley/shared/src/main/scala/parsley/token/predicate.scala b/parsley/shared/src/main/scala/parsley/token/predicate.scala index f449f37b5..83b90fde4 100644 --- a/parsley/shared/src/main/scala/parsley/token/predicate.scala +++ b/parsley/shared/src/main/scala/parsley/token/predicate.scala @@ -8,8 +8,9 @@ package parsley.token import scala.collection.immutable.NumericRange import parsley.Parsley, Parsley.empty -import parsley.character.{satisfy, satisfyUtf16} +import parsley.character.satisfy import parsley.exceptions.ParsleyException +import parsley.unicode.{satisfy => satisfyUtf16} // TODO: for parsley 5.0.0, make this a package? /** This module contains functionality to describe character predicates, which can diff --git a/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala b/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala index b60d11372..442960cbf 100644 --- a/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala +++ b/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala @@ -5,7 +5,7 @@ */ package parsley.token.symbol -import parsley.Parsley, Parsley.attempt +import parsley.Parsley, Parsley.atomic import parsley.character.{char, string} import parsley.token.descriptions.{NameDesc, SymbolDesc} import parsley.token.errors.ErrorConfig @@ -24,7 +24,7 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, require(name.nonEmpty, "Symbols may not be empty strings") if (symbolDesc.hardKeywords(name)) softKeyword(name) else if (symbolDesc.hardOperators(name)) softOperator(name) - else attempt(string(name)).void + else atomic(string(name)).void } override def apply(name: Char): Parsley[Unit] = char(name).void diff --git a/parsley/shared/src/main/scala/parsley/token/text/ConcreteString.scala b/parsley/shared/src/main/scala/parsley/token/text/ConcreteString.scala index c4dd5f1f6..18c097b83 100644 --- a/parsley/shared/src/main/scala/parsley/token/text/ConcreteString.scala +++ b/parsley/shared/src/main/scala/parsley/token/text/ConcreteString.scala @@ -7,7 +7,7 @@ package parsley.token.text import scala.Predef.{String => ScalaString, _} -import parsley.Parsley, Parsley.{attempt, fresh, pure} +import parsley.Parsley, Parsley.{atomic, fresh, pure} import parsley.character.{char, string} import parsley.combinator.{choice, skipManyUntil} import parsley.errors.combinator.ErrorMethods @@ -34,16 +34,18 @@ private [token] final class ConcreteString(ends: Set[ScalaString], stringChar: S val terminalInit = terminalStr.charAt(0) val strChar = stringChar(Character.letter(terminalInit, allowsAllSpace, isGraphic)) val pf = (sb: StringBuilder, cpo: Option[Int]) => { - for (cp <- cpo) parsley.character.addCodepoint(sb, cp) + for (cp <- cpo) parsley.unicode.addCodepoint(sb, cp) sb } - val content = valid(parsley.expr.infix.secretLeft1((sbReg.get, strChar).zipped(pf), strChar, pure(pf))) + // `content` is in a dropped position, so needs the unsafe to avoid the mutation + // TODO: this could be fixed better with registers and skipMany? + val content = valid(parsley.expr.infix.secretLeft1((sbReg.get, strChar).zipped(pf), strChar, pure(pf)).unsafe()) val terminal = string(terminalStr) // terminal should be first, to allow for a jump table on the main choice openLabel(allowsAllSpace, stringChar.isRaw)(terminal) *> // then only one string builder needs allocation sbReg.put(fresh(new StringBuilder)) *> - skipManyUntil(sbReg.modify(char(terminalInit).hide #> ((sb: StringBuilder) => sb += terminalInit)) <|> content, - closeLabel(allowsAllSpace, stringChar.isRaw)(attempt(terminal))) //is the attempt needed here? not sure + skipManyUntil(sbReg.modify(char(terminalInit).newHide.as((sb: StringBuilder) => sb += terminalInit)) <|> content, + closeLabel(allowsAllSpace, stringChar.isRaw)(atomic(terminal))) //is the atomic needed here? not sure } } diff --git a/parsley/shared/src/main/scala/parsley/token/text/StringCharacter.scala b/parsley/shared/src/main/scala/parsley/token/text/StringCharacter.scala index 70aec4e83..34b08d221 100644 --- a/parsley/shared/src/main/scala/parsley/token/text/StringCharacter.scala +++ b/parsley/shared/src/main/scala/parsley/token/text/StringCharacter.scala @@ -6,12 +6,13 @@ package parsley.token.text import parsley.Parsley, Parsley.empty -import parsley.character.{char, satisfy, satisfyUtf16} +import parsley.character.{char, satisfyMap} import parsley.combinator.skipSome import parsley.implicits.character.charLift import parsley.token.descriptions.text.EscapeDesc import parsley.token.errors.ErrorConfig import parsley.token.predicate.{Basic, CharPredicate, NotRequired, Unicode} +import parsley.unicode.{satisfyMap => satisfyMapUtf16} private [token] abstract class StringCharacter { def apply(isLetter: CharPredicate): Parsley[Option[Int]] @@ -23,8 +24,8 @@ private [token] abstract class StringCharacter { private [token] class RawCharacter(err: ErrorConfig) extends StringCharacter { override def isRaw: Boolean = true override def apply(isLetter: CharPredicate): Parsley[Option[Int]] = isLetter match { - case Basic(isLetter) => err.labelStringCharacter(satisfy(isLetter).map(c => Some(c.toInt))) <|> _checkBadChar(err) - case Unicode(isLetter) => err.labelStringCharacter(satisfyUtf16(isLetter).map(Some(_))) <|> _checkBadChar(err) + case Basic(isLetter) => err.labelStringCharacter(satisfyMap { case c if isLetter(c) => Some(c.toInt) }) <|> _checkBadChar(err) + case Unicode(isLetter) => err.labelStringCharacter(satisfyMapUtf16 { case c if isLetter(c) => Some(c) }) <|> _checkBadChar(err) case NotRequired => empty } } @@ -37,18 +38,18 @@ private [token] class EscapableCharacter(desc: EscapeDesc, escapes: Escape, spac else empty } private lazy val stringEscape: Parsley[Option[Int]] = - escapes.escapeBegin *> (escapeGap #> None - <|> escapeEmpty #> None + escapes.escapeBegin *> (escapeGap.as(None) + <|> escapeEmpty.as(None) <|> escapes.escapeCode.map(Some(_))) override def apply(isLetter: CharPredicate): Parsley[Option[Int]] = { isLetter match { case Basic(isLetter) => err.labelStringCharacter( - stringEscape <|> err.labelGraphicCharacter(satisfy(c => isLetter(c) && c != desc.escBegin).map(c => Some(c.toInt))) + stringEscape <|> err.labelGraphicCharacter(satisfyMap { case c if isLetter(c) && c != desc.escBegin => Some(c.toInt) }) <|> _checkBadChar(err) ) case Unicode(isLetter) => err.labelStringCharacter( - stringEscape <|> err.labelGraphicCharacter(satisfyUtf16(c => isLetter(c) && c != desc.escBegin.toInt).map(Some(_))) + stringEscape <|> err.labelGraphicCharacter(satisfyMapUtf16 { case c if isLetter(c) && c != desc.escBegin.toInt => Some(c) }) <|> _checkBadChar(err) ) case NotRequired => stringEscape diff --git a/parsley/shared/src/main/scala/parsley/unicode.scala b/parsley/shared/src/main/scala/parsley/unicode.scala new file mode 100644 index 000000000..774db5d74 --- /dev/null +++ b/parsley/shared/src/main/scala/parsley/unicode.scala @@ -0,0 +1,765 @@ +/* + * Copyright 2020 Parsley Contributors + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package parsley + +import parsley.Parsley.{empty, fresh, pure} +import parsley.combinator.{skipMany, skipSome} +import parsley.errors.combinator.ErrorMethods +import parsley.token.errors.{Label, LabelConfig, NotConfigured} + +import parsley.internal.deepembedding.singletons + +/** This module contains many parsers to do with reading one or more characters. + * + * In particular, this module contains: combinators that can read specific characters; combinators that represent character classes and their negations; + * combinators for reading specific strings; as well as a selection of pre-made parsers to parse specific kinds of character, like digits and letters. + * Unlike [[parsley.character `character`]], this module handles full utf-16 codepoints, which can be up to two 16-bit characters long. + * + * @since 4.4.0 + * + * @groupprio pred 100 + * @groupname pred Character Predicates + * @groupdesc pred + * These are useful for providing to the sub-descriptions of a [[token.descriptions.LexicalDesc]] to specify behaviour for the lexer. + * Other than that, they aren't ''particularly'' useful. + * + * @groupprio core 0 + * @groupname core Core Combinators and Parsers + * @groupdesc core + * These are the most primitive combinators for consuming input capable of any input reading tasks. + * + * @groupprio skip 75 + * @groupname skip Whitespace Skipping Parsers + * @groupdesc skip + * These parsers are designed to skip chunks of whitespace, for very rudimentary lexing tasks. It + * is probably better to use the functionality of [[parsley.token]]. + * + * @groupprio class 20 + * @groupname class Character Class Combinators + * @groupdesc class + * These combinators allow for working with ''character classes''. This means that a set, or range, of + * characters can be specified, and the combinator will return a parser that matches one of those characters + * (or conversely, any character that is ''not'' in that set). The parsed character is always returned. + * + * @groupprio spec 25 + * @groupname spec Specific Character Parsers + * @groupdesc spec + * These parsers are special cases of [[satisfy `satisfy`]] or [[char `char`]]. They are worth using, as they are given special error labelling, + * producing nicer error messages than their primitive counterparts. + * + * This documentation assumes JDK 17. + * JDK 17 is compliant with [[https://www.unicode.org/versions/Unicode13.0.0/UnicodeStandard-13.0.pdf Unicode® Specification 13.0]]. + * As such, the descriptions of the parsers in this section are accurate with respect to Unicode® Specification 13.0: + * using a different JDK may affect the ''precise'' definitions of the parsers below. If in doubt, check the documentation + * for `java.lang.Character` to see which Unicode version is supported by your JVM. A table of the Unicode versions + * up to JDK 17 can be found [[https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Character.html here]]. + * + * @groupprio string 22 + * @groupname string String Combinators + * @groupdesc string + * These combinators allow for working with, or building, strings. This means that they can + * parse specific strings, specific sets of strings, or can read codepoints repeatedly to + * generate strings. They are united in all returning `String` as their result. + * + * @define oneOf + * This combinator tries to parse any codepoint from supplied set of codepoints `cs`, returning it if successful. + * @define noneOf + * This combinator tries to parse any codepoint '''not''' from supplied set of codepoints `cs`, returning it if successful. + * + * @define categories + * ''The full list of codepoints found in a category can be found in the + * [[https://www.unicode.org/Public/13.0.0/ucd/extracted/DerivedGeneralCategory.txt Unicode Character Database]]''. + */ +object unicode { + /** This combinator tries to parse a single specific codepoint `c` from the input. + * + * Like [[character.char `character.char`]], except it may consume two characters from the input, + * in the case where the code-point is greater than `0xffff`. This is parsed ''atomically'' + * so that no input is consumed if the first half of the codepoint is parsed and the second + * is not. + * + * @example {{{ + * scala> import parsley.unicode.char + * scala> char(0x1f642).parse("") + * val res0 = Failure(..) + * scala> char(0x1f642).parse("🙂") + * val res1 = Success(0x1f642) + * scala> char(0x1f642).parse("b🙂") + * val res2 = Failure(..) + * }}} + * + * @param c the code-point to parse + * @return + * @group core + */ + def char(c: Int): Parsley[Int] = char(c, NotConfigured) + private def char(c: Int, label: String): Parsley[Int] = char(c, Label(label)) + private def char(c: Int, label: LabelConfig): Parsley[Int] = { + if (Character.isBmpCodePoint(c)) new Parsley(new singletons.CharTok(c.toChar, c, label)) + else new Parsley(new singletons.SupplementaryCharTok(c, c, label)) + } + + // TODO: test + /** This combinator tries to parse a single codepoint from the input that matches the given predicate. + * + * Attempts to read a codepoint from the input and tests it against the predicate `pred`. If a codepoint `c` + * can be read and `pred(c)` is true, then `c` is consumed and returned. Otherwise, no input is consumed + * and this combinator will fail. + * + * @example {{{ + * scala> import parsley.unicode.satisfy + * scala> satisfy(Character.isDigit(_)).parse("") + * val res0 = Failure(..) + * scala> satisfy(Character.isDigit(_)).parse("7") + * val res1 = Success(0x37) + * scala> satisfy(Character.isDigit(_)).parse("a5") + * val res2 = Failure(..) + * scala> def char(c: Int): Parsley[Int] = satisfy(_ == c) + * }}} + * + * @param pred the predicate to test the next codepoint against, should one exist. + * @return a parser that tries to read a single codepoint `c`, such that `pred(c)` is true, or fails. + * @group core + */ + def satisfy(pred: Int => Boolean): Parsley[Int] = satisfy(pred, NotConfigured) + private def satisfy(pred: Int => Boolean, label: String): Parsley[Int] = satisfy(pred, Label(label)) + private def satisfy(pred: Int => Boolean, label: LabelConfig) = new Parsley(new singletons.UniSatisfy(pred, label)) + + // TODO: test + /** This combinator tries to parse and process a codepoint from the input if it is defined for the given function. + * + * Attempts to read a codepoint from the input and tests to see if it is in the domain of `f`. If a codepoint + * `c` can be read and `f(c)` is defined, then `c` is consumed and `f(c)` is returned. Otherwise, no input is consumed + * and this combinator will fail. + * + * @example {{{ + * scala> import parsley.unicode.satisfyMap + * scala> val chars = satisfyMap { + * case c => Character.toChars(c) + * } + * scala> chars.parse("") + * val res0 = Failure(..) + * scala> chars.parse("7") + * val res1 = Success(Array('7')) + * scala> chars.parse("🙂") + * val res2 = Success(Array('\ud83d', '\ude42')) + * }}} + * + * @param f the function to test the next codepoint against and transform it with, should one exist. + * @return a parser that tries to read a single codepoint `c`, such that `f(c)` is defined, and returns `f(c)` if so, or fails. + * @since 4.4.0 + * @group core + */ + def satisfyMap[A](pred: PartialFunction[Int, A]): Parsley[A] = satisfy(pred.isDefinedAt(_)).map(pred) + + // This should always just match up, so no need to test + // $COVERAGE-OFF$ + /** This combinator attempts to parse a given string from the input, and fails otherwise. + * + * Attempts to read the given string ''completely'' from the input at the current position. + * If the string is present, then the parser succeeds, and the entire string is consumed + * from the input. Otherwise, if the input has too few characters remaining, or not all + * the characters matched, the parser fails. On failure, '''all''' the characters that were + * matched are consumed from the input. + * + * @example {{{ + * scala> import parsley.unicode.string + * scala> string("abc").parse("") + * val res0 = Failure(..) + * scala> string("abc").parse("abcd") + * val res1 = Success("abc") + * scala> string("abc").parse("xabc") + * val res2 = Failure(..) + * }}} + * + * @param s the string to be parsed from the input + * @return a parser that either parses the string `s` or fails at the first mismatched character. + * @note the error messages generated by `string` do not reflect how far into the input it managed + * to get: this is because the error being positioned at the start of the string is more + * natural. However, input '''will''' still be consumed for purposes of backtracking. + * @note just an alias for [[character.string `character.string`]], to allow for more ergonomic imports. + * @group string + */ + def string(s: String): Parsley[String] = character.string(s) + // $COVERAGE-ON$ + + /** $oneOf + * + * If the next codepoint in the input is a member of the set `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.codepoint.oneOf + * scala> val p = oneOf(Set(97, 98, 99)) + * scala> p.parse("a") + * val res0 = Success(97) + * scala> p.parse("c") + * val res1 = Success(99) + * scala> p.parse("xb") + * val res2 = Failure(..) + * }}} + * + * @param cs the set of codepoints to check. + * @return a parser that parses one of the member of the set `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def oneOf(cs: Set[Int]): Parsley[Int] = cs.size match { + case 0 => empty + case 1 => char(cs.head) + case _ => satisfy(cs, { + val Some(label) = parsley.errors.helpers.disjunct(cs.map(renderChar).toList, oxfordComma = true): @unchecked + s"one of $label" + }) + } + + /** $oneOf + * + * If the next codepoint in the input is an element of the list of codepoints `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.oneOf + * scala> val p = oneOf(97, 98, 99) + * scala> p.parse("a") + * val res0 = Success(97) + * scala> p.parse("c") + * val res1 = Success(99) + * scala> p.parse("xb") + * val res2 = Failure(..) + * }}} + * + * @param cs the codepoints to check. + * @return a parser that parses one of the elements of `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def oneOf(cs: Int*): Parsley[Int] = oneOf(cs.toSet) + + /** $oneOf + * + * If the next codepoint in the input is within the range of codepoints `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.oneOf + * scala> val p = oneOf(97 to 99) + * scala> p.parse("a") + * val res0 = Success(97) + * scala> p.parse("b") + * val res1 = Success(98) + * scala> p.parse("c") + * val res1 = Success(99) + * scala> p.parse("xb") + * val res2 = Failure(..) + * }}} + * + * @param cs the range of codepoints to check. + * @return a parser that parses a codepoint within the range `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def oneOf(cs: Range): Parsley[Int] = cs.size match { + case 0 => empty + case 1 => char(cs.head) + case _ if Math.abs(cs(0) - cs(1)) == 1 => satisfy(cs.contains(_), + s"one of ${renderChar(cs.min)} to ${renderChar(cs.max)}" + ) + case _ => satisfy(cs.contains(_)) + } + + /** $noneOf + * + * If the next codepoint in the input is not a member of the set `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.noneOf + * scala> val p = noneOf(Set('a', 'b', 'c')) + * scala> p.parse("a") + * val res0 = Failure(..) + * scala> p.parse("c") + * val res1 = Failure(..) + * scala> p.parse("xb") + * val res2 = Success('x') + * scala> p.parse("") + * val res3 = Failure(..) + * }}} + * + * @param cs the set of codepoints to check. + * @return a parser that parses one codepoint that is not a member of the set `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def noneOf(cs: Set[Int]): Parsley[Int] = cs.size match { + case 0 => item + case 1 => satisfy(cs.head != _, s"anything except ${renderChar(cs.head)}") + case _ => satisfy(!cs.contains(_), { + val Some(label) = parsley.errors.helpers.disjunct(cs.map(renderChar).toList, oxfordComma = true): @unchecked + s"anything except $label" + }) + } + + /** $noneOf + * + * If the next codepoint in the input is not an element of the list of codepoints `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.noneOf + * scala> val p = noneOf('a', 'b', 'c') + * scala> p.parse("a") + * val res0 = Failure(..) + * scala> p.parse("c") + * val res1 = Failure(..) + * scala> p.parse("xb") + * val res2 = Success('x') + * scala> p.parse("") + * val res3 = Failure(..) + * }}} + * + * @param cs the set of codepoints to check. + * @return a parser that parses one codepoint that is not an element of `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def noneOf(cs: Int*): Parsley[Int] = noneOf(cs.toSet) + + /** $noneOf + * + * If the next codepoint in the input is outside of the range of codepoints `cs`, it is consumed + * and returned. Otherwise, no input is consumed and the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.noneOf + * scala> val p = noneOf('a' to 'c') + * scala> p.parse("a") + * val res0 = Failure(..) + * scala> p.parse("b") + * val res1 = Failure(..) + * scala> p.parse("c") + * val res1 = Failure(..) + * scala> p.parse("xb") + * val res2 = Success('x') + * scala> p.parse("") + * val res3 = Failure(..) + * }}} + * + * @param cs the range of codepoints to check. + * @return a parser that parses a codepoint outside the range `cs`. + * @see [[satisfy `satisfy`]] + * @group class + */ + def noneOf(cs: Range): Parsley[Int] = cs.size match { + case 0 => item + case 1 => satisfy(cs.head != _, s"anything except ${renderChar(cs.head)}") + case _ if Math.abs(cs(0) - cs(1)) == 1 => satisfy(!cs.contains(_), { + s"anything outside of ${renderChar(cs.min)} to ${renderChar(cs.max)}" + }) + case _ => satisfy(!cs.contains(_)) + } + + // TODO: test? + /** This combinator parses `pc` '''zero''' or more times, collecting its results into a string. + * + * Parses `pc` repeatedly until it fails. The resulting codepoints are placed into a string, + * which is then returned. This is ''morally'' equivalent to `many(pc).flatMap(Character.chars(_)).map(_.mkString)`, but + * it uses `StringBuilder`, which makes it much more efficient. + * + * @example {{{ + * scala> import parsley.unicode.{letter, letterOrDigit, stringOfMany} + * scala> import parsley.implicits.zipped.Zipped2 + * scala> val ident = (letter, stringOfMany(letterOrDigit)).zipped((c, s) => s"${Character.toString(c)}$s") + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc9d") + * scala> ident.parse("a") + * val res1 = Success("a") + * scala> ident.parse("9") + * val res2 = Failure(..) + * }}} + * + * @param pc the parser whose results make up the string + * @return a parser that parses a string whose letters consist of results from `pc`. + * @since 4.4.0 + * @group string + */ + def stringOfMany(pc: Parsley[Int]): Parsley[String] = { + val pf = pure(addCodepoint(_, _)) + // Can't use the regular foldLeft here, because we need a fresh StringBuilder each time. + expr.infix.secretLeft1(fresh(new StringBuilder), pc, pf).map(_.toString) + } + + // TODO: test + /** This combinator parses codepoints matching the given predicate '''zero''' or more times, collecting + * the results into a string. + * + * Repeatly reads codepoints that satisfy the given predicate `pred`. When no more codepoints + * can be successfully read, the results are stitched together into a `String` and returned. + * This combinator can never fail, since `satisfy` can never fail having consumed input. + * + * @example {{{ + * scala> import parsley.unicode.{letter, stringOfMany} + * scala> import parsley.implicits.zipped.Zipped2 + * scala> val ident = (letter, stringOfMany(Character.isLetterOrDigit(_))).zipped((c, s) => s"${Character.toString(c)}$s") + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc9d") + * scala> ident.parse("a") + * val res1 = Success("a") + * scala> ident.parse("9") + * val res2 = Failure(..) + * }}} + * + * @param pred the predicate to test codepoints against. + * @return a parser that returns the span of codepoints satisfying `pred` + * @note this acts exactly like `stringOfMany(satisfy(pred))`, but may be more efficient. + * @note analogous to the `megaparsec` `takeWhileP` combinator. + * @since 4.4.0 + * @group string + */ + def stringOfMany(pred: Int => Boolean): Parsley[String] = skipMany(satisfy(pred)).span//stringOfMany(satisfy(pred)) + + // TODO: test? + /** This combinator parses `pc` '''one''' or more times, collecting its results into a string. + * + * Parses `pc` repeatedly until it fails. The resulting codepoints are placed into a string, + * which is then returned. This is ''morally'' equivalent to `some(pc).flatMap(Character.chars(_)).map(_.mkString)`, but + * it uses `StringBuilder`, which makes it much more efficient. The result string must have + * at least one codepoint in it. + * + * @example {{{ + * scala> import parsley.unicode.{letter, letterOrDigit, stringOfSome} + * scala> val ident = stringOfSome(letter) + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc") + * scala> ident.parse("") + * val res1 = Failure(..) + * }}} + * + * @param pc the parser whose results make up the string + * @return a parser that parses a string whose letters consist of results from `pc`. + * @since 4.4.0 + * @group string + */ + def stringOfSome(pc: Parsley[Int]): Parsley[String] = { + val pf = pure(addCodepoint(_, _)) + // Can't use the regular foldLeft1 here, because we need a fresh StringBuilder each time. + expr.infix.secretLeft1(pc.map(addCodepoint(new StringBuilder, _)), pc, pf).map(_.toString) + } + + // TODO: test + /** This combinator parses codepoints matching the given predicate '''one''' or more times, collecting + * the results into a string. + * + * Repeatly reads codepoints that satisfy the given predicate `pred`. When no more codepoints + * can be successfully read, the results are stitched together into a `String` and returned. + * This combinator can never fail having consumed input, since `satisfy` can never fail having + * consumed input. + * + * @example {{{ + * scala> import parsley.unicode.{letter, stringOfSome} + * scala> val ident = stringOfSome(Character.isLetter(_))) + * scala> ident.parse("abdc9d") + * val res0 = Success("abdc") + * scala> ident.parse("") + * val res1 = Failure(..) + * }}} + * + * @param pred the predicate to test codepoints against. + * @return a parser that returns the span of codepoints satisfying `pred` + * @note this acts exactly like `stringOfSome(satisfy(pred))`, but may be more efficient. + * @note analogous to the `megaparsec` `takeWhileP1` combinator. + * @since 4.4.0 + * @group string + */ + def stringOfSome(pred: Int => Boolean): Parsley[String] = skipSome(satisfy(pred)).span//stringOfSome(satisfy(pred)) + + // These should always just match up, so no need to test + // $COVERAGE-OFF$ + /** This combinator tries to parse each of the strings `strs` (and `str0`), until one of them succeeds. + * + * Unlike `choice`, or more accurately `atomicChoice`, this combinator will not + * necessarily parse the strings in the order provided. It will favour strings that have another string + * as a prefix first, so that it has ''Longest Match'' semantics. It will try to minimise backtracking + * too, making it a much more efficient option than `atomicChoice`. + * + * The longest succeeding string will be returned. If no strings match then the combinator fails. + * + * @example {{{ + * scala> import parsley.unicode.strings + * scala> val p = strings("hell", "hello", "goodbye", "g", "abc") + * scala> p.parse("hell") + * val res0 = Success("hell") + * scala> p.parse("hello") + * val res1 = Success("hello") + * scala> p.parse("good") + * val res2 = Success("g") + * scala> p.parse("goodbye") + * val res3 = Success("goodbye") + * scala> p.parse("a") + * val res4 = Failure(..) + * }}} + * + * @param str0 the first string to try to parse. + * @param strs the remaining strings to try to parse. + * @return a parser that tries to parse all the given strings returning the longest one that matches. + * @note just an alias for [[parsley.character.strings(str0* `character.strings`]], to allow for more ergonomic imports. + * @group string + */ + def strings(str0: String, strs: String*): Parsley[String] = character.strings(str0, strs: _*) + + /** This combinator tries to parse each of the key-value pairs `kvs` (and `kv0`), until one of them succeeds. + * + * Each argument to this combinator is a pair of a string and a parser to perform if that string can be parsed. + * `strings(s0 -> p0, ...)` can be thought of as `atomicChoice(string(s0) *> p0, ...)`, however, the given + * ordering of key-value pairs does not dictate the order in which the parses are tried. In particular, it + * will favour keys that are the prefix of another key first, so that it has ''Longest Match'' semantics. + * it will try to minimise backtracking too, making it a much more efficient option than `atomicChoice`. + * + * @example {{{ + * scala> import parsley.unicode.strings + * scala> val p = strings("hell" -> pure(4), "hello" -> pure(5), "goodbye" -> pure(7), "g" -> pure(1), "abc" -> pure(3)) + * scala> p.parse("hell") + * val res0 = Success(4) + * scala> p.parse("hello") + * val res1 = Success(5) + * scala> p.parse("good") + * val res2 = Success(1) + * scala> p.parse("goodbye") + * val res3 = Success(7) + * scala> p.parse("a") + * val res4 = Failure(..) + * }}} + * + * @note the scope of any backtracking performed is isolated to the key itself, as it is assumed that once a + * key parses correctly, the branch has been committed to. Putting an `attempt` around the values will not affect + * this behaviour. + * + * @param kv0 the first key-value pair to try to parse. + * @param kvs the remaining key-value pairs to try to parse. + * @return a parser that tries to parse all the given key-value pairs, returning the (possibly failing) result + * of the value that corresponds to the longest matching key. + * @note just an alias for [[parsley.character.strings[A](kv0* `character.strings`]], to allow for more ergonomic imports. + * @group string + */ + def strings[A](kv0: (String, Parsley[A]), kvs: (String, Parsley[A])*): Parsley[A] = character.strings(kv0, kvs: _*) + // $COVERAGE-ON$ + + /** This parser will parse '''any''' single codepoint from the input, failing if there is no input remaining. + * + * @group core + */ + val item: Parsley[Int] = satisfy(_ => true, "any character") + + /** This parser tries to parse a space or tab character, and returns it if successful + * + * @see [[isSpace `isSpace`]] + * @group spec + */ + val space: Parsley[Int] = satisfy(isSpace(_), "space/tab") + + /** This parser skips zero or more space characters using [[space `space`]]. + * + * @see [[combinator.skipMany `combinator.skipMany`]] + * @group skip + */ + val spaces: Parsley[Unit] = skipMany(space) + + /** This parser tries to parse a whitespace character, and returns it if successful. + * + * A whitespace character is one of: + * 1. a space (`' '`) + * 1. a tab (`'\t'`) + * 1. a line feed (`'\n'`) + * 1. a carriage return (`'\r'`) + * 1. a form feed (`'\f'`) + * 1. a vertical tab (`'\u000b'`) + * + * @group spec + */ + val whitespace: Parsley[Int] = satisfy(Character.isWhitespace(_), "whitespace") + + /** This parser skips zero or more space characters using [[whitespace `whitespace`]]. + * + * @see [[combinator.skipMany `combinator.skipMany`]] + * @group skip + */ + val whitespaces: Parsley[Unit] = skipMany(whitespace) + + /** This parser tries to parse a line feed newline (`'\n'`) character, and returns it if successful. + * + * This parser will not accept a carriage return (`CR`) character or `CRLF`. + * + * @group spec + */ + val newline: Parsley[Int] = char('\n', "newline") + + /** This parser tries to parse a `CRLF` newline character pair, returning `'\n'` if successful. + * + * A `CRLF` character is the pair of carriage return (`'\r'`) and line feed (`'\n'`). These + * two characters will be parsed together or not at all. The parser is made atomic using `attempt`. + * + * @group spec + */ + val crlf: Parsley[Int] = character.crlf.as(0x0a) + + /** This parser will parse either a line feed (`LF`) or a `CRLF` newline, returning `'\n'` if successful. + * + * @group spec + * @see [[crlf `crlf`]] + */ + val endOfLine: Parsley[Int] = (newline <|> crlf).label("end of line") + + /** This parser tries to parse a tab (`'\t'`) character, and returns it if successful. + * + * This parser does not recognise vertical tabs, only horizontal ones. + * + * @group spec + */ + val tab: Parsley[Int] = char('\t', "tab") + + /** This parser tries to parse an uppercase letter, and returns it if successful. + * + * An uppercase letter is any character whose Unicode ''Category Type'' is Uppercase Letter (`Lu`). + * Examples of characters within this category include: + * - the Latin letters `'A'` through `'Z'` + * - Latin special character such as `'Å'`, `'Ç'`, `'Õ'` + * - Cryillic letters + * - Greek letters + * - Coptic letters + * + * $categories + * + * @group spec + */ + val upper: Parsley[Int] = satisfy(Character.isUpperCase(_), "uppercase letter") + + /** This parser tries to parse a lowercase letter, and returns it if successful. + * + * A lowercase letter is any character whose Unicode ''Category Type'' is Lowercase Letter (`Ll`). + * Examples of characters within this category include: + * - the Latin letters `'a'` through `'z'` + * - Latin special character such as `'é'`, `'ß'`, `'ð'` + * - Cryillic letters + * - Greek letters + * - Coptic letters + * + * $categories + * + * @group spec + */ + val lower: Parsley[Int] = satisfy(Character.isLowerCase(_), "lowercase letter") + + /** This parser tries to parse either a letter or a digit, and returns it if successful. + * + * A letter or digit is anything that would parse in either `letter` or `digit`. + * + * @see documentation for [[letter `letter`]]. + * @see documentation for [[digit `digit`]]. + * @group spec + */ + val letterOrDigit: Parsley[Int] = satisfy(Character.isLetterOrDigit(_), "alpha-numeric character") + + /** This parser tries to parse a letter, and returns it if successful. + * + * A letter is any character whose Unicode ''Category Type'' is any of the following: + * 1. Uppercase Letter (`Lu`) + * 1. Lowercase Letter (`Ll`) + * 1. Titlecase Letter (`Lt`) + * 1. Modifier Letter (`Lm`) + * 1. Other Letter (`Lo`) + * + * $categories + * + * @group spec + */ + val letter: Parsley[Int] = satisfy(Character.isLetter(_), "letter") + + /** This parser tries to parse a digit, and returns it if successful. + * + * A digit is any character whose Unicode ''Category Type'' is Decimal Number (`Nd`). + * Examples of (inclusive) ranges within this category include: + * - the Latin digits `'0'` through `'9'` + * - the Arabic-Indic digits `'\u0660'` through `'\u0669'` + * - the Extended Arabic-Indic digits `'\u06F0'` through `'\u06F9'` + * - the Devangari digits `'\u0966'` through `'\u096F'` + * - the Fullwidth digits `'\uFF10'` through `'\uFF19'` + * + * $categories + * + * @group spec + */ + val digit: Parsley[Int] = satisfy(Character.isDigit(_), "digit") + + /** This parser tries to parse a hexadecimal digit, and returns it if successful. + * + * A hexadecimal digit is one of (all inclusive ranges): + * 1. the digits `'0'` through `'9'` + * 1. the letters `'a'` through `'f'` + * 1. the letters `'A'` through `'Z'` + * + * @see [[isHexDigit ``isHexDigit``]] + * @group spec + */ + val hexDigit: Parsley[Int] = satisfy(isHexDigit(_), "hexadecimal digit") + + /** This parser tries to parse an octal digit, and returns it if successful. + * + * An octal digit is one of `'0'` to `'7'` (inclusive). + * + * @see [[isOctDigit ``isOctDigit``]] + * @group spec + */ + val octDigit: Parsley[Int] = satisfy(isOctDigit(_), "octal digit") + + /** This parser tries to parse a bit and returns it if successful. + * + * A bit (binary digit) is either `'0'` or `'1'`. + * + * @group spec + */ + val bit: Parsley[Int] = satisfy(c => Character.digit(c, 2) != -1, "bit") + + // Functions + /** This function returns true if a character is a hexadecimal digit. + * + * A hexadecimal digit is one of (all inclusive ranges): + * 1. the digits `'0'` through `'9'` + * 1. the letters `'a'` through `'f'` + * 1. the letters `'A'` through `'Z'` + * 1. an equivalent from another charset + * + * @see [[hexDigit `hexDigit`]] + * @group pred + */ + def isHexDigit(c: Int): Boolean = Character.digit(c, 16) != -1 + + /** This function returns true if a character is an octal digit. + * + * An octal digit is one of `'0'` to `'7'` (inclusive). + * + * @group pred + * @see [[octDigit `octDigit`]] + */ + def isOctDigit(c: Int): Boolean = Character.digit(c, 8) != -1 + + /** This function returns true if a codepoint is either a space or a tab character. + * + * @group pred + * @see [[space `space`]] + */ + def isSpace(c: Int): Boolean = c == 0x20 || c == 0x09 + + // Sue me. + private def renderChar(c: Int): String = parsley.errors.helpers.renderRawString(Character.toChars(c).mkString) + + private [parsley] def addCodepoint(sb: StringBuilder, codepoint: Int): StringBuilder = { + if (Character.isSupplementaryCodePoint(codepoint)) { + sb += Character.highSurrogate(codepoint) + sb += Character.lowSurrogate(codepoint) + } + else sb += codepoint.toChar + } +} diff --git a/parsley/shared/src/test/scala/parsley/CharTests.scala b/parsley/shared/src/test/scala/parsley/CharTests.scala index 60a296b9c..a4cef725f 100644 --- a/parsley/shared/src/test/scala/parsley/CharTests.scala +++ b/parsley/shared/src/test/scala/parsley/CharTests.scala @@ -8,15 +8,15 @@ package parsley import Predef.{ArrowAssoc => _, _} import parsley.character._ -import parsley.implicits.character.charLift class CharTests extends ParsleyTest { + // TODO: property-based testing for this! "item" should "accept any character" in { - for (i <- 0 to 65535) item.parse(i.toChar.toString) should not be a [Failure[_]] + for (i <- ('\u0000' to '\u000a') ++ ('\u0040' to '\u00ef') ++ ('\uff00' to '\uff0a')) item.parse(i.toString) should not be a [Failure[_]] } it should "fail if the input has run out, expecting any character" in { inside(item.parse("")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (EndOfInput) exs should contain only (Named("any character")) rs shouldBe empty @@ -26,24 +26,19 @@ class CharTests extends ParsleyTest { "space" should "consume ' ' or '\t'" in cases(space)( " " -> Some(' '), "\t" -> Some('\t'), + "\u000b" -> None, ) - it should "expect space/tab otherwise" in { - for (i <- 0 to 65535; if i != ' ' && i != '\t') space.parse(i.toChar.toString) shouldBe a [Failure[_]] - } - "spaces" should "consume lots of spaces" in cases(spaces *> 'a')( - (" \t" * 100 + 'a') -> Some('a') + "spaces" should "consume lots of spaces" in cases(spaces *> char('a'))( + (" \t" * 5 + 'a') -> Some('a') ) - it should "never fail" in cases(spaces *> 'a')( + it should "never fail" in cases(spaces *> char('a'))( "a" -> Some('a') ) + // FIXME: this needs to be improved "whitespace" should "consume any whitespace chars" in { - (whitespaces *> 'a').parse(" \t\n\r\f\u000b" * 100 + 'a') should not be a [Failure[_]] - } - it should "fail otherwise" in { - val cs = " \t\n\r\f\u000b".toSet - for (i <- 0 to 65535; if !cs.contains(i.toChar)) whitespace.parse(i.toChar.toString) shouldBe a [Failure[_]] + (whitespaces *> char('a')).parse(" \t\n\r\f\u000b" * 2 + 'a') should not be a [Failure[_]] } "endOfLine" should "consume windows or unix line endings" in cases(endOfLine)( @@ -51,43 +46,79 @@ class CharTests extends ParsleyTest { "\r\n" -> Some('\n'), ) it should "fail otherwise" in { - for (i <- 0 to 65535; if i != 10) endOfLine.parse(i.toChar.toString) shouldBe a [Failure[_]] + endOfLine.parse("a") shouldBe a [Failure[_]] + endOfLine.parse("\r") shouldBe a [Failure[_]] + endOfLine.parse("\r ") shouldBe a [Failure[_]] + endOfLine.parse(" ") shouldBe a [Failure[_]] + } + + "letter" should "accept non-latin characters" in cases(letter)( + "ß" -> Some('ß'), + "ð" -> Some('ð'), + "é" -> Some('é'), + "Å" -> Some('Å'), + "λ" -> Some('λ'), + "Ω" -> Some('Ω'), + ) + it should "not accept high surrogates" in cases(letter)( + "\ud840" -> None, + "\ud87e\udc1a" -> None, + ) + + "satisfyMap" should "allow for simultaneous mapping and testing" in { + val toLower = satisfyMap { + case c if c.isUpper => c.toLower + } + for (c <- 'A' to 'Z') toLower.parse(c.toString) shouldBe Success(c + ('a' - 'A')) + toLower.parse("Ω") shouldBe Success('ω') + toLower.parse("Å") shouldBe Success('å') + toLower.parse("a") shouldBe a [Failure[_]] } "upper" should "only accept uppercase characters" in { for (c <- 'A' to 'Z') upper.parse(c.toString) shouldBe Success(c) + upper.parse("Ω") shouldBe Success('Ω') + upper.parse("Å") shouldBe Success('Å') } it should "fail otherwise" in { for (c <- 'a' to 'z') upper.parse(c.toString) shouldBe a [Failure[_]] + upper.parse("ß") shouldBe a [Failure[_]] + upper.parse("ð") shouldBe a [Failure[_]] + upper.parse("é") shouldBe a [Failure[_]] + upper.parse("λ") shouldBe a [Failure[_]] } "lower" should "only accept lowercase characters" in { for (c <- 'a' to 'z') lower.parse(c.toString) shouldBe Success(c) + lower.parse("ß") shouldBe Success('ß') + lower.parse("ð") shouldBe Success('ð') + lower.parse("é") shouldBe Success('é') + lower.parse("λ") shouldBe Success('λ') } it should "fail otherwise" in { for (c <- 'A' to 'Z') lower.parse(c.toString) shouldBe a [Failure[_]] + lower.parse("Ω") shouldBe a [Failure[_]] + lower.parse("Å") shouldBe a [Failure[_]] } "digit parsers" should "accept the appropriate characters" in { - for (c <- '0' to '9') { + for (c <- ('0' to '9') ++ ('\u0660' to '\u0669') ++ ('\uff10' to '\uff19')) { digit.parse(c.toString) shouldBe Success(c) + letterOrDigit.parse(c.toString) shouldBe Success(c) hexDigit.parse(c.toString) shouldBe Success(c) - if (c < '8') { - val _ = octDigit.parse(c.toString) shouldBe Success(c) - } + val d = c.asDigit + if (d >= 0 && d < 2) { val _ = bit.parse(c.toString) shouldBe Success(c) } + if (d >= 0 && d < 8) { val _ = octDigit.parse(c.toString) shouldBe Success(c) } } - for (c <- 'a' to 'f') hexDigit.parse(c.toString) shouldBe Success(c) - for (c <- 'A' to 'F') hexDigit.parse(c.toString) shouldBe Success(c) + for (c <- ('a' to 'f') ++ ('\uff41' to '\uff46') ++ ('A' to 'F') ++ ('\uff21' to '\uff26')) hexDigit.parse(c.toString) shouldBe Success(c) } they should "fail otherwise" in { - for (c <- 'a' to 'f') { - digit.parse(c.toString) shouldBe a [Failure[_]] - octDigit.parse(c.toString) shouldBe a [Failure[_]] - } - for (c <- 'A' to 'F') { + for (c <- ('a' to 'f') ++ ('A' to 'F')) { + bit.parse(c.toString) shouldBe a [Failure[_]] digit.parse(c.toString) shouldBe a [Failure[_]] octDigit.parse(c.toString) shouldBe a [Failure[_]] } + bit.parse("2") shouldBe a [Failure[_]] octDigit.parse("8") shouldBe a [Failure[_]] octDigit.parse("9") shouldBe a [Failure[_]] } @@ -119,17 +150,4 @@ class CharTests extends ParsleyTest { cases(character.noneOf('a')) ("a" -> None, "\n" -> Some('\n'), "b" -> Some('b')) cases(character.noneOf('a' to 'a'))("a" -> None, "\n" -> Some('\n'), "b" -> Some('b')) } - - "charUtf16" should "handle BMP characters" in { - cases(codePoint('a'))("a" -> Some('a')) - cases(codePoint('λ'))("λ" -> Some('λ')) - } - - it should "handle multi-character codepoints" in { - cases(codePoint(0x1F642))("🙂" -> Some(0x1F642)) - } - - it should "handle multi-character codepoints atomically on fail" in { - cases(codePoint(0x1F642) <|> codePoint(0x1F643))("🙃" -> Some(0x1F643)) - } } diff --git a/parsley/shared/src/test/scala/parsley/CombinatorTests.scala b/parsley/shared/src/test/scala/parsley/CombinatorTests.scala index a65cba8cc..e9ff6bd4d 100644 --- a/parsley/shared/src/test/scala/parsley/CombinatorTests.scala +++ b/parsley/shared/src/test/scala/parsley/CombinatorTests.scala @@ -8,6 +8,7 @@ package parsley import Predef.{ArrowAssoc => _, _} import parsley.combinator.{exactly => repeat, _} +import parsley.character.item import parsley.Parsley._ import parsley.registers.{forYieldP, forYieldP_, Reg} import parsley.implicits.character.{charLift, stringLift} @@ -27,8 +28,8 @@ class CombinatorTests extends ParsleyTest { choice("a", "b", "bc", "bcd").parse("c") shouldBe a [Failure[_]] } - "attemptChoice" should "correctly ensure the subparsers backtrack" in { - attemptChoice("ac", "aba", "abc").parse("abc") should be (Success("abc")) + "atomicChoice" should "correctly ensure the subparsers backtrack" in { + atomicChoice("ac", "aba", "abc").parse("abc") should be (Success("abc")) } "exactly" should "be pure(Nil) for n <= 0" in { @@ -134,7 +135,7 @@ class CombinatorTests extends ParsleyTest { "ab" -> None, ) it must "not corrupt the stack on sep hard-fail" in { - ('c' <::> attempt(sepEndBy('a', "bb")).getOrElse(List('d'))).parse("cab") should be (Success(List('c', 'd'))) + ('c' <::> atomic(sepEndBy('a', "bb")).getOrElse(List('d'))).parse("cab") should be (Success(List('c', 'd'))) } "sepEndBy1" must "require a p" in { @@ -208,4 +209,56 @@ class CombinatorTests extends ParsleyTest { abc.parse("aaabbbccc") should be (Success(List('c', 'c', 'c'))) abc.parse("aaaabc") shouldBe a [Failure[_]] } + + "count" should "report how many successful parses occurred" in { + val p = count("ab") + val q = count1("ab") + p.parse("") shouldBe Success(0) + q.parse("") shouldBe a [Failure[_]] + p.parse("ab") shouldBe Success(1) + q.parse("ab") shouldBe Success(1) + p.parse("ababab") shouldBe Success(3) + q.parse("ababab") shouldBe Success(3) + } + + it should "not allow partial results" in { + count("ab").parse("aba") shouldBe a [Failure[_]] + } + + it should "allow for ranges" in { + val p = count(min = 2, max = 5)("ab") + p.parse("ab") shouldBe a [Failure[_]] + p.parse("abab") shouldBe Success(2) + p.parse("ababab") shouldBe Success(3) + p.parse("abababab") shouldBe Success(4) + p.parse("ababababab") shouldBe Success(5) + p.parse("abababababab") shouldBe Success(5) + p.parse("ababababa") shouldBe a [Failure[_]] + val q = count(min = 2, max = 5)(atomic("ab")) + q.parse("ab") shouldBe a [Failure[_]] + q.parse("abab") shouldBe Success(2) + q.parse("ababab") shouldBe Success(3) + q.parse("abababab") shouldBe Success(4) + q.parse("ababababab") shouldBe Success(5) + q.parse("abababababab") shouldBe Success(5) + q.parse("ababababa") shouldBe Success(4) + } + + "range" should "collect results up instead of count" in { + val p = range(min = 2, max = 5)(item) + p.parse("a") shouldBe a [Failure[_]] + p.parse("ab") shouldBe Success(List('a', 'b')) + p.parse("abcd") shouldBe Success(List('a', 'b', 'c', 'd')) + p.parse("abcde") shouldBe Success(List('a', 'b', 'c', 'd', 'e')) + p.parse("abcdef") shouldBe Success(List('a', 'b', 'c', 'd', 'e')) + } + + "range_" should "perform a range with no results" in { + val p = range_(min = 2, max = 5)(item) + (p <~ eof).parse("a") shouldBe a [Failure[_]] + (p <~ eof).parse("ab") shouldBe Success(()) + (p <~ eof).parse("abcd") shouldBe Success(()) + (p <~ eof).parse("abcde") shouldBe Success(()) + (p <~ 'f').parse("abcdef") shouldBe Success(()) + } } diff --git a/parsley/shared/src/test/scala/parsley/CoreTests.scala b/parsley/shared/src/test/scala/parsley/CoreTests.scala index abac24973..b9f0df4e9 100644 --- a/parsley/shared/src/test/scala/parsley/CoreTests.scala +++ b/parsley/shared/src/test/scala/parsley/CoreTests.scala @@ -149,15 +149,15 @@ class CoreTests extends ParsleyTest { } it should "not be affected by an empty on the left" in { inside((Parsley.empty <|> 'a').parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("b")) exs should contain only (Raw("a")) rs shouldBe empty } } - "attempt" should "cause <|> to try second alternative even if input consumed" in { - attempt("ab").orElse("ac").parse("ac") should not be a [Failure[_]] + "atomic" should "cause <|> to try second alternative even if input consumed" in { + atomic("ab").orElse("ac").parse("ac") should not be a [Failure[_]] } "notFollowedBy" must "succeed if p fails" in { @@ -177,7 +177,7 @@ class CoreTests extends ParsleyTest { "lookAhead" should "consume no input on success" in { lookAhead('a').parse("a") should not be a [Failure[_]] inside((lookAhead('a') *> 'b').parse("ab")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("a")) exs should contain only (Raw("b")) rs shouldBe empty @@ -254,7 +254,7 @@ class CoreTests extends ParsleyTest { p.parse("") shouldBe Success(3) } they should "but not roll back if they hard fail" in { - val p = 3.makeReg(r1 => (attempt(r1.rollback('a' *> r1.put(2) *> Parsley.empty)) <|> unit) *> r1.get) + val p = 3.makeReg(r1 => (atomic(r1.rollback('a' *> r1.put(2) *> Parsley.empty)) <|> unit) *> r1.get) p.parse("a") shouldBe Success(2) } they should "not rollback if successful" in { @@ -281,7 +281,7 @@ class CoreTests extends ParsleyTest { } it should "also appear to create a fresh register even in the presence of a hard failure" in { - lazy val p: Parsley[Char] = item.fillReg(c => item *> (attempt(p) <|> c.get)) + lazy val p: Parsley[Char] = item.fillReg(c => item *> (atomic(p) <|> c.get)) p.parse("abc") shouldBe Success('a') } @@ -388,7 +388,7 @@ class CoreTests extends ParsleyTest { import parsley.combinator.{whileP, some, eof} val n = registers.Reg.make[Int] lazy val p: Parsley[Unit] = whileP(ifP(n.gets(_ % 2 == 0), some('a'), some('b')) *> n.modify(_ - 1) *> n.gets(_ != 0)) - val q = attempt(n.put(4) *> p <* eof) | n.put(2) *> p <* eof + val q = atomic(n.put(4) *> p <* eof) | n.put(2) *> p <* eof q.parse("aaaabbb") shouldBe a [Success[_]] } @@ -400,4 +400,9 @@ class CoreTests extends ParsleyTest { } (p *> p).parse("") shouldBe Success(4) } + + "span" should "return all the input parsed by a parser, exactly as it was" in { + import parsley.character.whitespaces + whitespaces.span.parse(" \n\n\t\ta") shouldBe Success(" \n\n\t\t") + } } diff --git a/parsley/shared/src/test/scala/parsley/ErrorTests.scala b/parsley/shared/src/test/scala/parsley/ErrorTests.scala index bc0c17e88..d4e8fae35 100644 --- a/parsley/shared/src/test/scala/parsley/ErrorTests.scala +++ b/parsley/shared/src/test/scala/parsley/ErrorTests.scala @@ -9,8 +9,9 @@ import parsley.combinator.{eof, optional, many} import parsley.Parsley._ import parsley.implicits.character.{charLift, stringLift} import parsley.character.{item, digit} -import parsley.errors.combinator.{fail => pfail, unexpected, amend, entrench, dislodge, amendThenDislodge, ErrorMethods} +import parsley.errors.combinator.{fail => pfail, unexpected, amend, partialAmend, entrench, dislodge, amendThenDislodge, /*partialAmendThenDislodge,*/ ErrorMethods} import parsley.errors.patterns._ +import parsley.errors.SpecialisedGen class ErrorTests extends ParsleyTest { "mzero parsers" should "always fail" in { @@ -24,7 +25,7 @@ class ErrorTests extends ParsleyTest { case c if c.isLower => s"'$c' should have been uppercase" } inside(p.parse("a")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex shouldBe empty exs shouldBe empty rs should contain only ("'a' should have been uppercase") @@ -34,13 +35,13 @@ class ErrorTests extends ParsleyTest { val q = item.guardAgainst { case c if c.isLower => Seq(s"'$c' is not uppercase") } - inside(q.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs))) => msgs should contain only ("'a' is not uppercase") } + inside(q.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => msgs should contain only ("'a' is not uppercase") } q.parse("A") shouldBe Success('A') val r = item.unexpectedWithReasonWhen { case c if c.isLower => ("lowercase letter", s"'$c' should have been uppercase") } - inside(r.parse("a")) { case Failure(TestError((1, 1), VanillaError(unex, exs, reasons))) => + inside(r.parse("a")) { case Failure(TestError((1, 1), VanillaError(unex, exs, reasons, 1))) => unex should contain (Named("lowercase letter")) exs shouldBe empty reasons should contain.only("'a' should have been uppercase") @@ -49,21 +50,21 @@ class ErrorTests extends ParsleyTest { val s = item.unexpectedWhen { case c if c.isLower => "lowercase letter" } - inside(s.parse("a")) { case Failure(TestError((1, 1), VanillaError(unex, exs, reasons))) => + inside(s.parse("a")) { case Failure(TestError((1, 1), VanillaError(unex, exs, reasons, 1))) => unex should contain (Named("lowercase letter")) exs shouldBe empty reasons shouldBe empty } } - "the collectMsg combinator" should "act like a filter then a map" in { + "the collect/mapFilter combinators" should "act like a filter then a map" in { val p = item.collectMsg("oops") { case '+' => 0 case c if c.isUpper => c - 'A' + 1 } p.parse("+") shouldBe Success(0) p.parse("C") shouldBe Success(3) - inside(p.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs))) => msgs should contain only ("oops") } + inside(p.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => msgs should contain only ("oops") } val q = item.collectMsg(c => Seq(s"$c is not appropriate")) { case '+' => 0 @@ -71,12 +72,22 @@ class ErrorTests extends ParsleyTest { } q.parse("+") shouldBe Success(0) q.parse("C") shouldBe Success(3) - inside(q.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs))) => msgs should contain only ("a is not appropriate") } + inside(q.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => msgs should contain only ("a is not appropriate") } + + val errGen = new SpecialisedGen[Char] { def messages(c: Char): Seq[String] = Seq(s"$c is not appropriate") } + val r = item.mapFilterWith(errGen) { + case '+' => Some(0) + case c if c.isUpper => Some(c - 'A' + 1) + case _ => None + } + r.parse("+") shouldBe Success(0) + r.parse("C") shouldBe Success(3) + inside(r.parse("a")) { case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => msgs should contain only ("a is not appropriate") } } // Issue #70 "filterOut" should "not corrupt the stack under a handler" in { - val p = attempt(item.filterOut { + val p = atomic(item.filterOut { case c if c.isLower => "no lowercase!" }) p.parse("a") shouldBe a [Failure[_]] @@ -84,23 +95,29 @@ class ErrorTests extends ParsleyTest { lazy val r: Parsley[List[String]] = "correct error message" <::> r "label" should "affect base error messages" in { - inside(('a'? "ay!").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + inside(('a' ? "ay!").parse("b")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("b")) exs should contain only (Named("ay!")) rs shouldBe empty } + inside(('a'.label("ay!", "see!")).parse("b")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => + unex should contain (Raw("b")) + exs should contain.only(Named("ay!"), Named("see!")) + rs shouldBe empty + } } it should "work across a recursion boundary" in { def p = r.label("nothing but this :)") inside(p.parse("")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (EndOfInput) exs should contain only (Named("nothing but this :)")) rs shouldBe empty } inside(p.parse("correct error message")) { - case Failure(TestError((1, 22), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 22), VanillaError(unex, exs, rs, 1))) => unex should contain (EndOfInput) exs should contain only (Raw("correct error message")) rs shouldBe empty @@ -110,46 +127,46 @@ class ErrorTests extends ParsleyTest { it should "replace everything under the label" in { val s = (optional('a') *> optional('b')).label("hi") *> 'c' inside(s.parse("e")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("hi"), Raw("c")) rs shouldBe empty } val t = (optional('a') *> optional('b').label("bee")).label("hi") *> 'c' inside(t.parse("e")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("hi"), Raw("c")) rs shouldBe empty } inside(t.parse("ae")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("bee"), Raw("c")) rs shouldBe empty } val u = (optional('a').hide *> optional('b')).label("hi") *> 'c' inside(u.parse("e")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("hi"), Raw("c")) rs shouldBe empty } inside(u.parse("ae")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Raw("b"), Raw("c")) rs shouldBe empty } val v = (optional('a').hide *> optional('b').label("bee")).label("hi") *> 'c' inside(v.parse("e")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("hi"), Raw("c")) rs shouldBe empty } inside(v.parse("ae")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("bee"), Raw("c")) rs shouldBe empty @@ -158,7 +175,7 @@ class ErrorTests extends ParsleyTest { it should "not replace hints if input is consumed" in { inside((many(digit).label("number") <* eof).parse("1e")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain.only(Named("digit"), EndOfInput) rs shouldBe empty @@ -167,22 +184,40 @@ class ErrorTests extends ParsleyTest { "hide" should "not produce any visible output" in { inside('a'.hide.parse("")) { - case Failure(TestError((1, 1), VanillaError(_, exs, _))) => + case Failure(TestError((1, 1), VanillaError(_, exs, _, 1))) => exs shouldBe empty } inside("a".hide.parse("")) { - case Failure(TestError((1, 1), VanillaError(_, exs, _))) => + case Failure(TestError((1, 1), VanillaError(_, exs, _, 1))) => exs shouldBe empty } inside(digit.hide.parse("")) { - case Failure(TestError((1, 1), VanillaError(_, exs, _))) => + case Failure(TestError((1, 1), VanillaError(_, exs, _, 1))) => + exs shouldBe empty + } + inside('a'.newHide.parse("")) { + case Failure(TestError((1, 1), VanillaError(_, exs, _, 0))) => + exs shouldBe empty + } + inside("a".newHide.parse("")) { + case Failure(TestError((1, 1), VanillaError(_, exs, _, 0))) => + exs shouldBe empty + } + inside(digit.newHide.parse("")) { + case Failure(TestError((1, 1), VanillaError(_, exs, _, 0))) => exs shouldBe empty } } it should "not replace hints if input is consumed" in { inside((many(digit).hide <* eof).parse("1e")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => + unex should contain (Raw("e")) + exs should contain only EndOfInput + rs shouldBe empty + } + inside((many(digit).newHide <* eof).parse("1e")) { + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain only EndOfInput rs shouldBe empty @@ -191,7 +226,13 @@ class ErrorTests extends ParsleyTest { it should "not allow hints to be unsuppressed by another label" in { inside((many(digit).hide.label("hey") <* eof).parse("1e")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => + unex should contain (Raw("e")) + exs should contain only EndOfInput + rs shouldBe empty + } + inside((many(digit).newHide.label("hey") <* eof).parse("1e")) { + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("e")) exs should contain only EndOfInput rs shouldBe empty @@ -200,25 +241,25 @@ class ErrorTests extends ParsleyTest { "explain" should "provide a message, but only on failure" in { inside(Parsley.empty.explain("oops!").parse("")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 0))) => unex shouldBe empty exs shouldBe empty rs should contain only ("oops!") } inside('a'.explain("requires an a").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("b")) exs should contain only (Raw("a")) rs should contain only ("requires an a") } inside(('a'.explain("an a") <|> 'b'.explain("a b")).parse("c")){ - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("c")) exs should contain.only(Raw("a"), Raw("b")) rs should contain.only("an a", "a b") } inside(('a'.explain("should be absent") *> 'b').parse("a")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (EndOfInput) exs should contain only (Raw("b")) rs shouldBe empty @@ -226,7 +267,7 @@ class ErrorTests extends ParsleyTest { } it should "not have any effect when more input has been consumed since it was added" in { inside(('a'.explain("should be absent") <|> ('b' *> digit)).parse("b")) { - case Failure(TestError((1, 2), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 2), VanillaError(unex, exs, rs, 1))) => unex should contain (EndOfInput) exs should contain only (Named("digit")) rs shouldBe empty @@ -234,12 +275,22 @@ class ErrorTests extends ParsleyTest { } "fail" should "yield a raw message" in { - inside(pfail("hi").parse("b")) { case Failure(TestError((1, 1), SpecialisedError(msgs))) => msgs should contain only ("hi") } + inside(pfail("hi").parse("b")) { case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => msgs should contain only ("hi") } + } + it should "be flexible when the width is unspecified" in { + inside(("abc" | pfail("hi")).parse("xyz")) { + case Failure(TestError((1, 1), SpecialisedError(msgs, 3))) => msgs should contain only ("hi") + } + } + it should "dominate otherwise" in { + inside(("abc" | pfail(2, "hi")).parse("xyz")) { + case Failure(TestError((1, 1), SpecialisedError(msgs, 2))) => msgs should contain only ("hi") + } } "unexpected" should "yield changes to unexpected messages" in { inside(unexpected("bee").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Named("bee")) exs shouldBe empty rs shouldBe empty @@ -247,31 +298,47 @@ class ErrorTests extends ParsleyTest { } it should "produce expected message under influence of ?, along with original message" in { inside(('a' <|> unexpected("bee") ? "something less cute").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Named("bee")) exs should contain.only(Raw("a"), Named("something less cute")) rs shouldBe empty } } + it should "be flexible when the width is unspecified" in { + inside(("abc" | unexpected("bee")).parse("xyz")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 3))) => + unex should contain (Named("bee")) + exs should contain.only(Raw("abc")) + rs shouldBe empty + } + } + it should "dominate otherwise" in { + inside(("abc" | unexpected(2, "bee")).parse("xyz")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 2))) => + unex should contain (Named("bee")) + exs should contain.only(Raw("abc")) + rs shouldBe empty + } + } "lookAhead" should "produce no hints following it" in { val p = 'a' <|> lookAhead(optional(digit) *> 'c') <|> 'b' inside(p.parse("d")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("d")) exs should contain.only(Raw("a"), Raw("c"), Raw("b"), Named("digit")) rs shouldBe empty } val q = 'a' <|> lookAhead(optional(digit)) *> 'c' <|> 'b' inside(q.parse("d")){ - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("d")) exs should contain.only(Raw("a"), Raw("b"), Raw("c")) rs shouldBe empty } val r = 'a' <|> lookAhead(digit) *> 'c' <|> 'b' inside(r.parse("d")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("d")) exs should contain.only(Raw("a"), Raw("b"), Named("digit")) rs shouldBe empty @@ -281,14 +348,14 @@ class ErrorTests extends ParsleyTest { "notFollowedBy" should "produce no hints" in { val p = 'a' <|> notFollowedBy(optional(digit)) *> 'c' <|> 'b' inside(p.parse("d")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("d")) exs should contain.only(Raw("a"), Raw("b")) rs shouldBe empty } val q = 'a' <|> notFollowedBy(digit) *> 'c' <|> 'b' inside(q.parse("d")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("d")) exs should contain.only(Raw("a"), Raw("c"), Raw("b")) rs shouldBe empty @@ -297,7 +364,7 @@ class ErrorTests extends ParsleyTest { "empty" should "produce unknown error messages" in { inside(Parsley.empty.parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 0))) => unex shouldBe empty exs shouldBe empty rs shouldBe empty @@ -305,7 +372,7 @@ class ErrorTests extends ParsleyTest { } it should "produce no unknown message under influence of ?" in { inside((Parsley.empty ? "something, at least").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 0))) => unex shouldBe empty exs should contain only (Named("something, at least")) rs shouldBe empty @@ -313,7 +380,7 @@ class ErrorTests extends ParsleyTest { } it should "not produce an error message at the end of <|> chain" in { inside(('a' <|> Parsley.empty).parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("b")) exs should contain only (Raw("a")) rs shouldBe empty @@ -321,16 +388,24 @@ class ErrorTests extends ParsleyTest { } it should "produce an expected error under influence of ? in <|> chain" in { inside(('a' <|> Parsley.empty ? "something, at least").parse("b")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("b")) exs should contain.only(Named("something, at least"), Raw("a")) rs shouldBe empty } } + it should "have an effect if it's caret is wider" in { + inside(('a' <|> Parsley.empty(3)).parse("bcd")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 3))) => + unex should contain (Raw("bcd")) + exs should contain.only(Raw("a")) + rs shouldBe empty + } + } "eof" should "produce expected end of input" in { inside(eof.parse("a")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("a")) exs should contain only (EndOfInput) rs shouldBe empty @@ -338,7 +413,7 @@ class ErrorTests extends ParsleyTest { } it should "change message under influence of ?" in { inside((eof ? "something more").parse("a")) { - case Failure(TestError((1, 1), VanillaError(unex, exs, rs))) => + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => unex should contain (Raw("a")) exs should contain only (Named("something more")) rs shouldBe empty @@ -371,7 +446,7 @@ class ErrorTests extends ParsleyTest { } "dislodge" should "undo an entrench so that amend works again" in { - val p = 'a' *> amend('b' *> dislodge(entrench('c')) *> 'd') + val p = 'a' *> amend('b' *> dislodge(entrench(entrench('c'))) *> 'd') inside(p.parse("ab")) { case Failure(TestError((1, 2), _)) => } inside(p.parse("abc")) { case Failure(TestError((1, 2), _)) => } } @@ -379,36 +454,84 @@ class ErrorTests extends ParsleyTest { val p = 'a' *> amend('b' *> entrench(dislodge(entrench('c')))) inside(p.parse("ab")) { case Failure(TestError((1, 3), _)) => } } + it should "only unwind as many as instructed if applicable" in { + val p = 'a' *> amend('b' *> dislodge(1)(entrench(entrench('c'))) *> 'd') + val q = 'a' *> amend('b' *> dislodge(2)(entrench(entrench('c'))) *> 'd') + inside(p.parse("abc")) { case Failure(TestError((1, 2), _)) => } + inside(p.parse("ab")) { case Failure(TestError((1, 3), _)) => } + inside(q.parse("abc")) { case Failure(TestError((1, 2), _)) => } + inside(q.parse("ab")) { case Failure(TestError((1, 2), _)) => } + } "amendThenDislodge" should "amend only non-entrenched messages and dislodge those that are" in { - val p = amend('a' *> amendThenDislodge('b' *> entrench('c'))) - inside(p.parse("ab")) { case Failure(TestError((1, 1), _)) => } + val p = 'a' *> amendThenDislodge('b' *> entrench(entrench('c')) *> 'd') + val q = amend(p) + inside(p.parse("ab")) { case Failure(TestError((1, 3), _)) => } + inside(p.parse("abc")) { case Failure(TestError((1, 2), _)) => } + inside(q.parse("ab")) { case Failure(TestError((1, 1), _)) => } + inside(q.parse("abc")) { case Failure(TestError((1, 1), _)) => } + } + it should "only unwind as many as instructed if applicable" in { + val p = 'a' *> amendThenDislodge(1)('b' *> entrench(entrench('c')) *> 'd') + val q = amend(p) + val r = 'a' *> amendThenDislodge(2)('b' *> entrench(entrench('c')) *> 'd') + val s = amend(r) + inside(p.parse("ab")) { case Failure(TestError((1, 3), _)) => } + inside(p.parse("abc")) { case Failure(TestError((1, 2), _)) => } + inside(q.parse("ab")) { case Failure(TestError((1, 3), _)) => } + inside(q.parse("abc")) { case Failure(TestError((1, 1), _)) => } + inside(r.parse("ab")) { case Failure(TestError((1, 3), _)) => } + inside(r.parse("abc")) { case Failure(TestError((1, 2), _)) => } + inside(s.parse("ab")) { case Failure(TestError((1, 1), _)) => } + inside(s.parse("abc")) { case Failure(TestError((1, 1), _)) => } + } + + "partialAmend" should "perform visual amendment but allow for domination" in { + def errorMaker(n: Int, msg: String) = atomic(combinator.exactly(n, 'a') *> ('b' <|> pfail(msg))) + val p = errorMaker(2, "small") <|> amend(errorMaker(3, "big")) + val q = errorMaker(2, "small") <|> partialAmend(errorMaker(3, "big")) + val r = errorMaker(3, "first") <|> partialAmend(errorMaker(3, "second")) + info("a regular amend should lose against even a shallower error") + inside(p.parse("a" * 4)) { + case Failure(TestError((1, 3), SpecialisedError(msgs, 1))) => + msgs should contain only "small" + } + info("a partial amend can win against an error at a lesser offset but greater presentation") + inside(q.parse("a" * 4)) { + case Failure(TestError((1, 1), SpecialisedError(msgs, 1))) => + msgs should contain only "big" + } + info("however, they do not win at equal underlying offset") + inside(r.parse("a" * 4)) { + case Failure(TestError((1, 4), SpecialisedError(msgs, 1))) => + msgs should contain only "first" + } } "oneOf" should "incorporate range notation into the error" in { inside(character.oneOf('0' to '9').parse("a")) { - case Failure(TestError(_, VanillaError(_, expecteds, _))) => + case Failure(TestError(_, VanillaError(_, expecteds, _, 1))) => expecteds should contain only Named("one of \"0\" to \"9\"") } } it should "incorporate sets of characters into the error" in { inside(character.oneOf(('0' to '9').toSet).parse("a")) { - case Failure(TestError(_, VanillaError(_, expecteds, _))) => + case Failure(TestError(_, VanillaError(_, expecteds, _, 1))) => expecteds should contain only Named("one of \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", or \"9\"") } } "noneOf" should "incorporate range notation into the error" in { inside(character.noneOf('0' to '9').parse("8")) { - case Failure(TestError(_, VanillaError(_, expecteds, _))) => + case Failure(TestError(_, VanillaError(_, expecteds, _, 1))) => expecteds should contain only Named("anything outside of \"0\" to \"9\"") } } it should "incorporate sets of characters into the error" in { inside(character.noneOf(('0' to '9').toSet).parse("8")) { - case Failure(TestError(_, VanillaError(_, expecteds, _))) => + case Failure(TestError(_, VanillaError(_, expecteds, _, _))) => expecteds should contain only Named("anything except \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", or \"9\"") } } @@ -416,7 +539,7 @@ class ErrorTests extends ParsleyTest { // Verified Errors "verifiedFail" should "fail having consumed input on the parser success" in { inside(optional("abc".verifiedFail(x => Seq("no, no", s"absolutely not $x"))).parse("abc")) { - case Failure(TestError((1, 1), SpecialisedError(msgs))) => + case Failure(TestError((1, 1), SpecialisedError(msgs, 3))) => msgs should contain.only("no, no", "absolutely not abc") } } @@ -425,26 +548,26 @@ class ErrorTests extends ParsleyTest { } it should "not produce any labels" in { inside("abc".verifiedFail("hi").parse("ab")) { - case Failure(TestError((1, 1), VanillaError(None, expecteds, _))) => + case Failure(TestError((1, 1), VanillaError(None, expecteds, _, 0))) => expecteds shouldBe empty } } "verifiedUnexpected" should "fail having consumed input on the parser success" in { inside(optional("abc".verifiedUnexpected(x => s"$x is not allowed")).parse("abc")) { - case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons))) => + case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons, 3))) => expecteds shouldBe empty unex should contain (Raw("abc")) reasons should contain only ("abc is not allowed") } inside(optional("abc".verifiedUnexpected(s"abc is not allowed")).parse("abc")) { - case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons))) => + case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons, 3))) => expecteds shouldBe empty unex should contain (Raw("abc")) reasons should contain only ("abc is not allowed") } inside(optional("abc".verifiedUnexpected).parse("abc")) { - case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons))) => + case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons, 3))) => expecteds shouldBe empty unex should contain (Raw("abc")) reasons shouldBe empty @@ -457,24 +580,61 @@ class ErrorTests extends ParsleyTest { } it should "not produce any labels" in { inside("abc".verifiedUnexpected.parse("ab")) { - case Failure(TestError((1, 1), VanillaError(None, expecteds, _))) => + case Failure(TestError((1, 1), VanillaError(None, expecteds, _, 0))) => expecteds shouldBe empty } } + // Preventative Errors + "preventativeFail" should "fail having consumed input on the parser success" in { + inside(optional("abc".preventativeFail(x => Seq("no, no", s"absolutely not $x"))).parse("abc")) { + case Failure(TestError((1, 1), SpecialisedError(msgs, 3))) => + msgs should contain.only("no, no", "absolutely not abc") + } + } + it should "not consume input if the parser did not succeed and not fail" in { + "abc".preventativeFail("no, no", "absolutely not").parse("ab") shouldBe Success(()) + } + + "preventativeExplain" should "fail having consumed input on the parser success" in { + inside(optional("abc".preventativeExplain(x => s"$x is not allowed")).parse("abc")) { + case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons, 3))) => + expecteds shouldBe empty + unex should contain (Raw("abc")) + reasons should contain only ("abc is not allowed") + } + inside(optional("abc".preventativeExplain(s"abc is not allowed")).parse("abc")) { + case Failure(TestError((1, 1), VanillaError(unex, expecteds, reasons, 3))) => + expecteds shouldBe empty + unex should contain (Raw("abc")) + reasons should contain only ("abc is not allowed") + } + } + it should "not consume input if the parser did not succeed and not fail" in { + "abc".preventativeExplain(x => s"$x is not allowed").parse("ab") shouldBe Success(()) + "abc".preventativeExplain(s"abc is not allowed").parse("ab") shouldBe Success(()) + } + it should "produce labels when specified" in { + inside("abc".preventativeExplain(x => s"$x is not allowed", "something else").parse("abc")) { + case Failure(TestError((1, 1), VanillaError(Some(Raw("abc")), expecteds, reasons, 3))) => + expecteds should contain only (Named("something else")) + reasons should contain only ("abc is not allowed") + } + } + // Issue 107 "hints" should "incorporate only with errors at the same offset depth" in { - val p = attempt('a' ~> digit) + val p = atomic('a' ~> digit) val parser = optional('b'.label("b")) ~> p.label("foo") inside(parser.parse("aa")) { - case Failure(TestError(_, VanillaError(_, expected, _))) => + case Failure(TestError(_, VanillaError(_, expected, _, 1))) => expected should contain only (Named("digit")) } val q = amend('a' ~> digit) val qarser = optional('b'.label("b")) ~> q.label("foo") inside(qarser.parse("aa")) { - case Failure(TestError(_, VanillaError(_, expected, _))) => + case Failure(TestError(_, VanillaError(_, expected, _, 1))) => expected should contain.allOf(Named("foo"), Named("b")) } } diff --git a/parsley/shared/src/test/scala/parsley/ExpressionParserTests.scala b/parsley/shared/src/test/scala/parsley/ExpressionParserTests.scala index 686a76353..334d641f7 100644 --- a/parsley/shared/src/test/scala/parsley/ExpressionParserTests.scala +++ b/parsley/shared/src/test/scala/parsley/ExpressionParserTests.scala @@ -29,7 +29,7 @@ class ExpressionParserTests extends ParsleyTest { chain.postfix('1' #> 1, "++" #> ((x: Int) => x + 1)).parseAll("1+++++++++++++") shouldBe a [Failure[_]] } it must "not leave the stack in an inconsistent state on failure" in { - val p = chain.postfix[Int]('1' #> 1, (col.#>[Int => Int](_ + 1)) <* '+') + val p = chain.postfix[Int]('1' #> 1, (col.as[Int => Int](_ + 1)) <* '+') val q = chain.left1(p, '*' #> ((x: Int, y: Int) => x * y)) noException should be thrownBy q.parse("1+*1+") } @@ -122,8 +122,8 @@ class ExpressionParserTests extends ParsleyTest { cases(chain.left1("11" #> 1, "++" #> ((x: Int, y: Int) => x + y)))("11+11+11+11+11" -> None, "11++11++11++1++11" -> None) } it must "not leave the stack in an inconsistent state on failure" in { - val p = chain.left1('1' #> 1, (col.#>[(Int, Int) => Int](_ + _)) <* '+') - val q = chain.left1(p, '*'.#>[(Int, Int) => Int](_ * _)) + val p = chain.left1('1' #> 1, (col.as[(Int, Int) => Int](_ + _)) <* '+') + val q = chain.left1(p, '*'.as[(Int, Int) => Int](_ * _)) noException shouldBe thrownBy (q.parse("1+1*1+1")) } it must "correctly accept the use of a wrapping function" in { diff --git a/parsley/shared/src/test/scala/parsley/ParsleyTest.scala b/parsley/shared/src/test/scala/parsley/ParsleyTest.scala index 5ec2ff9e6..77db0d4c4 100644 --- a/parsley/shared/src/test/scala/parsley/ParsleyTest.scala +++ b/parsley/shared/src/test/scala/parsley/ParsleyTest.scala @@ -17,8 +17,8 @@ import org.scalactic.source.Position case class TestError(pos: (Int, Int), lines: TestErrorLines) sealed trait TestErrorLines -case class VanillaError(unexpected: Option[TestErrorItem], expecteds: Set[TestErrorItem], reasons: Set[String]) extends TestErrorLines -case class SpecialisedError(msgs: Set[String]) extends TestErrorLines +case class VanillaError(unexpected: Option[TestErrorItem], expecteds: Set[TestErrorItem], reasons: Set[String], width: Int) extends TestErrorLines +case class SpecialisedError(msgs: Set[String], width: Int) extends TestErrorLines sealed trait TestErrorItem case class Raw(item: String) extends TestErrorItem @@ -36,9 +36,9 @@ abstract class TestErrorBuilder extends ErrorBuilder[TestError] { type ErrorInfoLines = TestErrorLines override def vanillaError(unexpected: UnexpectedLine, expected: ExpectedLine, reasons: Messages, line: LineInfo): ErrorInfoLines = { - VanillaError(unexpected, expected, reasons) + VanillaError(unexpected, expected, reasons, line) } - override def specialisedError(msgs: Messages, line: LineInfo): ErrorInfoLines = SpecialisedError(msgs) + override def specialisedError(msgs: Messages, line: LineInfo): ErrorInfoLines = SpecialisedError(msgs, line) type ExpectedItems = Set[Item] override def combineExpectedItems(alts: Set[Item]): ExpectedItems = alts @@ -55,8 +55,8 @@ abstract class TestErrorBuilder extends ErrorBuilder[TestError] { override def reason(reason: String): Message = reason override def message(msg: String): Message = msg - type LineInfo = Unit - override def lineInfo(line: String, linesBefore: Seq[String], linesAfter: Seq[String], errorPointsAt: Int, errorWidth: Int): Unit = () + type LineInfo = Int + override def lineInfo(line: String, linesBefore: Seq[String], linesAfter: Seq[String], errorPointsAt: Int, errorWidth: Int): Int = errorWidth override val numLinesBefore: Int = 2 override val numLinesAfter: Int = 2 diff --git a/parsley/shared/src/test/scala/parsley/PositionTests.scala b/parsley/shared/src/test/scala/parsley/PositionTests.scala index e290c1e26..d0f26b89d 100644 --- a/parsley/shared/src/test/scala/parsley/PositionTests.scala +++ b/parsley/shared/src/test/scala/parsley/PositionTests.scala @@ -5,8 +5,9 @@ */ package parsley +import Parsley.unit import parsley.position._ -import parsley.character.char +import parsley.character.{char, string} class PositionTests extends ParsleyTest { "line" should "start at 1" in { @@ -43,4 +44,13 @@ class PositionTests extends ParsleyTest { (char('\t') ~> offset).parse("\t") shouldBe Success(1) (char('a') ~> offset).parse("a") shouldBe Success(1) } + + "withWidth" should "return 0 for pure things" in { + withWidth(unit).parse("a") shouldBe Success(((), 0)) + (char('a') ~> withWidth(unit)).parse("a") shouldBe Success(((), 0)) + } + it should "correctly span input consumption" in { + withWidth(string("abc")).parse("abc") shouldBe Success(("abc", 3)) + (char('x') ~> withWidth(string("abc"))).parse("xabc") shouldBe Success(("abc", 3)) + } } diff --git a/parsley/shared/src/test/scala/parsley/ProfilerTests.scala b/parsley/shared/src/test/scala/parsley/ProfilerTests.scala new file mode 100644 index 000000000..94d6817ec --- /dev/null +++ b/parsley/shared/src/test/scala/parsley/ProfilerTests.scala @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Parsley Contributors + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package parsley + +import parsley.debug.Profiler + +class ProfilerTests extends ParsleyTest { + def tick(profiler: Profiler, name: String, in: Long, out: Long) = { + profiler.entriesFor(name) += in + profiler.exitsFor(name) += out + } + + "The profiler" should "handle sequential entries and exits" in { + val profiler = new Profiler + tick(profiler, "a", 0, 1) + tick(profiler, "b", 2, 3) + tick(profiler, "a", 4, 5) + tick(profiler, "c", 6, 10) + tick(profiler, "d", 11, 13) + val (selfTimes, invocations) = profiler.process + selfTimes shouldBe Map(("a", 2), ("b", 1), ("c", 4), ("d", 2)) + invocations shouldBe Map(("a", 2), ("b", 1), ("c", 1), ("d", 1)) + } + + it should "handle recursive entries" in { + val profiler = new Profiler + tick(profiler, "a", 0, 13) + tick(profiler, "b", 1, 12) + tick(profiler, "c", 4, 7) + tick(profiler, "d", 5, 6) + val (selfTimes, invocations) = profiler.process + selfTimes shouldBe Map(("a", 2), ("b", 8), ("c", 2), ("d", 1)) + invocations shouldBe Map(("a", 1), ("b", 1), ("c", 1), ("d", 1)) + } + + it should "handle mutually recursive entries" in { + val profiler = new Profiler + tick(profiler, "a", 0, 5) + tick(profiler, "a", 1, 4) + tick(profiler, "a", 2, 3) + val (selfTimes1, invocations1) = profiler.process + selfTimes1 shouldBe Map(("a", 5)) + invocations1 shouldBe Map(("a", 3)) + + profiler.reset() + tick(profiler, "a", 0, 11) + tick(profiler, "b", 1, 8) + tick(profiler, "a", 3, 6) + tick(profiler, "c", 4, 5) + val (selfTimes2, invocations2) = profiler.process + selfTimes2 shouldBe Map(("a", 6), ("b", 4), ("c", 1)) + invocations2 shouldBe Map(("a", 2), ("b", 1), ("c", 1)) + } + + it should "handle mixed iterative and recursive entries" in { + val profiler = new Profiler + tick(profiler, "a", 0, 30) + tick(profiler, "b", 5, 25) + tick(profiler, "c", 6, 9) + tick(profiler, "d", 10, 15) + tick(profiler, "e", 16, 21) + tick(profiler, "c", 17, 18) + tick(profiler, "d", 19, 20) + tick(profiler, "d", 22, 24) + tick(profiler, "b", 26, 28) + + val (selfTimes, invocations) = profiler.process + selfTimes shouldBe Map(("a", 8), ("b", 7), ("c", 4), ("d", 8), ("e", 3)) + invocations shouldBe Map(("a", 1), ("b", 2), ("c", 2), ("d", 3), ("e", 1)) + } + + it should "handle mixed iterative and self-recursive entries" in { + val profiler = new Profiler + tick(profiler, "a", 0, 30) + tick(profiler, "a", 5, 25) + tick(profiler, "a", 6, 9) + tick(profiler, "a", 10, 15) + tick(profiler, "a", 16, 21) + tick(profiler, "a", 17, 18) + tick(profiler, "a", 19, 20) + tick(profiler, "a", 22, 24) + tick(profiler, "a", 26, 28) + + val (selfTimes1, invocations1) = profiler.process + selfTimes1 shouldBe Map(("a", 30)) + invocations1 shouldBe Map(("a", 9)) + } +} diff --git a/parsley/shared/src/test/scala/parsley/StringTests.scala b/parsley/shared/src/test/scala/parsley/StringTests.scala index 85785d34c..3acb77eea 100644 --- a/parsley/shared/src/test/scala/parsley/StringTests.scala +++ b/parsley/shared/src/test/scala/parsley/StringTests.scala @@ -7,7 +7,7 @@ package parsley import Predef.{ArrowAssoc => _, _} -import parsley.character.{letter, string, strings, stringOfMany, stringOfSome} +import parsley.character.{string, strings, stringOfMany, stringOfSome} import parsley.implicits.character.stringLift import parsley.Parsley.{pos => _, _} import parsley.position.pos @@ -26,8 +26,8 @@ class StringTests extends ParsleyTest { it should "consume input if it fails mid-string" in { ("abc" <|> "ab").parse("ab") shouldBe a [Failure[_]] } - it should "not consume input if it fails mid-string when combined with attempt" in { - (attempt("abc") <|> "ab").parse("ab") should not be a [Failure[_]] + it should "not consume input if it fails mid-string when combined with atomic" in { + (atomic("abc") <|> "ab").parse("ab") should not be a [Failure[_]] } it should "update positions correctly" in { stringPositionCheck(0, "abc") shouldBe Success((1, 4)) @@ -47,6 +47,9 @@ class StringTests extends ParsleyTest { stringPositionCheck(2, "a\t\t") shouldBe Success((1, 9)) stringPositionCheck(2, "aa\t") shouldBe Success((1, 9)) } + it should "reject the empty string" in { + an [IllegalArgumentException] shouldBe thrownBy (string("")) + } "strings" should "have longest match behaviour" in cases(strings("hell", "hello", "h"), noEof = true)( "hello" -> Some("hello"), @@ -55,7 +58,7 @@ class StringTests extends ParsleyTest { ) it should "be extrinsically the same as a manual equivalent" in { val p = strings("hell", "hello", "abc", "g", "goodbye") - val q = string("abc") <|> attempt("goodbye") <|> string("g") <|> attempt(string("hello")) <|> string("hell") + val q = string("abc") <|> atomic("goodbye") <|> string("g") <|> atomic(string("hello")) <|> string("hell") info("parsing \"hello\"") p.parse("hello") shouldBe q.parse("hello") info("parsing \"hell\"") @@ -68,23 +71,44 @@ class StringTests extends ParsleyTest { p.parse("b") shouldBe q.parse("b") } - "stringOfMany" should "allow for no letters" in cases(stringOfMany(letter))("" -> Some("")) - it should "consume as many letters as it can" in cases(stringOfMany(letter), noEof = true)( + "character.stringOfMany" should "allow for no letters" in cases(stringOfMany(_.isLetter))("" -> Some("")) + it should "consume as many letters as it can" in cases(stringOfMany(_.isLetter), noEof = true)( "abc" -> Some("abc"), "ab4c" -> Some("ab"), ) - it should "fail if pc fails having consumed input" in cases(stringOfMany(string("ab") #> 'a'))( + it should "fail if pc fails having consumed input" in cases(stringOfMany(string("ab").as('a')))( "ababab" -> Some("aaa"), "aba" -> None, ) - "stringOfSome" should "not allow for no letters" in cases(stringOfSome(letter))("" -> None) - it should "consume as many letters as it can" in cases(stringOfSome(letter), noEof = true)( + "character.stringOfSome" should "not allow for no letters" in cases(stringOfSome(_.isLetter))("" -> None) + it should "consume as many letters as it can" in cases(stringOfSome(_.isLetter), noEof = true)( "abc" -> Some("abc"), "ab4c" -> Some("ab"), ) - it should "fail if pc fails having consumed input" in cases(stringOfSome(string("ab") #> 'a'))( + it should "fail if pc fails having consumed input" in cases(stringOfSome(string("ab").as('a')))( "ababab" -> Some("aaa"), "aba" -> None, ) + + private def isEmoji(c: Int) = c >= 0x1f600 && c <= 0x1f64f || c >= 0x1f900 && c <= 0x1f970 + "unicode.stringOfMany" should "allow for no letters" in cases(unicode.stringOfMany(isEmoji(_)))("" -> Some("")) + it should "consume as many letters as it can" in cases(unicode.stringOfMany(isEmoji(_)), noEof = true)( + "😀😅😂😇🥰😍🤩" -> Some("😀😅😂😇🥰😍🤩"), + "😀😅🌿😂😇🥰😍🤩" -> Some("😀😅"), + ) + it should "fail if pc fails having consumed input" in cases(unicode.stringOfMany(string("🌿🌿").as(0x1f33f)))( + "🌿🌿🌿🌿🌿🌿" -> Some("🌿🌿🌿"), + "🌿🌿🌿" -> None, + ) + + "unicode.stringOfSome" should "not allow for no letters" in cases(unicode.stringOfSome(isEmoji(_)))("" -> None) + it should "consume as many letters as it can" in cases(unicode.stringOfSome(isEmoji(_)), noEof = true)( + "😀😅😂😇🥰😍🤩" -> Some("😀😅😂😇🥰😍🤩"), + "😀😅🌿😂😇🥰😍🤩" -> Some("😀😅"), + ) + it should "fail if pc fails having consumed input" in cases(unicode.stringOfSome(string("🌿🌿").as(0x1f33f)))( + "🌿🌿🌿🌿🌿🌿" -> Some("🌿🌿🌿"), + "🌿🌿🌿" -> None, + ) } diff --git a/parsley/shared/src/test/scala/parsley/UnicodeTests.scala b/parsley/shared/src/test/scala/parsley/UnicodeTests.scala new file mode 100644 index 000000000..0fb3fc85c --- /dev/null +++ b/parsley/shared/src/test/scala/parsley/UnicodeTests.scala @@ -0,0 +1,158 @@ +/* + * Copyright 2020 Parsley Contributors + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package parsley + +import Predef.{ArrowAssoc => _, _} + +import parsley.unicode._ + +class UnicodeTests extends ParsleyTest { + // TODO: property-based testing for this! + "item" should "accept any character" in { + for (i <- (0x00000 to 0x0000a) ++ (0x00040 to 0x000ef) ++ (0x0ff00 to 0x0ff0a) ++ (0x1ff00 to 0x1ffff)) { + item.parse(Character.toChars(i).mkString) should not be a [Failure[_]] + } + } + it should "fail if the input has run out, expecting any character" in { + inside(item.parse("")) { + case Failure(TestError((1, 1), VanillaError(unex, exs, rs, 1))) => + unex should contain (EndOfInput) + exs should contain only (Named("any character")) + rs shouldBe empty + } + } + + "space" should "consume ' ' or '\t'" in cases(space)( + " " -> Some(' '), + "\t" -> Some('\t'), + "\u000b" -> None, + ) + + "spaces" should "consume lots of spaces" in cases(spaces *> char('a'))( + (" \t" * 5 + 'a') -> Some('a') + ) + it should "never fail" in cases(spaces *> char('a'))( + "a" -> Some('a') + ) + + // FIXME: this needs to be improved + "whitespace" should "consume any whitespace chars" in { + (whitespaces *> char('a')).parse(" \t\n\r\f\u000b" * 2 + 'a') should not be a [Failure[_]] + } + + "endOfLine" should "consume windows or unix line endings" in cases(endOfLine)( + "\n" -> Some('\n'), + "\r\n" -> Some('\n'), + ) + it should "fail otherwise" in { + endOfLine.parse("a") shouldBe a [Failure[_]] + endOfLine.parse("\r") shouldBe a [Failure[_]] + endOfLine.parse("\r ") shouldBe a [Failure[_]] + endOfLine.parse(" ") shouldBe a [Failure[_]] + } + + "letter" should "accept non-latin characters" in cases(letter)( + "ß" -> Some('ß'), + "ð" -> Some('ð'), + "é" -> Some('é'), + "Å" -> Some('Å'), + "λ" -> Some('λ'), + "Ω" -> Some('Ω'), + ) + it should "not accept raw high-surrogates but parse supplemental letters" in cases(letter)( + "\ud840" -> None, + "\ud87e\udc1a" -> Some(0x2f81a), + ) + + "upper" should "only accept uppercase characters" in { + for (c <- 'A' to 'Z') upper.parse(c.toString) shouldBe Success(c) + upper.parse("Ω") shouldBe Success('Ω') + upper.parse("Å") shouldBe Success('Å') + } + it should "fail otherwise" in { + for (c <- 'a' to 'z') upper.parse(c.toString) shouldBe a [Failure[_]] + upper.parse("ß") shouldBe a [Failure[_]] + upper.parse("ð") shouldBe a [Failure[_]] + upper.parse("é") shouldBe a [Failure[_]] + upper.parse("λ") shouldBe a [Failure[_]] + } + + "lower" should "only accept lowercase characters" in { + for (c <- 'a' to 'z') lower.parse(c.toString) shouldBe Success(c) + lower.parse("ß") shouldBe Success('ß') + lower.parse("ð") shouldBe Success('ð') + lower.parse("é") shouldBe Success('é') + lower.parse("λ") shouldBe Success('λ') + } + it should "fail otherwise" in { + for (c <- 'A' to 'Z') lower.parse(c.toString) shouldBe a [Failure[_]] + lower.parse("Ω") shouldBe a [Failure[_]] + lower.parse("Å") shouldBe a [Failure[_]] + } + + "digit parsers" should "accept the appropriate characters" in { + for (c <- ('0' to '9') ++ ('\u0660' to '\u0669') ++ ('\uff10' to '\uff19')) { + digit.parse(c.toString) shouldBe Success(c) + hexDigit.parse(c.toString) shouldBe Success(c) + letterOrDigit.parse(c.toString) shouldBe Success(c) + val d = c.asDigit + if (d >= 0 && d < 2) { val _ = bit.parse(c.toString) shouldBe Success(c) } + if (d >= 0 && d < 8) { val _ = octDigit.parse(c.toString) shouldBe Success(c) } + } + for (c <- ('a' to 'f') ++ ('\uff41' to '\uff46') ++ ('A' to 'F') ++ ('\uff21' to '\uff26')) hexDigit.parse(c.toString) shouldBe Success(c) + } + they should "fail otherwise" in { + for (c <- ('a' to 'f') ++ ('A' to 'F')) { + bit.parse(c.toString) shouldBe a [Failure[_]] + digit.parse(c.toString) shouldBe a [Failure[_]] + octDigit.parse(c.toString) shouldBe a [Failure[_]] + } + bit.parse("2") shouldBe a [Failure[_]] + octDigit.parse("8") shouldBe a [Failure[_]] + octDigit.parse("9") shouldBe a [Failure[_]] + } + + "oneOf" should "match any of the characters provided" in { + cases(unicode.oneOf('a', 'b', 'c')) ("a" -> Some('a'), "b" -> Some('b'), "c" -> Some('c'), "d" -> None) + cases(unicode.oneOf(97 to 99)) ("a" -> Some('a'), "b" -> Some('b'), "c" -> Some('c'), "d" -> None) + cases(unicode.oneOf(97 to 100 by 2))("a" -> Some('a'), "b" -> None, "c" -> Some('c'), "d" -> None) + } + it should "always fail if provided no characters" in { + cases(unicode.oneOf()) ("a" -> None, "\n" -> None, "0" -> None) + cases(unicode.oneOf(0 until 0))("a" -> None, "\n" -> None, "0" -> None) + } + it should "work for single character ranges too" in { + cases(unicode.oneOf('a')) ("a" -> Some('a'), "\n" -> None, "b" -> None) + cases(unicode.oneOf(97 to 97))("a" -> Some('a'), "\n" -> None, "b" -> None) + } + + "noneOf" should "match none of the characters provided" in { + cases(unicode.noneOf('a', 'b', 'c')) ("a" -> None, "b" -> None, "c" -> None, "d" -> Some('d')) + cases(unicode.noneOf(97 to 99)) ("a" -> None, "b" -> None, "c" -> None, "d" -> Some('d')) + cases(unicode.noneOf(97 to 100 by 2))("a" -> None, "b" -> Some('b'), "c" -> None, "d" -> Some('d')) + } + it should "match anything if provided no characters" in { + cases(unicode.noneOf()) ("a" -> Some('a'), "\n" -> Some('\n'), "0" -> Some('0')) + cases(unicode.noneOf(0 until 0))("a" -> Some('a'), "\n" -> Some('\n'), "0" -> Some('0')) + } + it should "work for single character ranges too" in { + cases(unicode.noneOf('a')) ("a" -> None, "\n" -> Some('\n'), "b" -> Some('b')) + cases(unicode.noneOf(97 to 97))("a" -> None, "\n" -> Some('\n'), "b" -> Some('b')) + } + + "char" should "handle BMP characters" in { + cases(char('a'))("a" -> Some('a')) + cases(char('λ'))("λ" -> Some('λ')) + } + + it should "handle multi-character codepoints" in { + cases(char(0x1F642))("🙂" -> Some(0x1F642)) + } + + it should "handle multi-character codepoints atomically on fail" in { + cases(char(0x1F642) <|> char(0x1F643))("🙃" -> Some(0x1F643)) + } +} diff --git a/parsley/shared/src/test/scala/parsley/internal/InternalTests.scala b/parsley/shared/src/test/scala/parsley/internal/InternalTests.scala index f4559fa35..2eb8d022f 100644 --- a/parsley/shared/src/test/scala/parsley/internal/InternalTests.scala +++ b/parsley/shared/src/test/scala/parsley/internal/InternalTests.scala @@ -8,7 +8,7 @@ package parsley.internal import parsley.{ParsleyTest, Success, Failure, TestError, VanillaError} import parsley.Parsley, Parsley._ import parsley.character.{char, satisfy, digit, string, stringOfSome} -import parsley.combinator.{attemptChoice, choice, some, optional} +import parsley.combinator.{atomicChoice, choice, some, optional} import parsley.expr._ import parsley.implicits.character.charLift import parsley.errors.combinator.ErrorMethods @@ -20,7 +20,7 @@ class InternalTests extends ParsleyTest { "subroutines" should "function correctly and be picked up" in { val p = satisfy(_ => true) *> satisfy(_ => true) *> satisfy(_ => true) val q = 'a' *> p <* 'b' <* p <* 'c' - q.internal.instrs.count(_ == instructions.Return) shouldBe 1 + q.internal.instrs.count(_ == instructions.Return) shouldBe 2 //one is in dropped position q.internal.instrs.last should be (instructions.Return) q.parse("a123b123c") should be (Success('3')) } @@ -28,40 +28,40 @@ class InternalTests extends ParsleyTest { they should "function correctly under error messages" in { val p = satisfy(_ => true) *> satisfy(_ => true) *> satisfy(_ => true) val q = p.label("err1") *> 'a' *> p.label("err1") <* 'b' <* p.label("err2") <* 'c' <* p.label("err2") <* 'd' - q.internal.instrs.count(_ == instructions.Return) shouldBe 1 + q.internal.instrs.count(_ == instructions.Return) shouldBe 2 //one is in dropped position q.parse("123a123b123c123d") should be (Success('3')) } they should "not duplicate subroutines when error label is the same" in { val p = satisfy(_ => true) *> satisfy(_ => true) *> satisfy(_ => true) val q = 'a' *> p.label("err1") <* 'b' <* p.label("err1") <* 'c' - q.internal.instrs.count(_ == instructions.Return) shouldBe 1 + q.internal.instrs.count(_ == instructions.Return) shouldBe 2 //one is in dropped position q.parse("a123b123c") should be (Success('3')) } they should "work in the precedence parser with one op" in { val atom = some(digit).map(_.mkString.toInt) val expr = precedence[Int](atom)( - Ops(InfixL)('+' #> (_ + _))) + Ops(InfixL)('+'.as(_ + _))) expr.internal.instrs.count(_ == instructions.Return) shouldBe 1 } they should "appear frequently inside expression parsers" in { val atom = some(digit).map(_.mkString.toInt) val expr = precedence[Int](atom)( - Ops(InfixL)('+' #> (_ + _)), - Ops(InfixL)('*' #> (_ * _)), - Ops(InfixL)('%' #> (_ % _))) + Ops(InfixL)('+'.as(_ + _)), + Ops(InfixL)('*'.as(_ * _)), + Ops(InfixL)('%'.as(_ % _))) expr.internal.instrs.count(_ == instructions.Return) shouldBe 3 } // Issue 118 "error alternatives for JumpTable" should "be complete across all branches" in { val strs = Seq("hello", "hi", "abc", "good", "g") - val p = attemptChoice(strs.map(string): _*) + val p = atomicChoice(strs.map(string): _*) assume(p.internal.instrs.count(_.isInstanceOf[instructions.JumpTable]) == 1) val dummy = Reg.make[Unit] - val q = attemptChoice(strs.map(s => dummy.put(()) *> string(s)): _*) + val q = atomicChoice(strs.map(s => dummy.put(()) *> string(s)): _*) assume(q.internal.instrs.count(_.isInstanceOf[instructions.JumpTable]) == 0) info("parsing 'h'") p.parse("h") shouldBe q.parse("h") @@ -71,10 +71,10 @@ class InternalTests extends ParsleyTest { p.parse("a") shouldBe q.parse("a") } they should "contain the default in case of no input" in { - val p = attemptChoice(string("abc"), string("a"), stringOfSome(digit), string("dead")) + val p = atomicChoice(string("abc"), string("a"), stringOfSome(digit), string("dead")) assume(p.internal.instrs.count(_.isInstanceOf[instructions.JumpTable]) == 1) val dummy = Reg.make[Unit] - val q = attemptChoice(dummy.put(()) *> string("abc"), dummy.put(()) *> string("a"), + val q = atomicChoice(dummy.put(()) *> string("abc"), dummy.put(()) *> string("a"), dummy.put(()) *> stringOfSome(digit), dummy.put(()) *> string("dead")) assume(q.internal.instrs.count(_.isInstanceOf[instructions.JumpTable]) == 0) p.parse("") shouldBe q.parse("") @@ -122,10 +122,10 @@ class InternalTests extends ParsleyTest { } "JumpTable" must "not try and commute branches" in { - val p = attempt(string("abc")) <|> string("b") <|> string("abd") <|> string("cbe") + val p = atomic(string("abc")) <|> string("b") <|> string("abd") <|> string("cbe") assume(p.internal.instrs.count(_.isInstanceOf[instructions.JumpTable]) >= 1) inside(p.parse("abe")) { - case Failure(TestError(_, VanillaError(_, es, _))) => es.size shouldBe 3 + case Failure(TestError(_, VanillaError(_, es, _, _))) => es.size shouldBe 3 } } diff --git a/parsley/shared/src/test/scala/parsley/internal/deepembedding/frontend/VisitorTests.scala b/parsley/shared/src/test/scala/parsley/internal/deepembedding/frontend/VisitorTests.scala index 0ce6e3965..94429f3e2 100644 --- a/parsley/shared/src/test/scala/parsley/internal/deepembedding/frontend/VisitorTests.scala +++ b/parsley/shared/src/test/scala/parsley/internal/deepembedding/frontend/VisitorTests.scala @@ -9,7 +9,7 @@ import org.scalatest.Assertion import org.typelevel.scalaccompat.annotation.unused import parsley.{Parsley, ParsleyTest} import parsley.debug.FullBreak -import parsley.errors.{DefaultErrorBuilder, ErrorBuilder, Token} +import parsley.errors, errors.{DefaultErrorBuilder, ErrorBuilder, Token} import parsley.internal.collection.immutable.Trie import parsley.internal.deepembedding.ContOps import parsley.internal.deepembedding.Sign @@ -53,7 +53,7 @@ class VisitorTests extends ParsleyTest { override protected def findLetsAux[M[_, +_] : ContOps, R](seen: Set[LazyParsley[_]])(implicit state: LetFinderState): M[R, Unit] = dontExecute() - override protected def preprocess[M[_, +_] : ContOps, R, A_ >: Nothing](implicit lets: LetMap, recs: RecMap): M[R, StrictParsley[A_]] = + override protected def preprocess[M[_, +_] : ContOps, R, A_ >: Nothing](implicit lets: LetMap): M[R, StrictParsley[A_]] = dontExecute() override def visit[T, U[+_]](visitor: LazyParsleyIVisitor[T, U], context: T): U[Nothing] = @@ -134,6 +134,8 @@ class VisitorTests extends ParsleyTest { new Chainl(dummyParser, dontEval, dontEval).testV new Chainr[Nothing, Nothing](dummyParser, dontEval, crash).testV new SepEndBy1(dummyParser, dontEval).testV + new Filter[Any](dummyParser, _ => false, dontEval).testV + new MapFilter[Any, Nothing](dummyParser, _ => None, dontEval).testV } they should "all return the constant unit object from the test visitor" in { @@ -150,15 +152,19 @@ class VisitorTests extends ParsleyTest { new Comment(SpaceDesc.plain, new ErrorConfig).testV new Sign(Sign.CombinedType, PlusSignPresence.Optional).testV new NonSpecific("foo", identity[String], _ => true, _ => true, _ => false).testV - new CharTok(' ', dummyLabelConfig).testV - new SupplementaryCharTok(0, dummyLabelConfig).testV - new StringTok("bar", dummyLabelConfig).testV + new CharTok(' ', ' ', dummyLabelConfig).testV + new SupplementaryCharTok(0, 0, dummyLabelConfig).testV + new StringTok("bar", 4, dummyLabelConfig).testV Eof.testV new UniSatisfy(_ => true, dummyLabelConfig).testV new Modify(dummyRegister(), identity[Unit]).testV Parsley.empty.internal.testV new Fail(dummyCaretWidth).testV new Unexpected("qux", dummyCaretWidth).testV + new VanillaGen(new errors.VanillaGen).testV + new SpecialisedGen[Any](new errors.SpecialisedGen[Any] { + def messages(x: Any) = Seq.empty + }).testV new EscapeMapped(Trie.empty[Int], Set("quux")).testV new EscapeAtMost(0, 0).testV new EscapeOneOfExactly(0, Nil, dummySFConfig[Int]()).testV @@ -168,25 +174,22 @@ class VisitorTests extends ParsleyTest { new Look(dummyParser).testV new NotFollowedBy(dummyParser).testV new Put(dummyRegister(), dummyParser).testV - new Debug(dummyParser, "fred", false, FullBreak).testV + new Debug(dummyParser, "fred", false, FullBreak, Seq.empty).testV new DebugError(dummyParser, "plugh", false, dummyErrorBuilder).testV - new Filter[Nothing](dummyParser, crash).testV - new MapFilter[Nothing, Nothing](dummyParser, crash).testV - new FilterOut[Nothing](dummyParser, crash).testV - new GuardAgainst[Nothing](dummyParser, crash).testV - new UnexpectedWhen[Nothing](dummyParser, crash) new <|>(dummyParser, dummyParser).testV new >>=[Nothing, Nothing](dummyParser, crash).testV new Many(dummyParser).testV - new SkipMany(dummyParser).testV + new ChainPre(dummyParser, dummyParser).testV + new Span(dummyParser).testV + new Profile(dummyParser, "", null).testV new ManyUntil(dummyParser).testV new SkipManyUntil(dummyParser).testV new ErrorLabel(dummyParser, Seq("bazola")).testV + new ErrorHide(dummyParser).testV new ErrorExplain(dummyParser, "ztesch").testV new ErrorAmend(dummyParser, false).testV new ErrorEntrench(dummyParser).testV new ErrorDislodge(0, dummyParser).testV new ErrorLexical(dummyParser).testV - new VerifiedError[Nothing](dummyParser, Left(crash)) } } diff --git a/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala b/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala index be5cbd75a..9688e3a7c 100644 --- a/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala +++ b/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala @@ -8,7 +8,7 @@ package parsley.token import Predef.{ArrowAssoc => _, _} import parsley.{Success, ParsleyTest} -import parsley.Parsley.attempt +import parsley.Parsley.atomic import descriptions.{SpaceDesc, LexicalDesc} import parsley.character.{string, char} @@ -278,7 +278,7 @@ class SpaceTests extends ParsleyTest { it should "not restore old whitespace if the given parser fails having consumed input" in { val space = makeSpace(basicDependent) - val p = space.init *> (attempt(space.alter(predicate.Basic(Set('a')))(char('b') *> space.whiteSpace <* char('b'))) <|> char('b') *> space.whiteSpace) + val p = space.init *> (atomic(space.alter(predicate.Basic(Set('a')))(char('b') *> space.whiteSpace <* char('b'))) <|> char('b') *> space.whiteSpace) cases(p)( "baaab" -> Some(()), "baaaa" -> Some(()), @@ -307,7 +307,7 @@ class SpaceTests extends ParsleyTest { val hintMaker = many(string("a")) val p = hintMaker <* hintKiller <* eof inside(p.parse("aaaa! :(")) { - case Failure(TestError(_, VanillaError(_, expecteds, _))) => + case Failure(TestError(_, VanillaError(_, expecteds, _, _))) => expecteds shouldNot contain (Raw("a")) } } diff --git a/parsley/shared/src/test/scala/parsley/token/names/NamesTests.scala b/parsley/shared/src/test/scala/parsley/token/names/NamesTests.scala index 8848311ec..ab8341081 100644 --- a/parsley/shared/src/test/scala/parsley/token/names/NamesTests.scala +++ b/parsley/shared/src/test/scala/parsley/token/names/NamesTests.scala @@ -90,7 +90,7 @@ class NamesTests extends ParsleyTest { it should "report the correct label" in { inside(plainNames.identifier.parseAll("HARD")) { - case Failure(TestError(_, VanillaError(unexpected, expecteds, reasons))) => + case Failure(TestError(_, VanillaError(unexpected, expecteds, reasons, 4))) => unexpected should contain (Named("keyword HARD")) expecteds should contain only (Named("identifier")) reasons shouldBe empty diff --git a/parsley/shared/src/test/scala/parsley/token/names/OriginalNames.scala b/parsley/shared/src/test/scala/parsley/token/names/OriginalNames.scala index ba2c5667d..13cd8d030 100644 --- a/parsley/shared/src/test/scala/parsley/token/names/OriginalNames.scala +++ b/parsley/shared/src/test/scala/parsley/token/names/OriginalNames.scala @@ -5,27 +5,28 @@ */ package parsley.token.names -import parsley.Parsley, Parsley.{attempt, empty, pure} -import parsley.character.{satisfy, satisfyUtf16, stringOfMany, stringOfManyUtf16} +import parsley.Parsley, Parsley.{atomic, empty, pure} +import parsley.character.{satisfy, stringOfMany} import parsley.errors.combinator.ErrorMethods import parsley.implicits.zipped.Zipped2 import parsley.token.descriptions.{NameDesc, SymbolDesc} import parsley.token.errors.ErrorConfig import parsley.token.predicate.{Basic, CharPredicate, NotRequired, Unicode} +import parsley.unicode.{satisfy => satisfyUtf16, stringOfMany => stringOfManyUtf16} // $COVERAGE-OFF$ private [token] class OriginalNames(nameDesc: NameDesc, symbolDesc: SymbolDesc, err: ErrorConfig) extends Names { private def keyOrOp(startImpl: CharPredicate, letterImpl: CharPredicate, illegal: String => Boolean, name: String, unexpectedIllegal: String => String) = { - attempt { + atomic { complete(startImpl, letterImpl).unexpectedWhen { case x if illegal(x) => unexpectedIllegal(x) } }.label(name) } private def trailer(impl: CharPredicate) = impl match { - case Basic(letter) => stringOfMany(satisfy(letter)) - case Unicode(letter) => stringOfManyUtf16(satisfyUtf16(letter)) + case Basic(letter) => stringOfMany(letter) + case Unicode(letter) => stringOfManyUtf16(letter) case NotRequired => pure("") } private def complete(start: CharPredicate, letter: CharPredicate) = start match { @@ -39,14 +40,14 @@ private [token] class OriginalNames(nameDesc: NameDesc, symbolDesc: SymbolDesc, override lazy val identifier: Parsley[String] = keyOrOp(nameDesc.identifierStart, nameDesc.identifierLetter, symbolDesc.isReservedName, err.labelNameIdentifier, err.unexpectedNameIllegalIdentifier) - override def identifier(startChar: CharPredicate): Parsley[String] = attempt { + override def identifier(startChar: CharPredicate): Parsley[String] = atomic { err.filterNameIllFormedIdentifier.filter(identifier)(startChar.startsWith) } override lazy val userDefinedOperator: Parsley[String] = keyOrOp(nameDesc.operatorStart, nameDesc.operatorLetter, symbolDesc.isReservedOp, err.labelNameOperator, err.unexpectedNameIllegalOperator) - def userDefinedOperator(startChar: CharPredicate, endChar: CharPredicate): Parsley[String] = attempt { + def userDefinedOperator(startChar: CharPredicate, endChar: CharPredicate): Parsley[String] = atomic { err.filterNameIllFormedOperator.filter(userDefinedOperator)(x => startChar.startsWith(x) && endChar.endsWith(x)) } } diff --git a/parsley/shared/src/test/scala/parsley/token/symbol/OriginalSymbol.scala b/parsley/shared/src/test/scala/parsley/token/symbol/OriginalSymbol.scala index e38ff4f66..580ca5c44 100644 --- a/parsley/shared/src/test/scala/parsley/token/symbol/OriginalSymbol.scala +++ b/parsley/shared/src/test/scala/parsley/token/symbol/OriginalSymbol.scala @@ -5,9 +5,9 @@ */ package parsley.token.symbol -import parsley.Parsley, Parsley.{attempt, notFollowedBy, unit} +import parsley.Parsley, Parsley.{atomic, empty, notFollowedBy, unit} import parsley.character.{char, codePoint, string, strings} -import parsley.errors.combinator.{ErrorMethods, empty, amend} +import parsley.errors.combinator.{ErrorMethods, amend} import parsley.token.descriptions.{NameDesc, SymbolDesc} import parsley.token.errors.ErrorConfig @@ -18,7 +18,7 @@ private [token] class OriginalSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, require(name.nonEmpty, "Symbols may not be empty strings") if (symbolDesc.hardKeywords(name)) softKeyword(name) else if (symbolDesc.hardOperators(name)) softOperator(name) - else attempt(string(name)).void + else atomic(string(name)).void } override def apply(name: Char): Parsley[Unit] = char(name).void @@ -36,12 +36,12 @@ private [token] class OriginalSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, p <~= caseChar(codepoint) offset += Character.charCount(codepoint) } - attempt(amend(p)) <|> empty(name.codePointCount(0, name.length)) + atomic(amend(p)) <|> empty(name.codePointCount(0, name.length)) }.label(name) } override def softKeyword(name: String): Parsley[Unit] = { require(name.nonEmpty, "Keywords may not be empty strings") - attempt { + atomic { err.labelSymbolKeyword(name)(caseString(name)) *> notFollowedBy(identLetter).label(err.labelSymbolEndOfKeyword(name)) } @@ -54,11 +54,11 @@ private [token] class OriginalSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, case op if op.startsWith(name) && op != name => op.substring(name.length) }.toList ends match { - case Nil => attempt { + case Nil => atomic { err.labelSymbolOperator(name)(string(name)) *> notFollowedBy(opLetter).label(err.labelSymbolEndOfOperator(name)) } - case end::ends => attempt { + case end::ends => atomic { err.labelSymbolOperator(name)(string(name)) *> notFollowedBy(opLetter <|> strings(end, ends: _*)).label(err.labelSymbolEndOfOperator(name)) } diff --git a/parsley/shared/src/test/scala/parsley/token/text/OriginalEscape.scala b/parsley/shared/src/test/scala/parsley/token/text/OriginalEscape.scala index b50ccfabe..b73756f28 100644 --- a/parsley/shared/src/test/scala/parsley/token/text/OriginalEscape.scala +++ b/parsley/shared/src/test/scala/parsley/token/text/OriginalEscape.scala @@ -5,7 +5,7 @@ */ package parsley.token.text -import parsley.Parsley, Parsley.{attempt, empty, pure} +import parsley.Parsley, Parsley.{atomic, empty, pure} import parsley.character.{bit, char, digit, hexDigit, octDigit, strings} import parsley.combinator.ensure import parsley.implicits.zipped.Zipped3 @@ -24,7 +24,7 @@ private [token] class OriginalEscape(desc: EscapeDesc, err: ErrorConfig, generic case e => e -> pure(desc.escTrie(e)) }.toList match { case Nil => empty - case x::xs => attempt(strings(x, xs: _*)) + case x::xs => atomic(strings(x, xs: _*)) } } @@ -59,8 +59,8 @@ private [token] class OriginalEscape(desc: EscapeDesc, err: ErrorConfig, generic case n :: ns => val theseDigits = exactly(digits, m, radix, digit, reqDigits) val restDigits = ( - (attempt(go(n-m, n, ns).map(Some(_)) <* digitsParsed.modify(_ + digits))) - <|> (digitsParsed.put(digits) #> None) + (atomic(go(n-m, n, ns).map(Some(_)) <* digitsParsed.modify(_ + digits))) + <|> (digitsParsed.put(digits).as(None)) ) (theseDigits, restDigits, digitsParsed.get).zipped[BigInt] { case (x, None, _) => x diff --git a/project/build.properties b/project/build.properties index 40b3b8e7b..27430827b 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.0 +sbt.version=1.9.6 diff --git a/project/mima.scala b/project/mima.scala index 388607453..b4dab7eac 100644 --- a/project/mima.scala +++ b/project/mima.scala @@ -26,6 +26,8 @@ object mima { ProblemFilters.exclude[DirectMissingMethodProblem]("parsley.token.errors.*.label"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("parsley.token.errors.*.this"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("parsley.errors.helpers#WhitespaceOrUnprintable.unapply"), + ProblemFilters.exclude[DirectMissingMethodProblem]("parsley.position.internalOffsetSpan"), + ProblemFilters.exclude[DirectMissingMethodProblem]("parsley.position.spanWith"), // Expression refactor ProblemFilters.exclude[ReversedMissingMethodProblem]("parsley.expr.Fixity.chain"), ProblemFilters.exclude[ReversedMissingMethodProblem]("parsley.expr.Ops.chain"), diff --git a/project/plugins.sbt b/project/plugins.sbt index 683ffb9c4..f89b6895e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -13,7 +13,7 @@ addSbtPlugin("org.typelevel" % "sbt-typelevel-site" % sbtTypelevelVersion) // CI Stuff addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.2") -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.14") +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.15") // This is here purely to enable the niceness settings addSbtPlugin("com.beautiful-scala" % "sbt-scalastyle" % "1.5.1")