Added flexible carets for unexpected/fail (#189)

parsley/shared/src/main/scala/parsley/errors/combinator.scala

@@ -6,6 +6,7 @@ package parsley.errors
 import parsley.Parsley
 
 import parsley.internal.deepembedding.{frontend, singletons}
+import parsley.internal.errors.{CaretWidth, FlexibleCaret, RigidCaret}
 
 /** This module contains combinators that can be used to directly influence error messages of parsers.
   *
@@ -52,7 +53,7 @@ object combinator {
       * @since 3.0.0
       * @group fail
       */
-    def fail(msg0: String, msgs: String*): Parsley[Nothing] = fail(1, msg0, msgs: _*)
+    def fail(msg0: String, msgs: String*): Parsley[Nothing] = fail(new FlexibleCaret(1), msg0, msgs: _*)
 
     /** This combinator consumes no input and fails immediately with the given error messages.
       *
@@ -70,7 +71,8 @@ object combinator {
       * @since 4.0.0
       * @group fail
       */
-    def fail(caretWidth: Int, msg0: String, msgs: String*): Parsley[Nothing] = new Parsley(new singletons.Fail(caretWidth, (msg0 +: msgs): _*))
+    def fail(caretWidth: Int, msg0: String, msgs: String*): Parsley[Nothing] = fail(new RigidCaret(caretWidth), msg0, msgs: _*)
+    private def fail(caretWidth: CaretWidth, msg0: String, msgs: String*): Parsley[Nothing] = new Parsley(new singletons.Fail(caretWidth, (msg0 +: msgs): _*))
 
     /** This combinator consumes no input and fails immediately, setting the unexpected component
       * to the given item.
@@ -83,7 +85,7 @@ object combinator {
       * @return a parser that fails producing an error with `item` as the unexpected token.
       * @group fail
       */
-    def unexpected(item: String): Parsley[Nothing] = unexpected(1, item)
+    def unexpected(item: String): Parsley[Nothing] = unexpected(new FlexibleCaret(1), item)
 
     /** This combinator consumes no input and fails immediately, setting the unexpected component
       * to the given item.
@@ -97,7 +99,8 @@ object combinator {
       * @return a parser that fails producing an error with `item` as the unexpected token.
       * @group fail
       */
-    def unexpected(caretWidth: Int, item: String): Parsley[Nothing] = new Parsley(new singletons.Unexpected(item, caretWidth))
+    def unexpected(caretWidth: Int, item: String): Parsley[Nothing] = unexpected(new RigidCaret(caretWidth), item)
+    private def unexpected(caretWidth: CaretWidth, item: String): Parsley[Nothing] = new Parsley(new singletons.Unexpected(item, caretWidth))
 
     /** This combinator adjusts any error messages generated by the given parser so that they
       * occur at the position recorded on entry to this combinator (effectively as if no

parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/ErrorEmbedding.scala

@@ -4,16 +4,17 @@
 package parsley.internal.deepembedding.singletons
 
 import parsley.internal.deepembedding.backend.MZero
+import parsley.internal.errors.CaretWidth
 import parsley.internal.machine.instructions
 
-private [parsley] final class Fail(width: Int, msgs: String*) extends Singleton[Nothing] with MZero {
+private [parsley] final class Fail(width: CaretWidth, msgs: String*) extends Singleton[Nothing] with MZero {
     // $COVERAGE-OFF$
     override def pretty: String = s"fail(${msgs.mkString(", ")})"
     // $COVERAGE-ON$
-    override def instr: instructions.Instr = new instructions.Fail(width: Int, msgs: _*)
+    override def instr: instructions.Instr = new instructions.Fail(width, msgs: _*)
 }
 
-private [parsley] final class Unexpected(msg: String, width: Int) extends Singleton[Nothing] with MZero {
+private [parsley] final class Unexpected(msg: String, width: CaretWidth) extends Singleton[Nothing] with MZero {
     // $COVERAGE-OFF$
     override def pretty: String = s"unexpected($msg)"
     // $COVERAGE-ON$

parsley/shared/src/main/scala/parsley/internal/errors/CaretWidth.scala

@@ -0,0 +1,15 @@
+/* SPDX-FileCopyrightText: © 2023 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors>
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package parsley.internal.errors
+
+private [parsley] sealed abstract class CaretWidth {
+    def width: Int
+    def isFlexible: Boolean
+}
+private [parsley] class FlexibleCaret(val width: Int) extends CaretWidth {
+    def isFlexible: Boolean = true
+}
+private [parsley] class RigidCaret(val width: Int) extends CaretWidth {
+    def isFlexible: Boolean = false
+}

parsley/shared/src/main/scala/parsley/internal/errors/ErrorItem.scala

@@ -7,17 +7,19 @@ import parsley.XAssert._
 import parsley.errors, errors.{ErrorBuilder, Token, TokenSpan}
 
 private [internal] sealed abstract class ErrorItem
-private [internal] sealed trait UnexpectItem extends ErrorItem {
+private [internal] sealed abstract class UnexpectItem extends ErrorItem {
     private [internal] def formatUnexpect(lexicalError: Boolean)(implicit builder: ErrorBuilder[_]): (builder.Item, TokenSpan)
     private [internal] def higherPriority(other: UnexpectItem): Boolean
     protected [errors] def lowerThanRaw(other: UnexpectRaw): Boolean
     protected [errors] def lowerThanDesc(other: UnexpectDesc): Boolean
+    private [internal] def isFlexible: Boolean
+    private [internal] def widen(caret: Int): UnexpectItem
 }
 private [parsley] sealed trait ExpectItem extends ErrorItem {
     private [internal] def formatExpect(implicit builder: ErrorBuilder[_]): builder.Item
 }
 
-private [internal] final case class UnexpectRaw(cs: Iterable[Char], amountOfInputParserWanted: Int) extends UnexpectItem {
+private [internal] final case class UnexpectRaw(val cs: Iterable[Char], val amountOfInputParserWanted: Int) extends UnexpectItem {
     assert(cs.nonEmpty, "we promise that unexpectedToken never receives empty input")
     private [internal] def formatUnexpect(lexicalError: Boolean)(implicit builder: ErrorBuilder[_]): (builder.Item, TokenSpan) = {
         builder.unexpectedToken(cs, amountOfInputParserWanted, lexicalError) match {
@@ -28,33 +30,43 @@ private [internal] final case class UnexpectRaw(cs: Iterable[Char], amountOfInpu
     private [internal] override def higherPriority(other: UnexpectItem): Boolean = other.lowerThanRaw(this)
     protected [errors] override def lowerThanRaw(other: UnexpectRaw): Boolean = this.amountOfInputParserWanted < other.amountOfInputParserWanted
     protected [errors] override def lowerThanDesc(other: UnexpectDesc): Boolean = true
+    private [internal] override def isFlexible: Boolean = true
+    private [internal] override def widen(caret: Int): UnexpectItem = this.copy(amountOfInputParserWanted = math.max(caret, amountOfInputParserWanted))
 }
 
 private [parsley] final case class ExpectRaw(cs: String) extends ExpectItem {
+    def this(c: Char) = this(s"$c")
     private [internal] def formatExpect(implicit builder: ErrorBuilder[_]): builder.Item = builder.raw(cs)
 }
-private [parsley] object ExpectRaw {
-    def apply(c: Char): ExpectRaw = new ExpectRaw(s"$c")
-}
 private [parsley] final case class ExpectDesc(msg: String) extends ExpectItem {
     assert(msg.nonEmpty, "Desc cannot contain empty things!")
     private [internal] def formatExpect(implicit builder: ErrorBuilder[_]): builder.Item = builder.named(msg)
 }
 
-private [parsley] final case class UnexpectDesc(msg: String, width: Int) extends UnexpectItem {
+private [parsley] final case class UnexpectDesc(msg: String, val width: CaretWidth) extends UnexpectItem {
     assert(msg.nonEmpty, "Desc cannot contain empty things!")
     // FIXME: When this is formatted, the width should really be normalised to the number of code points... this information is not readily available
     private [internal] def formatUnexpect(lexicalError: Boolean)(implicit builder: ErrorBuilder[_]): (builder.Item, TokenSpan) =
-        (builder.named(msg), TokenSpan.Width(width))
+        (builder.named(msg), TokenSpan.Width(width.width))
     private [internal] override def higherPriority(other: UnexpectItem): Boolean = other.lowerThanDesc(this)
     protected [errors] override def lowerThanRaw(other: UnexpectRaw): Boolean = false
-    protected [errors] override def lowerThanDesc(other: UnexpectDesc): Boolean = this.width < other.width
+    protected [errors] override def lowerThanDesc(other: UnexpectDesc): Boolean = {
+        if (this.isFlexible != other.isFlexible) !other.isFlexible
+        else this.width.width < other.width.width
+    }
+    private [internal] override def isFlexible: Boolean = width.isFlexible
+    private [internal] override def widen(caret: Int): UnexpectItem = {
+        assert(width.isFlexible, "can only widen flexible carets!")
+        this.copy(width = new FlexibleCaret(math.max(width.width, caret)))
+    }
 }
-private [internal] case object EndOfInput extends UnexpectItem with ExpectItem {
+private [internal] object EndOfInput extends UnexpectItem with ExpectItem {
     private [internal] def formatExpect(implicit builder: ErrorBuilder[_]): builder.Item = builder.endOfInput
     private [internal] def formatUnexpect(lexicalError: Boolean)(implicit builder: ErrorBuilder[_]): (builder.Item, TokenSpan) =
         (builder.endOfInput, TokenSpan.Width(1))
     private [internal] override def higherPriority(other: UnexpectItem): Boolean = true
     protected [errors] override def lowerThanRaw(other: UnexpectRaw): Boolean = false
     protected [errors] override def lowerThanDesc(other: UnexpectDesc): Boolean = false
+    private [internal] override def isFlexible: Boolean = false
+    private [internal] override def widen(caret: Int): UnexpectItem = this
 }

parsley/shared/src/main/scala/parsley/internal/machine/Context.scala

@@ -11,7 +11,7 @@ import parsley.Result
 import parsley.Success
 import parsley.errors.ErrorBuilder
 
-import parsley.internal.errors.{ExpectItem, LineBuilder, UnexpectDesc}
+import parsley.internal.errors.{CaretWidth, ExpectItem, LineBuilder, UnexpectDesc}
 import parsley.internal.machine.errors.{
     ClassicExpectedError, ClassicFancyError, ClassicUnexpectedError, DefuncError,
     DefuncHints, EmptyHints, ErrorItemBuilder, MultiExpectedError
@@ -200,7 +200,9 @@ private [parsley] final class Context(private [machine] var instrs: Array[Instr]
         }
     }
 
-    private [machine] def failWithMessage(caretWidth: Int, msgs: String*): Unit = this.fail(new ClassicFancyError(offset, line, col, caretWidth, msgs: _*))
+    private [machine] def failWithMessage(caretWidth: CaretWidth, msgs: String*): Unit = {
+        this.fail(new ClassicFancyError(offset, line, col, caretWidth, msgs: _*))
+    }
     private [machine] def unexpectedFail(expected: Option[ExpectItem], unexpected: UnexpectDesc): Unit = {
         this.fail(new ClassicUnexpectedError(offset, line, col, expected, unexpected))
     }

parsley/shared/src/main/scala/parsley/internal/machine/errors/Builders.scala

@@ -6,7 +6,7 @@ package parsley.internal.machine.errors
 import parsley.internal.errors.{UnexpectItem, UnexpectRaw}
 
 private [machine] abstract class ErrorItemBuilder {
-    final private [errors] def apply(offset: Int, size: Int): UnexpectItem = UnexpectRaw(iterableFrom(offset), size)
+    final private [errors] def apply(offset: Int, size: Int): UnexpectItem = new UnexpectRaw(iterableFrom(offset), size)
 
     private [errors] def inRange(offset: Int): Boolean
 

parsley/shared/src/main/scala/parsley/internal/machine/errors/DefuncBuilders.scala

@@ -5,7 +5,9 @@ package parsley.internal.machine.errors
 
 import scala.collection.mutable
 
-import parsley.internal.errors.{EndOfInput, ExpectItem, FancyError, TrivialError, UnexpectItem}
+import parsley.XAssert._
+
+import parsley.internal.errors.{CaretWidth, EndOfInput, ExpectItem, FancyError, TrivialError, UnexpectDesc, UnexpectItem}
 
 import TrivialErrorBuilder.{BuilderUnexpectItem, NoItem, Other, Raw}
 
@@ -48,7 +50,7 @@ private [errors] final class TrivialErrorBuilder(offset: Int, outOfRange: Boolea
       *
       * @param item
       */
-    def updateUnexpected(item: UnexpectItem): Unit = updateUnexpected(new Other(item))
+    def updateUnexpected(item: UnexpectDesc): Unit = updateUnexpected(new Other(item))
     /** Updates the unexpected (but empty) token generated by the error, so long as no other error item
       * has been used.
       * @param width
@@ -115,13 +117,16 @@ private [errors] object TrivialErrorBuilder {
     private [TrivialErrorBuilder] final class Raw(val size: Int) extends BuilderUnexpectItem {
         final def pickHigher(other: BuilderUnexpectItem): BuilderUnexpectItem = other.pickRaw(this)
         final override def pickRaw(other: Raw): Raw = if (this.size > other.size) this else other
-        final override def pickOther(other: Other): Other = other
+        final override def pickOther(other: Other): Other = other.pickRaw(this)
         final override def pickNoItem(other: NoItem): Raw = this
         def toErrorItem(offset: Int)(implicit builder: ErrorItemBuilder): Either[Int, UnexpectItem] = Right(builder(offset, size))
     }
     private [TrivialErrorBuilder] final class Other(val underlying: UnexpectItem) extends BuilderUnexpectItem {
         final def pickHigher(other: BuilderUnexpectItem): BuilderUnexpectItem = other.pickOther(this)
-        final override def pickRaw(other: Raw): Other = this
+        final override def pickRaw(other: Raw): Other = {
+            if (underlying.isFlexible) new Other(underlying.widen(other.size))
+            else this
+        }
         final override def pickOther(other: Other): Other = if (this.underlying.higherPriority(other.underlying)) this else other
         final override def pickNoItem(other: NoItem): Other = this
         def toErrorItem(offset: Int)(implicit builder: ErrorItemBuilder): Either[Int, UnexpectItem] = Right(underlying)
@@ -145,6 +150,7 @@ private [errors] final class FancyErrorBuilder(offset: Int, lexicalError: Boolea
     private var line: Int = _
     private var col: Int = _
     private var caretWidth: Int = 0
+    private var flexibleCaret: Boolean = true
     private val msgs = mutable.ListBuffer.empty[String]
 
     /** Updates the position of the error message.
@@ -157,7 +163,19 @@ private [errors] final class FancyErrorBuilder(offset: Int, lexicalError: Boolea
         this.col = col
     }
 
-    def updateCaretWidth(width: Int): Unit = this.caretWidth = math.max(this.caretWidth, width)
+    def updateCaretWidth(width: Int): Unit = {
+        assume(flexibleCaret, "if we are updating direct from a TrivialError, we better be flexible!")
+        this.caretWidth = math.max(this.caretWidth, width)
+    }
+    def updateCaretWidth(width: CaretWidth): Unit = {
+        // if they match, just take the max
+        if (width.isFlexible == this.flexibleCaret) this.caretWidth = math.max(this.caretWidth, width.width)
+        // if they don't match and we are rigid, then we override the flexible caret, otherwise do nothing
+        else if (!width.isFlexible) {
+            this.caretWidth = width.width
+            this.flexibleCaret = false
+        }
+    }
 
     /** Adds a collection of new error message lines into this error.
       *