From 8d9d48de1ae854d52bcbf13bf59356c19ce53023 Mon Sep 17 00:00:00 2001 From: Jamie Willis Date: Mon, 30 Jan 2023 21:23:00 +0000 Subject: [PATCH] Optimised `softOperator` and added `Trie` implementation (#159) * Broke out soft operator and shared code: we need a Trie to continue * Added Trie to complete SoftOperator implementation * Fixed issues * Removed Radix * Removed old instructions in comment * Improved completeness of Trie testing, added sonatype release resolver --- build.sbt | 4 +- .../internal/collection/immutable/Trie.scala | 51 ++++++++ .../internal/collection/mutable/Radix.scala | 122 ------------------ .../backend/AlternativeEmbedding.scala | 5 +- .../singletons/TokenEmbedding.scala | 9 -- .../singletons/token/SymbolEmbedding.scala | 15 ++- .../machine/instructions/TokenInstrs.scala | 58 --------- .../instructions/token/SymbolInstrs.scala | 87 +++++++++---- .../token/descriptions/SymbolDesc.scala | 3 + .../parsley/token/symbol/ConcreteSymbol.scala | 14 +- .../collection/immutable/TrieSpec.scala | 28 ++++ 11 files changed, 170 insertions(+), 226 deletions(-) create mode 100644 parsley/shared/src/main/scala/parsley/internal/collection/immutable/Trie.scala delete mode 100644 parsley/shared/src/main/scala/parsley/internal/collection/mutable/Radix.scala create mode 100644 parsley/shared/src/test/scala/parsley/internal/collection/immutable/TrieSpec.scala diff --git a/build.sbt b/build.sbt index dafd5f85a..5d17f4640 100644 --- a/build.sbt +++ b/build.sbt @@ -68,8 +68,10 @@ lazy val parsley = crossProject(JSPlatform, JVMPlatform, NativePlatform) .settings( name := projectName, + resolvers ++= Opts.resolver.sonatypeOssReleases, // Will speed up MiMA during fast back-to-back releases libraryDependencies ++= Seq( - "org.scalatest" %%% "scalatest" % "3.2.14" % Test, + "org.scalatest" %%% "scalatest" % "3.2.15" % Test, + "org.scalatestplus" %%% "scalacheck-1-17" % "3.2.15.0" % Test, ), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oI"), diff --git a/parsley/shared/src/main/scala/parsley/internal/collection/immutable/Trie.scala b/parsley/shared/src/main/scala/parsley/internal/collection/immutable/Trie.scala new file mode 100644 index 000000000..73aec419e --- /dev/null +++ b/parsley/shared/src/main/scala/parsley/internal/collection/immutable/Trie.scala @@ -0,0 +1,51 @@ +/* SPDX-FileCopyrightText: © 2023 Parsley Contributors + * SPDX-License-Identifier: BSD-3-Clause + */ +package parsley.internal.collection.immutable + +import scala.annotation.tailrec +import scala.collection.immutable.IntMap + +private [parsley] class Trie(private val present: Boolean, children: IntMap[Trie]) { + def contains(key: String): Boolean = suffixes(key).present/*contains(key, 0, key.length) + @tailrec private def contains(key: String, idx: Int, sz: Int): Boolean = { + if (idx == sz) present + else childAt(key, idx) match { + case None => false + case Some(t) => t.contains(key, idx + 1, sz) + } + }*/ + + def isEmpty: Boolean = this eq Trie.empty + def nonEmpty: Boolean = !isEmpty + + def suffixes(key: Char): Trie = children.getOrElse(key.toInt, Trie.empty) + def suffixes(key: String): Trie = suffixes(key, 0, key.length) + @tailrec private def suffixes(key: String, idx: Int, sz: Int): Trie = { + if (idx == sz) this + else childAt(key, idx) match { + case None => Trie.empty + case Some(t) => t.suffixes(key, idx + 1, sz) + } + } + + def incl(key: String): Trie = incl(key, 0, key.length) + private def incl(key: String, idx: Int, sz: Int): Trie = { + if (idx == sz && present) this + else if (idx == sz) new Trie(present = true, children) + else childAt(key, idx) match { + case None => new Trie(present, children.updated(key.charAt(idx).toInt, Trie.empty.incl(key, idx + 1, sz))) + case Some(t) => + val newT = t.incl(key, idx + 1, sz) + if (t eq newT) this + else new Trie(present, children.updated(key.charAt(idx).toInt, newT)) + } + } + + private def childAt(key: String, idx: Int) = children.get(key.charAt(idx).toInt) +} +private [parsley] object Trie { + val empty = new Trie(present = false, IntMap.empty) + + def apply(strs: Iterable[String]): Trie = strs.foldLeft(empty)(_.incl(_)) +} diff --git a/parsley/shared/src/main/scala/parsley/internal/collection/mutable/Radix.scala b/parsley/shared/src/main/scala/parsley/internal/collection/mutable/Radix.scala deleted file mode 100644 index ac2e9d63f..000000000 --- a/parsley/shared/src/main/scala/parsley/internal/collection/mutable/Radix.scala +++ /dev/null @@ -1,122 +0,0 @@ -/* SPDX-FileCopyrightText: © 2022 Parsley Contributors - * SPDX-License-Identifier: BSD-3-Clause - */ -package parsley.internal.collection.mutable - -import scala.annotation.tailrec -import scala.collection.{mutable, BufferedIterator} - -import Radix.{Entry, IteratorHelpers, StringHelpers} - -@deprecated("Radix is currently not being unit tested, if it is used again remove test coverage lines", "4.0.0") -// $COVERAGE-OFF$ -private [internal] class Radix[A] { - private [this] var x = Option.empty[A] - private val m = mutable.Map.empty[Char, Entry[A]] - - def get(key: String): Option[A] = { - if (key.isEmpty) x - else for { - e <- m.get(key.head) - if key.startsWith(e.prefix) - v <- e.radix.get(key.drop(e.prefix.length)) - } yield v - } - - def getMax(key: BufferedIterator[Char]): Option[A] = { - // If there are no forwards paths, and no value at this node - // we don't need to check for input: it will always return None - if (m.isEmpty && x.isEmpty) None - else { - (for { - k1 <- key.headOption - e <- m.get(k1) - if key.checkPrefixWhileConsuming(e.prefix) - v <- e.radix.getMax(key) - } yield v).orElse(x) - } - } - - def isEmpty: Boolean = x.isEmpty && m.isEmpty - def nonEmpty: Boolean = !isEmpty - - def suffixes(c: Char): Radix[A] = m.get(c) match { - case Some(e) => - // We have to form a new root - if (e.prefix.length > 1) Radix(new Entry(e.prefix.tail, e.radix)) - else e.radix - case None => Radix.empty - } - - def contains(key: String): Boolean = get(key).nonEmpty - def apply(key: String): A = get(key).getOrElse(throw new NoSuchElementException(key)) // scalastyle:ignore throw - - def update(key: String, value: A): Unit = - if (key.isEmpty) x = Some(value) - else { - val e = m.getOrElseUpdate(key.head, new Entry(key, Radix.empty[A])) - if (key.startsWith(e.prefix)) e.radix(key.drop(e.prefix.length)) = value - else { - // Need to split the tree: find their common prefix first - val common = key.commonPrefix(e.prefix) - e.dropInPlace(common.length) - val radix = Radix(e) - // Continue inserting the key - radix(key.drop(common.length)) = value - // Insert our new entry - m(common.head) = new Entry(common, radix) - } - } -} - -private [internal] object Radix { - type RadixSet = Radix[Unit] - - def empty[A]: Radix[A] = new Radix - - private def apply[A](e: Entry[A]): Radix[A] = { - val radix = empty[A] - radix.m(e.prefix.head) = e - radix - } - - def makeSet(ks: Iterable[String]): RadixSet = apply(ks.view.zip(units)) - def apply[A](kvs: (String, A)*): Radix[A] = apply(kvs) - def apply[A](kvs: Iterable[(String, A)]): Radix[A] = { - val r = Radix.empty[A] - for ((k, v) <- kvs) r(k) = v - r - } - - private class Entry[A](var prefix: String, val radix: Radix[A]) { - def dropInPlace(n: Int): Unit = prefix = prefix.drop(n) - } - - private [internal] implicit class IteratorHelpers(val it: BufferedIterator[Char]) extends AnyVal { - @tailrec private final def go(it: BufferedIterator[Char], itPre: BufferedIterator[Char]): Boolean = { - if (!itPre.hasNext) true - else if (!it.hasNext || it.head != itPre.head) false - else { - it.next() - itPre.next() - go(it, itPre) - } - } - - def checkPrefixWhileConsuming(prefix: Iterable[Char]): Boolean = go(it, prefix.iterator.buffered) - } - - private [internal] implicit class StringHelpers(val s1: String) extends AnyVal { - def commonPrefix(s2: String): String = s1.view.zip(s2).takeWhile(Function.tupled(_ == _)).map(_._1).mkString - } - - private val units = new Iterable[Unit] { - def iterator: Iterator[Unit] = unitIterator - } - - private val unitIterator = new Iterator[Unit] { - def hasNext = true - def next() = () - } -} -// $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala index 27fe3b58e..6dea4e986 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/backend/AlternativeEmbedding.scala @@ -11,7 +11,7 @@ import parsley.XAssert._ import parsley.internal.collection.mutable.SinglyLinkedList, SinglyLinkedList.LinkedListIterator import parsley.internal.deepembedding.ContOps, ContOps.{result, suspend, ContAdapter} import parsley.internal.deepembedding.singletons._ -import parsley.internal.errors.{ExpectDesc, ExpectItem} +import parsley.internal.errors.ExpectItem import parsley.internal.machine.instructions // scalastyle:off underscore.import @@ -233,7 +233,8 @@ private [backend] object Choice { //case op@MaxOp(o) => Some((o.head, Some(Desc(o)), o.size, backtracks)) //case _: StringLiteral | RawStringLiteral => Some(('"', Some(Desc("string")), 1, backtracks)) // TODO: This can be done for case insensitive things too, but with duplicated branching - case t@token.SoftKeyword(s) if t.caseSensitive => Some((s.head, Some(ExpectDesc(s)), s.codePointCount(0, s.length), backtracks)) + case t@token.SoftKeyword(s) if t.caseSensitive => Some((s.head, t.expected.asExpectDesc(s), s.codePointCount(0, s.length), backtracks)) + case t@token.SoftOperator(s) => Some((s.head, t.expected.asExpectDesc(s), s.codePointCount(0, s.length), backtracks)) case Attempt(t) => tablable(t, backtracks = true) case (_: Pure[_]) <*> t => tablable(t, backtracks) case Lift2(_, t, _) => tablable(t, backtracks) diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala index 48eeb74dc..64f2c246d 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/TokenEmbedding.scala @@ -42,12 +42,3 @@ private [parsley] class NonSpecific(name: String, unexpectedIllegal: String => S // $COVERAGE-ON$ override def instr: instructions.Instr = new instructions.TokenNonSpecific(name, unexpectedIllegal)(start, letter, illegal) } - -/* -private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: Set[String]) extends Singleton[Unit] { - // $COVERAGE-OFF$ - override def pretty: String = s"maxOp($operator)" - // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.TokenMaxOp(operator, ops) -} -*/ diff --git a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala index 4081825ee..f0b23cd3b 100644 --- a/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala +++ b/parsley/shared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala @@ -6,28 +6,31 @@ package parsley.internal.deepembedding.singletons.token import parsley.token.errors.LabelConfig import parsley.token.predicate.CharPredicate +import parsley.internal.collection.immutable.Trie import parsley.internal.deepembedding.singletons.Singleton import parsley.internal.machine.instructions private [parsley] final class SoftKeyword(private [SoftKeyword] val specific: String, letter: CharPredicate, val caseSensitive: Boolean, - expected: LabelConfig, expectedEnd: String) extends Singleton[Unit] { + val expected: LabelConfig, expectedEnd: String) extends Singleton[Unit] { // $COVERAGE-OFF$ override def pretty: String = s"softKeyword($specific)" // $COVERAGE-ON$ override def instr: instructions.Instr = new instructions.token.SoftKeyword(specific, letter, caseSensitive, expected, expectedEnd) } -/* -private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: Set[String]) extends Singleton[Unit] { +private [parsley] final class SoftOperator(private [SoftOperator] val specific: String, letter: CharPredicate, ops: Trie, + val expected: LabelConfig, expectedEnd: String) extends Singleton[Unit] { // $COVERAGE-OFF$ - override def pretty: String = s"maxOp($operator)" + override def pretty: String = s"softOperator($specific)" // $COVERAGE-ON$ - override def instr: instructions.Instr = new instructions.TokenMaxOp(operator, ops) + override def instr: instructions.Instr = new instructions.token.SoftOperator(specific, letter, ops, expected, expectedEnd) } -*/ // $COVERAGE-OFF$ private [deepembedding] object SoftKeyword { def unapply(self: SoftKeyword): Some[String] = Some(self.specific) } +private [deepembedding] object SoftOperator { + def unapply(self: SoftOperator): Some[String] = Some(self.specific) +} // $COVERAGE-ON$ diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala index 92393dae4..465ff4917 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala @@ -241,61 +241,3 @@ private [internal] final class TokenNonSpecific(name: String, unexpectedIllegal: override def toString: String = s"TokenNonSpecific($name)" // $COVERAGE-ON$ } - -/* -private [instructions] abstract class TokenSpecificAllowTrailing( - specific: String, expected: Option[ExpectDesc], protected final val expectedEnd: Option[ExpectDesc], caseSensitive: Boolean) extends Instr { - def this(specific: String, expected: LabelConfig, expectedEnd: String, caseSensitive: Boolean) = { - this(if (caseSensitive) specific else specific.toLowerCase, expected.asExpectDesc, Some(new ExpectDesc(expectedEnd)), caseSensitive) - } - private [this] final val strsz = specific.length - private [this] final val numCodePoints = specific.codePointCount(0, strsz) - protected def postprocess(ctx: Context, i: Int): Unit - - val readCharCaseHandled = { - if (caseSensitive) (ctx: Context, i: Int) => ctx.input.charAt(i) - else (ctx: Context, i: Int) => ctx.input.charAt(i).toLower - } - - @tailrec final private def readSpecific(ctx: Context, i: Int, j: Int): Unit = { - if (j < strsz && readCharCaseHandled(ctx, i) == specific.charAt(j)) readSpecific(ctx, i + 1, j + 1) - else if (j < strsz) ctx.expectedFail(expected, numCodePoints) - else { - ctx.saveState() - ctx.fastUncheckedConsumeChars(strsz) - postprocess(ctx, i) - } - } - - final override def apply(ctx: Context): Unit = { - if (ctx.inputsz >= ctx.offset + strsz) readSpecific(ctx, ctx.offset, 0) - else ctx.expectedFail(expected, numCodePoints) - } -} - -private [internal] final class TokenMaxOp(operator: String, _ops: Set[String]) extends TokenSpecificAllowTrailing(operator, true) { - private val ops = Radix.makeSet(_ops.collect { - case op if op.length > operator.length && op.startsWith(operator) => op.substring(operator.length) - }) - - @tailrec private def go(ctx: Context, i: Int, ops: RadixSet): Unit = { - lazy val ops_ = ops.suffixes(ctx.input.charAt(i)) - val possibleOpsRemain = i < ctx.inputsz && ops.nonEmpty - if (possibleOpsRemain && ops_.contains("")) { - ctx.expectedFail(expectedEnd) //This should only report a single token - ctx.restoreState() - } - else if (possibleOpsRemain) go(ctx, i + 1, ops_) - else { - ctx.states = ctx.states.tail - ctx.pushAndContinue(()) - } - } - - override def postprocess(ctx: Context, i: Int): Unit = go(ctx, i, ops) - - // $COVERAGE-OFF$ - override def toString: String = s"TokenMaxOp(${operator})" - // $COVERAGE-ON$ -} -*/ diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala index b0dd83f91..ab03cf7a5 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala @@ -3,26 +3,26 @@ */ package parsley.internal.machine.instructions.token +import scala.annotation.tailrec + import parsley.token.errors.LabelConfig import parsley.token.predicate +import parsley.internal.collection.immutable.Trie import parsley.internal.errors.ExpectDesc import parsley.internal.machine.Context import parsley.internal.machine.XAssert._ import parsley.internal.machine.instructions.Instr -private [internal] final class SoftKeyword( - specific: String, letter: CharPredicate, caseSensitive: Boolean, expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Instr { - def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = { - this(if (caseSensitive) specific else specific.toLowerCase, - letter.asInternalPredicate, - caseSensitive, - expected.asExpectDesc, Some(new ExpectDesc(expectedEnd))) - } - +private [token] abstract class Specific extends Instr { + protected val specific: String + protected val caseSensitive: Boolean + protected val expected: Option[ExpectDesc] private [this] final val strsz = specific.length private [this] final val numCodePoints = specific.codePointCount(0, strsz) + protected def postprocess(ctx: Context): Unit + final override def apply(ctx: Context): Unit = { ensureRegularInstruction(ctx) if (ctx.moreInput(strsz)) { @@ -32,23 +32,12 @@ private [internal] final class SoftKeyword( else ctx.expectedFail(expected, numCodePoints) } - private def postprocess(ctx: Context): Unit = { - if (letter.peek(ctx)) { - ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token - ctx.restoreState() - } - else { - ctx.states = ctx.states.tail - ctx.pushAndContinue(()) - } - } - - val readCharCaseHandledBMP = { + private val readCharCaseHandledBMP = { if (caseSensitive) (ctx: Context) => ctx.peekChar else (ctx: Context) => ctx.peekChar.toLower } - val readCharCaseHandledSupplementary = { + private val readCharCaseHandledSupplementary = { if (caseSensitive) (ctx: Context) => Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1)) else (ctx: Context) => Character.toLowerCase(Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1))) } @@ -71,8 +60,62 @@ private [internal] final class SoftKeyword( } else postprocess(ctx) } +} + +private [internal] final class SoftKeyword(protected val specific: String, letter: CharPredicate, protected val caseSensitive: Boolean, + protected val expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Specific { + def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = { + this(if (caseSensitive) specific else specific.toLowerCase, + letter.asInternalPredicate, + caseSensitive, + expected.asExpectDesc, Some(new ExpectDesc(expectedEnd))) + } + + protected def postprocess(ctx: Context): Unit = { + if (letter.peek(ctx)) { + ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token + ctx.restoreState() + } + else { + ctx.states = ctx.states.tail + ctx.pushAndContinue(()) + } + } // $COVERAGE-OFF$ override def toString: String = s"SoftKeyword($specific)" // $COVERAGE-ON$ } + +private [internal] final class SoftOperator(protected val specific: String, letter: CharPredicate, ops: Trie, + protected val expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Specific { + def this(specific: String, letter: predicate.CharPredicate, ops: Trie, expected: LabelConfig, expectedEnd: String) = { + this(specific, letter.asInternalPredicate, ops, expected.asExpectDesc, Some(new ExpectDesc(expectedEnd))) + } + protected val caseSensitive = true + private val ends = ops.suffixes(specific) + + // returns true if an end could be parsed from this point + @tailrec private def checkEnds(ctx: Context, ends: Trie, off: Int): Boolean = { + if (ends.nonEmpty && ctx.moreInput(off + 1)) { + val endsOfNext = ends.suffixes(ctx.peekChar(off)) + endsOfNext.contains("") || checkEnds(ctx, endsOfNext, off + 1) + } + else false + } + + protected def postprocess(ctx: Context): Unit = { + if (letter.peek(ctx) || checkEnds(ctx, ends, off = 0)) { + ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token + ctx.restoreState() + } + else { + ctx.states = ctx.states.tail + ctx.pushAndContinue(()) + } + } + + // $COVERAGE-OFF$ + override def toString: String = s"SoftOperator($specific)" + // $COVERAGE-ON$ +} diff --git a/parsley/shared/src/main/scala/parsley/token/descriptions/SymbolDesc.scala b/parsley/shared/src/main/scala/parsley/token/descriptions/SymbolDesc.scala index 18fda3d37..1e6eb177e 100644 --- a/parsley/shared/src/main/scala/parsley/token/descriptions/SymbolDesc.scala +++ b/parsley/shared/src/main/scala/parsley/token/descriptions/SymbolDesc.scala @@ -3,6 +3,8 @@ */ package parsley.token.descriptions +import parsley.internal.collection.immutable.Trie + /** This class describes how symbols (textual literals in a BNF) should be * processed lexically. * @@ -15,6 +17,7 @@ final case class SymbolDesc (hardKeywords: Set[String], hardOperators: Set[String], caseSensitive: Boolean) { require((hardKeywords & hardOperators).isEmpty, "there cannot be an intersection between keywords and operators") + private [parsley] val hardOperatorsTrie = Trie(hardOperators) private [parsley] def isReservedName(name: String): Boolean = theReservedNames.contains(if (caseSensitive) name else name.toLowerCase) private lazy val theReservedNames = if (caseSensitive) hardKeywords else hardKeywords.map(_.toLowerCase) diff --git a/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala b/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala index 5ae613c9c..0bc11e906 100644 --- a/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala +++ b/parsley/shared/src/main/scala/parsley/token/symbol/ConcreteSymbol.scala @@ -3,9 +3,8 @@ */ package parsley.token.symbol -import parsley.Parsley, Parsley.{attempt, notFollowedBy} -import parsley.character.{char, string, strings} -import parsley.errors.combinator.ErrorMethods +import parsley.Parsley, Parsley.attempt +import parsley.character.{char, string} import parsley.token.descriptions.{NameDesc, SymbolDesc} import parsley.token.errors.ErrorConfig @@ -49,14 +48,17 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, */ override def softKeyword(name: String): Parsley[Unit] = { + require(name.nonEmpty, "Keywords may not be empty strings") new Parsley(new token.SoftKeyword(name, nameDesc.identifierLetter, symbolDesc.caseSensitive, err.labelSymbolKeyword(name), err.labelSymbolEndOfKeyword(name))) } - private lazy val opLetter = nameDesc.operatorLetter.toNative + //private lazy val opLetter = nameDesc.operatorLetter.toNative override def softOperator(name: String): Parsley[Unit] = { require(name.nonEmpty, "Operators may not be empty strings") - val ends = symbolDesc.hardOperators.collect { + new Parsley(new token.SoftOperator(name, nameDesc.operatorLetter, symbolDesc.hardOperatorsTrie, + err.labelSymbolOperator(name), err.labelSymbolEndOfOperator(name))) + /*val ends = symbolDesc.hardOperators.collect { case op if op.startsWith(name) && op != name => op.substring(name.length) }.toList ends match { @@ -68,6 +70,6 @@ private [token] class ConcreteSymbol(nameDesc: NameDesc, symbolDesc: SymbolDesc, err.labelSymbolOperator(name)(string(name)) *> notFollowedBy(opLetter <|> strings(end, ends: _*)).label(err.labelSymbolEndOfOperator(name)) } - } + }*/ } } diff --git a/parsley/shared/src/test/scala/parsley/internal/collection/immutable/TrieSpec.scala b/parsley/shared/src/test/scala/parsley/internal/collection/immutable/TrieSpec.scala new file mode 100644 index 000000000..a65d0e4ea --- /dev/null +++ b/parsley/shared/src/test/scala/parsley/internal/collection/immutable/TrieSpec.scala @@ -0,0 +1,28 @@ +package parsley.internal.collection.immutable + +import org.scalatest.propspec.AnyPropSpec +import org.scalatest.matchers._ +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks + +class TrieSpec extends AnyPropSpec with ScalaCheckPropertyChecks with should.Matchers { + property("a Trie constructed from a set should contain all of its elements") { + forAll { (set: List[String]) => + val t = Trie(set) + set.forall(t.contains) + for (key <- set) { + t.contains(key) shouldBe true + } + } + } + + property("a Trie constructed from a set should not contain extra keys") { + forAll { (set: List[String]) => + val t = Trie(set) + forAll { (str: String) => + whenever(!set.contains(str)) { + t.contains(str) shouldBe false + } + } + } + } +}