Add DecodeString(ReadOnlySpan) (#144)

src/Parlot/Character.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Buffers;
 using System.Runtime.CompilerServices;
 
 namespace Parlot
@@ -19,40 +20,45 @@ public static partial class Character
  public static bool IsDecimalDigit(char ch) => IsInRange(ch, '0', '9');
 
  [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static bool IsInRange(char ch, char min, char max) => ch - (uint) min <= max - (uint) min;
+ public static bool IsInRange(char ch, char min, char max) => ch - (uint)min <= max - (uint)min;
 
  public static bool IsHexDigit(char ch) => HexConverter.IsHexChar(ch);
 
  public static bool IsIdentifierStart(char ch)
  {
- return (_characterData[ch] & (byte) CharacterMask.IdentifierStart) != 0;
+ return (_characterData[ch] & (byte)CharacterMask.IdentifierStart) != 0;
  }
 
  public static bool IsIdentifierPart(char ch)
  {
- return (_characterData[ch] & (byte) CharacterMask.IdentifierPart) != 0;
+ return (_characterData[ch] & (byte)CharacterMask.IdentifierPart) != 0;
  }
 
  public static bool IsWhiteSpace(char ch)
  {
- return (_characterData[ch] & (byte) CharacterMask.WhiteSpace) != 0;
+ return (_characterData[ch] & (byte)CharacterMask.WhiteSpace) != 0;
  }
 
  public static bool IsWhiteSpaceOrNewLine(char ch)
  {
- return (_characterData[ch] & (byte) CharacterMask.WhiteSpaceOrNewLine) != 0;
+ return (_characterData[ch] & (byte)CharacterMask.WhiteSpaceOrNewLine) != 0;
  }
 
  public static bool IsNewLine(char ch) => ch is '\n' or '\r' or '\v';
 
  public static char ScanHexEscape(string text, int index, out int length)
  {
- var lastIndex = Math.Min(4 + index, text.Length - 1);
+ return ScanHexEscape(text.AsSpan(index), out length);
+ }
+
+ public static char ScanHexEscape(ReadOnlySpan<char> text, out int length)
+ {
+ var lastIndex = Math.Min(4, text.Length - 1);
  var code = 0;
 
  length = 0;
 
- for (var i = index + 1; i < lastIndex + 1; i++)
+ for (var i = 1; i < lastIndex + 1; i++)
  {
  var d = text[i];
 
@@ -68,37 +74,32 @@ public static char ScanHexEscape(string text, int index, out int length)
  return (char)code;
  }
 
- public static TextSpan DecodeString(string s) => DecodeString(new TextSpan(s));
-
- public static TextSpan DecodeString(TextSpan span)
+ public static ReadOnlySpan<char> DecodeString(ReadOnlySpan<char> span)
  {
  // Nothing to do if the string doesn't have any escape char
- if (string.IsNullOrEmpty(span.Buffer) || span.Buffer.AsSpan(span.Offset, span.Length).IndexOf('\\') == -1)
+ if (span.IsEmpty || span.IndexOf('\\') == -1)
  {
  return span;
  }
 
-#if NET6_0_OR_GREATER
- var result = string.Create(span.Length, span, static (chars, source) =>
-#else
- var result = "".Create(span.Length, span, static (chars, source) =>
-#endif
- {
- // The assumption is that the new string will be shorter since escapes results are smaller than their source
+ // The assumption is that the new string will be shorter since escapes results are smaller than their source
+ char[]? rentedBuffer = null;
+ Span<char> buffer = span.Length <= 128
+ ? stackalloc char[span.Length]
+ : (rentedBuffer = ArrayPool<char>.Shared.Rent(span.Length));
 
+ try
+ {
  var dataIndex = 0;
- var buffer = source.Buffer!;
- var start = source.Offset;
- var end = source.Offset + source.Length;
 
- for (var i = start; i < end; i++)
+ for (var i = 0; i < span.Length; i++)
  {
- var c = buffer[i];
+ var c = span[i];
 
  if (c == '\\')
  {
  i++;
- c = buffer[i];
+ c = span[i];
 
  switch (c)
  {
@@ -114,31 +115,44 @@ public static TextSpan DecodeString(TextSpan span)
  case 't': c = '\t'; break;
  case 'v': c = '\v'; break;
  case 'u':
- c = Character.ScanHexEscape(buffer, i, out var length);
+ c = Character.ScanHexEscape(span[i..], out var length);
  i += length;
  break;
  case 'x':
- c = Character.ScanHexEscape(buffer, i, out length);
+ c = Character.ScanHexEscape(span[i..], out length);
  i += length;
  break;
  }
  }
 
- chars[dataIndex++] = c;
+ buffer[dataIndex++] = c;
  }
 
- chars[dataIndex++] = '\0';
- });
+ var result = buffer[..dataIndex].ToString().AsSpan();
 
- for (var i = result.Length - 1; i >= 0; i--)
+ return result;
+ }
+ finally
  {
- if (result[i] != '\0')
+ if (rentedBuffer != null)
  {
- return new TextSpan(result, 0, i + 1);
+ ArrayPool<char>.Shared.Return(rentedBuffer);
  }
  }
+ }
+
+ public static TextSpan DecodeString(string s) => DecodeString(new TextSpan(s));
+
+ public static TextSpan DecodeString(TextSpan span)
+ {
+ // Nothing to do if the string doesn't have any escape char
+ if (string.IsNullOrEmpty(span.Buffer) || span.Buffer.AsSpan(span.Offset, span.Length).IndexOf('\\') == -1)
+ {
+ return span;
+ }
+
+ return new TextSpan(DecodeString(span.Span).ToString());
 
- return new TextSpan(result);
  }
 
  private static int HexValue(char ch) => HexConverter.FromChar(ch);

src/Parlot/Fluent/StringLiteral.cs

@@ -20,19 +20,19 @@ public sealed class StringLiteral : Parser<TextSpan>, ICompilable, ISeekable
  static readonly char[] SingleQuotes = ['\''];
  static readonly char[] DoubleQuotes = ['\"'];
  static readonly char[] SingleOrDoubleQuotes = ['\'', '\"'];
- 
+
  private readonly StringLiteralQuotes _quotes;
 
  public StringLiteral(StringLiteralQuotes quotes)
  {
  _quotes = quotes;
 
- ExpectedChars = _quotes switch 
- { 
- StringLiteralQuotes.Single => SingleQuotes, 
- StringLiteralQuotes.Double => DoubleQuotes, 
- StringLiteralQuotes.SingleOrDouble => SingleOrDoubleQuotes, 
- _ => [] 
+ ExpectedChars = _quotes switch
+ {
+ StringLiteralQuotes.Single => SingleQuotes,
+ StringLiteralQuotes.Double => DoubleQuotes,
+ StringLiteralQuotes.SingleOrDouble => SingleOrDoubleQuotes,
+ _ => []
  };
  }
 
@@ -107,10 +107,10 @@ public CompilationResult Compile(CompilationContext context)
  [end],
  Expression.Assign(end, context.Offset()),
  Expression.Assign(result.Success, Expression.Constant(true, typeof(bool))),
- context.DiscardResult 
+ context.DiscardResult
  ? Expression.Empty()
- : Expression.Assign(result.Value, 
- Expression.Call(_decodeStringMethodInfo, 
+ : Expression.Assign(result.Value,
+ Expression.Call(_decodeStringMethodInfo,
  context.NewTextSpan(
  context.Buffer(),
  Expression.Add(start, Expression.Constant(1)),

src/Parlot/Fluent/TextLiteral.cs

@@ -64,7 +64,7 @@ public override bool Parse(ParseContext context, ref ParseResult<string> result)
  if (cursor.Match(Text.AsSpan(), _comparisonType))
  {
  var start = cursor.Offset;
- 
+
  if (_hasNewLines)
  {
  cursor.Advance(Text.Length);

src/Parlot/Scanner.cs

@@ -149,7 +149,7 @@ public bool ReadDecimal(bool allowLeadingSign, bool allowDecimalSeparator, bool
  {
  if (Cursor.Current == '-' || Cursor.Current == '+')
  {
- Cursor.AdvanceNoNewLines(1); 
+ Cursor.AdvanceNoNewLines(1);
  }
  }
 
@@ -259,7 +259,7 @@ public bool ReadInteger(out ReadOnlySpan<char> result)
  return false;
  }
 
- Cursor.AdvanceNoNewLines(next); 
+ Cursor.AdvanceNoNewLines(next);
  result = Buffer.AsSpan(Cursor.Offset - next, next);
 
  return true;
@@ -379,7 +379,7 @@ public bool ReadAnyOf(ReadOnlySpan<char> chars, StringComparison comparisonType,
  var current = Cursor.Buffer.AsSpan(Cursor.Offset, 1);
 
  var index = chars.IndexOf(current, comparisonType);
- 
+
  if (index == -1)
  {
  result = [];
@@ -522,17 +522,15 @@ public bool ReadQuotedString(out ReadOnlySpan<char> result)
  private bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
  {
  var startChar = Cursor.Current;
+ var start = Cursor.Position;
 
  if (startChar != quoteChar)
  {
  result = [];
  return false;
  }
 
- // Fast path if there aren't any escape char until next quote
- var startOffset = Cursor.Offset + 1;
-
- var nextQuote = Cursor.Buffer.AsSpan(startOffset).IndexOf(startChar);
+ var nextQuote = Cursor.Span.Slice(1).IndexOf(startChar);
 
  if (nextQuote == -1)
  {
@@ -541,18 +539,14 @@ private bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
  return false;
  }
 
- var start = Cursor.Position;
-
- Cursor.Advance();
-
- var nextEscape = Cursor.Buffer.AsSpan(startOffset, nextQuote).IndexOf('\\');
+ var nextEscape = Cursor.Span.IndexOf('\\');
 
- // If the next escape if not before the next quote, we can return the string as-is
- if (nextEscape == -1)
+ // If the next escape is not before the next quote, we can return the string as-is
+ if (nextEscape == -1 || nextEscape > nextQuote)
  {
- Cursor.Advance(nextQuote + 1);
+ Cursor.Advance(nextQuote + 2); // include start quote
 
- result = Buffer.AsSpan(start.Offset, Cursor.Offset - start.Offset);
+ result = Cursor.Buffer.AsSpan().Slice(start.Offset, nextQuote + 2);
  return true;
  }
 
@@ -672,7 +666,8 @@ private bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
 
  if (Cursor.Match(startChar))
  {
- Cursor.Advance(nextEscape + 1);
+ // Read end quote
+ Cursor.Advance(1);
  break;
  }
  else if (nextEscape == -1)
@@ -684,7 +679,7 @@ private bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
  }
  }
 
- result = Buffer.AsSpan(start.Offset, Cursor.Offset - start.Offset);
+ result = Cursor.Buffer.AsSpan()[start.Offset..Cursor.Offset];
 
  return true;
  }

test/Parlot.Tests/CharacterTests.cs

@@ -16,14 +16,13 @@ public class CharacterTests
  [InlineData(" \u03B2 ", " β ")]
  [InlineData(" \\a ", " \a ")]
  [InlineData(" \\0hello ", " \0hello ")]
-
- public void ShouldDescodeString(string text, string expected)
+ public void ShouldDecodeString(string text, string expected)
  {
  Assert.Equal(expected, Character.DecodeString(new TextSpan(text)).ToString());
  }
 
  [Fact]
- public void ShouldDescodeStringInBuffer()
+ public void ShouldDecodeStringInBuffer()
  {
  var span = new TextSpan(" a\\nbc ", 3, 5);
  Assert.Equal("a\nbc", Character.DecodeString(span).ToString());

test/Parlot.Tests/ScannerTests.cs

@@ -38,6 +38,7 @@ public void ShouldReadEscapedStringWithMatchingQuotes(string text, string expect
  [InlineData("'Lorem \\u1234 ipsum'", "'Lorem \\u1234 ipsum'")]
  [InlineData("'Lorem \\xabcd ipsum'", "'Lorem \\xabcd ipsum'")]
  [InlineData("'\\a ding'", "'\\a ding'")]
+ [InlineData("'Lorem ipsum' \\xabcd", "'Lorem ipsum'")]
  public void ShouldReadStringWithEscapes(string text, string expected)
  {
  Scanner s = new(text);