fix folded handling (#500)

goccy · Nov 2, 2024 · c83be7f · c83be7f
1 parent 9b2c456
commit c83be7f
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 0 deletions.
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -1815,6 +1815,41 @@ a: !!binary |
 				},
 			},
 		},
+		{
+			YAML: `
+a: >
+  Text`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "\na",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.FoldedType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockScalarIndicator,
+					Value:         ">",
+					Origin:        " >\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "Text",
+					Origin:        "  Text",
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.YAML, func(t *testing.T) {

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -554,6 +554,8 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) {
 	if ctx.isEOS() {
 		if ctx.isLiteral {
 			ctx.addBuf(c)
+		} else if ctx.isFolded && !s.isNewLineChar(c) {
+			ctx.addBuf(c)
 		}
 		value := ctx.bufferedSrc()
 		ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))