Tweak stringsx.Skeleton

stringsx/confusables.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"unicode"
 
 	"golang.org/x/text/unicode/norm"
 )
@@ -54,14 +55,19 @@ func Skeleton(s string) string {
 	var sb strings.Builder
 
 	for _, r := range norm.NFD.String(s) {
+		// TODO this is not the complete set of Default_Ignorable_Code_Point
+		if unicode.In(r, unicode.Other_Default_Ignorable_Code_Point) {
+			continue
+		}
+
 		if c, ok := confusables[r]; ok {
 			sb.WriteString(c)
 		} else {
 			sb.WriteRune(r)
 		}
 	}
 
-	return sb.String()
+	return norm.NFD.String(sb.String())
 }
 
 // Implements https://www.unicode.org/reports/tr39/#def-confusable

stringsx/confusables_test.go

@@ -16,6 +16,7 @@ func TestSkeleton(t *testing.T) {
 func TestConfusable(t *testing.T) {
 	assert.True(t, stringsx.Confusable("", ""))
 	assert.True(t, stringsx.Confusable("foo", "foo"))
+	assert.True(t, stringsx.Confusable("\uFFF1foo", "foo"))
 	assert.True(t, stringsx.Confusable("١", "۱"))     // 0x661 vs 0x6f1
 	assert.True(t, stringsx.Confusable("بلی", "بلى")) // 0x6cc vs 0x649
 	assert.True(t, stringsx.Confusable("nyaruka", "𝕟𝔂𝛼𝐫ᴜ𝞳𝕒"))