evalengine: properly handle bit values

Signed-off-by: Vicent Marti <[email protected]>

go/vt/sqlparser/ast_format.go

@@ -1318,8 +1318,8 @@ func (node *Literal) Format(buf *TrackedBuffer) {
 		buf.astPrintf(node, "%#s", node.Val)
 	case HexVal:
 		buf.astPrintf(node, "X'%#s'", node.Val)
-	case BitVal:
-		buf.astPrintf(node, "B'%#s'", node.Val)
+	case BitNum:
+		buf.astPrintf(node, "0b%#s", node.Val)
 	case DateVal:
 		buf.astPrintf(node, "date'%#s'", node.Val)
 	case TimeVal:

go/vt/sqlparser/ast_funcs.go

@@ -163,7 +163,7 @@ const (
 	FloatVal
 	HexNum
 	HexVal
-	BitVal
+	BitNum
 	DateVal
 	TimeVal
 	TimestampVal
@@ -515,9 +515,9 @@ func NewHexLiteral(in string) *Literal {
 	return &Literal{Type: HexVal, Val: in}
 }
 
-// NewBitLiteral builds a new BitVal containing a bit literal.
+// NewBitLiteral builds a new BitNum containing a bit literal.
 func NewBitLiteral(in string) *Literal {
-	return &Literal{Type: BitVal, Val: in}
+	return &Literal{Type: BitNum, Val: in}
 }
 
 // NewDateLiteral builds a new Date.
@@ -583,8 +583,8 @@ func (node *Literal) SQLType() sqltypes.Type {
 		return sqltypes.HexNum
 	case HexVal:
 		return sqltypes.HexVal
-	case BitVal:
-		return sqltypes.HexNum
+	case BitNum:
+		return sqltypes.BitNum
 	case DateVal:
 		return sqltypes.Date
 	case TimeVal:

go/vt/sqlparser/literal.go

@@ -77,7 +77,7 @@ func LiteralToValue(lit *Literal) (sqltypes.Value, error) {
 		return parseHexLiteral(b[1:])
 	case HexVal:
 		return parseHexLiteral(lit.Bytes())
-	case BitVal:
+	case BitNum:
 		return parseBitLiteral(lit.Bytes())
 	case DateVal:
 		d, ok := datetime.ParseDate(lit.Val)

go/vt/sqlparser/normalizer.go

@@ -18,10 +18,8 @@ package sqlparser
 
 import (
 	"bytes"
-	"math/big"
 
 	"vitess.io/vitess/go/mysql/datetime"
-	"vitess.io/vitess/go/mysql/hex"
 	"vitess.io/vitess/go/sqltypes"
 	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
 	"vitess.io/vitess/go/vt/vterrors"
@@ -365,19 +363,11 @@ func SQLToBindvar(node SQLNode) *querypb.BindVariable {
 			buf = append(buf, bytes.ToUpper(node.Bytes())...)
 			buf = append(buf, '\'')
 			v, err = sqltypes.NewValue(sqltypes.HexVal, buf)
-		case BitVal:
-			// Convert bit value to hex number in parameterized query format
-			var i big.Int
-			_, ok := i.SetString(string(node.Bytes()), 2)
-			if !ok {
-				return nil
-			}
-
-			buf := i.Bytes()
-			out := make([]byte, 0, (len(buf)*2)+2)
-			out = append(out, '0', 'x')
-			out = append(out, hex.EncodeBytes(buf)...)
-			v, err = sqltypes.NewValue(sqltypes.HexNum, out)
+		case BitNum:
+			out := make([]byte, 0, len(node.Bytes())+2)
+			out = append(out, '0', 'b')
+			out = append(out, node.Bytes()...)
+			v, err = sqltypes.NewValue(sqltypes.BitNum, out)
 		case DateVal:
 			v, err = sqltypes.NewValue(sqltypes.Date, node.Bytes())
 		case TimeVal:

go/vt/sqlparser/normalizer_test.go

@@ -191,23 +191,23 @@ func TestNormalize(t *testing.T) {
 	}, {
 		// Bin values work fine
 		in:      "select * from t where foo = b'11'",
-		outstmt: "select * from t where foo = :foo /* HEXNUM */",
+		outstmt: "select * from t where foo = :foo /* BITNUM */",
 		outbv: map[string]*querypb.BindVariable{
-			"foo": sqltypes.HexNumBindVariable([]byte("0x03")),
+			"foo": sqltypes.BitNumBindVariable([]byte("0b11")),
 		},
 	}, {
 		// Large bin values work fine
 		in:      "select * from t where foo = b'11101010100101010010101010101010101010101000100100100100100101001101010101010101000001'",
-		outstmt: "select * from t where foo = :foo /* HEXNUM */",
+		outstmt: "select * from t where foo = :foo /* BITNUM */",
 		outbv: map[string]*querypb.BindVariable{
-			"foo": sqltypes.HexNumBindVariable([]byte("0x3AA54AAAAAA24925355541")),
+			"foo": sqltypes.BitNumBindVariable([]byte("0b11101010100101010010101010101010101010101000100100100100100101001101010101010101000001")),
 		},
 	}, {
 		// Bin value does not convert for DMLs
 		in:      "update a set v1 = b'11'",
-		outstmt: "update a set v1 = :v1 /* HEXNUM */",
+		outstmt: "update a set v1 = :v1 /* BITNUM */",
 		outbv: map[string]*querypb.BindVariable{
-			"v1": sqltypes.HexNumBindVariable([]byte("0x03")),
+			"v1": sqltypes.BitNumBindVariable([]byte("0b11")),
 		},
 	}, {
 		// ORDER BY column_position
@@ -308,14 +308,14 @@ func TestNormalize(t *testing.T) {
 			"bv3": sqltypes.Int64BindVariable(3),
 		},
 	}, {
-		// BitVal should also be normalized
+		// BitNum should also be normalized
 		in:      `select b'1', 0b01, b'1010', 0b1111111`,
-		outstmt: `select :bv1 /* HEXNUM */, :bv2 /* HEXNUM */, :bv3 /* HEXNUM */, :bv4 /* HEXNUM */ from dual`,
+		outstmt: `select :bv1 /* BITNUM */, :bv2 /* BITNUM */, :bv3 /* BITNUM */, :bv4 /* BITNUM */ from dual`,
 		outbv: map[string]*querypb.BindVariable{
-			"bv1": sqltypes.HexNumBindVariable([]byte("0x01")),
-			"bv2": sqltypes.HexNumBindVariable([]byte("0x01")),
-			"bv3": sqltypes.HexNumBindVariable([]byte("0x0A")),
-			"bv4": sqltypes.HexNumBindVariable([]byte("0x7F")),
+			"bv1": sqltypes.BitNumBindVariable([]byte("0b1")),
+			"bv2": sqltypes.BitNumBindVariable([]byte("0b01")),
+			"bv3": sqltypes.BitNumBindVariable([]byte("0b1010")),
+			"bv4": sqltypes.BitNumBindVariable([]byte("0b1111111")),
 		},
 	}, {
 		// DateVal should also be normalized

go/vt/sqlparser/parse_test.go

@@ -527,7 +527,7 @@ var (
 		output: "select `name`, numbers from (select * from users) as x(`name`, numbers)",
 	}, {
 		input:  "select 0b010, 0b0111, b'0111', b'011'",
-		output: "select B'010', B'0111', B'0111', B'011' from dual",
+		output: "select 0b010, 0b0111, 0b0111, 0b011 from dual",
 	}, {
 		input:  "select 0x010, 0x0111, x'0111'",
 		output: "select 0x010, 0x0111, X'0111' from dual",
@@ -1120,9 +1120,10 @@ var (
 		input: "select /* hex caps */ X'F0a1' from t",
 	}, {
 		input:  "select /* bit literal */ b'0101' from t",
-		output: "select /* bit literal */ B'0101' from t",
+		output: "select /* bit literal */ 0b0101 from t",
 	}, {
-		input: "select /* bit literal caps */ B'010011011010' from t",
+		input:  "select /* bit literal caps */ B'010011011010' from t",
+		output: "select /* bit literal caps */ 0b010011011010 from t",
 	}, {
 		input: "select /* 0x */ 0xf0 from t",
 	}, {
@@ -5004,7 +5005,7 @@ func TestCreateTable(t *testing.T) {
 	` + "`" + `s3` + "`" + ` varchar default null,
 	s4 timestamp default current_timestamp(),
 	s41 timestamp default now(),
-	s5 bit(1) default B'0'
+	s5 bit(1) default 0b0
 )`,
 		}, {
 			// test non_reserved word in column name

go/vt/sqlparser/testdata/select_cases.txt

@@ -14654,7 +14654,7 @@ INPUT
 select hex(_utf8mb4 B'001111111111');
 END
 OUTPUT
-select hex(_utf8mb4 B'001111111111') from dual
+select hex(_utf8mb4 0b001111111111) from dual
 END
 INPUT
 select NULLIF(1,NULL), NULLIF(1.0, NULL), NULLIF("test", NULL);
@@ -18968,7 +18968,7 @@ INPUT
 select hex(_utf8 B'001111111111');
 END
 OUTPUT
-select hex(_utf8mb3 B'001111111111') from dual
+select hex(_utf8mb3 0b001111111111) from dual
 END
 INPUT
 select right('hello', -18446744073709551615);

go/vt/vtexplain/vtexplain_vttablet.go

@@ -846,7 +846,7 @@ func inferColTypeFromExpr(node sqlparser.Expr, tableColumnMap map[sqlparser.Iden
 			fallthrough
 		case sqlparser.HexVal:
 			fallthrough
-		case sqlparser.BitVal:
+		case sqlparser.BitNum:
 			colTypes = append(colTypes, querypb.Type_INT32)
 		case sqlparser.StrVal:
 			colTypes = append(colTypes, querypb.Type_VARCHAR)

go/vt/vtgate/evalengine/api_literal.go

@@ -28,6 +28,8 @@ import (
 	"vitess.io/vitess/go/mysql/fastparse"
 	"vitess.io/vitess/go/mysql/hex"
 	"vitess.io/vitess/go/sqltypes"
+	"vitess.io/vitess/go/vt/proto/vtrpc"
+	"vitess.io/vitess/go/vt/vterrors"
 )
 
 // NullExpr is just what you are lead to believe
@@ -156,11 +158,18 @@ func parseHexNumber(val []byte) ([]byte, error) {
 	return parseHexLiteral(val[1:])
 }
 
+func parseBitNum(val []byte) ([]byte, error) {
+	if val[0] != '0' || val[1] != 'b' {
+		return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "malformed Bit literal: %q (missing 0b prefix)", val)
+	}
+	return parseBitLiteral(val[2:])
+}
+
 func parseBitLiteral(val []byte) ([]byte, error) {
 	var i big.Int
-	_, ok := i.SetString(string(val), 2)
+	_, ok := i.SetString(hack.String(val), 2)
 	if !ok {
-		panic("malformed bit literal from parser")
+		return nil, vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "malformed Bit literal: %q (not base 2)", val)
 	}
 	return i.Bytes(), nil
 }

go/vt/vtgate/evalengine/arithmetic.go

@@ -133,12 +133,21 @@ func integerDivideConvert(arg eval) evalNumeric {
 		return dec
 	}
 
-	if b1, ok := arg.(*evalBytes); ok && b1.isHexLiteral() {
-		hex, ok := b1.toNumericHex()
-		if !ok {
-			return newEvalDecimal(decimal.Zero, 0, 0)
+	if b1, ok := arg.(*evalBytes); ok {
+		if b1.isHexLiteral() {
+			hex, ok := b1.toNumericHex()
+			if !ok {
+				return newEvalDecimal(decimal.Zero, 0, 0)
+			}
+			return hex
+		}
+		if b1.isBitLiteral() {
+			bit, ok := b1.toNumericBit()
+			if !ok {
+				return newEvalDecimal(decimal.Zero, 0, 0)
+			}
+			return bit
 		}
-		return hex
 	}
 	return evalToDecimal(arg, 0, 0)
 }

go/vt/vtgate/evalengine/compiler.go

@@ -91,9 +91,15 @@ func (c *compiler) compileToNumeric(ct ctype, offset int, fallback sqltypes.Type
 	if sqltypes.IsNumber(ct.Type) {
 		return ct
 	}
-	if ct.Type == sqltypes.VarBinary && (ct.Flag&flagHex) != 0 {
-		c.asm.Convert_hex(offset)
-		return ctype{sqltypes.Uint64, ct.Flag, collationNumeric}
+	if ct.Type == sqltypes.VarBinary {
+		if (ct.Flag & flagHex) != 0 {
+			c.asm.Convert_hex(offset)
+			return ctype{sqltypes.Uint64, ct.Flag, collationNumeric}
+		}
+		if (ct.Flag & flagBit) != 0 {
+			c.asm.Convert_bit(offset)
+			return ctype{sqltypes.Int64, ct.Flag, collationNumeric}
+		}
 	}
 
 	if sqltypes.IsDateOrTime(ct.Type) {

go/vt/vtgate/evalengine/compiler_asm.go

@@ -883,6 +883,17 @@ func (asm *assembler) Convert_hex(offset int) {
 	}, "CONV VARBINARY(SP-%d), HEX", offset)
 }
 
+func (asm *assembler) Convert_bit(offset int) {
+	asm.emit(func(env *ExpressionEnv) int {
+		var ok bool
+		env.vm.stack[env.vm.sp-offset], ok = env.vm.stack[env.vm.sp-offset].(*evalBytes).toNumericBit()
+		if !ok {
+			env.vm.err = errDeoptimize
+		}
+		return 1
+	}, "CONV VARBINARY(SP-%d), BIT", offset)
+}
+
 func (asm *assembler) Convert_Ti(offset int) {
 	asm.emit(func(env *ExpressionEnv) int {
 		v := env.vm.stack[env.vm.sp-offset].(*evalTemporal)
@@ -3094,6 +3105,14 @@ func (asm *assembler) Neg_hex() {
 	}, "NEG HEX(SP-1)")
 }
 
+func (asm *assembler) Neg_bit() {
+	asm.emit(func(env *ExpressionEnv) int {
+		arg := env.vm.stack[env.vm.sp-1].(*evalInt64)
+		env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalFloat(-float64(arg.i))
+		return 1
+	}, "NEG BIT(SP-1)")
+}
+
 func (asm *assembler) Neg_i() {
 	asm.emit(func(env *ExpressionEnv) int {
 		arg := env.vm.stack[env.vm.sp-1].(*evalInt64)

go/vt/vtgate/evalengine/compiler_asm_push.go

@@ -141,6 +141,26 @@ func (asm *assembler) PushBVar_f(key string) {
 	}, "PUSH FLOAT64(:%q)", key)
 }
 
+func push_bitnum(env *ExpressionEnv, raw []byte) int {
+	raw, env.vm.err = parseBitNum(raw)
+	env.vm.stack[env.vm.sp] = newEvalBytesBit(raw)
+	env.vm.sp++
+	return 1
+}
+
+func (asm *assembler) PushBVar_bitnum(key string) {
+	asm.adjustStack(1)
+
+	asm.emit(func(env *ExpressionEnv) int {
+		var bvar *querypb.BindVariable
+		bvar, env.vm.err = env.lookupBindVar(key)
+		if env.vm.err != nil {
+			return 0
+		}
+		return push_bitnum(env, bvar.Value)
+	}, "PUSH BITNUM(:%q)", key)
+}
+
 func push_hexnum(env *ExpressionEnv, raw []byte) int {
 	raw, env.vm.err = parseHexNumber(raw)
 	env.vm.stack[env.vm.sp] = newEvalBytesHex(raw)