From ea9c195c23fd38327ab3f07616560ea08ba1e41b Mon Sep 17 00:00:00 2001 From: Allan CORNET Date: Sun, 22 Oct 2023 21:01:31 +0200 Subject: [PATCH] extended numbers syntax about inspired by Rust (in progress) --- .../tests/test_swapbytes.m | 8 +- modules/integer/tests/test_integer_minus.m | 2 +- modules/integer/tests/test_integer_plus.m | 12 +- modules/integer/tests/test_intmax.m | 4 +- modules/integer/tests/test_uint64.m | 4 +- .../src/cpp/AbstractSyntaxTree.cpp | 4 +- .../src/cpp/AbstractSyntaxTreeHelpers.cpp | 7 + modules/interpreter/src/cpp/AsciiToDouble.cpp | 46 +++++- modules/interpreter/src/cpp/AsciiToDouble.hpp | 8 + modules/interpreter/src/cpp/Evaluator.cpp | 29 +++- modules/interpreter/src/cpp/NelsonLexer.cpp | 147 ++++++++++++++---- .../src/include/AbstractSyntaxTree.hpp | 13 +- .../interpreter/tests/test_number_syntax.m | 30 ++++ 13 files changed, 254 insertions(+), 60 deletions(-) create mode 100644 modules/interpreter/tests/test_number_syntax.m diff --git a/modules/elementary_functions/tests/test_swapbytes.m b/modules/elementary_functions/tests/test_swapbytes.m index 28f93e6010..9126b11a02 100644 --- a/modules/elementary_functions/tests/test_swapbytes.m +++ b/modules/elementary_functions/tests/test_swapbytes.m @@ -44,11 +44,11 @@ assert_isequal(swapbytes(int64(-1)),int64(-1)); %============================================================================= assert_isequal(swapbytes(uint64(12)), uint64(864691128455135232)); -assert_isequal(swapbytes(uint64(5648002115885334530u)),uint64(144115188088201550u)); +assert_isequal(swapbytes(5648002115885334530u64), 144115188088201550u64); %============================================================================= -R = swapbytes([1u, 2u, 3u; 4u, 5u, 6u]); -REF = [72057594037927936u, 144115188075855872u, 216172782113783808u; -288230376151711744u, 360287970189639680u, 432345564227567616u]; +R = swapbytes([1u64, 2u64, 3u64; 4u64, 5u64, 6u64]); +REF = [72057594037927936u64, 144115188075855872u64, 216172782113783808u64; +288230376151711744u64, 360287970189639680u64, 432345564227567616u64]; assert_isequal(R, REF); %============================================================================= msg = sprintf(_('Check for incorrect argument data type or missing argument in call to function ''%s''.'), 'swapbytes'); diff --git a/modules/integer/tests/test_integer_minus.m b/modules/integer/tests/test_integer_minus.m index ad461fc1b4..f2dd540c51 100644 --- a/modules/integer/tests/test_integer_minus.m +++ b/modules/integer/tests/test_integer_minus.m @@ -8,7 +8,7 @@ % LICENCE_BLOCK_END %============================================================================= R = (uint64(2^53) - 1); -REF = 9007199254740991u; +REF = 9007199254740991u64; assert_isequal(R, REF); %============================================================================= R = (uint64(2^53) - 1) - (uint64(2^53) - 1000); diff --git a/modules/integer/tests/test_integer_plus.m b/modules/integer/tests/test_integer_plus.m index c557494f23..d5732f2eb5 100644 --- a/modules/integer/tests/test_integer_plus.m +++ b/modules/integer/tests/test_integer_plus.m @@ -8,27 +8,27 @@ % LICENCE_BLOCK_END %============================================================================= R = (uint64(2^53) + 1) + (uint64(2^53) + 1000); -REF = 18014398509482985u; +REF = 18014398509482985u64; assert_isequal(R, REF); %============================================================================= R = (uint64(2^54) + 1) + (uint64(2^54) + 1000); -REF = 36028797018964969u; +REF = 36028797018964969u64; assert_isequal(R, REF); %============================================================================= R = (uint64(2^57) + 1) + (uint64(2^57) + 1000); -REF = 288230376151712745u; +REF = 288230376151712745u64; assert_isequal(R, REF); %============================================================================= R = (uint64(2^59) + 1) + (uint64(2^59) + 1000); -REF = 1152921504606847977u; +REF = 1152921504606847977u64; assert_isequal(R, REF); %============================================================================= R = (int64(2^53) + 1) + (int64(2^53) + 1000); -REF = int64(18014398509482985u); +REF = int64(18014398509482985u64); assert_isequal(R, REF); %============================================================================= R = (int64(2^54) + 1) + (int64(2^54) + 1000); -REF = int64(36028797018964969u); +REF = 36028797018964969i64; assert_isequal(R, REF); %============================================================================= R = int8(127) + int8(127); diff --git a/modules/integer/tests/test_intmax.m b/modules/integer/tests/test_intmax.m index 622cfde912..ab7c29b33d 100644 --- a/modules/integer/tests/test_intmax.m +++ b/modules/integer/tests/test_intmax.m @@ -17,8 +17,8 @@ assert_isequal(intmax('int16'), int16(32767)); assert_isequal(intmax('uint16'), uint16(65535)); assert_isequal(intmax('uint32'), uint32(4294967295)); -assert_isequal(intmax('int64'), int64(9223372036854775807U)); -assert_isequal(intmax('uint64'), uint64(18446744073709551615U)); +assert_isequal(intmax('int64'), int64(9223372036854775807i64)); +assert_isequal(intmax('uint64'), uint64(18446744073709551615u64)); %============================================================================= assert_checkerror('intmax(''uint64'', 3)', _('Wrong number of input arguments.')); assert_checkerror('intmax(3)', _('Wrong type for argument #1: string expected.')); diff --git a/modules/integer/tests/test_uint64.m b/modules/integer/tests/test_uint64.m index 06404d1568..c8b30e6970 100644 --- a/modules/integer/tests/test_uint64.m +++ b/modules/integer/tests/test_uint64.m @@ -7,12 +7,12 @@ % SPDX-License-Identifier: LGPL-3.0-or-later % LICENCE_BLOCK_END %============================================================================= -as_uint64 = 18446744073709551615U; +as_uint64 = 18446744073709551615u64; max_uint64 = intmax('uint64'); assert_isequal(as_uint64, max_uint64); %============================================================================= A = 1; -B = 1U; +B = 1u64; C = uint64(A); assert_isequal(class(B), class(C)); assert_isequal(class(B), 'uint64'); diff --git a/modules/interpreter/src/cpp/AbstractSyntaxTree.cpp b/modules/interpreter/src/cpp/AbstractSyntaxTree.cpp index 04ab91848b..cbf3b92720 100644 --- a/modules/interpreter/src/cpp/AbstractSyntaxTree.cpp +++ b/modules/interpreter/src/cpp/AbstractSyntaxTree.cpp @@ -49,7 +49,7 @@ AbstractSyntaxTree::getReferences() //============================================================================= AbstractSyntaxTreePtr -AbstractSyntaxTree::createNode(NODE_TYPE ntype, const char* name, int context) +AbstractSyntaxTree::createNode(NODE_TYPE ntype, const std::string& name, int context) { AbstractSyntaxTreePtr p; try { @@ -126,7 +126,7 @@ AbstractSyntaxTree::AbstractSyntaxTree() opNum = OP_NULL; } //============================================================================= -AbstractSyntaxTree::AbstractSyntaxTree(NODE_TYPE ntype, const char* name, int context) +AbstractSyntaxTree::AbstractSyntaxTree(NODE_TYPE ntype, const std::string& name, int context) : type(ntype), text(name), m_context(context) { tokenNumber = 0; diff --git a/modules/interpreter/src/cpp/AbstractSyntaxTreeHelpers.cpp b/modules/interpreter/src/cpp/AbstractSyntaxTreeHelpers.cpp index 837adddcdb..e663d757c9 100644 --- a/modules/interpreter/src/cpp/AbstractSyntaxTreeHelpers.cpp +++ b/modules/interpreter/src/cpp/AbstractSyntaxTreeHelpers.cpp @@ -50,6 +50,13 @@ expression(AbstractSyntaxTreePtr expr) } break; case const_int_node: case const_uint64_node: + case const_uint32_node: + case const_uint16_node: + case const_uint8_node: + case const_int64_node: + case const_int32_node: + case const_int16_node: + case const_int8_node: case const_float_node: case const_double_node: { res = expr->text; diff --git a/modules/interpreter/src/cpp/AsciiToDouble.cpp b/modules/interpreter/src/cpp/AsciiToDouble.cpp index 8fd4ecfa55..c811366784 100644 --- a/modules/interpreter/src/cpp/AsciiToDouble.cpp +++ b/modules/interpreter/src/cpp/AsciiToDouble.cpp @@ -10,7 +10,7 @@ #include #include #include -#include "fast_float/fast_float.h" +#include #include "AsciiToDouble.hpp" //============================================================================= namespace Nelson { @@ -19,9 +19,49 @@ double asciiToDouble(const std::string& str) { double value; - auto answer = fast_float::from_chars(str.data(), str.data() + str.size(), value); - return value; + auto answer = fast_float::from_chars(str.data(), str.data() + str.size(), value); + return abs(value); } //============================================================================= +int32 +asciiToInt32(const std::string& str) +{ + int32 res = static_cast(0); + int64 valuei64 = asciiToInt64(str); + if (valuei64 >= static_cast(std::numeric_limits::max())) { + res = std::numeric_limits::max(); + } else if (valuei64 <= static_cast(std::numeric_limits::min())) { + res = std::numeric_limits::min(); + } else { + res = abs(static_cast(valuei64)); + } + return res; +} +//============================================================================= +int64 +asciiToInt64(const std::string& str) +{ + int64 res = static_cast(0); + char* endptr = nullptr; + if (str[0] == '-') { + std::string withNeg = str.substr(1); + unsigned long long int v = strtoull(withNeg.c_str(), &endptr, 10); + if (v > -std::numeric_limits::min()) { + res = std::numeric_limits::min(); + } else { + res = static_cast(v); + } + } else { + unsigned long long int v = strtoull(str.c_str(), &endptr, 10); + if (v > std::numeric_limits::max()) { + res = std::numeric_limits::max(); + } else { + res = static_cast(v); + } + } + return res; +} +//============================================================================= + }; // namespace Nelson //============================================================================= diff --git a/modules/interpreter/src/cpp/AsciiToDouble.hpp b/modules/interpreter/src/cpp/AsciiToDouble.hpp index a5c7e4d8d9..f4866bed3e 100644 --- a/modules/interpreter/src/cpp/AsciiToDouble.hpp +++ b/modules/interpreter/src/cpp/AsciiToDouble.hpp @@ -11,11 +11,19 @@ //============================================================================= #include #include "nlsInterpreter_exports.h" +#include "Types.hpp" //============================================================================= namespace Nelson { //============================================================================= NLSINTERPRETER_IMPEXP double asciiToDouble(const std::string& str); + +NLSINTERPRETER_IMPEXP int32 +asciiToInt32(const std::string& str); + +NLSINTERPRETER_IMPEXP int64 +asciiToInt64(const std::string& str); + //============================================================================= }; // namespace Nelson //============================================================================= diff --git a/modules/interpreter/src/cpp/Evaluator.cpp b/modules/interpreter/src/cpp/Evaluator.cpp index 4da2256d65..863ab4d5ea 100644 --- a/modules/interpreter/src/cpp/Evaluator.cpp +++ b/modules/interpreter/src/cpp/Evaluator.cpp @@ -555,7 +555,21 @@ Evaluator::expression(AbstractSyntaxTreePtr t) { ArrayOf retval; switch (t->type) { - case const_double_node: + case const_double_node: { + callstack.pushID((size_t)t->getContext()); + retval = ArrayOf::doubleConstructor(asciiToDouble(t->text)); + callstack.popID(); + } break; + case const_int32_node: { + callstack.pushID((size_t)t->getContext()); + retval = ArrayOf::int32Constructor(asciiToInt32(t->text)); + callstack.popID(); + } break; + case const_int64_node: { + callstack.pushID((size_t)t->getContext()); + retval = ArrayOf::int64Constructor(asciiToInt64(t->text)); + callstack.popID(); + } break; case const_int_node: { callstack.pushID((size_t)t->getContext()); retval = ArrayOf::doubleConstructor(asciiToDouble(t->text)); @@ -576,8 +590,7 @@ Evaluator::expression(AbstractSyntaxTreePtr t) retval = ArrayOf::stringArrayConstructor(t->text); callstack.popID(); } break; - case const_dcomplex_node: - case const_complex_node: { + case const_dcomplex_node: { callstack.pushID((size_t)t->getContext()); double val = asciiToDouble(t->text); if (approximatelyEqual(val, 0, std::numeric_limits::epsilon())) { @@ -587,6 +600,16 @@ Evaluator::expression(AbstractSyntaxTreePtr t) } callstack.popID(); } break; + case const_complex_node: { + callstack.pushID((size_t)t->getContext()); + single val = asciiToDouble(t->text); + if (approximatelyEqual(val, 0, std::numeric_limits::epsilon())) { + retval = ArrayOf::singleConstructor(0.); + } else { + retval = ArrayOf::complexConstructor(0, val); + } + callstack.popID(); + } break; case const_uint64_node: { callstack.pushID((size_t)t->getContext()); char* endptr = nullptr; diff --git a/modules/interpreter/src/cpp/NelsonLexer.cpp b/modules/interpreter/src/cpp/NelsonLexer.cpp index 40571096ae..3ea221b0f9 100644 --- a/modules/interpreter/src/cpp/NelsonLexer.cpp +++ b/modules/interpreter/src/cpp/NelsonLexer.cpp @@ -512,11 +512,11 @@ lexIdentifier() int lexNumber() { + bool isNegative = (tokenType == '-'); int state = 0; indexType cp = 0; char buffer[DEFAULT_BUFFER_SIZE_LEXER]; int intonly = 1; - int vtype; // Initialize the state... state = 0; while (state != 7) { @@ -608,20 +608,74 @@ lexNumber() } } } - if ((datap[cp] == 'f') || (datap[cp] == 'F')) { - cp++; - vtype = 1; - } else if ((datap[cp] == 'u') || (datap[cp] == 'U')) { - cp++; - vtype = 4; //-V112 - } else if ((datap[cp] == 'd') || (datap[cp] == 'D')) { - cp++; - vtype = 2; - } else if (intonly == 0) { - vtype = 2; + + NODE_TYPE nodeType = null_node; + if (datap[cp] == 'f') { + // f32 --> single + if ((datap[cp + 1] == '3') && (datap[cp + 2] == '2')) { + cp = cp + 3; + nodeType = const_float_node; + } + // f64 --> double + else if ((datap[cp + 1] == '6') && (datap[cp + 2] == '4')) { + cp = cp + 3; + nodeType = const_double_node; + } else { + LexerException(_("Malformed floating point constant.")); + } + } else if (datap[cp] == 'i') { + // i8 --> int8 + if (datap[cp + 1] == '8') { + cp = cp + 2; + nodeType = const_int8_node; + } + // i16 --> int16 + else if ((datap[cp + 1] == '1') && (datap[cp + 2] == '6')) { + cp = cp + 3; + nodeType = const_int16_node; + } + // i32 --> int32 + else if ((datap[cp + 1] == '3') && (datap[cp + 2] == '2')) { + cp = cp + 3; + nodeType = const_int32_node; + } + // i64 --> int32 + else if ((datap[cp + 1] == '6') && (datap[cp + 2] == '4')) { + cp = cp + 3; + nodeType = const_int64_node; + } + } else if (datap[cp] == 'u') { + if (intonly == 0) { + LexerException(_("Malformed unsigned integer constant.")); + } + // u8 --> uint8 + if (datap[cp + 1] == '8') { + cp = cp + 2; + nodeType = const_uint8_node; + } + // u16 --> uint16 + else if ((datap[cp + 1] == '1') && (datap[cp + 2] == '6')) { + cp = cp + 3; + nodeType = const_uint16_node; + } + // u32 --> uint32 + else if ((datap[cp + 1] == '3') && (datap[cp + 2] == '2')) { + cp = cp + 3; + nodeType = const_uint32_node; + } + // u64 --> uint64 + else if ((datap[cp + 1] == '6') && (datap[cp + 2] == '4')) { + cp = cp + 3; + nodeType = const_uint64_node; + } else { + LexerException(_("Malformed unsigned integer constant.")); + } + } else if (intonly) { + nodeType = const_int_node; } else { - vtype = 3; + nodeType = const_double_node; } + for (indexType i = 0; i < cp; i++) { buffer[i] = datap[i]; } @@ -629,52 +683,77 @@ lexNumber() discardChar(); } buffer[cp] = '\0'; + std::string content = std::string(buffer); setTokenType(NUMERIC); - switch (vtype) { - case 1: + + switch (nodeType) { + case const_int_node: { tokenValue.isToken = false; - if ((currentChar() == 'i') || (currentChar() == 'I')) { + if (currentChar() == 'i') { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_complex_node, buffer, static_cast(ContextInt())); + const_dcomplex_node, content, static_cast(ContextInt())); discardChar(); } else { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_float_node, buffer, static_cast(ContextInt())); + const_double_node, content, static_cast(ContextInt())); } - break; - case 2: + } break; + case const_double_node: { tokenValue.isToken = false; - if ((currentChar() == 'i') || (currentChar() == 'I')) { + if (currentChar() == 'i') { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_dcomplex_node, buffer, static_cast(ContextInt())); + const_dcomplex_node, content, static_cast(ContextInt())); discardChar(); } else { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_double_node, buffer, static_cast(ContextInt())); + const_double_node, content, static_cast(ContextInt())); } - break; - case 3: + } break; + case const_float_node: { tokenValue.isToken = false; - if ((currentChar() == 'i') || (currentChar() == 'I')) { + if (currentChar() == 'i') { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_dcomplex_node, buffer, static_cast(ContextInt())); + const_complex_node, content, static_cast(ContextInt())); discardChar(); } else { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_int_node, buffer, static_cast(ContextInt())); + const_float_node, content, static_cast(ContextInt())); } - break; - case 4: + } break; + case const_int8_node: + case const_int16_node: + case const_int32_node: + case const_int64_node: { tokenValue.isToken = false; - if ((currentChar() == 'i') || (currentChar() == 'I')) { + if (isNegative) { + content = "-" + content; + } + tokenValue.v.p + = AbstractSyntaxTree::createNode(nodeType, content, static_cast(ContextInt())); + + } break; + case const_uint8_node: + case const_uint16_node: + case const_uint32_node: + case const_uint64_node: { + if (isNegative) { + LexerException(_("Malformed unsigned integer constant with unary operator '-'.")); + } + tokenValue.isToken = false; + tokenValue.v.p + = AbstractSyntaxTree::createNode(nodeType, content, static_cast(ContextInt())); + } break; + default: { + tokenValue.isToken = false; + if (currentChar() == 'i') { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_dcomplex_node, buffer, static_cast(ContextInt())); + const_dcomplex_node, content, static_cast(ContextInt())); discardChar(); } else { tokenValue.v.p = AbstractSyntaxTree::createNode( - const_uint64_node, buffer, static_cast(ContextInt())); + const_double_node, content, static_cast(ContextInt())); } - break; + } break; } return 1; } diff --git a/modules/interpreter/src/include/AbstractSyntaxTree.hpp b/modules/interpreter/src/include/AbstractSyntaxTree.hpp index 16dbe605e5..f6f42c3c25 100644 --- a/modules/interpreter/src/include/AbstractSyntaxTree.hpp +++ b/modules/interpreter/src/include/AbstractSyntaxTree.hpp @@ -24,12 +24,19 @@ enum NODE_TYPE { const_double_node = 0, const_int_node, + const_int8_node, + const_uint8_node, + const_int16_node, + const_uint16_node, + const_int32_node, + const_uint32_node, + const_int64_node, + const_uint64_node, const_float_node, const_character_array_node, const_string_node, const_dcomplex_node, const_complex_node, - const_uint64_node, reserved_node, non_terminal, id_node, @@ -113,7 +120,7 @@ class NLSINTERPRETER_IMPEXP AbstractSyntaxTree static AbstractSyntaxTreePtrVector astUsedAsVector; //============================================================================= static AbstractSyntaxTreePtr - createNode(NODE_TYPE ntype, const char* name, int context); + createNode(NODE_TYPE ntype, const std::string& name, int context); //============================================================================= static AbstractSyntaxTreePtr createNode(NODE_TYPE ntype, int token, int context); @@ -211,7 +218,7 @@ class NLSINTERPRETER_IMPEXP AbstractSyntaxTree * Creates a node of the specified type with the text field set to a copy * of the name argument. */ - AbstractSyntaxTree(NODE_TYPE ntype, const char* name, int context); + AbstractSyntaxTree(NODE_TYPE ntype, const std::string& name, int context); /** Token constructor * Creates a node of the specified type with the tokenNumber field set to * the token argument. This constructor is useful for nodes that are represented diff --git a/modules/interpreter/tests/test_number_syntax.m b/modules/interpreter/tests/test_number_syntax.m new file mode 100644 index 0000000000..3ea245e5bd --- /dev/null +++ b/modules/interpreter/tests/test_number_syntax.m @@ -0,0 +1,30 @@ +%============================================================================= +% Copyright (c) 2016-present Allan CORNET (Nelson) +%============================================================================= +% This file is part of the Nelson. +%============================================================================= +% LICENCE_BLOCK_BEGIN +% SPDX-License-Identifier: LGPL-3.0-or-later +% LICENCE_BLOCK_END +%============================================================================= +assert_isequal(9223372036854775808i32, intmax('int32')) +assert_isequal(9223372036854775807i32, intmax('int32')) +assert_isequal(-9223372036854775808i32, intmin('int32')) +assert_isequal(-9223372036854775809i32, intmin('int32')) +%============================================================================= +assert_isequal(9223372036854775808i64, intmax('int64')) +assert_isequal(9223372036854775807i64, intmax('int64')) +assert_isequal(-9223372036854775808i64, intmin('int64')) +assert_isequal(-9223372036854775809i64, intmin('int64')) +%============================================================================= +assert_isequal(18446744073709551616u64, intmax('uint64')) +assert_isequal(18446744073709551615u64, intmax('uint64')) +assert_isequal(0u64, intmin('uint64')) +assert_checkerror('-18446744073709551616u64', ... + _('Lexical error ''Malformed unsigned integer constant with unary operator ''-''.''')) +%============================================================================= +assert_isequal(3.22f32, single(3.22)) +assert_isequal(3.22f64, double(3.22)) + +assert_isequal(3.22f32i, single(3.22i)) +assert_isequal(3.22f64i, double(3.22i)) \ No newline at end of file