From 0ad72734d5174581e62215f3836f70c634ef1196 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 10 Nov 2023 06:49:01 -0600 Subject: [PATCH 1/6] Revert "GH-1461 Update base64 from upstream" This reverts commit 0842972b1edad772e03d12973d7e7b16e6845d47. --- libraries/libfc/include/fc/crypto/base64.hpp | 9 +- libraries/libfc/src/crypto/base64.cpp | 392 +++++-------------- libraries/libfc/test/test_base64.cpp | 134 +------ 3 files changed, 100 insertions(+), 435 deletions(-) diff --git a/libraries/libfc/include/fc/crypto/base64.hpp b/libraries/libfc/include/fc/crypto/base64.hpp index d57e2d11e5..9559214df1 100644 --- a/libraries/libfc/include/fc/crypto/base64.hpp +++ b/libraries/libfc/include/fc/crypto/base64.hpp @@ -1,15 +1,14 @@ #pragma once #include -#include namespace fc { std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len); inline std::string base64_encode(char const* bytes_to_encode, unsigned int in_len) { return base64_encode( (unsigned char const*)bytes_to_encode, in_len); } -std::string base64_encode( const std::string_view& enc ); -std::string base64_decode( const std::string_view& encoded_string); +std::string base64_encode( const std::string& enc ); +std::string base64_decode( const std::string& encoded_string); std::string base64url_encode(unsigned char const* bytes_to_encode, unsigned int in_len); inline std::string base64url_encode(char const* bytes_to_encode, unsigned int in_len) { return base64url_encode( (unsigned char const*)bytes_to_encode, in_len); } -std::string base64url_encode( const std::string_view& enc ); -std::string base64url_decode( const std::string_view& encoded_string); +std::string base64url_encode( const std::string& enc ); +std::string base64url_decode( const std::string& encoded_string); } // namespace fc diff --git a/libraries/libfc/src/crypto/base64.cpp b/libraries/libfc/src/crypto/base64.cpp index d5369dcbad..5ff4e68290 100644 --- a/libraries/libfc/src/crypto/base64.cpp +++ b/libraries/libfc/src/crypto/base64.cpp @@ -1,14 +1,10 @@ -/* +#include +#include +#include +/* base64.cpp and base64.h - base64 encoding and decoding with C++. - More information at - https://renenyffenegger.ch/notes/development/Base64/Encoding-and-decoding-base-64-with-cpp - https://github.com/ReneNyffenegger/cpp-base64 - - Version: 2.rc.09 (release candidate) - - Copyright (C) 2004-2017, 2020-2022 René Nyffenegger + Copyright (C) 2004-2008 René Nyffenegger This source code is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages @@ -32,332 +28,134 @@ */ -#include -#include - -#include -#include -#if __cplusplus >= 201703L -#include -#endif // __cplusplus >= 201703L - namespace fc { -// base64.hpp -// Added template return type - -std::string base64_encode (std::string const& s, bool url = false); -std::string base64_encode_pem (std::string const& s); -std::string base64_encode_mime(std::string const& s); - -std::string base64_decode(std::string const& s, bool remove_linebreaks = false); -std::string base64_encode(unsigned char const*, size_t len, bool url = false); - -#if __cplusplus >= 201703L -// -// Interface with std::string_view rather than const std::string& -// Requires C++17 -// Provided by Yannic Bonenberger (https://github.com/Yannic) -// -std::string base64_encode (std::string_view s, bool url = false); -std::string base64_encode_pem (std::string_view s); -std::string base64_encode_mime(std::string_view s); - -std::string base64_decode(std::string_view s, bool remove_linebreaks = false); -#endif // __cplusplus >= 201703L - -// base64.cpp - -// Includes performance improvement from unmerged PR: https://github.com/ReneNyffenegger/cpp-base64/pull/27 - - // - // Depending on the url parameter in base64_chars, one of - // two sets of base64 characters needs to be chosen. - // They differ in their last two characters. - // -static const char* base64_chars[2] = { - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "+/", - - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "-_"}; - -static const unsigned char from_base64_chars[256] = { - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 62, 64, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, - 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 63, - 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 -}; - -static unsigned int pos_of_char(const unsigned char chr) { - // - // Return the position of chr within base64_encode() - // - - auto c = from_base64_chars[chr]; - if (c != 64) return c; - - // - // 2020-10-23: Throw std::exception rather than const char* - //(Pablo Martin-Gomez, https://github.com/Bouska) - // - // throw std::runtime_error("Input is not valid base64-encoded data."); - - // - // FC_ASSERT instead of throw runtime_error - // - FC_ASSERT(false, "encountered non-base64 character"); -} - -static std::string insert_linebreaks(std::string str, size_t distance) { - // - // Provided by https://github.com/JomaCorpFX, adapted by me. - // - if (!str.length()) { - return std::string{}; - } - - size_t pos = distance; +static constexpr char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; +static constexpr char base64url_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789-_"; - while (pos < str.size()) { - str.insert(pos, "\n"); - pos += distance + 1; - } - - return str; -} - -template -static std::string encode_with_line_breaks(String s) { - return insert_linebreaks(base64_encode(s, false), line_length); -} - -template -static std::string encode_pem(String s) { - return encode_with_line_breaks(s); -} - -template -static std::string encode_mime(String s) { - return encode_with_line_breaks(s); -} +static_assert(sizeof(base64_chars) == sizeof(base64url_chars), "base64 and base64url must have the same amount of chars"); -template -static std::string encode(String s, bool url) { - return base64_encode(reinterpret_cast(s.data()), s.length(), url); +static inline void throw_on_nonbase64(unsigned char c, const char* const b64_chars) { + FC_ASSERT(isalnum(c) || (c == b64_chars[sizeof(base64_chars)-3]) || (c == b64_chars[sizeof(base64_chars)-2]), "encountered non-base64 character"); } -std::string base64_encode(unsigned char const* bytes_to_encode, size_t in_len, bool url) { - - size_t len_encoded = (in_len +2) / 3 * 4; - - unsigned char trailing_char = url ? '.' : '='; - - // - // Choose set of base64 characters. They differ - // for the last two positions, depending on the url - // parameter. - // A bool (as is the parameter url) is guaranteed - // to evaluate to either 0 or 1 in C++ therefore, - // the correct character set is chosen by subscripting - // base64_chars with url. - // - const char* base64_chars_ = base64_chars[url]; +std::string base64_encode_impl(unsigned char const* bytes_to_encode, unsigned int in_len, const char* const b64_chars) { - std::string ret; - ret.reserve(len_encoded); + std::string ret; + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; - unsigned int pos = 0; + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - while (pos < in_len) { - ret.push_back(base64_chars_[(bytes_to_encode[pos + 0] & 0xfc) >> 2]); - - if (pos+1 < in_len) { - ret.push_back(base64_chars_[((bytes_to_encode[pos + 0] & 0x03) << 4) + ((bytes_to_encode[pos + 1] & 0xf0) >> 4)]); - - if (pos+2 < in_len) { - ret.push_back(base64_chars_[((bytes_to_encode[pos + 1] & 0x0f) << 2) + ((bytes_to_encode[pos + 2] & 0xc0) >> 6)]); - ret.push_back(base64_chars_[ bytes_to_encode[pos + 2] & 0x3f]); - } - else { - ret.push_back(base64_chars_[(bytes_to_encode[pos + 1] & 0x0f) << 2]); - ret.push_back(trailing_char); - } - } - else { - - ret.push_back(base64_chars_[(bytes_to_encode[pos + 0] & 0x03) << 4]); - ret.push_back(trailing_char); - ret.push_back(trailing_char); - } - - pos += 3; + for(i = 0; (i <4) ; i++) + ret += b64_chars[char_array_4[i]]; + i = 0; } + } + if (i) + { + for(j = i; j < 3; j++) + char_array_3[j] = '\0'; - return ret; -} - -template -static std::string decode(String const& encoded_string, bool remove_linebreaks) { - // - // decode(…) is templated so that it can be used with String = const std::string& - // or std::string_view (requires at least C++17) - // - - if (encoded_string.empty()) return std::string{}; - - if (remove_linebreaks) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - std::string copy(encoded_string); + for (j = 0; (j < i + 1); j++) + ret += b64_chars[char_array_4[j]]; - copy.erase(std::remove(copy.begin(), copy.end(), '\n'), copy.end()); + while((i++ < 3)) + ret += '='; - return base64_decode(copy, false); - } + } - size_t length_of_string = encoded_string.length(); - size_t pos = 0; - - // - // The approximate length (bytes) of the decoded string might be one or - // two bytes smaller, depending on the amount of trailing equal signs - // in the encoded string. This approximation is needed to reserve - // enough space in the string to be returned. - // - size_t approx_length_of_decoded_string = length_of_string / 4 * 3; - std::string ret; - ret.reserve(approx_length_of_decoded_string); - - while (pos < length_of_string) { - // - // Iterate over encoded input string in chunks. The size of all - // chunks except the last one is 4 bytes. - // - // The last chunk might be padded with equal signs or dots - // in order to make it 4 bytes in size as well, but this - // is not required as per RFC 2045. - // - // All chunks except the last one produce three output bytes. - // - // The last chunk produces at least one and up to three bytes. - // - - size_t pos_of_char_1 = pos_of_char(encoded_string.at(pos+1) ); - - // - // Emit the first output byte that is produced in each chunk: - // - ret.push_back(static_cast( ( (pos_of_char(encoded_string.at(pos+0)) ) << 2 ) + ( (pos_of_char_1 & 0x30 ) >> 4))); - - if ( ( pos + 2 < length_of_string ) && // Check for data that is not padded with equal signs (which is allowed by RFC 2045) - encoded_string.at(pos+2) != '=' && - encoded_string.at(pos+2) != '.' // accept URL-safe base 64 strings, too, so check for '.' also. - ) - { - // - // Emit a chunk's second byte (which might not be produced in the last chunk). - // - unsigned int pos_of_char_2 = pos_of_char(encoded_string.at(pos+2) ); - ret.push_back(static_cast( (( pos_of_char_1 & 0x0f) << 4) + (( pos_of_char_2 & 0x3c) >> 2))); - - if ( ( pos + 3 < length_of_string ) && - encoded_string.at(pos+3) != '=' && - encoded_string.at(pos+3) != '.' - ) - { - // - // Emit a chunk's third byte (which might not be produced in the last chunk). - // - ret.push_back(static_cast( ( (pos_of_char_2 & 0x03 ) << 6 ) + pos_of_char(encoded_string.at(pos+3)) )); - } - } - - pos += 4; - } + return ret; - return ret; } -std::string base64_decode(std::string const& s, bool remove_linebreaks) { - return decode(s, remove_linebreaks); +std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { + return base64_encode_impl(bytes_to_encode, in_len, base64_chars); } -std::string base64_encode(std::string const& s, bool url) { - return encode(s, url); +std::string base64_encode( const std::string& enc ) { + char const* s = enc.c_str(); + return base64_encode( (unsigned char const*)s, enc.size() ); } -std::string base64_encode_pem (std::string const& s) { - return encode_pem(s); +std::string base64url_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { + return base64_encode_impl(bytes_to_encode, in_len, base64url_chars); } -std::string base64_encode_mime(std::string const& s) { - return encode_mime(s); +std::string base64url_encode( const std::string& enc ) { + char const* s = enc.c_str(); + return base64url_encode( (unsigned char const*)s, enc.size() ); } -#if __cplusplus >= 201703L -// -// Interface with std::string_view rather than const std::string& -// Requires C++17 -// Provided by Yannic Bonenberger (https://github.com/Yannic) -// +std::string base64_decode_impl(std::string const& encoded_string, const char* const b64_chars) { + int in_len = encoded_string.size(); + int i = 0; + int j = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::string ret; -std::string base64_encode(std::string_view s, bool url) { - return encode(s, url); -} + while (in_len-- && encoded_string[in_] != '=') { + throw_on_nonbase64(encoded_string[in_], b64_chars); + char_array_4[i++] = encoded_string[in_]; in_++; + if (i ==4) { + for (i = 0; i <4; i++) + char_array_4[i] = strchr(b64_chars, char_array_4[i]) - b64_chars; -std::string base64_encode_pem(std::string_view s) { - return encode_pem(s); -} + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; -std::string base64_encode_mime(std::string_view s) { - return encode_mime(s); -} + for (i = 0; (i < 3); i++) + ret += char_array_3[i]; + i = 0; + } + } -std::string base64_decode(std::string_view s, bool remove_linebreaks) { - return decode(s, remove_linebreaks); -} + if (i) { + for (j = i; j <4; j++) + char_array_4[j] = 0; -#endif // __cplusplus >= 201703L + for (j = 0; j <4; j++) + char_array_4[j] = strchr(b64_chars, char_array_4[j]) - b64_chars; -// end base64.cpp + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; -// fc interface + for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + } -std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { - return base64_encode(bytes_to_encode, in_len, false); -} -std::string base64_encode(const std::string_view& enc) { - return base64_encode(enc, false); -} -std::string base64_decode(const std::string_view& encoded_string) { - return base64_decode(encoded_string, false); + return ret; } -std::string base64url_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { - return base64_encode(bytes_to_encode, in_len, true); +std::string base64_decode(std::string const& encoded_string) { + return base64_decode_impl(encoded_string, base64_chars); } -std::string base64url_encode(const std::string_view& enc) { - return base64_encode(enc, true); -} -std::string base64url_decode(const std::string_view& encoded_string) { - return base64_decode(encoded_string, true); + +std::string base64url_decode(std::string const& encoded_string) { + return base64_decode_impl(encoded_string, base64url_chars); } } // namespace fc + diff --git a/libraries/libfc/test/test_base64.cpp b/libraries/libfc/test/test_base64.cpp index 38fb4e8cf8..ff6ac6a0ec 100644 --- a/libraries/libfc/test/test_base64.cpp +++ b/libraries/libfc/test/test_base64.cpp @@ -40,9 +40,7 @@ BOOST_AUTO_TEST_CASE(base64dec_extraequals) try { auto input = "YWJjMTIzJCYoKSc/tPUB+n5h========="s; auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; - BOOST_CHECK_EXCEPTION(base64_decode(input), fc::exception, [](const fc::exception& e) { - return e.to_detail_string().find("encountered non-base64 character") != std::string::npos; - }); + BOOST_CHECK_EQUAL(expected_output, base64_decode(input)); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(base64dec_bad_stuff) try { @@ -53,134 +51,4 @@ BOOST_AUTO_TEST_CASE(base64dec_bad_stuff) try { }); } FC_LOG_AND_RETHROW(); -// tests from https://github.com/ReneNyffenegger/cpp-base64/blob/master/test.cpp -BOOST_AUTO_TEST_CASE(base64_cpp_base64_tests) try { - // - // Note: this file must be encoded in UTF-8 - // for the following test, otherwise, the test item - // fails. - // - const std::string orig = - "René Nyffenegger\n" - "http://www.renenyffenegger.ch\n" - "passion for data\n"; - - std::string encoded = base64_encode(reinterpret_cast(orig.c_str()), orig.length()); - std::string decoded = base64_decode(encoded); - - BOOST_CHECK_EQUAL(encoded, "UmVuw6kgTnlmZmVuZWdnZXIKaHR0cDovL3d3dy5yZW5lbnlmZmVuZWdnZXIuY2gKcGFzc2lvbiBmb3IgZGF0YQo="); - BOOST_CHECK_EQUAL(decoded, orig); - - // Test all possibilites of fill bytes (none, one =, two ==) - // References calculated with: https://www.base64encode.org/ - - std::string rest0_original = "abc"; - std::string rest0_reference = "YWJj"; - - std::string rest0_encoded = base64_encode(reinterpret_cast(rest0_original.c_str()), - rest0_original.length()); - std::string rest0_decoded = base64_decode(rest0_encoded); - - BOOST_CHECK_EQUAL(rest0_decoded, rest0_original); - BOOST_CHECK_EQUAL(rest0_reference, rest0_encoded); - - std::string rest1_original = "abcd"; - std::string rest1_reference = "YWJjZA=="; - - std::string rest1_encoded = base64_encode(reinterpret_cast(rest1_original.c_str()), - rest1_original.length()); - std::string rest1_decoded = base64_decode(rest1_encoded); - - BOOST_CHECK_EQUAL(rest1_decoded, rest1_original); - BOOST_CHECK_EQUAL(rest1_reference, rest1_encoded); - - std::string rest2_original = "abcde"; - std::string rest2_reference = "YWJjZGU="; - - std::string rest2_encoded = base64_encode(reinterpret_cast(rest2_original.c_str()), - rest2_original.length()); - std::string rest2_decoded = base64_decode(rest2_encoded); - - BOOST_CHECK_EQUAL(rest2_decoded, rest2_original); - BOOST_CHECK_EQUAL(rest2_reference, rest2_encoded); - - // -------------------------------------------------------------- - // - // Data that is 17 bytes long requires one padding byte when - // base-64 encoded. Such an encoded string could not correctly - // be decoded when encoded with «url semantics». This bug - // was discovered by https://github.com/kosniaz. The following - // test checks if this bug was fixed: - // - std::string a17_orig = "aaaaaaaaaaaaaaaaa"; - std::string a17_encoded = base64_encode(a17_orig); - std::string a17_encoded_url = base64url_encode(a17_orig); - - BOOST_CHECK_EQUAL(a17_encoded, "YWFhYWFhYWFhYWFhYWFhYWE="); - BOOST_CHECK_EQUAL(a17_encoded_url, "YWFhYWFhYWFhYWFhYWFhYWE."); - BOOST_CHECK_EQUAL(base64_decode(a17_encoded_url), a17_orig); - BOOST_CHECK_EQUAL(base64_decode(a17_encoded), a17_orig); - - // -------------------------------------------------------------- - - // characters 63 and 64 / URL encoding - - std::string s_6364 = "\x03" "\xef" "\xff" "\xf9"; - - std::string s_6364_encoded = base64_encode(s_6364); - std::string s_6364_encoded_url = base64url_encode(s_6364); - - BOOST_CHECK_EQUAL(s_6364_encoded, "A+//+Q=="); - BOOST_CHECK_EQUAL(s_6364_encoded_url, "A-__-Q.."); - BOOST_CHECK_EQUAL(base64_decode(s_6364_encoded), s_6364); - BOOST_CHECK_EQUAL(base64_decode(s_6364_encoded_url), s_6364); - - // ---------------------------------------------- - - std::string unpadded_input = "YWJjZGVmZw"; // Note the 'missing' "==" - std::string unpadded_decoded = base64_decode(unpadded_input); - BOOST_CHECK_EQUAL(unpadded_decoded, "abcdefg"); - - unpadded_input = "YWJjZGU"; // Note the 'missing' "=" - unpadded_decoded = base64_decode(unpadded_input); - BOOST_CHECK_EQUAL(unpadded_decoded, "abcde"); - - unpadded_input = ""; - unpadded_decoded = base64_decode(unpadded_input); - BOOST_CHECK_EQUAL(unpadded_decoded, ""); - - unpadded_input = "YQ"; - unpadded_decoded = base64_decode(unpadded_input); - BOOST_CHECK_EQUAL(unpadded_decoded, "a"); - - unpadded_input = "YWI"; - unpadded_decoded = base64_decode(unpadded_input); - BOOST_CHECK_EQUAL(unpadded_decoded, "ab"); - - // -------------------------------------------------------------- - // - // 2022-11-01 - // Replace - // encoded_string[…] with encoded_sring.at(…) - // in - // decode() - // - std::string not_null_terminated = std::string(1, 'a'); - BOOST_CHECK_THROW(base64_decode(not_null_terminated), std::out_of_range); - - // -------------------------------------------------------------- - // - // Test the string_view interface (which required C++17) - // - std::string_view sv_orig = "foobarbaz"; - std::string sv_encoded = base64_encode(sv_orig); - - BOOST_CHECK_EQUAL(sv_encoded, "Zm9vYmFyYmF6"); - - std::string sv_decoded = base64_decode(sv_encoded); - - BOOST_CHECK_EQUAL(sv_decoded, sv_orig); - -} FC_LOG_AND_RETHROW(); - BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file From b0c12963b23c664327cea56f9619dc3779c1e8d1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 10 Nov 2023 07:14:54 -0600 Subject: [PATCH 2/6] Revert "GH-1461 Do not add or expect `=` at end of base64 encoded strings." This reverts commit 5c2f5077f5885db6f100a76ee9b17b72381526ef. --- libraries/libfc/src/variant.cpp | 7 +++---- libraries/libfc/test/variant/test_variant.cpp | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index 56705ad18e..da1e648da0 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -490,7 +490,7 @@ std::string variant::as_string()const return *reinterpret_cast(this) ? "true" : "false"; case blob_type: if( get_blob().data.size() ) - return base64_encode( get_blob().data.data(), get_blob().data.size() ); + return base64_encode( get_blob().data.data(), get_blob().data.size() ) + "="; return std::string(); case null_type: return std::string(); @@ -533,11 +533,10 @@ blob variant::as_blob()const { const std::string& str = get_string(); if( str.size() == 0 ) return blob(); - try { + if( str.back() == '=' ) + { std::string b64 = base64_decode( get_string() ); return blob( { std::vector( b64.begin(), b64.end() ) } ); - } catch(const std::exception&) { - // unable to decode, just return the raw chars } return blob( { std::vector( str.begin(), str.end() ) } ); } diff --git a/libraries/libfc/test/variant/test_variant.cpp b/libraries/libfc/test/variant/test_variant.cpp index cfef4e50be..827b420ed0 100644 --- a/libraries/libfc/test/variant/test_variant.cpp +++ b/libraries/libfc/test/variant/test_variant.cpp @@ -88,7 +88,7 @@ BOOST_AUTO_TEST_CASE(variant_format_string_limited) const string target_result = format_prefix + a_short_list + " " + "{" + "\"b\":\"" + b_short_list + "\",\"c\":\"" + c_short_list + "\"}" + " " + "[\"" + d_short_list + "\",\"" + e_short_list + "\"]" + " " + - base64_encode( a_blob.data.data(), a_blob.data.size() ) + " " + + base64_encode( a_blob.data.data(), a_blob.data.size() ) + "=" + " " + g_short_list; BOOST_CHECK_EQUAL( result, target_result); From 09e7fc2bd5a57d868e31022e646b207af4cdbc81 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 13 Nov 2023 08:11:54 -0600 Subject: [PATCH 3/6] GH-1461 Make WalletMgr honor --leave-running & --keep-logs --- tests/nodeos_run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nodeos_run_test.py b/tests/nodeos_run_test.py index 59b6fa68d9..8919ccce50 100755 --- a/tests/nodeos_run_test.py +++ b/tests/nodeos_run_test.py @@ -41,7 +41,7 @@ errFileName=f"{cluster.nodeosLogPath}/node_00/stderr.txt" if args.error_log_path: errFileName=args.error_log_path -walletMgr=WalletMgr(True, port=walletPort) +walletMgr=WalletMgr(True, port=walletPort, keepRunning=args.leave_running, keepLogs=args.keep_logs) testSuccessful=False dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started From 00b2f993f305ecb7510c927eb87c91bcdb730ac1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 13 Nov 2023 08:13:36 -0600 Subject: [PATCH 4/6] GH-1461 Remove addition of invalid `=` character --- libraries/libfc/src/variant.cpp | 17 ++-- libraries/libfc/test/variant/test_variant.cpp | 78 ++++++++++++++++++- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index da1e648da0..b882a35ed9 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -490,7 +490,7 @@ std::string variant::as_string()const return *reinterpret_cast(this) ? "true" : "false"; case blob_type: if( get_blob().data.size() ) - return base64_encode( get_blob().data.data(), get_blob().data.size() ) + "="; + return base64_encode( get_blob().data.data(), get_blob().data.size() ); return std::string(); case null_type: return std::string(); @@ -533,10 +533,14 @@ blob variant::as_blob()const { const std::string& str = get_string(); if( str.size() == 0 ) return blob(); - if( str.back() == '=' ) - { - std::string b64 = base64_decode( get_string() ); - return blob( { std::vector( b64.begin(), b64.end() ) } ); + try { + // pre-5.0 versions of variant added `=` to end of base64 encoded string in as_string() above. + // fc version of base64_decode allows for extra `=` at the end of the string. + // Other base64 decoders will not accept the extra `=`. + std::vector b64 = base64_decode( str ); + return blob( { std::move(b64) } ); + } catch(const std::exception&) { + // unable to decode, return the raw chars } return blob( { std::vector( str.begin(), str.end() ) } ); } @@ -758,8 +762,7 @@ void to_variant( const blob& b, variant& v ) { } void from_variant( const variant& v, blob& b ) { - std::string _s = base64_decode(v.as_string()); - b.data = std::vector(_s.begin(), _s.end()); + b.data = base64_decode(v.as_string()); } void to_variant( const UInt<8>& n, variant& v ) { v = uint64_t(n); } diff --git a/libraries/libfc/test/variant/test_variant.cpp b/libraries/libfc/test/variant/test_variant.cpp index 827b420ed0..8be5d99232 100644 --- a/libraries/libfc/test/variant/test_variant.cpp +++ b/libraries/libfc/test/variant/test_variant.cpp @@ -88,11 +88,87 @@ BOOST_AUTO_TEST_CASE(variant_format_string_limited) const string target_result = format_prefix + a_short_list + " " + "{" + "\"b\":\"" + b_short_list + "\",\"c\":\"" + c_short_list + "\"}" + " " + "[\"" + d_short_list + "\",\"" + e_short_list + "\"]" + " " + - base64_encode( a_blob.data.data(), a_blob.data.size() ) + "=" + " " + + base64_encode( a_blob.data.data(), a_blob.data.size() ) + " " + g_short_list; BOOST_CHECK_EQUAL( result, target_result); BOOST_CHECK_LT(result.size(), 1024 + 3 * mu.size()); } } + +BOOST_AUTO_TEST_CASE(variant_blob) +{ + // Some test cases from https://github.com/ReneNyffenegger/cpp-base64 + { + std::string a17_orig = "aaaaaaaaaaaaaaaaa"; + std::string a17_encoded = "YWFhYWFhYWFhYWFhYWFhYWE="; + fc::mutable_variant_object mu; + mu("blob", blob{{a17_orig.begin(), a17_orig.end()}}); + mu("str", a17_encoded); + + BOOST_CHECK_EQUAL(mu["blob"].as_string(), a17_encoded); + std::vector b64 = mu["str"].as_blob().data; + std::string_view b64_str(b64.data(), b64.size()); + BOOST_CHECK_EQUAL(b64_str, a17_orig); + } + { + std::string s_6364 = "\x03" "\xef" "\xff" "\xf9"; + std::string s_6364_encoded = "A+//+Q=="; + fc::mutable_variant_object mu; + mu("blob", blob{{s_6364.begin(), s_6364.end()}}); + mu("str", s_6364_encoded); + + BOOST_CHECK_EQUAL(mu["blob"].as_string(), s_6364_encoded); + std::vector b64 = mu["str"].as_blob().data; + std::string_view b64_str(b64.data(), b64.size()); + BOOST_CHECK_EQUAL(b64_str, s_6364); + } + { + std::string org = "abc"; + std::string encoded = "YWJj"; + + fc::mutable_variant_object mu; + mu("blob", blob{{org.begin(), org.end()}}); + mu("str", encoded); + + BOOST_CHECK_EQUAL(mu["blob"].as_string(), encoded); + std::vector b64 = mu["str"].as_blob().data; + std::string_view b64_str(b64.data(), b64.size()); + BOOST_CHECK_EQUAL(b64_str, org); + } +} + +BOOST_AUTO_TEST_CASE(variant_blob_backwards_compatibility) +{ + // pre-5.0 variant would add an additional `=` as a flag that the blob data was base64 encoded + // verify variant can process encoded data with the extra `=` + { + std::string a17_orig = "aaaaaaaaaaaaaaaaa"; + std::string a17_encoded = "YWFhYWFhYWFhYWFhYWFhYWE="; + std::string a17_encoded_old = a17_encoded + '='; + fc::mutable_variant_object mu; + mu("blob", blob{{a17_orig.begin(), a17_orig.end()}}); + mu("str", a17_encoded_old); + + BOOST_CHECK_EQUAL(mu["blob"].as_string(), a17_encoded); + std::vector b64 = mu["str"].as_blob().data; + std::string_view b64_str(b64.data(), b64.size()); + BOOST_CHECK_EQUAL(b64_str, a17_orig); + } + { + std::string org = "abc"; + std::string encoded = "YWJj"; + std::string encoded_old = encoded + '='; + + fc::mutable_variant_object mu; + mu("blob", blob{{org.begin(), org.end()}}); + mu("str", encoded_old); + + BOOST_CHECK_EQUAL(mu["blob"].as_string(), encoded); + std::vector b64 = mu["str"].as_blob().data; + std::string_view b64_str(b64.data(), b64.size()); + BOOST_CHECK_EQUAL(b64_str, org); + } +} + BOOST_AUTO_TEST_SUITE_END() From 7b43a4a46e7d0dd890dec140b46263df19cdeca6 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 13 Nov 2023 08:14:23 -0600 Subject: [PATCH 5/6] GH-1461 Optimize base64_decode for our use cases - Use string_view and return vector instead of string --- .../chain/include/eosio/chain/database_utils.hpp | 4 ++-- libraries/libfc/include/fc/crypto/base64.hpp | 6 ++++-- libraries/libfc/src/crypto/base64.cpp | 13 +++++++------ libraries/libfc/src/crypto/bigint.cpp | 4 ++-- libraries/libfc/src/crypto/elliptic_webauthn.cpp | 2 +- libraries/libfc/test/test_base64.cpp | 12 +++++++++--- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/libraries/chain/include/eosio/chain/database_utils.hpp b/libraries/chain/include/eosio/chain/database_utils.hpp index 0ee937e6e1..a3d6948080 100644 --- a/libraries/chain/include/eosio/chain/database_utils.hpp +++ b/libraries/chain/include/eosio/chain/database_utils.hpp @@ -182,8 +182,8 @@ namespace fc { inline void from_variant( const variant& v, eosio::chain::shared_blob& b ) { - std::string _s = base64_decode(v.as_string()); - b = eosio::chain::shared_blob(_s.begin(), _s.end(), b.get_allocator()); + std::vector b64 = base64_decode(v.as_string()); + b = eosio::chain::shared_blob(b64.begin(), b64.end(), b.get_allocator()); } template diff --git a/libraries/libfc/include/fc/crypto/base64.hpp b/libraries/libfc/include/fc/crypto/base64.hpp index 9559214df1..34dd35ad0b 100644 --- a/libraries/libfc/include/fc/crypto/base64.hpp +++ b/libraries/libfc/include/fc/crypto/base64.hpp @@ -1,14 +1,16 @@ #pragma once #include +#include +#include namespace fc { std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len); inline std::string base64_encode(char const* bytes_to_encode, unsigned int in_len) { return base64_encode( (unsigned char const*)bytes_to_encode, in_len); } std::string base64_encode( const std::string& enc ); -std::string base64_decode( const std::string& encoded_string); +std::vector base64_decode( std::string_view encoded_string); std::string base64url_encode(unsigned char const* bytes_to_encode, unsigned int in_len); inline std::string base64url_encode(char const* bytes_to_encode, unsigned int in_len) { return base64url_encode( (unsigned char const*)bytes_to_encode, in_len); } std::string base64url_encode( const std::string& enc ); -std::string base64url_decode( const std::string& encoded_string); +std::vector base64url_decode( std::string_view encoded_string); } // namespace fc diff --git a/libraries/libfc/src/crypto/base64.cpp b/libraries/libfc/src/crypto/base64.cpp index 5ff4e68290..ae6669ffdd 100644 --- a/libraries/libfc/src/crypto/base64.cpp +++ b/libraries/libfc/src/crypto/base64.cpp @@ -107,13 +107,14 @@ std::string base64url_encode( const std::string& enc ) { return base64url_encode( (unsigned char const*)s, enc.size() ); } -std::string base64_decode_impl(std::string const& encoded_string, const char* const b64_chars) { +std::vector base64_decode_impl(std::string_view encoded_string, const char* const b64_chars) { int in_len = encoded_string.size(); int i = 0; int j = 0; int in_ = 0; unsigned char char_array_4[4], char_array_3[3]; - std::string ret; + std::vector ret; + ret.reserve(in_len / 4 * 3); while (in_len-- && encoded_string[in_] != '=') { throw_on_nonbase64(encoded_string[in_], b64_chars); @@ -127,7 +128,7 @@ std::string base64_decode_impl(std::string const& encoded_string, const char* co char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; for (i = 0; (i < 3); i++) - ret += char_array_3[i]; + ret.push_back(char_array_3[i]); i = 0; } } @@ -143,17 +144,17 @@ std::string base64_decode_impl(std::string const& encoded_string, const char* co char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + for (j = 0; (j < i - 1); j++) ret.push_back(char_array_3[j]); } return ret; } -std::string base64_decode(std::string const& encoded_string) { +std::vector base64_decode(std::string_view encoded_string) { return base64_decode_impl(encoded_string, base64_chars); } -std::string base64url_decode(std::string const& encoded_string) { +std::vector base64url_decode(std::string_view encoded_string) { return base64_decode_impl(encoded_string, base64url_chars); } diff --git a/libraries/libfc/src/crypto/bigint.cpp b/libraries/libfc/src/crypto/bigint.cpp index 3e7ab67047..8574fa7022 100644 --- a/libraries/libfc/src/crypto/bigint.cpp +++ b/libraries/libfc/src/crypto/bigint.cpp @@ -221,8 +221,8 @@ namespace fc { else { std::string b64 = v.as_string(); - std::string bin = base64_decode(b64); - bi = bigint(bin.c_str(), bin.size() ); + std::vector bin = base64_decode(b64); + bi = bigint(bin.data(), bin.size() ); } } diff --git a/libraries/libfc/src/crypto/elliptic_webauthn.cpp b/libraries/libfc/src/crypto/elliptic_webauthn.cpp index ae08f5e5fc..de635b09b4 100644 --- a/libraries/libfc/src/crypto/elliptic_webauthn.cpp +++ b/libraries/libfc/src/crypto/elliptic_webauthn.cpp @@ -225,7 +225,7 @@ public_key::public_key(const signature& c, const fc::sha256& digest, bool) { FC_ASSERT(handler.found_type == "webauthn.get", "webauthn signature type not an assertion"); - std::string challenge_bytes = fc::base64url_decode(handler.found_challenge); + std::vector challenge_bytes = fc::base64url_decode(handler.found_challenge); FC_ASSERT(fc::sha256(challenge_bytes.data(), challenge_bytes.size()) == digest, "Wrong webauthn challenge"); char required_origin_scheme[] = "https://"; diff --git a/libraries/libfc/test/test_base64.cpp b/libraries/libfc/test/test_base64.cpp index ff6ac6a0ec..526d7c75d0 100644 --- a/libraries/libfc/test/test_base64.cpp +++ b/libraries/libfc/test/test_base64.cpp @@ -26,21 +26,27 @@ BOOST_AUTO_TEST_CASE(base64dec) try { auto input = "YWJjMTIzJCYoKSc/tPUB+n5h"s; auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; - BOOST_CHECK_EQUAL(expected_output, base64_decode(input)); + std::vector b64 = base64_decode(input); + std::string b64_str(b64.begin(), b64.end()); + BOOST_CHECK_EQUAL(expected_output, b64_str); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(base64urldec) try { auto input = "YWJjMTIzJCYoKSc_tPUB-n5h"s; auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; - BOOST_CHECK_EQUAL(expected_output, base64url_decode(input)); + std::vector b64 = base64url_decode(input); + std::string b64_str(b64.begin(), b64.end()); + BOOST_CHECK_EQUAL(expected_output, b64_str); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(base64dec_extraequals) try { auto input = "YWJjMTIzJCYoKSc/tPUB+n5h========="s; auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; - BOOST_CHECK_EQUAL(expected_output, base64_decode(input)); + std::vector b64 = base64_decode(input); + std::string b64_str(b64.begin(), b64.end()); + BOOST_CHECK_EQUAL(expected_output, b64_str); } FC_LOG_AND_RETHROW(); BOOST_AUTO_TEST_CASE(base64dec_bad_stuff) try { From 3ccf0235e5ed0e0d10798fac9213f1c8b95b6614 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 13 Nov 2023 12:19:30 -0600 Subject: [PATCH 6/6] GH-1461 Removed unneeded explicit type --- libraries/libfc/src/variant.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/libfc/src/variant.cpp b/libraries/libfc/src/variant.cpp index b882a35ed9..326a507d08 100644 --- a/libraries/libfc/src/variant.cpp +++ b/libraries/libfc/src/variant.cpp @@ -538,7 +538,7 @@ blob variant::as_blob()const // fc version of base64_decode allows for extra `=` at the end of the string. // Other base64 decoders will not accept the extra `=`. std::vector b64 = base64_decode( str ); - return blob( { std::move(b64) } ); + return { std::move(b64) }; } catch(const std::exception&) { // unable to decode, return the raw chars }