Skip to content

Commit

Permalink
Introduced basic_storage_backend interface (#33)
Browse files Browse the repository at this point in the history
Refactored 'binsrv::filesystem_storage' class. Instead, we now have a new class
named 'binsrv::storage', that incorporate all the binlog storage logic
(iterating over existing objects and initialisation, analysing the content of
binlog index, updating binlog index, opening/closing event stream and writing
event data). It now uses 'binsrv::basic_storage_backend' abstract interface to
perform backend-specific operations.

Added one concrete 'binsrv::filesystem_storage_backend' class inherited from the
'binsrv::basic_storage_backend' abstract class that implements the interface
using the local filesystem storage (a dedicated directory on a filesystem).

Another planned implementation is an "AWS S3"-based storage backend (not
included in this commit).

'binsrv::master_config' class renamed to 'binsrv::main_config'.

'master_config.json' sample configuration file renamed into 'main_config.json'.

'binsrv::storage_config' class extended with additional data field "type" which
is supposed to determine the type of the backend storage to be used.
'main_config.json' and 'binlog_streaming.binsrv' MTR test case updated
correspondingly.

Introduced 'binsrv::storage_backend_factory' class that based on the value of
the 'type' field in the 'binsrv::storage_config' will return concrete
implementation of the 'binsrv::basic_storage_backend' interface. Currently,
only "fs" value is supported.

Revisited exception types thrown via
'util::exception_location().raise<xxx>(...)' helper:
* std::invalid_argument - one of the provided argument is incorrect;
* std::logic_error - invariant / pre-condition / post-condition violation;
* std::runtime_error - an error related to hardware, resource unavailability,
                       etc. where simply repeating the call may resolve the
                       problem (like IO errors);
* std::out_of_range - size/limits-related errors.
  • Loading branch information
percona-ysorokin authored Mar 1, 2024
1 parent b9aa9f8 commit 997ed77
Show file tree
Hide file tree
Showing 26 changed files with 616 additions and 309 deletions.
24 changes: 17 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,15 @@ set(source_files
src/binsrv/event/unknown_post_header.hpp
src/binsrv/event/unknown_post_header.cpp

# billog files
# binlog files
src/binsrv/basic_logger_fwd.hpp
src/binsrv/basic_logger.hpp
src/binsrv/basic_logger.cpp

src/binsrv/basic_storage_backend_fwd.hpp
src/binsrv/basic_storage_backend.hpp
src/binsrv/basic_storage_backend.cpp

src/binsrv/cout_logger.hpp
src/binsrv/cout_logger.cpp

Expand All @@ -134,9 +138,8 @@ set(source_files
src/binsrv/file_logger.hpp
src/binsrv/file_logger.cpp

src/binsrv/filesystem_storage_fwd.hpp
src/binsrv/filesystem_storage.hpp
src/binsrv/filesystem_storage.cpp
src/binsrv/filesystem_storage_backend.hpp
src/binsrv/filesystem_storage_backend.cpp

src/binsrv/log_severity_fwd.hpp
src/binsrv/log_severity.hpp
Expand All @@ -147,9 +150,16 @@ set(source_files
src/binsrv/logger_factory.hpp
src/binsrv/logger_factory.cpp

src/binsrv/master_config_fwd.hpp
src/binsrv/master_config.hpp
src/binsrv/master_config.cpp
src/binsrv/main_config_fwd.hpp
src/binsrv/main_config.hpp
src/binsrv/main_config.cpp

src/binsrv/storage_fwd.hpp
src/binsrv/storage.hpp
src/binsrv/storage.cpp

src/binsrv/storage_backend_factory.hpp
src/binsrv/storage_backend_factory.cpp

src/binsrv/storage_config_fwd.hpp
src/binsrv/storage_config.hpp
Expand Down
1 change: 1 addition & 0 deletions master_config.json → main_config.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"password": ""
},
"storage": {
"type": "fs",
"path": "./storage"
}
}
1 change: 1 addition & 0 deletions mtr/binlog_streaming/r/binsrv.result
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ SET @binsrv_config_json = JSON_OBJECT(
'password', ''
),
'storage', JSON_OBJECT(
'type', 'fs',
'path', @storage_path
)
);
Expand Down
3 changes: 2 additions & 1 deletion mtr/binlog_streaming/t/binsrv.test
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ INSERT INTO t1 VALUES(DEFAULT);
--replace_result $binsrv_storage_path <BINSRV_STORAGE_PATH>
eval SET @storage_path = '$binsrv_storage_path';

--let $binsrv_log_path = $MYSQL_TMP_DIR/binsrv.log
--let $binsrv_log_path = $MYSQL_TMP_DIR/binsrv_utility.log
--replace_result $binsrv_log_path <BINSRV_LOG_PATH>
eval SET @log_path = '$binsrv_log_path';

Expand All @@ -64,6 +64,7 @@ eval SET @binsrv_config_json = JSON_OBJECT(
'password', ''
),
'storage', JSON_OBJECT(
'type', 'fs',
'path', @storage_path
)
);
Expand Down
42 changes: 25 additions & 17 deletions src/app.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,18 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>

#include <boost/lexical_cast.hpp>

#include "binsrv/basic_logger.hpp"
#include "binsrv/basic_storage_backend.hpp"
#include "binsrv/exception_handling_helpers.hpp"
#include "binsrv/filesystem_storage.hpp"
#include "binsrv/log_severity.hpp"
#include "binsrv/logger_factory.hpp"
#include "binsrv/master_config.hpp"
#include "binsrv/main_config.hpp"
#include "binsrv/storage.hpp"
#include "binsrv/storage_backend_factory.hpp"

#include "binsrv/event/code_type.hpp"
#include "binsrv/event/event.hpp"
Expand Down Expand Up @@ -75,7 +78,7 @@ void log_span_dump(binsrv::basic_logger &logger,

void receive_binlog_events(binsrv::basic_logger &logger,
easymysql::binlog &binlog,
binsrv::filesystem_storage &storage) {
binsrv::storage &storage) {
// Network streams are requested with COM_BINLOG_DUMP and
// each Binlog Event response is prepended with 00 OK-byte.
static constexpr std::byte expected_event_packet_prefix{'\0'};
Expand All @@ -86,7 +89,7 @@ void receive_binlog_events(binsrv::basic_logger &logger,

while (!(portion = binlog.fetch()).empty()) {
if (portion[0] != expected_event_packet_prefix) {
util::exception_location().raise<std::invalid_argument>(
util::exception_location().raise<std::runtime_error>(
"unexpected event prefix");
}
portion = portion.subspan(1U);
Expand Down Expand Up @@ -138,7 +141,7 @@ int main(int argc, char *argv[]) {
const auto number_of_cmd_args = std::size(cmd_args);
const auto executable_name = util::extract_executable_name(cmd_args);

if (number_of_cmd_args != binsrv::master_config::flattened_size + 1 &&
if (number_of_cmd_args != binsrv::main_config::flattened_size + 1 &&
number_of_cmd_args != 2) {
std::cerr
<< "usage: " << executable_name
Expand All @@ -162,17 +165,16 @@ int main(int argc, char *argv[]) {
logger->log(binsrv::log_severity::delimiter,
util::get_readable_command_line_arguments(cmd_args));

binsrv::master_config_ptr config;
binsrv::main_config_ptr config;
if (number_of_cmd_args == 2U) {
logger->log(binsrv::log_severity::delimiter,
"Reading connection configuration from the JSON file.");
config = std::make_shared<binsrv::master_config>(cmd_args[1]);
} else if (number_of_cmd_args ==
binsrv::master_config::flattened_size + 1) {
config = std::make_shared<binsrv::main_config>(cmd_args[1]);
} else if (number_of_cmd_args == binsrv::main_config::flattened_size + 1) {
logger->log(binsrv::log_severity::delimiter,
"Reading connection configuration from the command line "
"arguments.");
config = std::make_shared<binsrv::master_config>(cmd_args.subspan(1U));
config = std::make_shared<binsrv::main_config>(cmd_args.subspan(1U));
} else {
assert(false);
}
Expand All @@ -197,22 +199,28 @@ int main(int argc, char *argv[]) {

std::string msg;
const auto &storage_config = config->root().get<"storage">();
binsrv::filesystem_storage storage(storage_config.get<"path">());
logger->log(binsrv::log_severity::info,
"created filesystem binlog storage");
msg = "filesystem binlog storage root path: ";
msg += storage.get_root_path();
auto storage_backend{
binsrv::storage_backend_factory::create(storage_config)};
logger->log(binsrv::log_severity::info, "created binlog storage backend");
msg = "type: ";
msg += storage_config.get<"type">();
msg += ", path: ";
msg += storage_config.get<"path">();
logger->log(binsrv::log_severity::info, msg);

binsrv::storage storage{std::move(storage_backend)};
logger->log(binsrv::log_severity::info,
"created binlog storage with the provided backend");

const auto last_binlog_name{storage.get_binlog_name()};
// if storage position is detected to be 0 (empty data directory), we
// start streaming from the position magic_binlog_offset (4)
const auto last_binlog_position{
std::max(binsrv::event::magic_binlog_offset, storage.get_position())};
if (last_binlog_name.empty()) {
msg = "filesystem binlog storage initialized on an empty directory";
msg = "binlog storage initialized on an empty directory";
} else {
msg = "filesystem binlog storage initialized at \"";
msg = "binlog storage initialized at \"";
msg += last_binlog_name;
msg += "\":";
msg += std::to_string(last_binlog_position);
Expand Down
54 changes: 54 additions & 0 deletions src/binsrv/basic_storage_backend.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#include "binsrv/basic_storage_backend.hpp"

#include <stdexcept>
#include <string>
#include <string_view>

#include "util/byte_span_fwd.hpp"
#include "util/exception_location_helpers.hpp"

namespace binsrv {

[[nodiscard]] storage_object_name_container
basic_storage_backend::list_objects() {
return do_list_objects();
}

[[nodiscard]] std::string
basic_storage_backend::get_object(std::string_view name) {
return do_get_object(name);
}

void basic_storage_backend::put_object(std::string_view name,
util::const_byte_span content) {
return do_put_object(name, content);
}

void basic_storage_backend::open_stream(std::string_view name) {
if (stream_opened_) {
util::exception_location().raise<std::logic_error>(
"cannot open a new stream as the previous one has not been closed");
}

do_open_stream(name);
stream_opened_ = true;
}

void basic_storage_backend::write_data_to_stream(util::const_byte_span data) {
if (!stream_opened_) {
util::exception_location().raise<std::logic_error>(
"cannot write to the stream as it has not been opened");
}
do_write_data_to_stream(data);
}

void basic_storage_backend::close_stream() {
if (!stream_opened_) {
util::exception_location().raise<std::logic_error>(
"cannot close the stream as it has not been opened");
}
do_close_stream();
stream_opened_ = false;
}

} // namespace binsrv
46 changes: 46 additions & 0 deletions src/binsrv/basic_storage_backend.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#ifndef BINSRV_BASIC_STORAGE_BACKEND_HPP
#define BINSRV_BASIC_STORAGE_BACKEND_HPP

#include "binsrv/basic_storage_backend_fwd.hpp" // IWYU pragma: export

#include <string>
#include <string_view>

#include "util/byte_span_fwd.hpp"

namespace binsrv {

class basic_storage_backend {
public:
basic_storage_backend() = default;
basic_storage_backend(const basic_storage_backend &) = delete;
basic_storage_backend(basic_storage_backend &&) noexcept = delete;
basic_storage_backend &operator=(const basic_storage_backend &) = delete;
basic_storage_backend &operator=(basic_storage_backend &&) = delete;

virtual ~basic_storage_backend() = default;

[[nodiscard]] storage_object_name_container list_objects();
[[nodiscard]] std::string get_object(std::string_view name);
void put_object(std::string_view name, util::const_byte_span content);

void open_stream(std::string_view name);
void write_data_to_stream(util::const_byte_span data);
void close_stream();

private:
bool stream_opened_{false};

[[nodiscard]] virtual storage_object_name_container do_list_objects() = 0;
[[nodiscard]] virtual std::string do_get_object(std::string_view name) = 0;
virtual void do_put_object(std::string_view name,
util::const_byte_span content) = 0;

virtual void do_open_stream(std::string_view name) = 0;
virtual void do_write_data_to_stream(util::const_byte_span data) = 0;
virtual void do_close_stream() = 0;
};

} // namespace binsrv

#endif // BINSRV_BASIC_STORAGE_BACKEND_HPP
38 changes: 38 additions & 0 deletions src/binsrv/basic_storage_backend_fwd.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#ifndef BINSRV_BASIC_STORAGE_BACKEND_FWD_HPP
#define BINSRV_BASIC_STORAGE_BACKEND_FWD_HPP

#include <cstdint>
#include <memory>
#include <string>
#include <unordered_map>

namespace binsrv {

class basic_storage_backend;

using basic_storage_backend_ptr = std::unique_ptr<basic_storage_backend>;

// the following hash calculation helper allows to look for keys represented
// by either const char*, std::string_view of const std::string in unordered
// containers with std::string key transparently without converting the value
// being searched to std::string
struct transparent_string_like_hash {
using is_transparent = void;
std::size_t operator()(const char *key) const noexcept {
return std::hash<std::string_view>{}(key);
}
std::size_t operator()(std::string_view key) const noexcept {
return std::hash<std::string_view>{}(key);
}
std::size_t operator()(const std::string &key) const noexcept {
return std::hash<std::string>{}(key);
}
};
// the container thatr uses transparent_string_like_hash also needs a
// transparent version of KeyEqual template argument (std::equal_to<void>)
using storage_object_name_container =
std::unordered_map<std::string, std::uint64_t, transparent_string_like_hash,
std::equal_to<>>;
} // namespace binsrv

#endif // BINSRV_BASIC_STORAGE_BACKEND_FWD_HPP
4 changes: 3 additions & 1 deletion src/binsrv/event/protocol_traits_fwd.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef BINSRV_EVENT_PROTOCOL_TRAITS_FWD_HPP
#define BINSRV_EVENT_PROTOCOL_TRAITS_FWD_HPP

#include <array>
#include <cstddef>
#include <cstdint>
#include <limits>
Expand All @@ -17,7 +18,8 @@ inline constexpr std::size_t unspecified_post_header_length{

// https://github.com/mysql/mysql-server/blob/trunk/sql/log_event.h#L211
// 4 bytes which all binlogs should begin with
inline constexpr std::string_view magic_binlog_payload{"\xFE\x62\x69\x6E"};
inline constexpr std::array<std::byte, 4> magic_binlog_payload{
std::byte{0xFE}, std::byte{0x62}, std::byte{0x69}, std::byte{0x6E}};
inline constexpr std::uint64_t magic_binlog_offset{4ULL};

} // namespace binsrv::event
Expand Down
2 changes: 1 addition & 1 deletion src/binsrv/file_logger.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace binsrv {
file_logger::file_logger(log_severity min_level, std::string_view file_name)
: basic_logger{min_level}, stream_{std::filesystem::path{file_name}} {
if (!stream_.is_open()) {
util::exception_location().raise<std::invalid_argument>(
util::exception_location().raise<std::runtime_error>(
"unable to create \"" + std::string(file_name) + "\" file for logging");
}
}
Expand Down
Loading

0 comments on commit 997ed77

Please sign in to comment.