Skip to content

Commit

Permalink
Refactor modules (#430)
Browse files Browse the repository at this point in the history 
* Refactor third_party module

Signed-off-by: jinhai <[email protected]>

* Refactor third_party module

Signed-off-by: jinhai <[email protected]>

* Refactor stl: std::move and std::forward

Signed-off-by: jinhai <[email protected]>

* Refactor min/max

Signed-off-by: jinhai <[email protected]>

* Refactor fill/copy/str related

Signed-off-by: jinhai <[email protected]>

* More std migrated

Signed-off-by: jinhai <[email protected]>

* std::numeric_limits

Signed-off-by: jinhai <[email protected]>

* std::numeric_limits part2

Signed-off-by: jinhai <[email protected]>

* Refactor from_chars and memory order

Signed-off-by: jinhai <[email protected]>

* Refactor boost wrapper

Signed-off-by: jinhai <[email protected]>

* Remove unused declaration

Signed-off-by: jinhai <[email protected]>

* Update benchmark check

Signed-off-by: jinhai <[email protected]>

---------

Signed-off-by: jinhai <[email protected]>
  • Loading branch information
@JinHai-CN
JinHai-CN authored Jan 8, 2024
1 parent 902f65e commit 5b5d14d
Show file tree
Hide file tree
Showing 412 changed files with 2,537 additions and 2,706 deletions.
2 changes: 1 addition & 1 deletion benchmark/local_infinity/fulltext/fulltext_query_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ bool ReadJsonl(const std::string &query_str, size_t &start_pos, size_t query_siz
std::string_view json_sv(query_str.data() + start_pos, end_pos - start_pos);
start_pos = end_pos + 1;

Json json = Json::parse(json_sv);
nlohmann::json json = nlohmann::json::parse(json_sv);
result = {json["_id"], json["text"]};
return true;
}
Expand Down
40 changes: 20 additions & 20 deletions benchmark/local_infinity/infinity_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,28 +91,28 @@ int main() {
auto tims_costing_second = Measurement("Get Database", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default");
});
results.push_back(Format("-> Get Database QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Get Database QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second = Measurement("List Databases", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->ListDatabases();
});
results.push_back(Format("-> List Databases QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> List Databases QPS: {}", total_times / tims_costing_second));
}
{
CreateDatabaseOptions create_db_opts;
auto tims_costing_second = Measurement("Create Database", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->CreateDatabase(ToStr(i), create_db_opts);
__attribute__((unused)) auto ignored = infinity->CreateDatabase(std::to_string(i), create_db_opts);
});
results.push_back(Format("-> Create Database QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Create Database QPS: {}", total_times / tims_costing_second));
}

{
DropDatabaseOptions drop_db_opts;
auto tims_costing_second = Measurement("Drop Database", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->DropDatabase(ToStr(i), drop_db_opts);
__attribute__((unused)) auto ignored = infinity->DropDatabase(std::to_string(i), drop_db_opts);
});
results.push_back(Format("-> Drop Database QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Drop Database QPS: {}", total_times / tims_costing_second));
}
// Table
{
Expand Down Expand Up @@ -144,19 +144,19 @@ int main() {
// thread_id) {
// __attribute__((unused)) auto ignored = infinity->GetDatabase("default")->ListTables();
// });
// results.push_back(Format("-> List Tables QPS: {}", total_times / tims_costing_second));
// results.push_back(fmt::format("-> List Tables QPS: {}", total_times / tims_costing_second));
// }
{
auto tims_costing_second = Measurement("Get Tables", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_test");
});
results.push_back(Format("-> Get Tables QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Get Tables QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second = Measurement("Describe Tables", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->DescribeTable("benchmark_test");
});
results.push_back(Format("-> Describe Tables QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Describe Tables QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second = Measurement("Create Tables", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
Expand All @@ -175,15 +175,15 @@ int main() {
column_definitions.emplace_back(col_def_2);

__attribute__((unused)) auto ignored =
infinity->GetDatabase("default")->CreateTable(ToStr(i), column_definitions, Vector<TableConstraint *>(), create_table_opts);
infinity->GetDatabase("default")->CreateTable(std::to_string(i), column_definitions, Vector<TableConstraint *>(), create_table_opts);
});
results.push_back(Format("-> Create Table QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Create Table QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second = Measurement("Drop Table", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->DropTable(ToStr(i), drop_table_options);
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->DropTable(std::to_string(i), drop_table_options);
});
results.push_back(Format("-> Drop Table QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Drop Table QPS: {}", total_times / tims_costing_second));
}
{
{
Expand All @@ -201,7 +201,7 @@ int main() {
__attribute__((unused)) auto ignored =
infinity->GetDatabase("default")->GetTable("benchmark_test")->Search(nullptr, nullptr, output_columns);
});
results.push_back(Format("-> Select QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Select QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second =
Expand All @@ -223,7 +223,7 @@ int main() {

__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_test")->Insert(columns, values);
});
results.push_back(Format("-> Insert QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Insert QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second =
Expand Down Expand Up @@ -252,14 +252,14 @@ int main() {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_test")->Update(nullptr,
values);
});
results.push_back(Format("-> Update QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Update QPS: {}", total_times / tims_costing_second));
}
{
auto tims_costing_second =
Measurement("Delete", thread_num, total_times, [&](SizeT i, SharedPtr<Infinity> infinity, std::thread::id thread_id) {
__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_test")->Delete(nullptr);
});
results.push_back(Format("-> Delete QPS: {}", total_times / tims_costing_second));
results.push_back(fmt::format("-> Delete QPS: {}", total_times / tims_costing_second));
}
}
}
Expand Down Expand Up @@ -311,7 +311,7 @@ int main() {

__attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_sort")->Insert(columns, values);
});
results.push_back(Format("-> Insert for Sort Time: {}s", tims_costing_second));
results.push_back(fmt::format("-> Insert for Sort Time: {}s", tims_costing_second));
}
{
auto tims_costing_second =
Expand All @@ -327,7 +327,7 @@ int main() {

__attribute__((unused)) auto ignored = infinity->Query("select c1, c2 from benchmark_sort order by c1");
});
results.push_back(Format("-> Select Sort Time QPS: {}", sort_times / tims_costing_second));
results.push_back(fmt::format("-> Select Sort Time QPS: {}", sort_times / tims_costing_second));
}
}

Expand Down Expand Up @@ -396,7 +396,7 @@ int main() {
// __attribute__((unused)) auto ignored = infinity->GetDatabase("default")->GetTable("benchmark_test")->Search(nullptr, nullptr,
// nullptr);
// });
// results.push_back(Format("-> SEARCH QPS: {}", total_times / tims_costing_second));
// results.push_back(fmt::format("-> SEARCH QPS: {}", total_times / tims_costing_second));
// std::cout << "OK" << std::endl;
// }

Expand Down
8 changes: 4 additions & 4 deletions benchmark/local_infinity/knn/knn_query_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ int main(int argc, char *argv[]) {
if (times >= 2) {
auto elapsed_ns = profiler.Elapsed();
auto elapsed_s = elapsed_ns / (1'000'000'000.0);
results.push_back(Format("Total cost : {} s", elapsed_s));
results.push_back(fmt::format("Total cost : {} s", elapsed_s));
}
{
size_t correct_1 = 0, correct_10 = 0, correct_100 = 0;
Expand All @@ -257,9 +257,9 @@ int main(int argc, char *argv[]) {
}
}
}
results.push_back(Format("R@1: {:.3f}", float(correct_1) / float(query_count * 1)));
results.push_back(Format("R@10: {:.3f}", float(correct_10) / float(query_count * 10)));
results.push_back(Format("R@100: {:.3f}", float(correct_100) / float(query_count * 100)));
results.push_back(fmt::format("R@1: {:.3f}", float(correct_1) / float(query_count * 1)));
results.push_back(fmt::format("R@10: {:.3f}", float(correct_10) / float(query_count * 10)));
results.push_back(fmt::format("R@100: {:.3f}", float(correct_100) / float(query_count * 100)));
}
}

Expand Down
8 changes: 4 additions & 4 deletions benchmark/remote_infinity/remote_query_benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ int main() {
ParallelFor(0, query_count, thread_num, query_function);
profiler.End();

results.push_back(infinity::Format("Total cost: {}", profiler.ElapsedToString(1000)));
results.push_back(fmt::format("Total cost: {}", profiler.ElapsedToString(1000)));
{
size_t correct_1 = 0, correct_10 = 0, correct_100 = 0;
for (size_t query_idx = 0; query_idx < query_count; ++query_idx) {
Expand All @@ -265,9 +265,9 @@ int main() {
}
}
}
results.push_back(infinity::Format("R@1: {:.3f}", float(correct_1) / float(query_count * 1)));
results.push_back(infinity::Format("R@10: {:.3f}", float(correct_10) / float(query_count * 10)));
results.push_back(infinity::Format("R@100: {:.3f}", float(correct_100) / float(query_count * 100)));
results.push_back(fmt::format("R@1: {:.3f}", float(correct_1) / float(query_count * 1)));
results.push_back(fmt::format("R@10: {:.3f}", float(correct_10) / float(query_count * 10)));
results.push_back(fmt::format("R@100: {:.3f}", float(correct_100) / float(query_count * 100)));
}
} while (--total_times);

Expand Down
67 changes: 33 additions & 34 deletions src/bin/infinity_main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include "network/thrift_server.h"
#include <csignal>
#include <cstdlib>
#include "network/thrift_server.h"

import compilation_config;
import stl;
Expand All @@ -27,25 +27,25 @@ namespace {

infinity::DBServer db_server;

//infinity::Thread threaded_thrift_thread;
//infinity::ThreadedThriftServer threaded_thrift_server;
// infinity::Thread threaded_thrift_thread;
// infinity::ThreadedThriftServer threaded_thrift_server;

infinity::Thread pool_thrift_thread;
infinity::PoolThriftServer pool_thrift_server;
//infinity::NonBlockPoolThriftServer non_block_pool_thrift_server;
// infinity::NonBlockPoolThriftServer non_block_pool_thrift_server;

infinity::Mutex server_mutex;
infinity::CondVar server_cv;
std::mutex server_mutex;
std::condition_variable server_cv;

bool server_running = false;

infinity::Thread shut_down_thread;

void ShutdownServer() {

infinity::UniqueLock<infinity::Mutex> lock(server_mutex);
std::unique_lock<std::mutex> lock(server_mutex);
server_running = true;
server_cv.wait(lock, [&]{ return !server_running; });
server_cv.wait(lock, [&] { return !server_running; });

// threaded_thrift_server.Shutdown();
// threaded_thrift_thread.join();
Expand All @@ -56,7 +56,6 @@ void ShutdownServer() {
// non_block_pool_thrift_server.Shutdown();

db_server.Shutdown();

}

void SignalHandler(int signal_number, siginfo_t *, void *) {
Expand All @@ -65,7 +64,7 @@ void SignalHandler(int signal_number, siginfo_t *, void *) {
case SIGQUIT:
case SIGTERM: {

infinity::UniqueLock<infinity::Mutex> lock(server_mutex);
std::unique_lock<std::mutex> lock(server_mutex);
server_running = false;
server_cv.notify_one();

Expand All @@ -82,7 +81,7 @@ void SignalHandler(int signal_number, siginfo_t *, void *) {
printf("Other type of signal: %d\n", signal_number);
}
}
// exit(0);
// exit(0);
}

void RegisterSignal() {
Expand All @@ -101,18 +100,18 @@ void RegisterSignal() {
namespace infinity {

void ParseArguments(int argc, char **argv, StartupParameter &parameters) {
CxxOptions options("./infinity_main", "");
cxxopts::Options options("./infinity_main", "");

options.add_options()("h,help", "Display this help and exit") // NOLINT
("f,config",
"Specify the config file path. No default config file",
cxx_value<String>()->default_value("")) // NOLINT
MakeShared<cxxopts::values::standard_value<String>>()->default_value("")) // NOLINT
;

ParseResult result = options.parse(argc, argv);
cxxopts::ParseResult result = options.parse(argc, argv);

if (result.count("help")) {
Printf("{}", options.help());
fmt::print("{}", options.help());
return;
}

Expand All @@ -127,21 +126,21 @@ void ParseArguments(int argc, char **argv, StartupParameter &parameters) {
auto main(int argc, char **argv) -> int {
using namespace infinity;

Printf(" __ .__ __. _______ __ .__ __. __ .___________.____ ____ \n"
"| | | \\ | | | ____|| | | \\ | | | | | |\\ \\ / / \n"
"| | | \\| | | |__ | | | \\| | | | `---| |----` \\ \\/ / \n"
"| | | . ` | | __| | | | . ` | | | | | \\_ _/ \n"
"| | | |\\ | | | | | | |\\ | | | | | | | \n"
"|__| |__| \\__| |__| |__| |__| \\__| |__| |__| |__| \n");

Printf("Infinity, version: {}.{}.{} build on {} with {} mode from branch: {}, commit-id: {}\n",
version_major(),
version_minor(),
version_patch(),
current_system_time(),
build_type(),
git_branch_name(),
git_commit_id());
fmt::print(" __ .__ __. _______ __ .__ __. __ .___________.____ ____ \n"
"| | | \\ | | | ____|| | | \\ | | | | | |\\ \\ / / \n"
"| | | \\| | | |__ | | | \\| | | | `---| |----` \\ \\/ / \n"
"| | | . ` | | __| | | | . ` | | | | | \\_ _/ \n"
"| | | |\\ | | | | | | |\\ | | | | | | | \n"
"|__| |__| \\__| |__| |__| |__| \\__| |__| |__| |__| \n");

fmt::print("Infinity, version: {}.{}.{} build on {} with {} mode from branch: {}, commit-id: {}\n",
version_major(),
version_minor(),
version_patch(),
current_system_time(),
build_type(),
git_branch_name(),
git_commit_id());

StartupParameter parameters;
ParseArguments(argc, argv, parameters);
Expand All @@ -152,15 +151,15 @@ auto main(int argc, char **argv) -> int {

InfinityContext::instance().config()->PrintAll();

// threaded_thrift_server.Init(9090);
// threaded_thrift_thread = infinity::Thread([&]() { threaded_thrift_server.Start(); });
// threaded_thrift_server.Init(9090);
// threaded_thrift_thread = infinity::Thread([&]() { threaded_thrift_server.Start(); });
u32 thrift_server_port = InfinityContext::instance().config()->sdk_port();

pool_thrift_server.Init(thrift_server_port, 128);
pool_thrift_thread = infinity::Thread([&]() { pool_thrift_server.Start(); });

// non_block_pool_thrift_server.Init(9070, 64);
// non_block_pool_thrift_server.Start();
// non_block_pool_thrift_server.Init(9070, 64);
// non_block_pool_thrift_server.Start();
shut_down_thread = infinity::Thread([&]() { ShutdownServer(); });
db_server.Run();

Expand Down
8 changes: 4 additions & 4 deletions src/common/analyzer/analyzer_pool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,18 @@ void AnalyzerPool::Set(const StringView &name) {
switch (Str2Int(name.data())) {
case Str2Int(CHINESE.data()): {
String path = InfinityContext::instance().config()->resource_dict_path();
UniquePtr<ChineseAnalyzer> analyzer = MakeUnique<ChineseAnalyzer>(Move(path));
UniquePtr<ChineseAnalyzer> analyzer = MakeUnique<ChineseAnalyzer>(std::move(path));
if (analyzer->Load())
cache_[CHINESE] = Move(analyzer);
cache_[CHINESE] = std::move(analyzer);
} break;
case Str2Int(STANDARD.data()): {
UniquePtr<StandardAnalyzer> analyzer = MakeUnique<StandardAnalyzer>();
cache_[STANDARD] = Move(analyzer);
cache_[STANDARD] = std::move(analyzer);
} break;
case Str2Int(NGRAM.data()): {
u32 ngram = 2; /// TODO config
UniquePtr<NGramAnalyzer> analyzer = MakeUnique<NGramAnalyzer>(ngram);
cache_[NGRAM] = Move(analyzer);
cache_[NGRAM] = std::move(analyzer);
} break;
default:
break;
Expand Down
10 changes: 5 additions & 5 deletions src/common/analyzer/chinese_analyzer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,23 +60,23 @@ bool ChineseAnalyzer::Load() {
fs::path stopwords_path(root / STOP_WORD_PATH);

if (!fs::exists(dict_path)) {
LOG_INFO(Format("Invalid jieba config {} dict for jieba_analyzer does not exist", dict_path.string()));
LOG_INFO(fmt::format("Invalid jieba config {} dict for jieba_analyzer does not exist", dict_path.string()));
return false;
}
if (!fs::exists(hmm_path)) {
LOG_INFO(Format("Invalid jieba config {} hmm for jieba_analyzer does not exist", hmm_path.string()));
LOG_INFO(fmt::format("Invalid jieba config {} hmm for jieba_analyzer does not exist", hmm_path.string()));
return false;
}
if (!fs::exists(userdict_path)) {
LOG_INFO(Format("Invalid jieba config {} user_dict for jieba_analyzer does not exist", userdict_path.string()));
LOG_INFO(fmt::format("Invalid jieba config {} user_dict for jieba_analyzer does not exist", userdict_path.string()));
return false;
}
if (!fs::exists(idf_path)) {
LOG_INFO(Format("Invalid jieba config {} idf for jieba_analyzer does not exist", idf_path.string()));
LOG_INFO(fmt::format("Invalid jieba config {} idf for jieba_analyzer does not exist", idf_path.string()));
return false;
}
if (!fs::exists(stopwords_path)) {
LOG_INFO(Format("Invalid jieba config {} stopword for jieba_analyzer does not exist", stopwords_path.string()));
LOG_INFO(fmt::format("Invalid jieba config {} stopword for jieba_analyzer does not exist", stopwords_path.string()));
return false;
}

Expand Down
Loading

0 comments on commit 5b5d14d

Please sign in to comment.