diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..56ccf19 --- /dev/null +++ b/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +ReflowComments: false diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..a1fc134 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,32 @@ +Checks: > + *, + -altera-*, + -bugprone-easily-swappable-parameters, + -bugprone-reserved-identifier, + -cert-*, + -cppcoreguidelines-*, + -fuchsia-*, + -hicpp-*, + -llvm-*, + -llvmlibc-*, + -google-objc-function-naming, + -misc-const-correctness, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -misc-unused-parameters, + -modernize-avoid-c-arrays, + -modernize-deprecated-headers, + -modernize-use-nodiscard, + -modernize-use-trailing-return-type, + -modernize-use-using, + -performance-no-int-to-ptr, + -readability-function-cognitive-complexity, + -readability-identifier-length, + -readability-implicit-bool-conversion, + -readability-magic-numbers, + +WarningsAsErrors: '' +HeaderFilterRegex: '' +AnalyzeTemporaryDtors: false +FormatStyle: none +User: sockfuzzer diff --git a/.gitignore b/.gitignore index c7a425d..eaf78ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,13 @@ __pycache__ *.d *.filelist /build + +### Automatically added by Hedron's Bazel Compile Commands Extractor: https://github.com/hedronvision/bazel-compile-commands-extractor +# Ignore the `external` link (that is added by `bazel-compile-commands-extractor`). The link differs between macOS/Linux and Windows, so it shouldn't be checked in. The pattern must not end with a trailing `/` because it's a symlink on macOS/Linux. +/external +# Ignore links to Bazel's output. The pattern needs the `*` because people can change the name of the directory into which your repository is cloned (changing the `bazel-` symlink), and must not end with a trailing `/` because it's a symlink on macOS/Linux. +/bazel-* +# Ignore generated output. Although valuable (after all, the primary purpose of `bazel-compile-commands-extractor` is to produce `compile_commands.json`!), it should not be checked in. +/compile_commands.json +# Ignore the directory in which `clangd` stores its local index. +/.cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..31dac66 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# third_party ignored +files: ^((?!third_party).)*$ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-merge-conflict + - id: requirements-txt-fixer +- repo: https://github.com/psf/black + rev: '22.8.0' + hooks: + - id: black +- repo: https://github.com/jlebar/pre-commit-hooks.git + rev: 62ca83ba4958da48ea44d9f24cd0aa58633376c7 + hooks: + - id: bazel-buildifier + - id: clang-format-whole-file + types_or: [c++, c, proto] +- repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.7.2 + hooks: + - id: shellcheck + args: ["--severity=warning"] diff --git a/Dockerfile b/Dockerfile index ea9ea8f..9ebcb32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This Dockerfile prepares a simple Debian build environment. # docker build --pull -t builder . diff --git a/README.md b/README.md index 4fa093b..507d77a 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ for fuzzing the network stack in userland on macOS and Linux-based hosts. # Building and Using the Fuzzer +NOTE: The project is moving to a Bazel-based build system. The following steps +work for now but will be updated once the new build system is published. + Build the fuzzer the same way you would typically build a project using CMake for your platform. For example: diff --git a/fuzz/executor/BUILD.bazel b/fuzz/executor/BUILD.bazel new file mode 100644 index 0000000..7213f3d --- /dev/null +++ b/fuzz/executor/BUILD.bazel @@ -0,0 +1,51 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cc_library( + name = "headers", + srcs = ["coroutine_executor.h"], + visibility = [ + "//fuzz/host:__pkg__", + ], + deps = [ + "//third_party/libco:co", + ], +) + +cc_library( + name = "coroutine_executor", + srcs = ["coroutine_executor.cc"], + hdrs = ["coroutine_executor.h"], + visibility = [ + "//fuzz/host:__pkg__", + ], + deps = [ + ":headers", + "//executor", + "//scheduler:fuzzed_scheduler", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + +cc_test( + name = "coroutine_executor_test", + size = "small", + srcs = ["coroutine_executor_test.cc"], + deps = [ + ":coroutine_executor", + "//executor:executor_test_template", + "//scheduler:fuzzed_scheduler_test_template", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/fuzz/executor/coroutine_executor.cc b/fuzz/executor/coroutine_executor.cc new file mode 100644 index 0000000..0a4fbf7 --- /dev/null +++ b/fuzz/executor/coroutine_executor.cc @@ -0,0 +1,137 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// #define _GNU_SOURCE +#include "fuzz/executor/coroutine_executor.h" + +#include +#include +#include + +#include "absl/meta/type_traits.h" + +#if __has_feature(address_sanitizer) +#include +#endif + +const int KERNEL_STACK_SIZE = 4096 * 16; + +#include "third_party/libco/libco.h" +#include +#include +#include + +static CoroutineExecutor *g_coroutine_executor; + +static void ThreadStart() { + g_coroutine_executor->CallPendingFunctionThenSwap(); +} + +CoroutineExecutor::CoroutineExecutor() : main_thread_(co_active()) { + g_current_thread = reinterpret_cast(co_active()); + g_coroutine_executor = this; +} + +CoroutineExecutor::~CoroutineExecutor() { g_coroutine_executor = nullptr; } + +ThreadHandle CoroutineExecutor::CreateThread(std::function target) { + void *mapping = + mmap(nullptr, 0x1000 + KERNEL_STACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) { + abort(); + } + if (mprotect(mapping, 0x1000, PROT_NONE)) { + abort(); + } + cothread_t thread = co_derive(static_cast(mapping) + 0x1000, + KERNEL_STACK_SIZE, ThreadStart); + pending_functions_[thread] = std::move(target); + return reinterpret_cast(thread); +} + +void CoroutineExecutor::CallPendingFunctionThenSwap() { +#if __has_feature(address_sanitizer) + __sanitizer_finish_switch_fiber(nullptr, nullptr, nullptr); +#endif + cothread_t active = co_active(); + g_current_thread = reinterpret_cast(active); + if (active == main_thread_) { + abort(); + } + + auto it = pending_functions_.find(active); + if (it == pending_functions_.end()) { + abort(); + } + + auto pending_function = std::move(it->second); + pending_functions_.erase(it); + + pending_function(); + + // TODO(nedwill): only notify destroyed once the thread actually gets + // destroyed explicitly + callbacks()->ThreadDestroyed(reinterpret_cast(active)); + SwitchToMainThread(); +} + +void CoroutineExecutor::DeleteThread(ThreadHandle handle) { + // co_delete(cothread); + DeleteBacktrace(handle); + munmap(reinterpret_cast(handle - 0x1000), + KERNEL_STACK_SIZE + 0x1000); +} + +void CoroutineExecutor::SwitchToMainThread() { + SwitchTo(reinterpret_cast(main_thread_)); +} + +void CoroutineExecutor::SwitchTo(ThreadHandle handle) { + auto *cothread = reinterpret_cast(handle); + + if (co_active() == cothread) { + // Already on requested thread + return; + } + +#if __has_feature(address_sanitizer) + // TODO(nedwill): track first argument to support stack use after return + // detection + __sanitizer_start_switch_fiber(nullptr, cothread, KERNEL_STACK_SIZE); +#endif + co_switch(cothread); + g_current_thread = reinterpret_cast(co_active()); +#if __has_feature(address_sanitizer) + __sanitizer_finish_switch_fiber(nullptr, nullptr, nullptr); +#endif +} + +ThreadHandle CoroutineExecutor::GetCurrentThreadHandle() { + // For performance, prefer to access g_current_thread directly. + // This global is set in SwitchTo. + return g_current_thread; +} + +ThreadHandle CoroutineExecutor::GetMainThreadHandle() { + return reinterpret_cast(main_thread_); +} + +// TODO(nedwill): support naming fibers +void CoroutineExecutor::SetThreadName(ThreadHandle handle, + const std::string &name) {} + +std::string CoroutineExecutor::DescribeThreadHandle(ThreadHandle handle) { + return {}; +} diff --git a/fuzz/executor/coroutine_executor.h b/fuzz/executor/coroutine_executor.h new file mode 100644 index 0000000..dca61f4 --- /dev/null +++ b/fuzz/executor/coroutine_executor.h @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COROUTINE_EXECUTOR_H_ +#define COROUTINE_EXECUTOR_H_ + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "executor/executor.h" +#include "third_party/libco/libco.h" + +class CoroutineExecutor : public Executor { + public: + explicit CoroutineExecutor(); + ~CoroutineExecutor() override; + + ThreadHandle CreateThread(std::function target) override; + void DeleteThread(ThreadHandle handle) override; + + void SwitchToMainThread() override; + void SwitchTo(ThreadHandle handle) override; + + ThreadHandle GetCurrentThreadHandle() override; + ThreadHandle GetMainThreadHandle() override; + + void SetThreadName(ThreadHandle handle, const std::string &name) override; + + std::string DescribeThreadHandle(ThreadHandle handle) override; + + void CallPendingFunctionThenSwap(); + + private: + cothread_t main_thread_; + + // Map from cothread_t to target functions + absl::flat_hash_map> pending_functions_; +}; + +#endif /* COROUTINE_EXECUTOR_H_ */ diff --git a/fuzz/executor/coroutine_executor_test.cc b/fuzz/executor/coroutine_executor_test.cc new file mode 100644 index 0000000..18f9fc0 --- /dev/null +++ b/fuzz/executor/coroutine_executor_test.cc @@ -0,0 +1,49 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fuzz/executor/coroutine_executor.h" + +#include "executor/executor_test_template.h" +#include "gtest/gtest.h" +#include "scheduler/fuzzed_scheduler.h" +#include "scheduler/fuzzed_scheduler_test_template.h" + +class Scheduler; + +Executor *g_executor; + +Scheduler *g_scheduler; + +template <> +Executor *CreateExecutor() { + return new CoroutineExecutor; +} + +typedef ::testing::Types ExecutorImplementation; + +// To create a downstream executor, replicate this line with your types +INSTANTIATE_TYPED_TEST_SUITE_P(CoroutineExecutorTests, ExecutorTest, + ExecutorImplementation); + +template <> +Scheduler *CreateScheduler() { + return new FuzzedScheduler(new CoroutineExecutor, + new EmptySchedulerCallbacks); +} + +typedef ::testing::Types SchedulerImplementation; + +// To create a downstream executor, replicate this line with your types +INSTANTIATE_TYPED_TEST_SUITE_P(CoroutineSchedulerTest, SchedulerTest, + SchedulerImplementation); diff --git a/third_party/concurrence/.bazelrc b/third_party/concurrence/.bazelrc new file mode 100644 index 0000000..71d7b1e --- /dev/null +++ b/third_party/concurrence/.bazelrc @@ -0,0 +1,13 @@ +build --action_env=CC=clang +build --action_env=CXX=clang++ +build --action_env=BAZEL_CXXOPTS="-std=c++20" + +build:asan --copt=-fsanitize=address +build:asan --linkopt=-fsanitize=address +# Support fast unwind +build:asan --copt=-fno-omit-frame-pointer + +# TODO(nedwill): Investigate bugs in ASAN fiber support when used with libco; +# We should only enable these options when selecting the libco backend. +build:asan --copt=-mllvm +build:asan --copt=-asan-stack=0 diff --git a/third_party/concurrence/LICENSE b/third_party/concurrence/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/third_party/concurrence/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/third_party/concurrence/README.md b/third_party/concurrence/README.md new file mode 100644 index 0000000..9accab3 --- /dev/null +++ b/third_party/concurrence/README.md @@ -0,0 +1,42 @@ +# Concurrence + +Concurrence is a library for fuzzing multi-threaded targets. It schedules one +thread at a time cooperatively and selects threads using fuzzed input data. This +project is still in early stages and this repository serves as a reference. + +There are three main components. The executor handles how contexts (threads) get +created, deleted, and defines how to switch between them. The scheduler depends +on the abstract executor interface. We provide a fuzzed scheduler implementation +to expose the scheduling decisions to fuzzed inputs. The sync primitives depend +on the scheduler and provide an example of how to integrate with the scheduler. + +In order to support developers' goals, every component is interchangeable. You +could have your own notion of thread creation and switching and write a custom +executor implementation. You could also opt for another scheduler implementation. +Finally, the sync primitives might not match the behavior of your codebase. You +could use your existing sync primitives without an issue so long as you can +Block() or Yield() appropriately so the scheduler knows which contexts are runnable. + +# Kernel vs. Userspace + +Currently, the project targets a kernel which means it has to handle things +like interrupt masking and deleting a thread that's waiting to return from a +context switch. These aspects are incompatible with userspace target fuzzing. +A future version of this library will attempt to bridge this gap so it can be +used for arbitrary targets. Until then, do not expect the API to remain stable. + +# Documentation + +See presentations/catch_me_if_you_can.pdf to read the slides from my Black Hat 2022 +presentation on this project. It covers the design and motivation and provides some +real world examples of bugs found using this project. + +# Build + +This project is built using Bazel. The //scheduler:fuzzed_scheduler target, when paired +with a suitable executor for your codebase, is the core class of this project and is +probably what you're looking for if you're not using this as a generic execution library. + +You can use or review //sync:mutex and //sync:rwlock to see how to interact with the +scheduler from your target code to explicitly yield or block in code that was designed +for multi-threaded environments. diff --git a/third_party/concurrence/WORKSPACE.bazel b/third_party/concurrence/WORKSPACE.bazel new file mode 100644 index 0000000..bc6c85b --- /dev/null +++ b/third_party/concurrence/WORKSPACE.bazel @@ -0,0 +1,24 @@ +workspace(name = "concurrence") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "bazel_skylib", + sha256 = "20ee473b16cc22958a408e83e816a352857e825b39f9d3666a99e058edc53623", + strip_prefix = "bazel-skylib-bc112d41fd71a5abf51ff827f19a6af9ed81af43", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/bc112d41fd71a5abf51ff827f19a6af9ed81af43.zip"], +) + +http_archive( + name = "com_google_absl", + sha256 = "e43a13c52747d056ac565f208087fca79dc294cf90795382be1217e22b80c076", + strip_prefix = "abseil-cpp-8317b9a01cbc32594ad4bf971709c97cb13ec921", + urls = ["https://github.com/abseil/abseil-cpp/archive/8317b9a01cbc32594ad4bf971709c97cb13ec921.zip"], +) + +http_archive( + name = "com_google_googletest", + sha256 = "bb38e3f07350fa48c49309d4e26dd19c00f7f76dde65d0b5bda75af3234f0e20", + strip_prefix = "googletest-67e264834a45c3d543aa2fa11bcc41c4ba90316b", + urls = ["https://github.com/google/googletest/archive/67e264834a45c3d543aa2fa11bcc41c4ba90316b.zip"], +) diff --git a/third_party/concurrence/backtrace/BUILD.bazel b/third_party/concurrence/backtrace/BUILD.bazel new file mode 100644 index 0000000..cb238b5 --- /dev/null +++ b/third_party/concurrence/backtrace/BUILD.bazel @@ -0,0 +1,45 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cc_library( + name = "backtrace", + srcs = [ + "backtrace.cc", + ], + hdrs = [ + "backtrace.h", + ], + visibility = [ + "//fuzz/host:__pkg__", + "//fuzz/host/hypercall:__pkg__", + "//:__subpackages__", + ], + deps = [ + "@com_google_absl//absl/debugging:stacktrace", + "@com_google_absl//absl/debugging:symbolize", + "@com_google_absl//absl/log", + ], +) + +cc_test( + name = "backtrace_test", + size = "small", + srcs = [ + "backtrace_test.cc", + ], + deps = [ + "//backtrace", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/concurrence/backtrace/backtrace.cc b/third_party/concurrence/backtrace/backtrace.cc new file mode 100644 index 0000000..e5fba48 --- /dev/null +++ b/third_party/concurrence/backtrace/backtrace.cc @@ -0,0 +1,42 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "backtrace.h" + +#include +#include + +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/log/log.h" +#include "absl/strings/str_format.h" + + +void GetBacktrace(std::vector &stack) { + stack.clear(); + std::array result{}; + std::array sizes{}; + int depth = + absl::GetStackFrames(result.data(), sizes.data(), result.size(), 1); + stack = std::vector(result.data(), result.data() + depth); +} + +void PrintBacktraceFromStack(const std::vector &stack) { + for (const void *pc : stack) { + std::array symbol{}; + absl::Symbolize(const_cast(pc), symbol.data(), symbol.size()); + + LOG(INFO) << absl::StrFormat("pc=%p %s\n", pc, symbol.data()); + } +} diff --git a/third_party/concurrence/backtrace/backtrace.h b/third_party/concurrence/backtrace/backtrace.h new file mode 100644 index 0000000..1d8144d --- /dev/null +++ b/third_party/concurrence/backtrace/backtrace.h @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONCURRENCE_BACKTRACE_H_ +#define CONCURRENCE_BACKTRACE_H_ + +#include + +void GetBacktrace(std::vector &stack); +void PrintBacktraceFromStack(const std::vector &stack); + +#endif diff --git a/third_party/concurrence/backtrace/backtrace_test.cc b/third_party/concurrence/backtrace/backtrace_test.cc new file mode 100644 index 0000000..229e186 --- /dev/null +++ b/third_party/concurrence/backtrace/backtrace_test.cc @@ -0,0 +1,29 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "backtrace/backtrace.h" + +#include + +#include "gtest/gtest.h" + +TEST(BacktraceTest, PopulatesStackAndDoesntCrash) { +#ifdef NDEBUG + GTEST_SKIP(); +#endif + std::vector stack; + GetBacktrace(stack); + EXPECT_FALSE(stack.empty()); + PrintBacktraceFromStack(stack); +} diff --git a/third_party/concurrence/executor/BUILD.bazel b/third_party/concurrence/executor/BUILD.bazel new file mode 100644 index 0000000..4f45c33 --- /dev/null +++ b/third_party/concurrence/executor/BUILD.bazel @@ -0,0 +1,82 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cc_library( + name = "headers", + hdrs = [ + "executor.h", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "executor", + srcs = [ + "executor.cc", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + ":headers", + "//backtrace", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:check", + ], +) + +cc_library( + name = "stdthread_executor", + srcs = [ + "stdthread/thread.cc", + "stdthread/thread_executor.cc", + ], + hdrs = [ + "stdthread/thread.h", + "stdthread/thread_executor.h", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + ":executor", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/synchronization", + ], +) + +cc_library( + name = "executor_test_template", + hdrs = ["executor_test_template.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":headers", + ], +) + +cc_test( + name = "executor_test", + size = "small", + srcs = [ + "executor_test.cc", + ], + deps = [ + ":executor_test_template", + ":stdthread_executor", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/concurrence/executor/executor.cc b/third_party/concurrence/executor/executor.cc new file mode 100644 index 0000000..6bb639b --- /dev/null +++ b/third_party/concurrence/executor/executor.cc @@ -0,0 +1,37 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "executor.h" + +#include "backtrace/backtrace.h" + +ThreadHandle g_current_thread; + +Executor::~Executor() = default; + +void Executor::SetBacktrace(ThreadHandle handle) { + GetBacktrace(backtraces_[handle]); +} + +void Executor::PrintBacktrace(ThreadHandle handle) { + if (!backtraces_.count(handle)) { + return; + } + + PrintBacktraceFromStack(backtraces_[handle]); +} + +void Executor::DeleteBacktrace(ThreadHandle handle) { + backtraces_.erase(handle); +} diff --git a/third_party/concurrence/executor/executor.h b/third_party/concurrence/executor/executor.h new file mode 100644 index 0000000..93d7124 --- /dev/null +++ b/third_party/concurrence/executor/executor.h @@ -0,0 +1,75 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef EXECUTOR_H_ +#define EXECUTOR_H_ + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/hash/hash.h" + +// Use pointer type as we use pointers the the corresponding C++ objects +// as handles +typedef uintptr_t ThreadHandle; + +// Implementations must not use 1 for child thread handles +// Thread handle 0 is disallowed because it implies null thread +const ThreadHandle MAIN_THREAD_HANDLE = 1; + +extern ThreadHandle g_current_thread; + +class Executor { + public: + virtual ~Executor(); + + virtual ThreadHandle CreateThread(std::function target) = 0; + virtual void DeleteThread(ThreadHandle handle) = 0; + + virtual void SwitchToMainThread() = 0; + virtual void SwitchTo(ThreadHandle handle) = 0; + + virtual ThreadHandle GetCurrentThreadHandle() = 0; + virtual ThreadHandle GetMainThreadHandle() = 0; + + virtual void SetThreadName(ThreadHandle handle, const std::string &name) = 0; + virtual std::string DescribeThreadHandle(ThreadHandle handle) = 0; + + void SetBacktrace(ThreadHandle handle); + void PrintBacktrace(ThreadHandle handle); + void DeleteBacktrace(ThreadHandle handle); + + class CallbackInterface { + public: + virtual ~CallbackInterface() = default; + virtual void ThreadDestroyed(ThreadHandle) = 0; + }; + + void set_callbacks(Executor::CallbackInterface *callbacks) { + callbacks_ = callbacks; + } + + Executor::CallbackInterface *callbacks() { return callbacks_; } + + private: + Executor::CallbackInterface *callbacks_{}; + absl::flat_hash_map> backtraces_; +}; + +#endif /* EXECUTOR_H_ */ diff --git a/third_party/concurrence/executor/executor_test.cc b/third_party/concurrence/executor/executor_test.cc new file mode 100644 index 0000000..c494b61 --- /dev/null +++ b/third_party/concurrence/executor/executor_test.cc @@ -0,0 +1,33 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "executor/executor.h" + +#include "gtest/gtest.h" + +#include "executor/executor_test_template.h" +#include "executor/stdthread/thread_executor.h" + +Executor* g_executor; + +template <> +Executor* CreateExecutor() { + return new ThreadExecutor; +} + +typedef ::testing::Types BuiltinImplementations; + +// To create a downstream executor, replicate this line with your types +INSTANTIATE_TYPED_TEST_SUITE_P(BuiltinExecutorTests, ExecutorTest, + BuiltinImplementations); diff --git a/third_party/concurrence/executor/executor_test_template.h b/third_party/concurrence/executor/executor_test_template.h new file mode 100644 index 0000000..43a17e0 --- /dev/null +++ b/third_party/concurrence/executor/executor_test_template.h @@ -0,0 +1,78 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONCURRENCE_EXECUTOR_TEST_H_ +#define CONCURRENCE_EXECUTOR_TEST_H_ + +#include +#include + +#include "executor/executor.h" + +class EmptyExecutorCallbacks : public Executor::CallbackInterface { + void ThreadDestroyed(ThreadHandle handle) override {} +}; + +template +Executor* CreateExecutor(); + +template +class ExecutorTest : public ::testing::Test { + protected: + // The ctor calls the factory function to create a prime table + // implemented by T. + ExecutorTest() : executor_(CreateExecutor()) { + executor_->set_callbacks(&callbacks_); + } + + ~ExecutorTest() override { delete executor_; } + + // Note that we test an implementation via the base interface + // instead of the actual implementation class. This is important + // for keeping the tests close to the real world scenario, where the + // implementation is invoked via the base interface. It avoids + // got-yas where the implementation class has a method that shadows + // a method with the same name (but slightly different argument + // types) in the base interface, for example. + EmptyExecutorCallbacks callbacks_; + Executor* const executor_; +}; + +TYPED_TEST_SUITE_P(ExecutorTest); + +// Switch round robin between threads [ main -> 1 -> 2 -> main ] +TYPED_TEST_P(ExecutorTest, SwitchTo) { + std::vector order; + + ThreadHandle thread2 = this->executor_->CreateThread([&] { + order.push_back(2); + this->executor_->SwitchToMainThread(); + }); + + ThreadHandle thread1 = this->executor_->CreateThread([&] { + order.push_back(1); + this->executor_->SwitchTo(thread2); + }); + + this->executor_->SwitchTo(thread1); + order.push_back(3); + + EXPECT_THAT(order, ::testing::ElementsAre(1, 2, 3)); +} + +REGISTER_TYPED_TEST_SUITE_P(ExecutorTest, SwitchTo); + +#endif /* CONCURRENCE_EXECUTOR_TEST_H_ */ diff --git a/third_party/concurrence/executor/stdthread/thread.cc b/third_party/concurrence/executor/stdthread/thread.cc new file mode 100644 index 0000000..1c4e9e3 --- /dev/null +++ b/third_party/concurrence/executor/stdthread/thread.cc @@ -0,0 +1,89 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "thread.h" + +#include +#include +#include + +#include "absl/log/check.h" +#include "thread_executor.h" +#include "absl/synchronization/mutex.h" +#include "executor/executor.h" + +void CooperativeThread::Start() { + tid_ = gettid(); + // wait to be scheduled + execution_mutex_->LockWhen(absl::Condition(&running_)); + + target_(); + + ExitThread(); +} + +// Execution mutex held by this thread +void CooperativeThread::ExitThread() { + execution_mutex_->AssertHeld(); + thread_executor_->NotifyDestroyed(reinterpret_cast(this)); + joinable_ = true; + execution_mutex_->Unlock(); + pthread_exit(nullptr); +} + +void CooperativeThread::Wait() { + execution_mutex_->AssertHeld(); + running_ = false; + execution_mutex_->Await(absl::Condition(&running_)); + if (terminate_) { + ExitThread(); + } +} + +void CooperativeThread::Signal() { + CHECK_NE(gettid(), tid_); + running_ = true; +} + +std::thread::id CooperativeThread::internal_id() { + return thread_->get_id(); +} + +CooperativeThread::CooperativeThread(ThreadExecutor* thread_executor, + std::function target) + : thread_executor_(thread_executor), + execution_mutex_(thread_executor->mutex()), + target_(std::move(target)), + tid_(-1) { + thread_ = std::make_unique([this] { Start(); }); +} + +CooperativeThread::~CooperativeThread() { + // Threads are destroyed from the main thread so shouldn't destroy themselves. + CHECK_NE(gettid(), tid_); + + if (!joinable_) { + // Notify thread must terminate then wake it up so it can pthread_exit + terminate_ = true; + Signal(); + execution_mutex_->Await(absl::Condition(&joinable_)); + } + + thread_->join(); +} + +void CooperativeThread::SetName(const std::string& name) { + pthread_t handle = thread_->native_handle(); + pthread_setname_np(handle, name.c_str()); +} diff --git a/third_party/concurrence/executor/stdthread/thread.h b/third_party/concurrence/executor/stdthread/thread.h new file mode 100644 index 0000000..e73353b --- /dev/null +++ b/third_party/concurrence/executor/stdthread/thread.h @@ -0,0 +1,68 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COOPERATIVE_THREAD_H_ +#define COOPERATIVE_THREAD_H_ + +#include +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" + +class ThreadExecutor; + +namespace absl { +class Mutex; +} // namespace absl + +class ABSL_LOCKABLE CooperativeThread { + public: + CooperativeThread(ThreadExecutor *thread_executor, + std::function target); + ~CooperativeThread(); + + // From another thread, begin running this one + void Start(); + + // In this thread, wait to be signalled + void Wait() ABSL_ASSERT_EXCLUSIVE_LOCK(*execution_mutex_); + + // Inform this thread is ready to run from another thread + void Signal(); + + void ExitThread() ABSL_UNLOCK_FUNCTION(*execution_mutex_); + + void SetName(const std::string &name); + + std::thread::id internal_id(); + pid_t tid() const { return tid_; } + + private: + ThreadExecutor *thread_executor_; + absl::Mutex *execution_mutex_; + bool running_ = false; + bool joinable_ = false; + bool terminate_ ABSL_GUARDED_BY(*execution_mutex_) = false; + std::function target_; + std::unique_ptr thread_; + // The number shown in GDB is [tid_] + pid_t tid_; +}; + +#endif diff --git a/third_party/concurrence/executor/stdthread/thread_executor.cc b/third_party/concurrence/executor/stdthread/thread_executor.cc new file mode 100644 index 0000000..fcd4ab9 --- /dev/null +++ b/third_party/concurrence/executor/stdthread/thread_executor.cc @@ -0,0 +1,115 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "thread_executor.h" + +#include + +#include "absl/log/check.h" +#include "absl/strings/str_format.h" +#include "executor/executor.h" +#include "absl/hash/hash.h" +#include "executor/stdthread/thread.h" + +ThreadExecutor::ThreadExecutor() { + g_current_thread = MAIN_THREAD_HANDLE; + executor_mutex_.Lock(); +} + +ThreadExecutor::~ThreadExecutor() = default; + +ThreadHandle ThreadExecutor::CreateThread(std::function target) { + std::unique_ptr thread( + new CooperativeThread(this, target)); + + auto handle = reinterpret_cast(thread.get()); + threads_[handle] = std::move(thread); + return handle; +} + +ThreadHandle ThreadExecutor::GetThreadHandle(CooperativeThread *thread) { + auto handle = reinterpret_cast(thread); + auto it = threads_.find(handle); + if (it == threads_.end()) { + return MAIN_THREAD_HANDLE; + } + + return handle; +} + +void ThreadExecutor::NotifyDestroyed(ThreadHandle handle) { + DCHECK(callbacks()); + callbacks()->ThreadDestroyed(handle); + SwitchToMainThread(); +} + +void ThreadExecutor::DeleteThread(ThreadHandle handle) { + DeleteBacktrace(handle); + threads_.erase(handle); +} + +void ThreadExecutor::SwitchToMainThread() { + SwitchTo(MAIN_THREAD_HANDLE); +} + +void ThreadExecutor::SwitchTo(ThreadHandle handle) { + auto current_handle = GetCurrentThreadHandle(); + if (handle == current_handle) { + // Already on requested thread + return; + } + + // Resume target + g_current_thread = handle; + if (handle == MAIN_THREAD_HANDLE) { + CHECK(!main_thread_running_); + main_thread_running_ = true; + } else { + auto *thread = reinterpret_cast(handle); + thread->Signal(); + } + + // Pause current thread + if (current_handle == MAIN_THREAD_HANDLE) { + main_thread_running_ = false; + executor_mutex_.Await(absl::Condition(&main_thread_running_)); + } else { + auto *current_thread = + reinterpret_cast(current_handle); + current_thread->Wait(); + } +} + +ThreadHandle ThreadExecutor::GetCurrentThreadHandle() { + // For performance, prefer to access g_current_thread directly. + return g_current_thread; +} + +ThreadHandle ThreadExecutor::GetMainThreadHandle() { + return MAIN_THREAD_HANDLE; +} + +std::string ThreadExecutor::DescribeThreadHandle(ThreadHandle handle) { + if (handle == MAIN_THREAD_HANDLE) { + return "main_thread"; + } + auto *thread = reinterpret_cast(handle); + return absl::StrFormat("tid%d", thread->tid()); +} + +void ThreadExecutor::SetThreadName(ThreadHandle handle, + const std::string &name) { + auto *thread = reinterpret_cast(handle); + thread->SetName(name); +} diff --git a/third_party/concurrence/executor/stdthread/thread_executor.h b/third_party/concurrence/executor/stdthread/thread_executor.h new file mode 100644 index 0000000..8fe24f9 --- /dev/null +++ b/third_party/concurrence/executor/stdthread/thread_executor.h @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef THREAD_EXECUTOR_H_ +#define THREAD_EXECUTOR_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/synchronization/mutex.h" + +#include "executor/executor.h" + +class CooperativeThread; + +class ThreadExecutor : public Executor { + public: + ThreadExecutor(); + ~ThreadExecutor() override; + + ThreadHandle CreateThread(std::function target) override; + void DeleteThread(ThreadHandle handle) override; + void SwitchToMainThread() override; + void SwitchTo(ThreadHandle handle) override; + + ThreadHandle GetCurrentThreadHandle() override; + ThreadHandle GetMainThreadHandle() override; + + void SetThreadName(ThreadHandle handle, const std::string &name) override; + std::string DescribeThreadHandle(ThreadHandle handle) override; + + void NotifyDestroyed(ThreadHandle handle); + + absl::Mutex *mutex() { return &executor_mutex_; } + + private: + ThreadHandle GetThreadHandle(CooperativeThread *thread); + void AcquireLock(); + void ReleaseLock(); + + // Token that must be held for any thread including scheduler to be runnable + absl::Mutex executor_mutex_; + bool main_thread_running_ = true; + + absl::flat_hash_map> + threads_; +}; + +#endif /* THREAD_EXECUTOR_H_ */ diff --git a/third_party/concurrence/presentations/catch_me_if_you_can.pdf b/third_party/concurrence/presentations/catch_me_if_you_can.pdf new file mode 100644 index 0000000..059dfb8 Binary files /dev/null and b/third_party/concurrence/presentations/catch_me_if_you_can.pdf differ diff --git a/third_party/concurrence/scheduler/BUILD.bazel b/third_party/concurrence/scheduler/BUILD.bazel new file mode 100644 index 0000000..1a669fd --- /dev/null +++ b/third_party/concurrence/scheduler/BUILD.bazel @@ -0,0 +1,79 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cc_library( + name = "headers", + hdrs = [ + "fuzzed_scheduler.h", + "scheduler.h", + ], + visibility = ["//visibility:public"], + deps = ["//executor:headers"], +) + +cc_library( + name = "fuzzed_scheduler", + srcs = [ + "fuzzed_scheduler.cc", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + ":headers", + "//executor", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/strings:str_format", + ], +) + +cc_library( + name = "fuzzed_scheduler_test_template", + hdrs = [ + "fuzzed_scheduler_test_template.h", + ], + visibility = [ + "//visibility:public", + ], + deps = [ + ":headers", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +cc_test( + name = "fuzzed_scheduler_test", + size = "small", + srcs = [ + "fuzzed_scheduler_test.cc", + ], + deps = [ + ":fuzzed_scheduler", + ":fuzzed_scheduler_test_template", + "//executor:stdthread_executor", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "mock_scheduler", + testonly = 1, + hdrs = ["mock_scheduler.h"], + visibility = ["//visibility:public"], + deps = [ + ":headers", + "@com_google_googletest//:gtest", + ], +) diff --git a/third_party/concurrence/scheduler/fuzzed_scheduler.cc b/third_party/concurrence/scheduler/fuzzed_scheduler.cc new file mode 100644 index 0000000..861098c --- /dev/null +++ b/third_party/concurrence/scheduler/fuzzed_scheduler.cc @@ -0,0 +1,329 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fuzzed_scheduler.h" + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/str_format.h" +#include "executor/executor.h" +#include "scheduler/scheduler.h" + +FuzzedScheduler::FuzzedScheduler( + Executor *executor, Scheduler::CallbackInterface *callback_interface) + : executor_(executor), + enable_debug_checks_(false), + callbacks_(callback_interface), + thread_choices_(nullptr), + interrupts_enabled_(true), + bytes_needed_(0) { + SetRandSeed(0); + executor_->set_callbacks(this); +} + +// TODO(nedwill): consider using lock state as part of fuzzer feedback +// Is it typical to run the same code path with different locks? Perhaps +// the state of locks in other threads is important. + +ThreadHandle FuzzedScheduler::CreateThread(std::function target, + bool runnable) { + ThreadHandle handle = executor_->CreateThread(target); + // printf("Created thread 0x%zx\n", handle); + live_set_.insert(handle); + if (runnable) { + MakeRunnable(handle); + } + return handle; +} + +ThreadHandle FuzzedScheduler::GetMainThread() { + return executor_->GetMainThreadHandle(); +} + +void FuzzedScheduler::ThreadDestroyed(ThreadHandle handle) { + live_set_.erase(handle); + pending_deletion_.insert(handle); + callbacks_->ThreadDestroyed(handle); +} + +void FuzzedScheduler::HandleDeadlock(const std::deque &handles) { + callbacks_->ReportDeadlock(handles); +} + +std::string FuzzedScheduler::DescribeThreadHandle(ThreadHandle handle) { + std::string host_description = callbacks_->DescribeThreadHandle(handle); + + std::string executor_description = executor_->DescribeThreadHandle(handle); + if (executor_description.empty()) { + return host_description; + } + + return absl::StrFormat("%s/%s", host_description, executor_description); +} + +void FuzzedScheduler::SetThreadName(ThreadHandle handle, + const std::string &name) { + executor_->SetThreadName(handle, name); +} + +void FuzzedScheduler::Yield() { + // Called from child thread + + // Yielding in main thread is a nop + if (g_current_thread == executor_->GetMainThreadHandle()) { + return; + } + + if (!interrupts_enabled_) { + return; + } + + if (runnable_.empty()) { + return; + } + + MakeRunnable(g_current_thread); + Block(); +} + +void FuzzedScheduler::Block() { + if (g_current_thread == executor_->GetMainThreadHandle()) { + std::cout + << "FuzzedScheduler::Block: blocking main thread is not allowed\n"; + abort(); + } + ThreadHandle main_handle = executor_->GetMainThreadHandle(); + callbacks_->ThreadWillBlock(); + SwitchTo(main_handle); +} + +ThreadHandle FuzzedScheduler::GetRandomRunnableHandle( + bool can_choose_existing) { + // for speed, default to removing from the end of the runnables_ vector + size_t idx = runnable_.size() - 1; + if (idx == 0) { + // there is only one runnable handle. + ThreadHandle tid = runnable_.at(0); + runnable_.erase(runnable_.begin()); + return tid; + } + + if (thread_choices_) { + size_t upper_bound = idx; + if (can_choose_existing) { + upper_bound++; + } + if (thread_choices_->remaining_bytes()) { + idx = thread_choices_->ConsumeIntegralInRange(0, upper_bound); + } else { + idx = (static_cast(std::rand())) % (upper_bound + 1); + } + bytes_needed_++; + if (can_choose_existing && idx >= runnable_.size()) { + MakeNotRunnable(g_current_thread); + return g_current_thread; + } + } + ThreadHandle tid = runnable_.at(idx); + runnable_.erase(runnable_.begin() + idx); + return tid; +} + +void FuzzedScheduler::CleanupDeadThreads() { + for (const ThreadHandle &handle : pending_deletion_) { + executor_->DeleteThread(handle); + } + pending_deletion_.clear(); +} + +void FuzzedScheduler::SwitchTo(ThreadHandle handle) { + if (executor_->GetMainThreadHandle() != handle && + !live_set_.contains(handle)) { + return; + } + + if (enable_debug_checks_) { + executor_->SetBacktrace(g_current_thread); + } + executor_->SwitchTo(handle); + callbacks_->ThreadResumed(); +} + +bool FuzzedScheduler::RunUntilEmpty() { + // ASSERT in scheduler thread + + if (runnable_.empty()) { + return false; + } + + while (!runnable_.empty()) { + ThreadHandle handle = GetRandomRunnableHandle(false); + SwitchTo(handle); + } + + CleanupDeadThreads(); + return true; +} + +void FuzzedScheduler::MakeRunnable(ThreadHandle handle) { + if (handle == executor_->GetMainThreadHandle()) { + // Main thread is always runnable + return; + } + + if (IsRunnable(handle)) { + // TODO(nwach): seems like this is called twice for every XNU thread... + return; + } + +#ifndef NDEBUG + if (!handle) { + LOG(FATAL) << "missing handle\n"; + } + + if (pending_deletion_.contains(handle)) { + LOG(FATAL) << "pending deletion\n"; + } +#endif + + runnable_.push_back(handle); +} + +void FuzzedScheduler::MakeNotRunnable(ThreadHandle handle) { + if (handle == executor_->GetMainThreadHandle()) { + // Main thread is always runnable + abort(); + } + + auto it = std::find(runnable_.begin(), runnable_.end(), handle); + if (it == runnable_.end()) { + // Thread is already not runnable + // TODO(nwach): how can this happen? + return; + } + + runnable_.erase(it); +} + +void FuzzedScheduler::MakeAllRunnable( + const absl::flat_hash_set &runnable) { + for (const ThreadHandle &handle : runnable) { + MakeRunnable(handle); + } +} + +bool FuzzedScheduler::IsRunnable(ThreadHandle handle) { + return std::find(runnable_.begin(), runnable_.end(), handle) != + runnable_.end(); +} + +void FuzzedScheduler::SetRandSeed(uint32_t rand_seed) { + std::srand(rand_seed); +} + +void FuzzedScheduler::SetThreadChoices(FuzzedDataProvider *thread_choices) { + thread_choices_ = thread_choices; +} + +ThreadHandle FuzzedScheduler::GetCurrentThreadHandle() { + return g_current_thread; +} + +bool FuzzedScheduler::SetInterruptsEnabled(bool enabled) { + bool original_state = interrupts_enabled_; + interrupts_enabled_ = enabled; + return original_state; +} + +bool FuzzedScheduler::GetInterruptsEnabled() { + return interrupts_enabled_; +} + +size_t FuzzedScheduler::NumLiveThreads() { + return live_set_.size(); +} + +size_t FuzzedScheduler::NumRunnableThreads() { + return runnable_.size(); +} + +ThreadHandle FuzzedScheduler::ChooseThread(bool can_choose_existing) { + while (!runnable_.empty()) { + ThreadHandle handle = GetRandomRunnableHandle(can_choose_existing); + // Ownership of runnable state goes back to the scheduler + if (can_choose_existing && handle == g_current_thread) { + return handle; + } + if (pending_deletion_.contains(handle) || !live_set_.contains(handle)) { + continue; + } + return handle; + } + + if (can_choose_existing) { + return g_current_thread; + } + + return 0; +} + +void FuzzedScheduler::PrintThreadState(ThreadHandle handle) { + LOG(INFO) << "; "; + bool printed = false; + if (IsRunnable(handle)) { + LOG(INFO) << "RUNNABLE"; + printed = true; + } + if (live_set_.contains(handle)) { + if (printed) { + LOG(INFO) << ", "; + } + LOG(INFO) << "ALIVE"; + printed = true; + } + if (pending_deletion_.contains(handle)) { + if (printed) { + LOG(INFO) << ", "; + } + LOG(INFO) << "PENDING DELETION"; + printed = true; + } + if (printed) { + LOG(INFO) << "\n"; + } +} + +void FuzzedScheduler::PrintBacktrace(ThreadHandle handle) { + executor_->PrintBacktrace(handle); +} + +void FuzzedScheduler::ResetBytesNeeded() { + bytes_needed_ = 0; +} + +size_t FuzzedScheduler::BytesNeeded() { + return bytes_needed_; +} + +size_t FuzzedScheduler::RandomDataRemaining() { + if (!thread_choices_) { + return 0; + } + return thread_choices_->remaining_bytes(); +} diff --git a/third_party/concurrence/scheduler/fuzzed_scheduler.h b/third_party/concurrence/scheduler/fuzzed_scheduler.h new file mode 100644 index 0000000..1842274 --- /dev/null +++ b/third_party/concurrence/scheduler/fuzzed_scheduler.h @@ -0,0 +1,107 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SCHEDULER_IMPL_H_ +#define SCHEDULER_IMPL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" + +#include "executor/executor.h" +#include "scheduler/scheduler.h" + +class FuzzedDataProvider; + +class FuzzedScheduler : public Scheduler { + public: + FuzzedScheduler(Executor *executor, + Scheduler::CallbackInterface *callback_interface); + ~FuzzedScheduler() override = default; + + // External scheduler support + ThreadHandle CreateThread(std::function target, + bool runnable) override; + ThreadHandle ChooseThread(bool can_choose_existing) override; + ThreadHandle GetRandomRunnableHandle(bool can_choose_existing); + + // Yielding/blocking + void Yield() override; + void Block() override; + void SwitchTo(ThreadHandle handle) override; + + // Scheduler state + bool SetInterruptsEnabled(bool enable) override; + bool GetInterruptsEnabled() override; + + // Thread state + void MakeRunnable(ThreadHandle handle) override; + void MakeNotRunnable(ThreadHandle handle) override; + void MakeAllRunnable( + const absl::flat_hash_set &runnables) override; + bool IsRunnable(ThreadHandle handle) override; + void CleanupDeadThreads() override; + + // Executor callbacks + void ThreadDestroyed(ThreadHandle handle) override; + + // Task processing + bool RunUntilEmpty() override; + + // Util + void SetRandSeed(uint32_t rand_seed) override; + void SetThreadChoices(FuzzedDataProvider *thread_choices) override; + ThreadHandle GetMainThread() override; + ThreadHandle GetCurrentThreadHandle() override; + const std::set &GetLiveThreads() override { return live_set_; } + size_t NumLiveThreads() override; + size_t NumRunnableThreads() override; + std::string DescribeThreadHandle(ThreadHandle handle) override; + void SetThreadName(ThreadHandle handle, const std::string &name) override; + void PrintThreadState(ThreadHandle handle) override; + void PrintBacktrace(ThreadHandle handle) override; + void HandleDeadlock(const std::deque &handles) override; + + // Benchmarking + void ResetBytesNeeded() override; + size_t BytesNeeded() override; + size_t RandomDataRemaining() override; + + private: + std::unique_ptr executor_; + bool enable_debug_checks_; + + Scheduler::CallbackInterface *callbacks_; + + FuzzedDataProvider *thread_choices_; + + // Track all runnable threads so we don't double-enqueue + std::vector runnable_; + // Set of threads that are not deleted or pending deletion + std::set live_set_; + std::set pending_deletion_; + bool interrupts_enabled_; + size_t bytes_needed_; +}; + +#endif /* SCHEDULER_IMPL_H_ */ diff --git a/third_party/concurrence/scheduler/fuzzed_scheduler_test.cc b/third_party/concurrence/scheduler/fuzzed_scheduler_test.cc new file mode 100644 index 0000000..484f55b --- /dev/null +++ b/third_party/concurrence/scheduler/fuzzed_scheduler_test.cc @@ -0,0 +1,33 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fuzzed_scheduler.h" + +#include "fuzzed_scheduler_test_template.h" +#include "gtest/gtest.h" + +#include "executor/stdthread/thread_executor.h" + +Scheduler* g_scheduler; + +template <> +Scheduler* CreateScheduler() { + return new FuzzedScheduler(new ThreadExecutor, new EmptySchedulerCallbacks); +} + +typedef ::testing::Types BuiltinImplementations; + +// To create a downstream executor, replicate this line with your types +INSTANTIATE_TYPED_TEST_SUITE_P(BuiltinSchedulerTest, SchedulerTest, + BuiltinImplementations); diff --git a/third_party/concurrence/scheduler/fuzzed_scheduler_test_template.h b/third_party/concurrence/scheduler/fuzzed_scheduler_test_template.h new file mode 100644 index 0000000..691743a --- /dev/null +++ b/third_party/concurrence/scheduler/fuzzed_scheduler_test_template.h @@ -0,0 +1,123 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONCURRENCE_SCHEDULER_FUZZED_SCHEDULER_TEST_TEMPLATE_H_ +#define CONCURRENCE_SCHEDULER_FUZZED_SCHEDULER_TEST_TEMPLATE_H_ + +#include +#include + +#include "scheduler/fuzzed_scheduler.h" + +extern Scheduler *g_scheduler; + +class EmptySchedulerCallbacks : public Scheduler::CallbackInterface { + void ThreadWillBlock() override {} + void ThreadResumed() override {} + void ThreadDestroyed(ThreadHandle handle) override {} + std::string DescribeThreadHandle(ThreadHandle handle) override { return {}; } + void ReportDeadlock(const std::deque &handles) override {} +}; + +template +Scheduler *CreateScheduler(); + +template +class SchedulerTest : public testing::Test { + protected: + // The ctor calls the factory function to create a prime table + // implemented by T. + SchedulerTest() : scheduler_(CreateScheduler()) {} + + ~SchedulerTest() override { delete scheduler_; } + + Scheduler *const scheduler_; +}; + +TYPED_TEST_SUITE_P(SchedulerTest); + +inline void yield() { + g_scheduler->Yield(); +} + +inline void Store123(std::vector *results) { + yield(); + results->push_back(1); + yield(); + results->push_back(2); + yield(); + results->push_back(3); +} + +inline void Store456(std::vector *results) { + yield(); + results->push_back(4); + yield(); + results->push_back(5); + yield(); + results->push_back(6); +} + +TYPED_TEST_P(SchedulerTest, TwoThreads) { + g_scheduler = this->scheduler_; + + std::vector results; + + this->scheduler_->CreateThread([&results]() { Store123(&results); }, true); + this->scheduler_->CreateThread([&results]() { Store456(&results); }, true); + + this->scheduler_->RunUntilEmpty(); + EXPECT_THAT(results, ::testing::SizeIs(6)); +} + +TYPED_TEST_P(SchedulerTest, TwoThreadsAlternating) { + std::vector data = {0, 0, 0, 0, 0, 0, 0, 0}; + FuzzedDataProvider fdp(data.data(), data.size()); + this->scheduler_->SetThreadChoices(&fdp); + g_scheduler = this->scheduler_; + std::vector results; + this->scheduler_->CreateThread([&results]() { Store123(&results); }, true); + this->scheduler_->CreateThread([&results]() { Store456(&results); }, true); + + this->scheduler_->RunUntilEmpty(); + EXPECT_THAT(results, ::testing::ElementsAre(1, 4, 2, 5, 3, 6)); +} + +inline void CreateSubThreads(std::vector *results) { + g_scheduler->CreateThread([results]() { Store123(results); }, true); + g_scheduler->CreateThread([results]() { Store456(results); }, true); +} + +// Nested thread creation +TYPED_TEST_P(SchedulerTest, NestedThreadCreation) { + g_scheduler = this->scheduler_; + std::vector results; + this->scheduler_->CreateThread([&results]() { CreateSubThreads(&results); }, + true); + this->scheduler_->RunUntilEmpty(); + EXPECT_THAT(results, ::testing::SizeIs(6)); +} + +// Test "yielding" doesn't crash when called from the main thread +TYPED_TEST_P(SchedulerTest, YieldNop) { + g_scheduler = this->scheduler_; + yield(); +} + +REGISTER_TYPED_TEST_SUITE_P(SchedulerTest, TwoThreads, TwoThreadsAlternating, + NestedThreadCreation, YieldNop); + +#endif /* CONCURRENCE_SCHEDULER_FUZZED_SCHEDULER_TEST_TEMPLATE_H_ */ diff --git a/third_party/concurrence/scheduler/mock_scheduler.h b/third_party/concurrence/scheduler/mock_scheduler.h new file mode 100644 index 0000000..4dc9035 --- /dev/null +++ b/third_party/concurrence/scheduler/mock_scheduler.h @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gmock/gmock.h" +#include "scheduler.h" + +class MockScheduler : public Scheduler { + public: + MOCK_METHOD(ThreadHandle, CreateThread, + (std::function target, bool runnable), (override)); + MOCK_METHOD(ThreadHandle, ChooseThread, (bool can_choose_existing), + (override)); + + MOCK_METHOD(void, Yield, (), (override)); + MOCK_METHOD(void, Block, (), (override)); + MOCK_METHOD(void, SwitchTo, (ThreadHandle handle), (override)); + + MOCK_METHOD(bool, SetInterruptsEnabled, (bool enable), (override)); + MOCK_METHOD(bool, GetInterruptsEnabled, (), (override)); + + MOCK_METHOD(void, MakeRunnable, (ThreadHandle id), (override)); + MOCK_METHOD(void, MakeNotRunnable, (ThreadHandle id), (override)); + MOCK_METHOD(void, MakeAllRunnable, + (const absl::flat_hash_set &runnable), (override)); + MOCK_METHOD(bool, IsRunnable, (ThreadHandle handle), (override)); + MOCK_METHOD(void, CleanupDeadThreads, (), (override)); + + MOCK_METHOD(void, ThreadDestroyed, (ThreadHandle handle), (override)); + + MOCK_METHOD(bool, RunUntilEmpty, (), (override)); + + MOCK_METHOD(void, SetRandSeed, (uint32_t rand_seed), (override)); + MOCK_METHOD(void, SetThreadChoices, (FuzzedDataProvider * fuzzed_data), + (override)); + MOCK_METHOD(void, SetThreadName, + (ThreadHandle handle, const std::string &name), (override)); + MOCK_METHOD(std::string, DescribeThreadHandle, (ThreadHandle), (override)); + MOCK_METHOD(ThreadHandle, GetMainThread, (), (override)); + MOCK_METHOD(ThreadHandle, GetCurrentThreadHandle, (), (override)); + MOCK_METHOD(const std::set &, GetLiveThreads, (), (override)); + MOCK_METHOD(size_t, NumLiveThreads, (), (override)); + MOCK_METHOD(size_t, NumRunnableThreads, (), (override)); + MOCK_METHOD(void, PrintThreadState, (ThreadHandle handle), (override)); + MOCK_METHOD(void, PrintBacktrace, (ThreadHandle handle), (override)); + MOCK_METHOD(void, HandleDeadlock, (const std::deque &handles), + (override)); + + MOCK_METHOD(void, ResetBytesNeeded, (), (override)); + MOCK_METHOD(size_t, BytesNeeded, (), (override)); + MOCK_METHOD(size_t, RandomDataRemaining, (), (override)); +}; diff --git a/third_party/concurrence/scheduler/scheduler.h b/third_party/concurrence/scheduler/scheduler.h new file mode 100644 index 0000000..ee96f5e --- /dev/null +++ b/third_party/concurrence/scheduler/scheduler.h @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SCHEDULER_H_ +#define SCHEDULER_H_ + +#include + +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "executor/executor.h" + +class Scheduler : public Executor::CallbackInterface { + public: + ~Scheduler() override = default; + + // External scheduler support + // TODO(nedwill): remove these + virtual ThreadHandle CreateThread(std::function target, + bool runnable) = 0; + virtual ThreadHandle ChooseThread(bool can_choose_existing) = 0; + + // Yielding/blocking + virtual void Yield() = 0; + virtual void Block() = 0; + virtual void SwitchTo(ThreadHandle handle) = 0; + + // Scheduler state + virtual bool SetInterruptsEnabled(bool enable) = 0; + virtual bool GetInterruptsEnabled() = 0; + + // Thread state + virtual void MakeRunnable(ThreadHandle id) = 0; + virtual void MakeNotRunnable(ThreadHandle id) = 0; + virtual void MakeAllRunnable( + const absl::flat_hash_set &runnable) = 0; + virtual bool IsRunnable(ThreadHandle handle) = 0; + virtual void CleanupDeadThreads() = 0; + + // Executor callbacks + void ThreadDestroyed(ThreadHandle handle) override = 0; + + // Task processing + virtual bool RunUntilEmpty() = 0; + + // Util + virtual void SetRandSeed(uint32_t rand_seed) = 0; + virtual void SetThreadChoices(FuzzedDataProvider *thread_choices) = 0; + virtual ThreadHandle GetMainThread() = 0; + virtual ThreadHandle GetCurrentThreadHandle() = 0; + virtual const std::set &GetLiveThreads() = 0; + virtual size_t NumLiveThreads() = 0; + virtual size_t NumRunnableThreads() = 0; + virtual void PrintThreadState(ThreadHandle handle) = 0; + virtual void PrintBacktrace(ThreadHandle handle) = 0; + virtual void HandleDeadlock(const std::deque &handles) = 0; + virtual void SetThreadName(ThreadHandle handle, const std::string &name) = 0; + virtual std::string DescribeThreadHandle(ThreadHandle) = 0; + + // Benchmarking + virtual void ResetBytesNeeded() = 0; + virtual size_t BytesNeeded() = 0; + virtual size_t RandomDataRemaining() = 0; + + class CallbackInterface { + public: + virtual ~CallbackInterface() = default; + virtual void ThreadWillBlock() = 0; + virtual void ThreadResumed() = 0; + virtual void ThreadDestroyed(ThreadHandle) = 0; + virtual std::string DescribeThreadHandle(ThreadHandle) = 0; + virtual void ReportDeadlock(const std::deque &handles) = 0; + }; +}; + +#endif /* SCHEDULER_H_ */ diff --git a/third_party/concurrence/sync/BUILD.bazel b/third_party/concurrence/sync/BUILD.bazel new file mode 100644 index 0000000..da36496 --- /dev/null +++ b/third_party/concurrence/sync/BUILD.bazel @@ -0,0 +1,55 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cc_library( + name = "sync", + srcs = [ + "mutex.cc", + "rwlock.cc", + "sync.cc", + "tracker.cc", + ], + hdrs = [ + "mutex.h", + "rwlock.h", + "sync.h", + "tracker.h", + ], + visibility = [ + "//fuzz/host:__pkg__", + "//fuzz/host/hypercall:__pkg__", + "//fuzz/sync:__pkg__", + "//fuzz/xnu/osfmk:__pkg__", + ], + deps = [ + "//executor", + "//scheduler:headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +cc_test( + name = "sync_test", + size = "small", + srcs = [ + "mutex_test.cc", + "rwlock_test.cc", + ], + deps = [ + ":sync", + "//scheduler:mock_scheduler", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/concurrence/sync/mutex.cc b/third_party/concurrence/sync/mutex.cc new file mode 100644 index 0000000..0081a0f --- /dev/null +++ b/third_party/concurrence/sync/mutex.cc @@ -0,0 +1,96 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mutex.h" + +#include + +#include "backtrace/backtrace.h" +#include "executor/executor.h" +#include "absl/container/flat_hash_map.h" +#include "absl/hash/hash.h" +#include "scheduler/scheduler.h" +#include "sync/sync.h" + +VirtualMutex::VirtualMutex(SyncTracker *sync_tracker) + : Sync(sync_tracker), owner_(0) {} + +VirtualMutex::~VirtualMutex() = default; + +void VirtualMutex::Lock() { + while (owner_) { + AddWaiter(); + scheduler()->Block(); + } + + owner_ = g_current_thread; + + if (enable_debug_checks_) { + GetBacktrace(owner_backtraces()[owner_]); + } +} + +bool VirtualMutex::TryLock() { + // Trying to locking something we already hold + if (Held()) { + HandleNestedWait(); + } + if (owner_) { + return false; + } + + owner_ = g_current_thread; + + if (enable_debug_checks_) { + GetBacktrace(owner_backtraces()[owner_]); + } + + return true; +} + +void VirtualMutex::Unlock() { + AssertHeld(); + if (enable_debug_checks_) { + owner_backtraces().erase(owner_); + } + owner_ = 0; + ClearAndUnblockWaiters(); +} + +bool VirtualMutex::Held() { + return owner_ == g_current_thread; +} + +void VirtualMutex::AssertHeld() { + if (!Held()) { + abort(); + } +} + +bool VirtualMutex::IsOwned() { + return owner_ != 0; +} + +bool VirtualMutex::IsOwnedBy(ThreadHandle handle) { + return owner_ == handle; +} + +const absl::flat_hash_set &VirtualMutex::owners() { + if (owner_) { + owners_ = {owner_}; + } else { + owners_ = {}; + } + return owners_; +} diff --git a/third_party/concurrence/sync/mutex.h b/third_party/concurrence/sync/mutex.h new file mode 100644 index 0000000..fc92e33 --- /dev/null +++ b/third_party/concurrence/sync/mutex.h @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIRTUAL_MUTEX_H_ +#define VIRTUAL_MUTEX_H_ + +#include "absl/container/flat_hash_set.h" +#include "sync.h" + +#include "executor/executor.h" + +class SyncTracker; + +class VirtualMutex : public Sync { + public: + explicit VirtualMutex(SyncTracker *sync_tracker); + ~VirtualMutex() override; + + void Lock(); + bool TryLock(); + void Unlock(); + + bool Held(); + void AssertHeld(); + + bool IsOwned() override; + bool IsOwnedBy(ThreadHandle handle) override; + const absl::flat_hash_set &owners() override; + + private: + // owners_ is only populated when requested. + absl::flat_hash_set owners_; + ThreadHandle owner_; +}; + +#endif /* VIRTUAL_MUTEX_H_ */ diff --git a/third_party/concurrence/sync/mutex_test.cc b/third_party/concurrence/sync/mutex_test.cc new file mode 100644 index 0000000..ed7a5d5 --- /dev/null +++ b/third_party/concurrence/sync/mutex_test.cc @@ -0,0 +1,57 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mutex.h" + +#include + +#include "scheduler/mock_scheduler.h" +#include "sync/tracker.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::ElementsAre; +using ::testing::IsEmpty; + +TEST(MutexTest, LockUnlock) { + MockScheduler mock_scheduler; + SyncTracker tracker(&mock_scheduler); + VirtualMutex mutex(&tracker); + + // Thread 1 acquires + g_current_thread = 1; + mutex.Lock(); + + // Prepare thread 2 to block + EXPECT_CALL(mock_scheduler, Block()).WillOnce([&]() { + // Prepare MakeAllRunnable call from scheduler + EXPECT_CALL(mock_scheduler, MakeAllRunnable(ElementsAre(2))); + // Thread 1 releases while thread 2 is blocked + g_current_thread = 1; + mutex.Unlock(); + g_current_thread = 2; + return true; + }); + + // Thread 2 blocks, triggering thread 1 unlock callback + g_current_thread = 2; + mutex.Lock(); + + // When thread 2 unlocks there should be no waiters + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + + g_current_thread = 2; + // Thread 2 releases + mutex.Unlock(); +} diff --git a/third_party/concurrence/sync/rwlock.cc b/third_party/concurrence/sync/rwlock.cc new file mode 100644 index 0000000..afeb8a9 --- /dev/null +++ b/third_party/concurrence/sync/rwlock.cc @@ -0,0 +1,153 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rwlock.h" + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/hash/hash.h" +#include "absl/log/log.h" + +#include "scheduler/scheduler.h" + +class SyncTracker; + +void GetBacktrace(std::vector &stack); + +VirtualRwLock::VirtualRwLock(SyncTracker *sync_tracker) + : Sync(sync_tracker), mutex_(sync_tracker) {} +VirtualRwLock::~VirtualRwLock() = default; + +bool VirtualRwLock::TryLockExclusive() { + if (!owners_.empty()) { + return false; + } + if (mutex_.TryLock()) { + return true; + } + return false; +} + +bool VirtualRwLock::TryLockShared() { + if (!mutex_.TryLock()) { + return false; + } + + AddOwner(); + + mutex_.Unlock(); + + return true; +} + +void VirtualRwLock::LockExclusive() { + while (true) { + while (!owners_.empty()) { + AddWaiter(); + scheduler()->Block(); + } + + // mutex_.Lock can block and we might lose the race after someone else + // took it added a reader + mutex_.Lock(); + if (owners_.empty()) { + return; + } + mutex_.Unlock(); + } +} + +void VirtualRwLock::UnlockExclusive() { + if (!mutex_.Held()) { + abort(); + } + if (!owners_.empty()) { + abort(); + } + mutex_.Unlock(); +} + +void VirtualRwLock::LockShared() { + mutex_.Lock(); + AddOwner(); + mutex_.Unlock(); +} + +void VirtualRwLock::UnlockShared() { + mutex_.Lock(); + RemoveOwner(); + mutex_.Unlock(); +} + +void VirtualRwLock::ExclusiveToShared() { + mutex_.AssertHeld(); + if (!owners_.empty()) { + abort(); + } + + AddOwner(); + mutex_.Unlock(); +} + +void VirtualRwLock::Unlock() { + if (!IsShared()) { + UnlockExclusive(); + } else { + UnlockShared(); + } +} + +bool VirtualRwLock::IsShared() { + return !owners_.empty(); +} + +bool VirtualRwLock::SharedToExclusive() { + RemoveOwner(); + // TODO(nedwill): testing non-trylock version for now + return TryLockExclusive(); +} + +void VirtualRwLock::AddOwner() { + if (IsOwnedBy(g_current_thread)) { + LOG(ERROR) << "VirtualRwLock is already owned by current thread!\n"; + abort(); + } + owners_.insert(g_current_thread); + if (enable_debug_checks_) { + GetBacktrace(owner_backtraces()[g_current_thread]); + } +} + +void VirtualRwLock::RemoveOwner() { + size_t erased = owners_.erase(g_current_thread); + if (!erased) { + abort(); + } + if (enable_debug_checks_) { + owner_backtraces().erase(g_current_thread); + } + if (owners_.empty()) { + ClearAndUnblockWaiters(); + } +} + +bool VirtualRwLock::IsOwned() { + return !owners_.empty(); +} + +bool VirtualRwLock::IsOwnedBy(ThreadHandle handle) { + return owners_.count(handle); +} diff --git a/third_party/concurrence/sync/rwlock.h b/third_party/concurrence/sync/rwlock.h new file mode 100644 index 0000000..a2ce272 --- /dev/null +++ b/third_party/concurrence/sync/rwlock.h @@ -0,0 +1,62 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIRTUAL_RWLOCK_H_ +#define VIRTUAL_RWLOCK_H_ + +#include "absl/container/flat_hash_set.h" + +#include "executor/executor.h" +#include "sync/mutex.h" +#include "sync/sync.h" + +class SyncTracker; + +class VirtualRwLock : public Sync { + public: + explicit VirtualRwLock(SyncTracker *sync_tracker); + ~VirtualRwLock() override; + + bool TryLockExclusive(); + void LockExclusive(); + void UnlockExclusive(); + + bool TryLockShared(); + void LockShared(); + void UnlockShared(); + + void Unlock(); + + bool IsShared(); + void ExclusiveToShared(); + bool SharedToExclusive(); + + bool IsOwned() override; + bool IsOwnedBy(ThreadHandle handle) override; + const absl::flat_hash_set &owners() override { + return owners_; + }; + + void AddOwner(); + void RemoveOwner(); + + private: + absl::flat_hash_set owners_; + // Used to lock access to owners (readers) + VirtualMutex mutex_; +}; + +#endif /* VIRTUAL_RWLOCK_H_ */ diff --git a/third_party/concurrence/sync/rwlock_test.cc b/third_party/concurrence/sync/rwlock_test.cc new file mode 100644 index 0000000..1d2b504 --- /dev/null +++ b/third_party/concurrence/sync/rwlock_test.cc @@ -0,0 +1,93 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sync/rwlock.h" + +#include +#include + +#include "scheduler/mock_scheduler.h" +#include "sync/tracker.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::StrictMock; + +TEST(RwLockTest, MultipleReaders) { + StrictMock mock_scheduler; + SyncTracker tracker(&mock_scheduler); + VirtualRwLock rwlock(&tracker); + + g_current_thread = 1; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + + rwlock.LockShared(); + + g_current_thread = 2; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + + rwlock.LockShared(); + + g_current_thread = 3; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + + rwlock.LockShared(); + + g_current_thread = 1; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + rwlock.UnlockShared(); + + g_current_thread = 2; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())); + rwlock.UnlockShared(); + + g_current_thread = 3; + // Lock is released, waiters signalled + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())).Times(2); + rwlock.UnlockShared(); +} + +TEST(RwLockTest, ReadersWaitForWriter) { + MockScheduler mock_scheduler; + SyncTracker tracker(&mock_scheduler); + VirtualRwLock rwlock(&tracker); + + // Exclusive lock with thread 1 + g_current_thread = 1; + rwlock.LockExclusive(); + + // Try to shared lock with thread 2 + g_current_thread = 2; + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())).Times(1); + EXPECT_CALL(mock_scheduler, MakeAllRunnable(ElementsAre(2))); + + EXPECT_CALL(mock_scheduler, Block()).WillRepeatedly([&]() { + // Thread 1 releases + g_current_thread = 1; + rwlock.UnlockExclusive(); + g_current_thread = 2; + return true; + }); + + rwlock.LockShared(); + + // No one will be waiting + EXPECT_CALL(mock_scheduler, MakeAllRunnable(IsEmpty())).Times(2); + g_current_thread = 2; + rwlock.UnlockShared(); +} + +// TEST(RwLockTest, WriterWaitsForReaders) diff --git a/third_party/concurrence/sync/sync.cc b/third_party/concurrence/sync/sync.cc new file mode 100644 index 0000000..62edac0 --- /dev/null +++ b/third_party/concurrence/sync/sync.cc @@ -0,0 +1,110 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sync.h" + +#include +#include + +#include "absl/hash/hash.h" +#include "absl/log/log.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/str_format.h" +#include "backtrace/backtrace.h" +#include "scheduler/scheduler.h" +#include "sync/tracker.h" + +Sync::Sync(SyncTracker *sync_tracker) + : sync_tracker_(sync_tracker), scheduler_(sync_tracker->scheduler()) { + if (enable_debug_checks_) { + GetBacktrace(construction_backtrace_); + } +} + +Sync::~Sync() = default; + +void Sync::AddWaiter() { + if (IsOwnedBy(g_current_thread)) { + HandleNestedWait(); + } + waiters_.insert(g_current_thread); + if (enable_debug_checks_) { + DetectDeadlock(); + } +} + +void Sync::ClearAndUnblockWaiters() { + scheduler_->MakeAllRunnable(waiters_); + waiters_.clear(); +} + +// The current thread is now a waiter for this sync object. Check for deadlocks. +void Sync::DetectDeadlock() { + ThreadHandle handle = g_current_thread; + if (handle == scheduler_->GetMainThread()) { + HandleBlockedMainThread(); + } + std::deque path; + for (ThreadHandle owner : owners()) { + // If any owner has a dependency chain leading back to us we have a + // deadlock. + if (sync_tracker_->FindDependencyPath(owner, handle, path)) { + scheduler_->HandleDeadlock(path); + } + } +} + +void Sync::HandleNestedWait() { + if (enable_debug_checks_) { + std::vector backtrace; + GetBacktrace(backtrace); + + std::vector original_backtrace = + owner_backtraces_[g_current_thread]; + + LOG(INFO) << absl::StrFormat("Nested wait for sync primitive %p\n", this); + LOG(INFO) << "\nOriginal ownership acquired:\n"; + PrintBacktraceFromStack(original_backtrace); + LOG(INFO) << "\nTried to wait again on the same object here:\n"; + PrintBacktraceFromStack(backtrace); + } + abort(); +} + +void Sync::HandleBlockedMainThread() { + LOG(INFO) << absl::StrFormat( + "\nMain thread %s blocked\n", + scheduler_->DescribeThreadHandle(g_current_thread)); + std::vector backtrace; + GetBacktrace(backtrace); + PrintBacktraceFromStack(backtrace); + Sync *current = this; + while (current && !current->owners().empty()) { + // Only consider first owner since we only need one example dependency path + ThreadHandle owner = *current->owners().begin(); + Sync *blocking_object = sync_tracker_->GetObjectWithWaiter(owner); + LOG(INFO) << absl::StrFormat( + "\nBlocked by thread %s (IsRunnable %d) on object %p\n", + scheduler_->DescribeThreadHandle(owner), scheduler_->IsRunnable(owner), + blocking_object); + scheduler_->PrintBacktrace(owner); + LOG(INFO) << "\nBacktrace where ownership was acquired\n"; + auto it = owner_backtraces_.find(owner); + if (it != owner_backtraces_.end()) { + PrintBacktraceFromStack(it->second); + } + current = blocking_object; + } + abort(); +} diff --git a/third_party/concurrence/sync/sync.h b/third_party/concurrence/sync/sync.h new file mode 100644 index 0000000..9342f89 --- /dev/null +++ b/third_party/concurrence/sync/sync.h @@ -0,0 +1,75 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONCURRENCE_SYNC_H_ +#define CONCURRENCE_SYNC_H_ + +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +#include "executor/executor.h" + +class Scheduler; +class SyncTracker; + +class Sync { + public: + explicit Sync(SyncTracker *sync_tracker); + virtual ~Sync(); + + void AddWaiter(); + void ClearAndUnblockWaiters(); + + Scheduler *scheduler() { return scheduler_; } + + const std::vector &construction_backtrace() { + return construction_backtrace_; + } + + virtual bool IsOwned() = 0; + virtual bool IsOwnedBy(ThreadHandle handle) = 0; + // Some sync primitives have multiple owners + virtual const absl::flat_hash_set &owners() = 0; + + const absl::flat_hash_set &waiters() { return waiters_; } + + absl::flat_hash_map> &owner_backtraces() { + return owner_backtraces_; + } + + void HandleNestedWait(); + void HandleBlockedMainThread(); + + protected: + // Logs stacktraces and checks for deadlocks and nested locking + bool enable_debug_checks_; + + private: + void DetectDeadlock(); + + SyncTracker *sync_tracker_; + // Scheduler can be referenced through SyncTracker, but we maintain a direct + // reference for a slight performance boost. + Scheduler *scheduler_; + + absl::flat_hash_set waiters_; + std::vector construction_backtrace_; + absl::flat_hash_map> owner_backtraces_; +}; + +#endif diff --git a/third_party/concurrence/sync/tracker.cc b/third_party/concurrence/sync/tracker.cc new file mode 100644 index 0000000..13efdbb --- /dev/null +++ b/third_party/concurrence/sync/tracker.cc @@ -0,0 +1,145 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sync/tracker.h" + +#include +#include + +#include "sync/mutex.h" +#include "sync/rwlock.h" +#include "sync/sync.h" + +class Scheduler; + +SyncTracker::SyncTracker(Scheduler *scheduler) : scheduler_(scheduler) {} +SyncTracker::~SyncTracker() = default; + +VirtualMutex *SyncTracker::AllocateMutex() { + auto mutex = std::make_unique(this); + VirtualMutex *ptr = mutex.get(); + all_primitives_[ptr] = std::move(mutex); +#ifdef NDEBUG + session_tracked_primitives_.insert(ptr); +#endif + return ptr; +} + +void SyncTracker::FreeMutex(VirtualMutex *mutex) { + all_primitives_.erase(mutex); +#ifdef NDEBUG + session_tracked_primitives_.erase(mutex); +#endif +} + +VirtualRwLock *SyncTracker::AllocateRwLock() { + auto rwlock = std::make_unique(this); + VirtualRwLock *ptr = rwlock.get(); + all_primitives_[ptr] = std::move(rwlock); +#ifdef NDEBUG + session_tracked_primitives_.insert(ptr); +#endif + return ptr; +} + +void SyncTracker::FreeRwLock(VirtualRwLock *rwlock) { + all_primitives_.erase(rwlock); +#ifdef NDEBUG + session_tracked_primitives_.erase(rwlock); +#endif +} + +// Assistance routines to help debug blocked threads +absl::flat_hash_set SyncTracker::Owned(ThreadHandle handle) { + absl::flat_hash_set owned; + for (const auto &it : all_primitives_) { + Sync *sync = it.first; + if (sync->IsOwnedBy(handle)) { + owned.insert(sync); + } + } + return owned; +} + +Sync *SyncTracker::GetObjectWithWaiter(ThreadHandle handle) { + for (const auto &it : all_primitives_) { + Sync *sync = it.first; + if (sync->waiters().count(handle)) { + if (sync->IsOwnedBy(handle)) { + abort(); + } + return sync; + } + } + return nullptr; +} + +bool SyncTracker::FindDependencyPath(ThreadHandle source, + ThreadHandle destination, + std::deque &path) { + if (source == destination) { + if (!path.empty()) { + abort(); + } + path.push_front(destination); + return true; + } + + Sync *wanted = GetObjectWithWaiter(source); + // Not waiting on an object + if (!wanted) { + return false; + } + + for (ThreadHandle neighbor : wanted->owners()) { + if (FindDependencyPath(neighbor, destination, path)) { + path.push_front(source); + return true; + } + } + + return false; +} + +void SyncTracker::BeginSession() { +#ifdef NDEBUG + session_tracked_primitives_.clear(); +#endif +} + +void SyncTracker::ReportSession() { +#ifdef NDEBUG + if (!enable_debug_checks_ || session_tracked_primitives_.empty()) { + return; + } + printf( + "SyncTracker Report: there are %lu sync primitives that were never " + "freed.\n", + session_tracked_primitives_.size()); + + for (auto *sync : session_tracked_primitives_) { + printf("Sync primitive was never freed. Created here:\n"); + PrintBacktraceFromStack(sync->construction_backtrace()); + } +#endif +} + +void SyncTracker::FreeSession() { +#ifdef NDEBUG + for (auto *sync : session_tracked_primitives_) { + all_primitives_.erase(sync); + } + session_tracked_primitives_.clear(); +#endif +} diff --git a/third_party/concurrence/sync/tracker.h b/third_party/concurrence/sync/tracker.h new file mode 100644 index 0000000..0b610a9 --- /dev/null +++ b/third_party/concurrence/sync/tracker.h @@ -0,0 +1,65 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SYNC_TRACKER_H_ +#define SYNC_TRACKER_H_ + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +#include "executor/executor.h" + +class Scheduler; +class Sync; +class VirtualMutex; +class VirtualRwLock; + +class SyncTracker { + public: + explicit SyncTracker(Scheduler *scheduler); + ~SyncTracker(); + + VirtualMutex *AllocateMutex(); + void FreeMutex(VirtualMutex *mutex); + + VirtualRwLock *AllocateRwLock(); + void FreeRwLock(VirtualRwLock *rwlock); + + // Assistance routines to help debug blocked threads + absl::flat_hash_set Owned(ThreadHandle handle); + Sync *GetObjectWithWaiter(ThreadHandle handle); + + bool FindDependencyPath(ThreadHandle source, ThreadHandle destination, + std::deque &path); + + Scheduler *scheduler() { return scheduler_; }; + + void BeginSession(); + void ReportSession(); + void FreeSession(); + + private: + Scheduler *scheduler_; + absl::flat_hash_map> all_primitives_; +#ifdef NDEBUG + absl::flat_hash_set session_tracked_primitives_; +#endif +}; + +#endif diff --git a/third_party/libco/BUILD.bazel b/third_party/libco/BUILD.bazel new file mode 100644 index 0000000..2cb1539 --- /dev/null +++ b/third_party/libco/BUILD.bazel @@ -0,0 +1,13 @@ +cc_library( + name = "co", + srcs = [ + "libco.c", + ], + hdrs = [ + "amd64.c", + "libco.h", + "settings.h", + ], + includes = ["."], + visibility = ["//fuzz/executor:__pkg__"], +) diff --git a/third_party/libco/LICENSE b/third_party/libco/LICENSE new file mode 100644 index 0000000..7719049 --- /dev/null +++ b/third_party/libco/LICENSE @@ -0,0 +1,7 @@ +ISC License (ISC) + +Copyright byuu and the higan team + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/third_party/libco/README.md b/third_party/libco/README.md new file mode 100644 index 0000000..fabc5a1 --- /dev/null +++ b/third_party/libco/README.md @@ -0,0 +1,29 @@ +# libco + +libco is a cooperative multithreading library written in C89. + +Although cooperative multithreading is limited to a single CPU core, it scales substantially better than preemptive multithreading. + +For applications that need 100,000 or more context switches per second, the kernel overhead involved in preemptive multithreading can end up becoming the bottleneck in the application. libco can easily scale to 10,000,000 or more context switches per second. + +Ideal use cases include servers (HTTP, RDBMS) and emulators (CPU cores, etc.) + +It currently includes backends for: + +* x86 CPUs +* amd64 CPUs +* PowerPC CPUs +* PowerPC64 ELFv1 CPUs +* PowerPC64 ELFv2 CPUs +* ARM 32-bit CPUs +* ARM 64-bit (AArch64) CPUs +* POSIX platforms (setjmp) +* Windows platforms (fibers) + +See [doc/targets.md] for details. + +See [doc/usage.md] for documentation. + +## License + +libco is released under the ISC license. diff --git a/third_party/libco/aarch64.c b/third_party/libco/aarch64.c new file mode 100644 index 0000000..980686b --- /dev/null +++ b/third_party/libco/aarch64.c @@ -0,0 +1,106 @@ +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#include +#ifdef LIBCO_MPROTECT + #include + #include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +static thread_local unsigned long co_active_buffer[64]; +static thread_local cothread_t co_active_handle = 0; +static void (*co_swap)(cothread_t, cothread_t) = 0; + +#ifdef LIBCO_MPROTECT + alignas(4096) +#else + section(text) +#endif +static const uint32_t co_swap_function[1024] = { + 0x910003f0, /* mov x16,sp */ + 0xa9007830, /* stp x16,x30,[x1] */ + 0xa9407810, /* ldp x16,x30,[x0] */ + 0x9100021f, /* mov sp,x16 */ + 0xa9015033, /* stp x19,x20,[x1, 16] */ + 0xa9415013, /* ldp x19,x20,[x0, 16] */ + 0xa9025835, /* stp x21,x22,[x1, 32] */ + 0xa9425815, /* ldp x21,x22,[x0, 32] */ + 0xa9036037, /* stp x23,x24,[x1, 48] */ + 0xa9436017, /* ldp x23,x24,[x0, 48] */ + 0xa9046839, /* stp x25,x26,[x1, 64] */ + 0xa9446819, /* ldp x25,x26,[x0, 64] */ + 0xa905703b, /* stp x27,x28,[x1, 80] */ + 0xa945701b, /* ldp x27,x28,[x0, 80] */ + 0xf900303d, /* str x29, [x1, 96] */ + 0xf940301d, /* ldr x29, [x0, 96] */ + 0x6d072428, /* stp d8, d9, [x1,112] */ + 0x6d472408, /* ldp d8, d9, [x0,112] */ + 0x6d082c2a, /* stp d10,d11,[x1,128] */ + 0x6d482c0a, /* ldp d10,d11,[x0,128] */ + 0x6d09342c, /* stp d12,d13,[x1,144] */ + 0x6d49340c, /* ldp d12,d13,[x0,144] */ + 0x6d0a3c2e, /* stp d14,d15,[x1,160] */ + 0x6d4a3c0e, /* ldp d14,d15,[x0,160] */ + 0xd61f03c0, /* br x30 */ +}; + +static void co_init() { + #ifdef LIBCO_MPROTECT + unsigned long addr = (unsigned long)co_swap_function; + unsigned long base = addr - (addr % sysconf(_SC_PAGESIZE)); + unsigned long size = (addr - base) + sizeof co_swap_function; + mprotect((void*)base, size, PROT_READ | PROT_EXEC); + #endif +} + +cothread_t co_active() { + if(!co_active_handle) co_active_handle = &co_active_buffer; + return co_active_handle; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*entrypoint)(void)) { + unsigned long* handle; + if(!co_swap) { + co_init(); + co_swap = (void (*)(cothread_t, cothread_t))co_swap_function; + } + if(!co_active_handle) co_active_handle = &co_active_buffer; + + if(handle = (unsigned long*)memory) { + unsigned int offset = (size & ~15); + unsigned long* p = (unsigned long*)((unsigned char*)handle + offset); + handle[0] = (unsigned long)p; /* x16 (stack pointer) */ + handle[1] = (unsigned long)entrypoint; /* x30 (link register) */ + handle[12] = (unsigned long)p; /* x29 (frame pointer) */ + } + + return handle; +} + +cothread_t co_create(unsigned int size, void (*entrypoint)(void)) { + void* memory = LIBCO_MALLOC(size); + if(!memory) return (cothread_t)0; + return co_derive(memory, size, entrypoint); +} + +void co_delete(cothread_t handle) { + LIBCO_FREE(handle); +} + +void co_switch(cothread_t handle) { + cothread_t co_previous_handle = co_active_handle; + co_swap(co_active_handle = handle, co_previous_handle); +} + +int co_serializable() { + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/amd64.c b/third_party/libco/amd64.c new file mode 100644 index 0000000..fa45480 --- /dev/null +++ b/third_party/libco/amd64.c @@ -0,0 +1,162 @@ +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#ifdef __cplusplus +extern "C" { +#endif + +static thread_local long long co_active_buffer[64]; +static thread_local cothread_t co_active_handle = 0; +static void (*co_swap)(cothread_t, cothread_t) = 0; + +#ifdef LIBCO_MPROTECT + alignas(4096) +#else + section(text) +#endif +#ifdef _WIN32 + /* ABI: Win64 */ + static const unsigned char co_swap_function[4096] = { + 0x48, 0x89, 0x22, /* mov [rdx],rsp */ + 0x48, 0x8b, 0x21, /* mov rsp,[rcx] */ + 0x58, /* pop rax */ + 0x48, 0x89, 0x6a, 0x08, /* mov [rdx+ 8],rbp */ + 0x48, 0x89, 0x72, 0x10, /* mov [rdx+16],rsi */ + 0x48, 0x89, 0x7a, 0x18, /* mov [rdx+24],rdi */ + 0x48, 0x89, 0x5a, 0x20, /* mov [rdx+32],rbx */ + 0x4c, 0x89, 0x62, 0x28, /* mov [rdx+40],r12 */ + 0x4c, 0x89, 0x6a, 0x30, /* mov [rdx+48],r13 */ + 0x4c, 0x89, 0x72, 0x38, /* mov [rdx+56],r14 */ + 0x4c, 0x89, 0x7a, 0x40, /* mov [rdx+64],r15 */ + #if !defined(LIBCO_NO_SSE) + 0x0f, 0x29, 0x72, 0x50, /* movaps [rdx+ 80],xmm6 */ + 0x0f, 0x29, 0x7a, 0x60, /* movaps [rdx+ 96],xmm7 */ + 0x44, 0x0f, 0x29, 0x42, 0x70, /* movaps [rdx+112],xmm8 */ + 0x48, 0x83, 0xc2, 0x70, /* add rdx,112 */ + 0x44, 0x0f, 0x29, 0x4a, 0x10, /* movaps [rdx+ 16],xmm9 */ + 0x44, 0x0f, 0x29, 0x52, 0x20, /* movaps [rdx+ 32],xmm10 */ + 0x44, 0x0f, 0x29, 0x5a, 0x30, /* movaps [rdx+ 48],xmm11 */ + 0x44, 0x0f, 0x29, 0x62, 0x40, /* movaps [rdx+ 64],xmm12 */ + 0x44, 0x0f, 0x29, 0x6a, 0x50, /* movaps [rdx+ 80],xmm13 */ + 0x44, 0x0f, 0x29, 0x72, 0x60, /* movaps [rdx+ 96],xmm14 */ + 0x44, 0x0f, 0x29, 0x7a, 0x70, /* movaps [rdx+112],xmm15 */ + #endif + 0x48, 0x8b, 0x69, 0x08, /* mov rbp,[rcx+ 8] */ + 0x48, 0x8b, 0x71, 0x10, /* mov rsi,[rcx+16] */ + 0x48, 0x8b, 0x79, 0x18, /* mov rdi,[rcx+24] */ + 0x48, 0x8b, 0x59, 0x20, /* mov rbx,[rcx+32] */ + 0x4c, 0x8b, 0x61, 0x28, /* mov r12,[rcx+40] */ + 0x4c, 0x8b, 0x69, 0x30, /* mov r13,[rcx+48] */ + 0x4c, 0x8b, 0x71, 0x38, /* mov r14,[rcx+56] */ + 0x4c, 0x8b, 0x79, 0x40, /* mov r15,[rcx+64] */ + #if !defined(LIBCO_NO_SSE) + 0x0f, 0x28, 0x71, 0x50, /* movaps xmm6, [rcx+ 80] */ + 0x0f, 0x28, 0x79, 0x60, /* movaps xmm7, [rcx+ 96] */ + 0x44, 0x0f, 0x28, 0x41, 0x70, /* movaps xmm8, [rcx+112] */ + 0x48, 0x83, 0xc1, 0x70, /* add rcx,112 */ + 0x44, 0x0f, 0x28, 0x49, 0x10, /* movaps xmm9, [rcx+ 16] */ + 0x44, 0x0f, 0x28, 0x51, 0x20, /* movaps xmm10,[rcx+ 32] */ + 0x44, 0x0f, 0x28, 0x59, 0x30, /* movaps xmm11,[rcx+ 48] */ + 0x44, 0x0f, 0x28, 0x61, 0x40, /* movaps xmm12,[rcx+ 64] */ + 0x44, 0x0f, 0x28, 0x69, 0x50, /* movaps xmm13,[rcx+ 80] */ + 0x44, 0x0f, 0x28, 0x71, 0x60, /* movaps xmm14,[rcx+ 96] */ + 0x44, 0x0f, 0x28, 0x79, 0x70, /* movaps xmm15,[rcx+112] */ + #endif + 0xff, 0xe0, /* jmp rax */ + }; + + #include + + static void co_init() { + #ifdef LIBCO_MPROTECT + DWORD old_privileges; + VirtualProtect((void*)co_swap_function, sizeof co_swap_function, PAGE_EXECUTE_READ, &old_privileges); + #endif + } +#else + /* ABI: SystemV */ + static const unsigned char co_swap_function[4096] = { + 0x48, 0x89, 0x26, /* mov [rsi],rsp */ + 0x48, 0x8b, 0x27, /* mov rsp,[rdi] */ + 0x58, /* pop rax */ + 0x48, 0x89, 0x6e, 0x08, /* mov [rsi+ 8],rbp */ + 0x48, 0x89, 0x5e, 0x10, /* mov [rsi+16],rbx */ + 0x4c, 0x89, 0x66, 0x18, /* mov [rsi+24],r12 */ + 0x4c, 0x89, 0x6e, 0x20, /* mov [rsi+32],r13 */ + 0x4c, 0x89, 0x76, 0x28, /* mov [rsi+40],r14 */ + 0x4c, 0x89, 0x7e, 0x30, /* mov [rsi+48],r15 */ + 0x48, 0x8b, 0x6f, 0x08, /* mov rbp,[rdi+ 8] */ + 0x48, 0x8b, 0x5f, 0x10, /* mov rbx,[rdi+16] */ + 0x4c, 0x8b, 0x67, 0x18, /* mov r12,[rdi+24] */ + 0x4c, 0x8b, 0x6f, 0x20, /* mov r13,[rdi+32] */ + 0x4c, 0x8b, 0x77, 0x28, /* mov r14,[rdi+40] */ + 0x4c, 0x8b, 0x7f, 0x30, /* mov r15,[rdi+48] */ + 0xff, 0xe0, /* jmp rax */ + }; + + #ifdef LIBCO_MPROTECT + #include + #include + #endif + + static void co_init() { + #ifdef LIBCO_MPROTECT + unsigned long long addr = (unsigned long long)co_swap_function; + unsigned long long base = addr - (addr % sysconf(_SC_PAGESIZE)); + unsigned long long size = (addr - base) + sizeof co_swap_function; + mprotect((void*)base, size, PROT_READ | PROT_EXEC); + #endif + } +#endif + +static void crash() { + LIBCO_ASSERT(0); /* called only if cothread_t entrypoint returns */ +} + +cothread_t co_active() { + if(!co_active_handle) co_active_handle = &co_active_buffer; + return co_active_handle; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*entrypoint)(void)) { + cothread_t handle; + if(!co_swap) { + co_init(); + co_swap = (void (*)(cothread_t, cothread_t))co_swap_function; + } + if(!co_active_handle) co_active_handle = &co_active_buffer; + + if(handle = (cothread_t)memory) { + unsigned int offset = (size & ~15) - 32; + long long *p = (long long*)((char*)handle + offset); /* seek to top of stack */ + *--p = (long long)crash; /* crash if entrypoint returns */ + *--p = (long long)entrypoint; /* start of function */ + *(long long*)handle = (long long)p; /* stack pointer */ + } + + return handle; +} + +cothread_t co_create(unsigned int size, void (*entrypoint)(void)) { + void* memory = LIBCO_MALLOC(size); + if(!memory) return (cothread_t)0; + return co_derive(memory, size, entrypoint); +} + +void co_delete(cothread_t handle) { + LIBCO_FREE(handle); +} + +void co_switch(cothread_t handle) { + register cothread_t co_previous_handle = co_active_handle; + co_swap(co_active_handle = handle, co_previous_handle); +} + +int co_serializable() { + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/arm.c b/third_party/libco/arm.c new file mode 100644 index 0000000..6f0aa24 --- /dev/null +++ b/third_party/libco/arm.c @@ -0,0 +1,82 @@ +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#ifdef LIBCO_MPROTECT + #include + #include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +static thread_local unsigned long co_active_buffer[64]; +static thread_local cothread_t co_active_handle = 0; +static void (*co_swap)(cothread_t, cothread_t) = 0; + +#ifdef LIBCO_MPROTECT + alignas(4096) +#else + section(text) +#endif +static const unsigned long co_swap_function[1024] = { + 0xe8a16ff0, /* stmia r1!, {r4-r11,sp,lr} */ + 0xe8b0aff0, /* ldmia r0!, {r4-r11,sp,pc} */ + 0xe12fff1e, /* bx lr */ +}; + +static void co_init() { + #ifdef LIBCO_MPROTECT + unsigned long addr = (unsigned long)co_swap_function; + unsigned long base = addr - (addr % sysconf(_SC_PAGESIZE)); + unsigned long size = (addr - base) + sizeof co_swap_function; + mprotect((void*)base, size, PROT_READ | PROT_EXEC); + #endif +} + +cothread_t co_active() { + if(!co_active_handle) co_active_handle = &co_active_buffer; + return co_active_handle; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*entrypoint)(void)) { + unsigned long* handle; + if(!co_swap) { + co_init(); + co_swap = (void (*)(cothread_t, cothread_t))co_swap_function; + } + if(!co_active_handle) co_active_handle = &co_active_buffer; + + if(handle = (unsigned long*)memory) { + unsigned int offset = (size & ~15); + unsigned long* p = (unsigned long*)((unsigned char*)handle + offset); + handle[8] = (unsigned long)p; + handle[9] = (unsigned long)entrypoint; + } + + return handle; +} + +cothread_t co_create(unsigned int size, void (*entrypoint)(void)) { + void* memory = LIBCO_MALLOC(size); + if(!memory) return (cothread_t)0; + return co_derive(memory, size, entrypoint); +} + +void co_delete(cothread_t handle) { + LIBCO_FREE(handle); +} + +void co_switch(cothread_t handle) { + cothread_t co_previous_handle = co_active_handle; + co_swap(co_active_handle = handle, co_previous_handle); +} + +int co_serializable() { + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/doc/examples/.gitignore b/third_party/libco/doc/examples/.gitignore new file mode 100644 index 0000000..d3db256 --- /dev/null +++ b/third_party/libco/doc/examples/.gitignore @@ -0,0 +1,4 @@ +test_args +test_serialization +test_timing +*.o diff --git a/third_party/libco/doc/examples/build.bat b/third_party/libco/doc/examples/build.bat new file mode 100755 index 0000000..f8fecb2 --- /dev/null +++ b/third_party/libco/doc/examples/build.bat @@ -0,0 +1,8 @@ +cc -O3 -fomit-frame-pointer -I../.. -o libco.o -c ../../libco.c +c++ -O3 -fomit-frame-pointer -I../.. -c test_timing.cpp +c++ -O3 -fomit-frame-pointer -o test_timing libco.o test_timing.o +c++ -O3 -fomit-frame-pointer -I../.. -c test_args.cpp +c++ -O3 -fomit-frame-pointer -o test_args libco.o test_args.o +c++ -O3 -fomit-frame-pointer -I../.. -c test_serialization.cpp +c++ -O3 -fomit-frame-pointer -o test_serialization libco.o test_serialization.o +@del *.o diff --git a/third_party/libco/doc/examples/build.sh b/third_party/libco/doc/examples/build.sh new file mode 100755 index 0000000..dd187a9 --- /dev/null +++ b/third_party/libco/doc/examples/build.sh @@ -0,0 +1,8 @@ +cc -O3 -fomit-frame-pointer -I../.. -o libco.o -c ../../libco.c +c++ -O3 -fomit-frame-pointer -I../.. -c test_timing.cpp +c++ -O3 -fomit-frame-pointer -o test_timing libco.o test_timing.o +c++ -O3 -fomit-frame-pointer -I../.. -c test_args.cpp +c++ -O3 -fomit-frame-pointer -o test_args libco.o test_args.o +c++ -O3 -fomit-frame-pointer -I../.. -c test_serialization.cpp +c++ -O3 -fomit-frame-pointer -o test_serialization libco.o test_serialization.o +rm -f *.o diff --git a/third_party/libco/doc/examples/test.h b/third_party/libco/doc/examples/test.h new file mode 100644 index 0000000..753ea52 --- /dev/null +++ b/third_party/libco/doc/examples/test.h @@ -0,0 +1,6 @@ +#include +#include +#include +#include + +#include diff --git a/third_party/libco/doc/examples/test_args.cpp b/third_party/libco/doc/examples/test_args.cpp new file mode 100644 index 0000000..1e6e2bb --- /dev/null +++ b/third_party/libco/doc/examples/test_args.cpp @@ -0,0 +1,76 @@ +/***** + * cothread parameterized function example + ***** + * entry point to cothreads cannot take arguments. + * this is due to portability issues: each processor, + * operating system, programming language and compiler + * can use different parameter passing methods, so + * arguments to the cothread entry points were omitted. + * + * however, the behavior can easily be simulated by use + * of a specialized co_switch to set global parameters to + * be used as function arguments. + * + * in this way, with a bit of extra red tape, one gains + * even more flexibility than would be possible with a + * fixed argument list entry point, such as void (*)(void*), + * as any number of arguments can be used. + * + * this also eliminates race conditions where a pointer + * passed to co_create may have changed or become invalidated + * before call to co_switch, as said pointer would now be set + * when calling co_switch, instead. + *****/ + +#include "test.h" + +cothread_t thread[3]; + +namespace co_arg { + int param_x; + int param_y; +}; + +//one could also call this co_init or somesuch if they preferred ... +void co_switch(cothread_t thread, int param_x, int param_y) { + co_arg::param_x = param_x; + co_arg::param_y = param_y; + co_switch(thread); +} + +void co_entrypoint() { +int param_x = co_arg::param_x; +int param_y = co_arg::param_y; + printf("co_entrypoint(%d, %d)\n", param_x, param_y); + co_switch(thread[0]); + +//co_arg::param_x will change here (due to co_switch(cothread_t, int, int) call changing values), +//however, param_x and param_y will persist as they are thread local + + printf("co_entrypoint(%d, %d)\n", param_x, param_y); + co_switch(thread[0]); + throw; +} + +int main() { + printf("cothread parameterized function example\n\n"); + + thread[0] = co_active(); + thread[1] = co_create(65536, co_entrypoint); + thread[2] = co_create(65536, co_entrypoint); + +//use specialized co_switch(cothread_t, int, int) for initial co_switch call + co_switch(thread[1], 1, 2); + co_switch(thread[2], 4, 8); + +//after first call, entry point arguments have been initialized, standard +//co_switch(cothread_t) can be used from now on + co_switch(thread[2]); + co_switch(thread[1]); + + printf("\ndone\n"); +#if defined(_MSC_VER) || defined(__DJGPP__) + getch(); +#endif + return 0; +} diff --git a/third_party/libco/doc/examples/test_serialization.cpp b/third_party/libco/doc/examples/test_serialization.cpp new file mode 100644 index 0000000..15fbdb1 --- /dev/null +++ b/third_party/libco/doc/examples/test_serialization.cpp @@ -0,0 +1,117 @@ +#include "test.h" +#include +#include + +namespace Thread { + cothread_t host; + cothread_t cpu; + cothread_t apu; +} + +namespace Buffer { + uint8_t cpu[65536]; + uint8_t apu[65536]; +} + +namespace Memory { + uint8_t* buffer; +} + +struct CPU { + static auto Enter() -> void; + auto main() -> void; + auto sub() -> void; + auto leaf() -> void; +} cpu; + +struct APU { + static auto Enter() -> void; + auto main() -> void; + auto sub() -> void; + auto leaf() -> void; +} apu; + +auto CPU::Enter() -> void { + while(true) cpu.main(); +} + +auto CPU::main() -> void { + printf("2\n"); + sub(); +} + +auto CPU::sub() -> void { + co_switch(Thread::apu); + printf("4\n"); + leaf(); +} + +auto CPU::leaf() -> void { + int x = 42; + co_switch(Thread::host); + printf("6\n"); + co_switch(Thread::apu); + printf("8 (%d)\n", x); + co_switch(Thread::host); +} + +auto APU::Enter() -> void { + while(true) apu.main(); +} + +auto APU::main() -> void { + printf("3\n"); + sub(); +} + +auto APU::sub() -> void { + co_switch(Thread::cpu); + printf("7\n"); + leaf(); +} + +auto APU::leaf() -> void { + co_switch(Thread::cpu); +} + +auto main() -> int { + if(!co_serializable()) { + printf("This implementation does not support serialization\n"); + return 1; + } + + Memory::buffer = (uint8_t*)mmap( + (void*)0x10'0000'0000, 2 * 65536, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 + ); + Memory::buffer[0] = 42; + printf("%p (%u)\n", Memory::buffer, Memory::buffer[0]); + + Thread::host = co_active(); + Thread::cpu = co_derive((void*)(Memory::buffer + 0 * 65536), 65536, CPU::Enter); + Thread::apu = co_derive((void*)(Memory::buffer + 1 * 65536), 65536, APU::Enter); + + printf("1\n"); + co_switch(Thread::cpu); + + printf("5\n"); + memcpy(Buffer::cpu, Thread::cpu, 65536); + memcpy(Buffer::apu, Thread::apu, 65536); + co_switch(Thread::cpu); + + Thread::cpu = nullptr; + Thread::apu = nullptr; + Thread::cpu = co_derive((void*)(Memory::buffer + 0 * 65536), 65536, CPU::Enter); + Thread::apu = co_derive((void*)(Memory::buffer + 1 * 65536), 65536, APU::Enter); + + printf("9\n"); + memcpy(Thread::cpu, Buffer::cpu, 65536); + memcpy(Thread::apu, Buffer::apu, 65536); + co_switch(Thread::cpu); + + Thread::cpu = nullptr; + Thread::apu = nullptr; + munmap((void*)0x900000000, 2 * 65536); + return 0; +} diff --git a/third_party/libco/doc/examples/test_timing.cpp b/third_party/libco/doc/examples/test_timing.cpp new file mode 100644 index 0000000..0232e47 --- /dev/null +++ b/third_party/libco/doc/examples/test_timing.cpp @@ -0,0 +1,52 @@ +#include "test.h" +enum { Iterations = 500000000 }; + +namespace thread { + cothread_t x; + cothread_t y; + volatile int counter; +} + +void co_timingtest() { + for(;;) { + thread::counter++; + co_switch(thread::x); + } +} + +void sub_timingtest() { + thread::counter++; +} + +int main() { + printf("context-switching timing test\n\n"); + time_t start, end; + int i, t1, t2; + + start = clock(); + for(thread::counter = 0, i = 0; i < Iterations; i++) { + sub_timingtest(); + } + end = clock(); + + t1 = (int)difftime(end, start); + printf("%2.3f seconds per 50 million subroutine calls (%d iterations)\n", (float)t1 / CLOCKS_PER_SEC, thread::counter); + + thread::x = co_active(); + thread::y = co_create(65536, co_timingtest); + + start = clock(); + for(thread::counter = 0, i = 0; i < Iterations; i++) { + co_switch(thread::y); + } + end = clock(); + + co_delete(thread::y); + + t2 = (int)difftime(end, start); + printf("%2.3f seconds per 100 million co_switch calls (%d iterations)\n", (float)t2 / CLOCKS_PER_SEC, thread::counter); + + printf("co_switch skew = %fx\n\n", (double)t2 / (double)t1); + return 0; +} + diff --git a/third_party/libco/doc/targets.md b/third_party/libco/doc/targets.md new file mode 100644 index 0000000..d01dcbe --- /dev/null +++ b/third_party/libco/doc/targets.md @@ -0,0 +1,68 @@ +# Supported targets +In the following lists, supported targets are only those that have been tested +and confirmed working. It is quite possible that libco will work on more +processors, compilers and operating systems than those listed below. + +The "Overhead" is the cost of switching co-routines, as compared to an ordinary +C function call. + +## libco.x86 +* **Overhead:** ~5x +* **Supported processor(s):** 32-bit x86 +* **Supported compiler(s):** any +* **Supported operating system(s):** + * Windows + * Mac OS X + * Linux + * BSD + +## libco.amd64 +* **Overhead:** ~10x (Windows), ~6x (all other platforms) +* **Supported processor(s):** 64-bit amd64 +* **Supported compiler(s):** any +* **Supported operating system(s):** + * Windows + * Mac OS X + * Linux + * BSD + +## libco.ppc +* **Overhead:** ~20x +* **Supported processor(s):** 32-bit PowerPC, 64-bit PowerPC +* **Supported compiler(s):** GNU GCC +* **Supported operating system(s):** + * Mac OS X + * Linux + * BSD + * Playstation 3 + +**Note:** this module contains compiler flags to enable/disable FPU and Altivec +support. + +## libco.fiber +This uses Windows' "fibers" API. +* **Overhead:** ~15x +* **Supported processor(s):** Processor independent +* **Supported compiler(s):** any +* **Supported operating system(s):** + * Windows + +## libco.sjlj +This uses the C standard library's `setjump`/`longjmp` APIs. +* **Overhead:** ~30x +* **Supported processor(s):** Processor independent +* **Supported compiler(s):** any +* **Supported operating system(s):** + * Mac OS X + * Linux + * BSD + * Solaris + +## libco.ucontext +This uses the POSIX "ucontext" API. +* **Overhead:** ***~300x*** +* **Supported processor(s):** Processor independent +* **Supported compiler(s):** any +* **Supported operating system(s):** + * Linux + * BSD diff --git a/third_party/libco/doc/usage.md b/third_party/libco/doc/usage.md new file mode 100644 index 0000000..a3b0f04 --- /dev/null +++ b/third_party/libco/doc/usage.md @@ -0,0 +1,150 @@ +# License +libco is released under the ISC license. + +# Foreword +libco is a cross-platform, permissively licensed implementation of +cooperative-multithreading; a feature that is sorely lacking from the ISO C/C++ +standard. + +The library is designed for maximum speed and portability, and not for safety or +features. If safety or extra functionality is desired, a wrapper API can easily +be written to encapsulate all library functions. + +Behavior of executing operations that are listed as not permitted below result +in undefined behavior. They may work anyway, they may cause undesired / unknown +behavior, or they may crash the program entirely. + +The goal of this library was to simplify the base API as much as possible, +implementing only that which cannot be implemented using pure C. Additional +functionality after this would only complicate ports of this library to new +platforms. + +# Porting +This document is included as a reference for porting libco. Please submit any +ports you create to me, so that libco can become more useful. Please note that +since libco is permissively licensed, you must submit your code as a work of the +public domain in order for it to be included in the official distribution. + +Full credit will be given in the source code of the official release. Please +do not bother submitting code to me under any other license -- including GPL, +LGPL, BSD or CC -- I am not interested in creating a library with multiple +different licenses depending on which targets are used. + +Note that there are a variety of compile-time options in `settings.h`, +so if you want to use libco on a platform where it is not supported by default, +you may be able to configure the implementation appropriately without having +to make a whole new port. + +# Synopsis +```c +typedef void* cothread_t; + +cothread_t co_active(); +cothread_t co_create(unsigned int heapsize, void (*coentry)(void)); +void co_delete(cothread_t cothread); +void co_switch(cothread_t cothread); +``` + +# Usage +## cothread_t +```c +typedef void* cothread_t; +``` +Handle to cothread. + +Handle must be of type `void*`. + +A value of null (0) indicates an uninitialized or invalid handle, whereas a +non-zero value indicates a valid handle. A valid handle is backed by execution +state to which the execution can be co_switch()ed to. + +## co_active +```c +cothread_t co_active(); +``` +Return handle to current cothread. + +Note that the handle is valid even if the function is called from a non-cothread +context. To achieve this, we save the execution state in an internal buffer, +instead of using the user-provided memory. Since this handle is valid, it can +be used to co_switch to this context from another cothread. In multi-threaded +applications, make sure to not switch non-cothread context across CPU cores, +to prevent any possible conflicts with the OS scheduler. + +## co_derive +```c +cothread_t co_derive(void* memory, + unsigned int heapsize, + void (*coentry)(void)); +``` +Initializes new cothread. + +This function is identical to `co_create`, only it attempts to use the provided +memory instead of allocating new memory on the heap. Please note that certain +implementations (currently only Windows Fibers) cannot be created using existing +memory, and as such, this function will fail. + +## co_create +```c +cothread_t co_create(unsigned int heapsize, + void (*coentry)(void)); +``` +Create new cothread. + +`heapsize` is the amount of memory allocated for the cothread stack, specified +in bytes. This is unfortunately impossible to make fully portable. It is +recommended to specify sizes using `n * sizeof(void*)`. It is better to err +on the side of caution and allocate more memory than will be needed to ensure +compatibility with other platforms, within reason. A typical heapsize for a +32-bit architecture is ~1MB. + +When the new cothread is first called, program execution jumps to coentry. +This function does not take any arguments, due to portability issues with +passing function arguments. However, arguments can be simulated by the use +of global variables, which can be set before the first call to each cothread. + +`coentry()` must not return, and should end with an appropriate `co_switch()` +statement. Behavior is undefined if entry point returns normally. + +Library is responsible for allocating cothread stack memory, to free +the user from needing to allocate special memory capable of being used +as program stack memory on platforms where this is required. + +User is always responsible for deleting cothreads with `co_delete()`. + +Return value of `null` (0) indicates cothread creation failed. + +## co_delete +```c +void co_delete(cothread_t cothread); +``` +Delete specified cothread. + +`null` (0) or invalid cothread handle is not allowed. + +Passing handle of active cothread to this function is not allowed. + +Passing handle of primary cothread is not allowed. + +## co_serializable + +```c +int co_serializable(void); +``` + +Returns non-zero if the implementation keeps the entire coroutine state in the +buffer passed to `co_derive()`. That is, if `co_serializable()` returns +non-zero, and if your cothread does not modify the heap or any process-wide +state, then you can "snapshot" the cothread's state by taking a copy of the +buffer originally passed to `co_derive()`, and "restore" a previous state +by copying the snapshot back into the buffer it came from. + +## co_switch +```c +void co_switch(cothread_t cothread); +``` +Switch to specified cothread. + +`null` (0) or invalid cothread handle is not allowed. + +Passing handle of active cothread to this function is not allowed. diff --git a/third_party/libco/fiber.c b/third_party/libco/fiber.c new file mode 100644 index 0000000..3bd2c4e --- /dev/null +++ b/third_party/libco/fiber.c @@ -0,0 +1,55 @@ +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#define WINVER 0x0400 +#define _WIN32_WINNT 0x0400 +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static thread_local cothread_t co_active_ = 0; + +static void __stdcall co_thunk(void* coentry) { + ((void (*)(void))coentry)(); +} + +cothread_t co_active() { + if(!co_active_) { + ConvertThreadToFiber(0); + co_active_ = GetCurrentFiber(); + } + return co_active_; +} + +cothread_t co_derive(void* memory, unsigned int heapsize, void (*coentry)(void)) { + /* Windows fibers do not allow users to supply their own memory */ + return (cothread_t)0; +} + +cothread_t co_create(unsigned int heapsize, void (*coentry)(void)) { + if(!co_active_) { + ConvertThreadToFiber(0); + co_active_ = GetCurrentFiber(); + } + return (cothread_t)CreateFiber(heapsize, co_thunk, (void*)coentry); +} + +void co_delete(cothread_t cothread) { + DeleteFiber(cothread); +} + +void co_switch(cothread_t cothread) { + co_active_ = cothread; + SwitchToFiber(cothread); +} + +int co_serializable() { + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/libco.c b/third_party/libco/libco.c new file mode 100644 index 0000000..21fe4ca --- /dev/null +++ b/third_party/libco/libco.c @@ -0,0 +1,37 @@ +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wparentheses" + + /* placing code in section(text) does not mark it executable with Clang. */ + #undef LIBCO_MPROTECT + #define LIBCO_MPROTECT +#endif + +#if defined(__clang__) || defined(__GNUC__) + #if defined(__i386__) + #include "x86.c" + #elif defined(__amd64__) + #include "amd64.c" + #elif defined(__arm__) + #include "arm.c" + #elif defined(__aarch64__) + #include "aarch64.c" + #elif defined(__powerpc64__) && defined(_CALL_ELF) && _CALL_ELF == 2 + #include "ppc64v2.c" + #elif defined(_ARCH_PPC) && !defined(__LITTLE_ENDIAN__) + #include "ppc.c" + #elif defined(_WIN32) + #include "fiber.c" + #else + #include "sjlj.c" + #endif +#elif defined(_MSC_VER) + #if defined(_M_IX86) + #include "x86.c" + #elif defined(_M_AMD64) + #include "amd64.c" + #else + #include "fiber.c" + #endif +#else + #error "libco: unsupported processor, compiler or operating system" +#endif diff --git a/third_party/libco/libco.h b/third_party/libco/libco.h new file mode 100644 index 0000000..633cd54 --- /dev/null +++ b/third_party/libco/libco.h @@ -0,0 +1,28 @@ +/* + libco v20 (2019-10-16) + author: byuu + license: ISC +*/ + +#ifndef LIBCO_H +#define LIBCO_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cothread_t; + +cothread_t co_active(void); +cothread_t co_derive(void*, unsigned int, void (*)(void)); +cothread_t co_create(unsigned int, void (*)(void)); +void co_delete(cothread_t); +void co_switch(cothread_t); +int co_serializable(void); + +#ifdef __cplusplus +} +#endif + +/* ifndef LIBCO_H */ +#endif diff --git a/third_party/libco/ppc.c b/third_party/libco/ppc.c new file mode 100644 index 0000000..a39d558 --- /dev/null +++ b/third_party/libco/ppc.c @@ -0,0 +1,430 @@ +/* ppc64le (ELFv2) is not currently supported */ + +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#include +#include + +#ifdef LIBCO_MPROTECT + #include + #include +#endif + +/* state format (offsets in 32-bit words) + + +0 pointer to swap code + rest of function descriptor for entry function + +8 PC ++10 SP + special registers + GPRs + FPRs + VRs + stack +*/ + +enum { state_size = 1024 }; +enum { above_stack = 2048 }; +enum { stack_align = 256 }; + +static thread_local cothread_t co_active_handle = 0; + +/* determine environment */ + +#define LIBCO_PPC64 (_ARCH_PPC64 || __PPC64__ || __ppc64__ || __powerpc64__) + +/* whether function calls are indirect through a descriptor, or are directly to function */ +#ifndef LIBCO_PPCDESC + #if !_CALL_SYSV && (_CALL_AIX || _CALL_AIXDESC || (LIBCO_PPC64 && (!defined(_CALL_ELF) || _CALL_ELF == 1))) + #define LIBCO_PPCDESC 1 + #endif +#endif + +#ifdef LIBCO_MPROTECT + alignas(4096) +#else + section(text) +#endif +static const uint32_t libco_ppc_code[1024] = { + #if LIBCO_PPC64 + 0x7d000026, /* mfcr r8 */ + 0xf8240028, /* std r1,40(r4) */ + 0x7d2802a6, /* mflr r9 */ + 0xf9c40048, /* std r14,72(r4) */ + 0xf9e40050, /* std r15,80(r4) */ + 0xfa040058, /* std r16,88(r4) */ + 0xfa240060, /* std r17,96(r4) */ + 0xfa440068, /* std r18,104(r4) */ + 0xfa640070, /* std r19,112(r4) */ + 0xfa840078, /* std r20,120(r4) */ + 0xfaa40080, /* std r21,128(r4) */ + 0xfac40088, /* std r22,136(r4) */ + 0xfae40090, /* std r23,144(r4) */ + 0xfb040098, /* std r24,152(r4) */ + 0xfb2400a0, /* std r25,160(r4) */ + 0xfb4400a8, /* std r26,168(r4) */ + 0xfb6400b0, /* std r27,176(r4) */ + 0xfb8400b8, /* std r28,184(r4) */ + 0xfba400c0, /* std r29,192(r4) */ + 0xfbc400c8, /* std r30,200(r4) */ + 0xfbe400d0, /* std r31,208(r4) */ + 0xf9240020, /* std r9,32(r4) */ + 0xe8e30020, /* ld r7,32(r3) */ + 0xe8230028, /* ld r1,40(r3) */ + 0x48000009, /* bl 1 */ + 0x7fe00008, /* trap */ + 0x91040030, /*1:stw r8,48(r4) */ + 0x80c30030, /* lwz r6,48(r3) */ + 0x7ce903a6, /* mtctr r7 */ + 0xe9c30048, /* ld r14,72(r3) */ + 0xe9e30050, /* ld r15,80(r3) */ + 0xea030058, /* ld r16,88(r3) */ + 0xea230060, /* ld r17,96(r3) */ + 0xea430068, /* ld r18,104(r3) */ + 0xea630070, /* ld r19,112(r3) */ + 0xea830078, /* ld r20,120(r3) */ + 0xeaa30080, /* ld r21,128(r3) */ + 0xeac30088, /* ld r22,136(r3) */ + 0xeae30090, /* ld r23,144(r3) */ + 0xeb030098, /* ld r24,152(r3) */ + 0xeb2300a0, /* ld r25,160(r3) */ + 0xeb4300a8, /* ld r26,168(r3) */ + 0xeb6300b0, /* ld r27,176(r3) */ + 0xeb8300b8, /* ld r28,184(r3) */ + 0xeba300c0, /* ld r29,192(r3) */ + 0xebc300c8, /* ld r30,200(r3) */ + 0xebe300d0, /* ld r31,208(r3) */ + 0x7ccff120, /* mtcr r6 */ + #else + 0x7d000026, /* mfcr r8 */ + 0x90240028, /* stw r1,40(r4) */ + 0x7d2802a6, /* mflr r9 */ + 0x91a4003c, /* stw r13,60(r4) */ + 0x91c40040, /* stw r14,64(r4) */ + 0x91e40044, /* stw r15,68(r4) */ + 0x92040048, /* stw r16,72(r4) */ + 0x9224004c, /* stw r17,76(r4) */ + 0x92440050, /* stw r18,80(r4) */ + 0x92640054, /* stw r19,84(r4) */ + 0x92840058, /* stw r20,88(r4) */ + 0x92a4005c, /* stw r21,92(r4) */ + 0x92c40060, /* stw r22,96(r4) */ + 0x92e40064, /* stw r23,100(r4) */ + 0x93040068, /* stw r24,104(r4) */ + 0x9324006c, /* stw r25,108(r4) */ + 0x93440070, /* stw r26,112(r4) */ + 0x93640074, /* stw r27,116(r4) */ + 0x93840078, /* stw r28,120(r4) */ + 0x93a4007c, /* stw r29,124(r4) */ + 0x93c40080, /* stw r30,128(r4) */ + 0x93e40084, /* stw r31,132(r4) */ + 0x91240020, /* stw r9,32(r4) */ + 0x80e30020, /* lwz r7,32(r3) */ + 0x80230028, /* lwz r1,40(r3) */ + 0x48000009, /* bl 1 */ + 0x7fe00008, /* trap */ + 0x91040030, /*1:stw r8,48(r4) */ + 0x80c30030, /* lwz r6,48(r3) */ + 0x7ce903a6, /* mtctr r7 */ + 0x81a3003c, /* lwz r13,60(r3) */ + 0x81c30040, /* lwz r14,64(r3) */ + 0x81e30044, /* lwz r15,68(r3) */ + 0x82030048, /* lwz r16,72(r3) */ + 0x8223004c, /* lwz r17,76(r3) */ + 0x82430050, /* lwz r18,80(r3) */ + 0x82630054, /* lwz r19,84(r3) */ + 0x82830058, /* lwz r20,88(r3) */ + 0x82a3005c, /* lwz r21,92(r3) */ + 0x82c30060, /* lwz r22,96(r3) */ + 0x82e30064, /* lwz r23,100(r3) */ + 0x83030068, /* lwz r24,104(r3) */ + 0x8323006c, /* lwz r25,108(r3) */ + 0x83430070, /* lwz r26,112(r3) */ + 0x83630074, /* lwz r27,116(r3) */ + 0x83830078, /* lwz r28,120(r3) */ + 0x83a3007c, /* lwz r29,124(r3) */ + 0x83c30080, /* lwz r30,128(r3) */ + 0x83e30084, /* lwz r31,132(r3) */ + 0x7ccff120, /* mtcr r6 */ + #endif + + #ifndef LIBCO_PPC_NOFP + 0xd9c400e0, /* stfd f14,224(r4) */ + 0xd9e400e8, /* stfd f15,232(r4) */ + 0xda0400f0, /* stfd f16,240(r4) */ + 0xda2400f8, /* stfd f17,248(r4) */ + 0xda440100, /* stfd f18,256(r4) */ + 0xda640108, /* stfd f19,264(r4) */ + 0xda840110, /* stfd f20,272(r4) */ + 0xdaa40118, /* stfd f21,280(r4) */ + 0xdac40120, /* stfd f22,288(r4) */ + 0xdae40128, /* stfd f23,296(r4) */ + 0xdb040130, /* stfd f24,304(r4) */ + 0xdb240138, /* stfd f25,312(r4) */ + 0xdb440140, /* stfd f26,320(r4) */ + 0xdb640148, /* stfd f27,328(r4) */ + 0xdb840150, /* stfd f28,336(r4) */ + 0xdba40158, /* stfd f29,344(r4) */ + 0xdbc40160, /* stfd f30,352(r4) */ + 0xdbe40168, /* stfd f31,360(r4) */ + 0xc9c300e0, /* lfd f14,224(r3) */ + 0xc9e300e8, /* lfd f15,232(r3) */ + 0xca0300f0, /* lfd f16,240(r3) */ + 0xca2300f8, /* lfd f17,248(r3) */ + 0xca430100, /* lfd f18,256(r3) */ + 0xca630108, /* lfd f19,264(r3) */ + 0xca830110, /* lfd f20,272(r3) */ + 0xcaa30118, /* lfd f21,280(r3) */ + 0xcac30120, /* lfd f22,288(r3) */ + 0xcae30128, /* lfd f23,296(r3) */ + 0xcb030130, /* lfd f24,304(r3) */ + 0xcb230138, /* lfd f25,312(r3) */ + 0xcb430140, /* lfd f26,320(r3) */ + 0xcb630148, /* lfd f27,328(r3) */ + 0xcb830150, /* lfd f28,336(r3) */ + 0xcba30158, /* lfd f29,344(r3) */ + 0xcbc30160, /* lfd f30,352(r3) */ + 0xcbe30168, /* lfd f31,360(r3) */ + #endif + + #ifdef __ALTIVEC__ + 0x7ca042a6, /* mfvrsave r5 */ + 0x39040180, /* addi r8,r4,384 */ + 0x39240190, /* addi r9,r4,400 */ + 0x70a00fff, /* andi. r0,r5,4095 */ + 0x90a40034, /* stw r5,52(r4) */ + 0x4182005c, /* beq- 2 */ + 0x7e8041ce, /* stvx v20,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7ea049ce, /* stvx v21,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7ec041ce, /* stvx v22,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7ee049ce, /* stvx v23,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f0041ce, /* stvx v24,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7f2049ce, /* stvx v25,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f4041ce, /* stvx v26,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7f6049ce, /* stvx v27,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f8041ce, /* stvx v28,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7fa049ce, /* stvx v29,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7fc041ce, /* stvx v30,r0,r8 */ + 0x7fe049ce, /* stvx v31,r0,r9 */ + 0x80a30034, /*2:lwz r5,52(r3) */ + 0x39030180, /* addi r8,r3,384 */ + 0x39230190, /* addi r9,r3,400 */ + 0x70a00fff, /* andi. r0,r5,4095 */ + 0x7ca043a6, /* mtvrsave r5 */ + 0x4d820420, /* beqctr */ + 0x7e8040ce, /* lvx v20,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7ea048ce, /* lvx v21,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7ec040ce, /* lvx v22,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7ee048ce, /* lvx v23,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f0040ce, /* lvx v24,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7f2048ce, /* lvx v25,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f4040ce, /* lvx v26,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7f6048ce, /* lvx v27,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7f8040ce, /* lvx v28,r0,r8 */ + 0x39080020, /* addi r8,r8,32 */ + 0x7fa048ce, /* lvx v29,r0,r9 */ + 0x39290020, /* addi r9,r9,32 */ + 0x7fc040ce, /* lvx v30,r0,r8 */ + 0x7fe048ce, /* lvx v31,r0,r9 */ + #endif + + 0x4e800420, /* bctr */ +}; + +#if LIBCO_PPCDESC + /* function call goes through indirect descriptor */ + #define CO_SWAP_ASM(x, y) ((void (*)(cothread_t, cothread_t))(uintptr_t)x)(x, y) +#else + /* function call goes directly to code */ + #define CO_SWAP_ASM(x, y) ((void (*)(cothread_t, cothread_t))(uintptr_t)libco_ppc_code)(x, y) +#endif + +static uint32_t* co_derive_(void* memory, unsigned size, uintptr_t entry) { + (void)entry; + + uint32_t* t = (uint32_t*)memory; + + #if LIBCO_PPCDESC + if(t) { + memcpy(t, (void*)entry, sizeof(void*) * 3); /* copy entry's descriptor */ + *(const void**)t = libco_ppc_code; /* set function pointer to swap routine */ + } + #endif + + return t; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*entry_)(void)) { + uintptr_t entry = (uintptr_t)entry_; + uint32_t* t = 0; + + /* be sure main thread was successfully allocated */ + if(co_active()) { + t = co_derive_(memory, size, entry); + } + + if(t) { + uintptr_t sp; + int shift; + + /* save current registers into new thread, so that any special ones will have proper values when thread is begun */ + CO_SWAP_ASM(t, t); + + #if LIBCO_PPCDESC + entry = (uintptr_t)*(void**)entry; /* get real address */ + #endif + + /* put stack near end of block, and align */ + sp = (uintptr_t)t + size - above_stack; + sp -= sp % stack_align; + + /* on PPC32, we save and restore GPRs as 32 bits. for PPC64, we + save and restore them as 64 bits, regardless of the size the ABI + uses. so, we manually write pointers at the proper size. we always + save and restore at the same address, and since PPC is big-endian, + we must put the low byte first on PPC32. */ + + /* if uintptr_t is 32 bits, >>32 is undefined behavior, + so we do two shifts and don't have to care how many bits uintptr_t is. */ + #if LIBCO_PPC64 + shift = 16; + #else + shift = 0; + #endif + + /* set up so entry will be called on next swap */ + t[ 8] = (uint32_t)(entry >> shift >> shift); + t[ 9] = (uint32_t)entry; + + t[10] = (uint32_t)(sp >> shift >> shift); + t[11] = (uint32_t)sp; + } + + return t; +} + +static uint32_t* co_create_(unsigned size, uintptr_t entry) { + (void)entry; + + uint32_t* t = (uint32_t*)LIBCO_MALLOC(size); + + #if LIBCO_PPCDESC + if(t) { + memcpy(t, (void*)entry, sizeof(void*) * 3); /* copy entry's descriptor */ + *(const void**)t = libco_ppc_code; /* set function pointer to swap routine */ + } + #endif + + return t; +} + +cothread_t co_create(unsigned int size, void (*entry_)(void)) { + uintptr_t entry = (uintptr_t)entry_; + uint32_t* t = 0; + + /* be sure main thread was successfully allocated */ + if(co_active()) { + size += state_size + above_stack + stack_align; + t = co_create_(size, entry); + } + + if(t) { + uintptr_t sp; + int shift; + + /* save current registers into new thread, so that any special ones will have proper values when thread is begun */ + CO_SWAP_ASM(t, t); + + #if LIBCO_PPCDESC + entry = (uintptr_t)*(void**)entry; /* get real address */ + #endif + + /* put stack near end of block, and align */ + sp = (uintptr_t)t + size - above_stack; + sp -= sp % stack_align; + + /* on PPC32, we save and restore GPRs as 32 bits. for PPC64, we + save and restore them as 64 bits, regardless of the size the ABI + uses. so, we manually write pointers at the proper size. we always + save and restore at the same address, and since PPC is big-endian, + we must put the low byte first on PPC32. */ + + /* if uintptr_t is 32 bits, >>32 is undefined behavior, + so we do two shifts and don't have to care how many bits uintptr_t is. */ + #if LIBCO_PPC64 + shift = 16; + #else + shift = 0; + #endif + + /* set up so entry will be called on next swap */ + t[ 8] = (uint32_t)(entry >> shift >> shift); + t[ 9] = (uint32_t)entry; + + t[10] = (uint32_t)(sp >> shift >> shift); + t[11] = (uint32_t)sp; + } + + return t; +} + +void co_delete(cothread_t t) { + LIBCO_FREE(t); +} + +static void co_init_(void) { + #if LIBCO_MPROTECT + long page_size = sysconf(_SC_PAGESIZE); + if(page_size > 0) { + uintptr_t align = page_size; + uintptr_t begin = (uintptr_t)libco_ppc_code; + uintptr_t end = begin + sizeof libco_ppc_code; + + /* align beginning and end */ + end += align - 1; + end -= end % align; + begin -= begin % align; + + mprotect((void*)begin, end - begin, PROT_READ | PROT_EXEC); + } + #endif + + co_active_handle = co_create_(state_size, (uintptr_t)&co_switch); +} + +cothread_t co_active() { + if(!co_active_handle) co_init_(); + + return co_active_handle; +} + +void co_switch(cothread_t t) { + cothread_t old = co_active_handle; + co_active_handle = t; + + CO_SWAP_ASM(t, old); +} + +int co_serializable() { + return 0; +} diff --git a/third_party/libco/ppc64v2.c b/third_party/libco/ppc64v2.c new file mode 100644 index 0000000..349f87b --- /dev/null +++ b/third_party/libco/ppc64v2.c @@ -0,0 +1,278 @@ +/* author: Shawn Anastasio */ + +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct ppc64_context { + /* GPRs */ + uint64_t gprs[32]; + uint64_t lr; + uint64_t ccr; + + /* FPRs */ + uint64_t fprs[32]; + + #ifdef __ALTIVEC__ + /* Altivec (VMX) */ + uint64_t vmx[12 * 2]; + uint32_t vrsave; + #endif +}; + +static thread_local struct ppc64_context* co_active_handle = 0; + +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#define ALIGN(p, x) ((void*)((uintptr_t)(p) & ~((x) - 1))) + +#define MIN_STACK 0x10000lu +#define MIN_STACK_FRAME 0x20lu +#define STACK_ALIGN 0x10lu + +void swap_context(struct ppc64_context* read, struct ppc64_context* write); +__asm__( + ".text\n" + ".align 4\n" + ".type swap_context @function\n" + "swap_context:\n" + ".cfi_startproc\n" + + /* save GPRs */ + "std 1, 8(4)\n" + "std 2, 16(4)\n" + "std 12, 96(4)\n" + "std 13, 104(4)\n" + "std 14, 112(4)\n" + "std 15, 120(4)\n" + "std 16, 128(4)\n" + "std 17, 136(4)\n" + "std 18, 144(4)\n" + "std 19, 152(4)\n" + "std 20, 160(4)\n" + "std 21, 168(4)\n" + "std 22, 176(4)\n" + "std 23, 184(4)\n" + "std 24, 192(4)\n" + "std 25, 200(4)\n" + "std 26, 208(4)\n" + "std 27, 216(4)\n" + "std 28, 224(4)\n" + "std 29, 232(4)\n" + "std 30, 240(4)\n" + "std 31, 248(4)\n" + + /* save LR */ + "mflr 5\n" + "std 5, 256(4)\n" + + /* save CCR */ + "mfcr 5\n" + "std 5, 264(4)\n" + + /* save FPRs */ + "stfd 14, 384(4)\n" + "stfd 15, 392(4)\n" + "stfd 16, 400(4)\n" + "stfd 17, 408(4)\n" + "stfd 18, 416(4)\n" + "stfd 19, 424(4)\n" + "stfd 20, 432(4)\n" + "stfd 21, 440(4)\n" + "stfd 22, 448(4)\n" + "stfd 23, 456(4)\n" + "stfd 24, 464(4)\n" + "stfd 25, 472(4)\n" + "stfd 26, 480(4)\n" + "stfd 27, 488(4)\n" + "stfd 28, 496(4)\n" + "stfd 29, 504(4)\n" + "stfd 30, 512(4)\n" + "stfd 31, 520(4)\n" + + #ifdef __ALTIVEC__ + /* save VMX */ + "li 5, 528\n" + "stvxl 20, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 21, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 22, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 23, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 24, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 25, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 26, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 27, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 28, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 29, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 30, 4, 5\n" + "addi 5, 5, 16\n" + "stvxl 31, 4, 5\n" + "addi 5, 5, 16\n" + + /* save VRSAVE */ + "mfvrsave 5\n" + "stw 5, 736(4)\n" + #endif + + /* restore GPRs */ + "ld 1, 8(3)\n" + "ld 2, 16(3)\n" + "ld 12, 96(3)\n" + "ld 13, 104(3)\n" + "ld 14, 112(3)\n" + "ld 15, 120(3)\n" + "ld 16, 128(3)\n" + "ld 17, 136(3)\n" + "ld 18, 144(3)\n" + "ld 19, 152(3)\n" + "ld 20, 160(3)\n" + "ld 21, 168(3)\n" + "ld 22, 176(3)\n" + "ld 23, 184(3)\n" + "ld 24, 192(3)\n" + "ld 25, 200(3)\n" + "ld 26, 208(3)\n" + "ld 27, 216(3)\n" + "ld 28, 224(3)\n" + "ld 29, 232(3)\n" + "ld 30, 240(3)\n" + "ld 31, 248(3)\n" + + /* restore LR */ + "ld 5, 256(3)\n" + "mtlr 5\n" + + /* restore CCR */ + "ld 5, 264(3)\n" + "mtcr 5\n" + + /* restore FPRs */ + "lfd 14, 384(3)\n" + "lfd 15, 392(3)\n" + "lfd 16, 400(3)\n" + "lfd 17, 408(3)\n" + "lfd 18, 416(3)\n" + "lfd 19, 424(3)\n" + "lfd 20, 432(3)\n" + "lfd 21, 440(3)\n" + "lfd 22, 448(3)\n" + "lfd 23, 456(3)\n" + "lfd 24, 464(3)\n" + "lfd 25, 472(3)\n" + "lfd 26, 480(3)\n" + "lfd 27, 488(3)\n" + "lfd 28, 496(3)\n" + "lfd 29, 504(3)\n" + "lfd 30, 512(3)\n" + "lfd 31, 520(3)\n" + + #ifdef __ALTIVEC__ + /* restore VMX */ + "li 5, 528\n" + "lvxl 20, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 21, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 22, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 23, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 24, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 25, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 26, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 27, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 28, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 29, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 30, 3, 5\n" + "addi 5, 5, 16\n" + "lvxl 31, 3, 5\n" + "addi 5, 5, 16\n" + + /* restore VRSAVE */ + "lwz 5, 720(3)\n" + "mtvrsave 5\n" + #endif + + /* branch to LR */ + "blr\n" + + ".cfi_endproc\n" + ".size swap_context, .-swap_context\n" +); + +cothread_t co_active() { + if(!co_active_handle) { + co_active_handle = (struct ppc64_context*)LIBCO_MALLOC(MIN_STACK + sizeof(struct ppc64_context)); + } + return (cothread_t)co_active_handle; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*coentry)(void)) { + uint8_t* sp; + struct ppc64_context* context = (struct ppc64_context*)memory; + + /* save current context into new context to initialize it */ + swap_context(context, context); + + /* align stack */ + sp = (uint8_t*)memory + size - STACK_ALIGN; + sp = (uint8_t*)ALIGN(sp, STACK_ALIGN); + + /* write 0 for initial backchain */ + *(uint64_t*)sp = 0; + + /* create new frame with backchain */ + sp -= MIN_STACK_FRAME; + *(uint64_t*)sp = (uint64_t)(sp + MIN_STACK_FRAME); + + /* update context with new stack (r1) and entrypoint (r12, lr) */ + context->gprs[ 1] = (uint64_t)sp; + context->gprs[12] = (uint64_t)coentry; + context->lr = (uint64_t)coentry; + + return (cothread_t)memory; +} + +cothread_t co_create(unsigned int size, void (*coentry)(void)) { + void* memory = LIBCO_MALLOC(size); + if(!memory) return (cothread_t)0; + return co_derive(memory, size, coentry); +} + +void co_delete(cothread_t handle) { + LIBCO_FREE(handle); +} + +void co_switch(cothread_t to) { + struct ppc64_context* from = co_active_handle; + co_active_handle = (struct ppc64_context*)to; + swap_context((struct ppc64_context*)to, from); +} + +int co_serializable() { + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/settings.h b/third_party/libco/settings.h new file mode 100644 index 0000000..721c1ac --- /dev/null +++ b/third_party/libco/settings.h @@ -0,0 +1,128 @@ +#if defined(LIBCO_C) + +/*[amd64, arm, ppc, x86]: + by default, co_swap_function is marked as a text (code) section + if not supported, uncomment the below line to use mprotect instead */ +/* #define LIBCO_MPROTECT */ + +/*[amd64]: + Win64 only: provides a substantial speed-up, but will thrash XMM regs + do not use this unless you are certain your application won't use SSE */ +/* #define LIBCO_NO_SSE */ + +#if !defined(thread_local) /* User can override thread_local for obscure compilers */ + #if !defined(LIBCO_MP) /* Running in single-threaded environment */ + #define thread_local + #else /* Running in multi-threaded environment */ + #if defined(__STDC__) /* Compiling as C Language */ + #if defined(_MSC_VER) /* Don't rely on MSVC's C11 support */ + #define thread_local __declspec(thread) + #elif __STDC_VERSION__ < 201112L /* If we are on C90/99 */ + #if defined(__clang__) || defined(__GNUC__) /* Clang and GCC */ + #define thread_local __thread + #else /* Otherwise, we ignore the directive (unless user provides their own) */ + #define thread_local + #endif + #else /* C11 and newer define thread_local in threads.h */ + #include + #endif + #elif defined(__cplusplus) /* Compiling as C++ Language */ + #if __cplusplus < 201103L /* thread_local is a C++11 feature */ + #if defined(_MSC_VER) + #define thread_local __declspec(thread) + #elif defined(__clang__) || defined(__GNUC__) + #define thread_local __thread + #else /* Otherwise, we ignore the directive (unless user provides their own) */ + #define thread_local + #endif + #else /* In C++ >= 11, thread_local in a builtin keyword */ + /* Don't do anything */ + #endif + #endif + #endif +#endif + +/* In alignas(a), 'a' should be a power of two that is at least the type's + alignment and at most the implementation's alignment limit. This limit is + 2**13 on MSVC. To be portable to MSVC through at least version 10.0, + 'a' should be an integer constant, as MSVC does not support expressions + such as 1 << 3. + + The following C11 requirements are NOT supported on MSVC: + + - If 'a' is zero, alignas has no effect. + - alignas can be used multiple times; the strictest one wins. + - alignas (TYPE) is equivalent to alignas (alignof (TYPE)). +*/ +#if !defined(alignas) + #if defined(__STDC__) /* C Language */ + #if defined(_MSC_VER) /* Don't rely on MSVC's C11 support */ + #define alignas(bytes) __declspec(align(bytes)) + #elif __STDC_VERSION__ >= 201112L /* C11 and above */ + #include + #elif defined(__clang__) || defined(__GNUC__) /* C90/99 on Clang/GCC */ + #define alignas(bytes) __attribute__ ((aligned (bytes))) + #else /* Otherwise, we ignore the directive (user should provide their own) */ + #define alignas(bytes) + #endif + #elif defined(__cplusplus) /* C++ Language */ + #if __cplusplus < 201103L + #if defined(_MSC_VER) + #define alignas(bytes) __declspec(align(bytes)) + #elif defined(__clang__) || defined(__GNUC__) /* C++98/03 on Clang/GCC */ + #define alignas(bytes) __attribute__ ((aligned (bytes))) + #else /* Otherwise, we ignore the directive (unless user provides their own) */ + #define alignas(bytes) + #endif + #else /* C++ >= 11 has alignas keyword */ + /* Do nothing */ + #endif + #endif /* = !defined(__STDC_VERSION__) && !defined(__cplusplus) */ +#endif + +#if !defined(LIBCO_ASSERT) + #include + #define LIBCO_ASSERT assert +#endif + +#if defined (__OpenBSD__) + #if !defined(LIBCO_MALLOC) || !defined(LIBCO_FREE) + #include + #include + + static void* malloc_obsd(size_t size) { + long pagesize = sysconf(_SC_PAGESIZE); + char* memory = (char*)mmap(NULL, size + pagesize, PROT_READ|PROT_WRITE, MAP_STACK|MAP_PRIVATE|MAP_ANON, -1, 0); + if (memory == MAP_FAILED) return NULL; + *(size_t*)memory = size + pagesize; + memory += pagesize; + return (void*)memory; + } + + static void free_obsd(void *ptr) { + char* memory = (char*)ptr - sysconf(_SC_PAGESIZE); + munmap(memory, *(size_t*)memory); + } + + #define LIBCO_MALLOC malloc_obsd + #define LIBCO_FREE free_obsd + #endif +#endif + +#if !defined(LIBCO_MALLOC) || !defined(LIBCO_FREE) + #include + #define LIBCO_MALLOC malloc + #define LIBCO_FREE free +#endif + +#if defined(_MSC_VER) + #define section(name) __declspec(allocate("." #name)) +#elif defined(__APPLE__) + #define section(name) __attribute__((section("__TEXT,__" #name))) +#else + #define section(name) __attribute__((section("." #name "#"))) +#endif + + +/* if defined(LIBCO_C) */ +#endif diff --git a/third_party/libco/sjlj.c b/third_party/libco/sjlj.c new file mode 100644 index 0000000..5af1472 --- /dev/null +++ b/third_party/libco/sjlj.c @@ -0,0 +1,145 @@ +/* + note this was designed for UNIX systems. Based on ideas expressed in a paper by Ralf Engelschall. + for SJLJ on other systems, one would want to rewrite springboard() and co_create() and hack the jmb_buf stack pointer. +*/ + +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#define _BSD_SOURCE +#define _XOPEN_SOURCE 500 +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + sigjmp_buf context; + void (*coentry)(void); + void* stack; +} cothread_struct; + +static thread_local cothread_struct co_primary; +static thread_local cothread_struct* creating; +static thread_local cothread_struct* co_running = 0; + +static void springboard(int ignored) { + if(sigsetjmp(creating->context, 0)) { + co_running->coentry(); + } +} + +cothread_t co_active() { + if(!co_running) co_running = &co_primary; + return (cothread_t)co_running; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*coentry)(void)) { + if(!co_running) co_running = &co_primary; + + cothread_struct* thread = (cothread_struct*)memory; + memory = (unsigned char*)memory + sizeof(cothread_struct); + size -= sizeof(cothread_struct); + if(thread) { + struct sigaction handler; + struct sigaction old_handler; + + stack_t stack; + stack_t old_stack; + + thread->coentry = thread->stack = 0; + + stack.ss_flags = 0; + stack.ss_size = size; + thread->stack = stack.ss_sp = memory; + if(stack.ss_sp && !sigaltstack(&stack, &old_stack)) { + handler.sa_handler = springboard; + handler.sa_flags = SA_ONSTACK; + sigemptyset(&handler.sa_mask); + creating = thread; + + if(!sigaction(SIGUSR1, &handler, &old_handler)) { + if(!raise(SIGUSR1)) { + thread->coentry = coentry; + } + sigaltstack(&old_stack, 0); + sigaction(SIGUSR1, &old_handler, 0); + } + } + + if(thread->coentry != coentry) { + co_delete(thread); + thread = 0; + } + } + + return (cothread_t)thread; +} + +cothread_t co_create(unsigned int size, void (*coentry)(void)) { + if(!co_running) co_running = &co_primary; + + cothread_struct* thread = (cothread_struct*)malloc(sizeof(cothread_struct)); + if(thread) { + struct sigaction handler; + struct sigaction old_handler; + + stack_t stack; + stack_t old_stack; + + thread->coentry = thread->stack = 0; + + stack.ss_flags = 0; + stack.ss_size = size; + thread->stack = stack.ss_sp = malloc(size); + if(stack.ss_sp && !sigaltstack(&stack, &old_stack)) { + handler.sa_handler = springboard; + handler.sa_flags = SA_ONSTACK; + sigemptyset(&handler.sa_mask); + creating = thread; + + if(!sigaction(SIGUSR1, &handler, &old_handler)) { + if(!raise(SIGUSR1)) { + thread->coentry = coentry; + } + sigaltstack(&old_stack, 0); + sigaction(SIGUSR1, &old_handler, 0); + } + } + + if(thread->coentry != coentry) { + co_delete(thread); + thread = 0; + } + } + + return (cothread_t)thread; +} + +void co_delete(cothread_t cothread) { + if(cothread) { + if(((cothread_struct*)cothread)->stack) { + free(((cothread_struct*)cothread)->stack); + } + free(cothread); + } +} + +void co_switch(cothread_t cothread) { + if(!sigsetjmp(co_running->context, 0)) { + co_running = (cothread_struct*)cothread; + siglongjmp(co_running->context, 1); + } +} + +int co_serializable() { + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/ucontext.c b/third_party/libco/ucontext.c new file mode 100644 index 0000000..5ff76af --- /dev/null +++ b/third_party/libco/ucontext.c @@ -0,0 +1,86 @@ +/* + WARNING: the overhead of POSIX ucontext is very high, + assembly versions of libco or libco_sjlj should be much faster + + this library only exists for two reasons: + 1: as an initial test for the viability of a ucontext implementation + 2: to demonstrate the power and speed of libco over existing implementations, + such as pth (which defaults to wrapping ucontext on unix targets) + + use this library only as a *last resort* +*/ + +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#define _BSD_SOURCE +#define _XOPEN_SOURCE 500 +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static thread_local ucontext_t co_primary; +static thread_local ucontext_t* co_running = 0; + +cothread_t co_active() { + if(!co_running) co_running = &co_primary; + return (cothread_t)co_running; +} + +cothread_t co_derive(void* memory, unsigned int heapsize, void (*coentry)(void)) { + if(!co_running) co_running = &co_primary; + ucontext_t* thread = (ucontext_t*)memory; + memory = (unsigned char*)memory + sizeof(ucontext_t); + heapsize -= sizeof(ucontext_t); + if(thread) { + if((!getcontext(thread) && !(thread->uc_stack.ss_sp = 0)) && (thread->uc_stack.ss_sp = memory)) { + thread->uc_link = co_running; + thread->uc_stack.ss_size = heapsize; + makecontext(thread, coentry, 0); + } else { + thread = 0; + } + } + return (cothread_t)thread; +} + +cothread_t co_create(unsigned int heapsize, void (*coentry)(void)) { + if(!co_running) co_running = &co_primary; + ucontext_t* thread = (ucontext_t*)malloc(sizeof(ucontext_t)); + if(thread) { + if((!getcontext(thread) && !(thread->uc_stack.ss_sp = 0)) && (thread->uc_stack.ss_sp = malloc(heapsize))) { + thread->uc_link = co_running; + thread->uc_stack.ss_size = heapsize; + makecontext(thread, coentry, 0); + } else { + co_delete((cothread_t)thread); + thread = 0; + } + } + return (cothread_t)thread; +} + +void co_delete(cothread_t cothread) { + if(cothread) { + if(((ucontext_t*)cothread)->uc_stack.ss_sp) { free(((ucontext_t*)cothread)->uc_stack.ss_sp); } + free(cothread); + } +} + +void co_switch(cothread_t cothread) { + ucontext_t* old_thread = co_running; + co_running = (ucontext_t*)cothread; + swapcontext(old_thread, co_running); +} + +int co_serializable() { + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/third_party/libco/x86.c b/third_party/libco/x86.c new file mode 100644 index 0000000..d31b4ac --- /dev/null +++ b/third_party/libco/x86.c @@ -0,0 +1,116 @@ +#define LIBCO_C +#include "libco.h" +#include "settings.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__clang__) || defined(__GNUC__) + #define fastcall __attribute__((fastcall)) +#elif defined(_MSC_VER) + #define fastcall __fastcall +#else + #error "libco: please define fastcall macro" +#endif + +static thread_local long co_active_buffer[64]; +static thread_local cothread_t co_active_handle = 0; +static void (fastcall *co_swap)(cothread_t, cothread_t) = 0; + +#ifdef LIBCO_MPROTECT + alignas(4096) +#else + section(text) +#endif +/* ABI: fastcall */ +static const unsigned char co_swap_function[4096] = { + 0x89, 0x22, /* mov [edx],esp */ + 0x8b, 0x21, /* mov esp,[ecx] */ + 0x58, /* pop eax */ + 0x89, 0x6a, 0x04, /* mov [edx+ 4],ebp */ + 0x89, 0x72, 0x08, /* mov [edx+ 8],esi */ + 0x89, 0x7a, 0x0c, /* mov [edx+12],edi */ + 0x89, 0x5a, 0x10, /* mov [edx+16],ebx */ + 0x8b, 0x69, 0x04, /* mov ebp,[ecx+ 4] */ + 0x8b, 0x71, 0x08, /* mov esi,[ecx+ 8] */ + 0x8b, 0x79, 0x0c, /* mov edi,[ecx+12] */ + 0x8b, 0x59, 0x10, /* mov ebx,[ecx+16] */ + 0xff, 0xe0, /* jmp eax */ +}; + +#ifdef _WIN32 + #include + + static void co_init() { + #ifdef LIBCO_MPROTECT + DWORD old_privileges; + VirtualProtect((void*)co_swap_function, sizeof co_swap_function, PAGE_EXECUTE_READ, &old_privileges); + #endif + } +#else + #ifdef LIBCO_MPROTECT + #include + #include + #endif + + static void co_init() { + #ifdef LIBCO_MPROTECT + unsigned long addr = (unsigned long)co_swap_function; + unsigned long base = addr - (addr % sysconf(_SC_PAGESIZE)); + unsigned long size = (addr - base) + sizeof co_swap_function; + mprotect((void*)base, size, PROT_READ | PROT_EXEC); + #endif + } +#endif + +static void crash() { + LIBCO_ASSERT(0); /* called only if cothread_t entrypoint returns */ +} + +cothread_t co_active() { + if(!co_active_handle) co_active_handle = &co_active_buffer; + return co_active_handle; +} + +cothread_t co_derive(void* memory, unsigned int size, void (*entrypoint)(void)) { + cothread_t handle; + if(!co_swap) { + co_init(); + co_swap = (void (fastcall*)(cothread_t, cothread_t))co_swap_function; + } + if(!co_active_handle) co_active_handle = &co_active_buffer; + + if(handle = (cothread_t)memory) { + unsigned int offset = (size & ~15) - 32; + long *p = (long*)((char*)handle + offset); /* seek to top of stack */ + *--p = (long)crash; /* crash if entrypoint returns */ + *--p = (long)entrypoint; /* start of function */ + *(long*)handle = (long)p; /* stack pointer */ + } + + return handle; +} + +cothread_t co_create(unsigned int size, void (*entrypoint)(void)) { + void* memory = LIBCO_MALLOC(size); + if(!memory) return (cothread_t)0; + return co_derive(memory, size, entrypoint); +} + +void co_delete(cothread_t handle) { + LIBCO_FREE(handle); +} + +void co_switch(cothread_t handle) { + register cothread_t co_previous_handle = co_active_handle; + co_swap(co_active_handle = handle, co_previous_handle); +} + +int co_serializable() { + return 1; +} + +#ifdef __cplusplus +} +#endif