From b8a7d2d3942dc8830d279e7430e23820823737c2 Mon Sep 17 00:00:00 2001 From: Bob Weinand Date: Wed, 2 Oct 2024 15:52:33 +0200 Subject: [PATCH] Live debugger parsing, expression evaluation, sender, redaction and FFI (#497) All stuff for live debugging. --- .github/workflows/lint.yml | 2 +- Cargo.lock | 59 +- Cargo.toml | 3 + LICENSE-3rdparty.yml | 468 +++++- ddcommon-ffi/src/option.rs | 7 + ddcommon-ffi/src/slice.rs | 7 + ddcommon-ffi/src/vec.rs | 12 +- ddcommon/src/rate_limiter.rs | 20 +- ipc/src/platform/mem_handle.rs | 12 +- ipc/src/rate_limiter.rs | 187 ++- live-debugger-ffi/Cargo.toml | 29 + live-debugger-ffi/build.rs | 10 + live-debugger-ffi/cbindgen.toml | 35 + live-debugger-ffi/src/data.rs | 322 ++++ live-debugger-ffi/src/evaluator.rs | 325 ++++ live-debugger-ffi/src/lib.rs | 8 + live-debugger-ffi/src/parse.rs | 40 + live-debugger-ffi/src/send_data.rs | 455 ++++++ live-debugger-ffi/src/sender.rs | 170 ++ live-debugger/Cargo.toml | 25 + live-debugger/src/debugger_defs.rs | 156 ++ live-debugger/src/expr_defs.rs | 252 +++ live-debugger/src/expr_eval.rs | 1373 +++++++++++++++++ live-debugger/src/lib.rs | 16 + live-debugger/src/parse_json.rs | 845 ++++++++++ live-debugger/src/probe_defs.rs | 158 ++ live-debugger/src/redacted_names.rs | 209 +++ live-debugger/src/sender.rs | 265 ++++ remote-config/Cargo.toml | 1 + remote-config/src/fetch/fetcher.rs | 40 +- remote-config/src/fetch/multitarget.rs | 94 +- remote-config/src/fetch/shared.rs | 30 +- remote-config/src/parse.rs | 6 +- sidecar-ffi/Cargo.toml | 1 + sidecar-ffi/cbindgen.toml | 2 +- sidecar-ffi/src/lib.rs | 42 + sidecar-ffi/tests/sidecar.rs | 2 + sidecar/Cargo.toml | 2 + sidecar/src/config.rs | 8 + sidecar/src/entry.rs | 4 + sidecar/src/lib.rs | 3 +- sidecar/src/log.rs | 6 +- sidecar/src/service/blocking.rs | 101 +- .../service/exception_hash_rate_limiter.rs | 113 ++ sidecar/src/service/mod.rs | 2 + sidecar/src/service/remote_configs.rs | 15 +- sidecar/src/service/runtime_info.rs | 68 +- sidecar/src/service/session_info.rs | 143 +- sidecar/src/service/sidecar_interface.rs | 28 + sidecar/src/service/sidecar_server.rs | 92 +- sidecar/src/shm_remote_config.rs | 158 +- sidecar/src/tokio_util.rs | 13 + sidecar/src/tracer.rs | 14 + sidecar/src/watchdog.rs | 2 +- spawn_worker/src/unix/spawn.rs | 1 + tools/docker/Dockerfile.build | 2 + 56 files changed, 6315 insertions(+), 148 deletions(-) create mode 100644 live-debugger-ffi/Cargo.toml create mode 100644 live-debugger-ffi/build.rs create mode 100644 live-debugger-ffi/cbindgen.toml create mode 100644 live-debugger-ffi/src/data.rs create mode 100644 live-debugger-ffi/src/evaluator.rs create mode 100644 live-debugger-ffi/src/lib.rs create mode 100644 live-debugger-ffi/src/parse.rs create mode 100644 live-debugger-ffi/src/send_data.rs create mode 100644 live-debugger-ffi/src/sender.rs create mode 100644 live-debugger/Cargo.toml create mode 100644 live-debugger/src/debugger_defs.rs create mode 100644 live-debugger/src/expr_defs.rs create mode 100644 live-debugger/src/expr_eval.rs create mode 100644 live-debugger/src/lib.rs create mode 100644 live-debugger/src/parse_json.rs create mode 100644 live-debugger/src/probe_defs.rs create mode 100644 live-debugger/src/redacted_names.rs create mode 100644 live-debugger/src/sender.rs create mode 100644 sidecar/src/service/exception_hash_rate_limiter.rs create mode 100644 sidecar/src/tokio_util.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 91ab05694..4ce24b427 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -49,7 +49,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.76.0 ] && echo -Aunknown-lints -Ainvalid_reference_casting) + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.76.0 ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" diff --git a/Cargo.lock b/Cargo.lock index 6fc6af77a..3d1b0a8b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,6 +1125,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "constcat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5cd0c57ef83705837b1cb872c973eff82b070846d3e23668322b2c0f8246d0" + [[package]] name = "core-foundation" version = "0.9.4" @@ -1486,6 +1492,43 @@ dependencies = [ "syn 2.0.70", ] +[[package]] +name = "datadog-live-debugger" +version = "0.0.1" +dependencies = [ + "anyhow", + "constcat", + "ddcommon", + "hyper 0.14.28", + "json", + "lazy_static", + "percent-encoding", + "regex", + "regex-automata 0.4.6", + "serde", + "serde_json", + "smallvec", + "sys-info", + "tokio", + "uuid", +] + +[[package]] +name = "datadog-live-debugger-ffi" +version = "0.0.1" +dependencies = [ + "build_common", + "datadog-live-debugger", + "ddcommon", + "ddcommon-ffi", + "log", + "percent-encoding", + "serde_json", + "tokio", + "tokio-util 0.7.11", + "uuid", +] + [[package]] name = "datadog-profiling" version = "13.0.0" @@ -1576,6 +1619,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "datadog-dynamic-configuration", + "datadog-live-debugger", "datadog-trace-protobuf", "ddcommon", "futures", @@ -1624,6 +1668,7 @@ dependencies = [ "datadog-dynamic-configuration", "datadog-ipc", "datadog-ipc-macros", + "datadog-live-debugger", "datadog-remote-config", "datadog-sidecar-macros", "datadog-trace-normalization", @@ -1668,6 +1713,7 @@ dependencies = [ "uuid", "winapi 0.3.9", "windows 0.51.1", + "windows-sys 0.52.0", "zwohash", ] @@ -1676,6 +1722,7 @@ name = "datadog-sidecar-ffi" version = "0.0.1" dependencies = [ "datadog-ipc", + "datadog-live-debugger", "datadog-remote-config", "datadog-sidecar", "datadog-trace-utils", @@ -3002,6 +3049,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -3054,9 +3107,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -5554,6 +5607,8 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", + "hashbrown 0.14.3", "pin-project-lite", "slab", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 8f7d7687f..747a82034 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,9 @@ members = [ "tools", "ipc", "ipc/macros", + "live-debugger", + "live-debugger-ffi", + "remote-config", "sidecar", "sidecar/macros", "sidecar-ffi", diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 4c4303438..2f7daf602 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,4 +1,4 @@ -root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, datadog-trace-utils, ddcommon, tinybytes, dogstatsd-client, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent +root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, datadog-trace-utils, ddcommon, tinybytes, dogstatsd-client, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-live-debugger, datadog-live-debugger-ffi, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent third_party_libraries: - package_name: addr2line package_version: 0.21.0 @@ -7586,6 +7586,233 @@ third_party_libraries: OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +- package_name: constcat + package_version: 0.4.1 + repository: https://github.com/rossmacarthur/constcat + license: MIT OR Apache-2.0 + licenses: + - license: MIT + text: | + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + - license: Apache-2.0 + text: |2 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. - package_name: core-foundation package_version: 0.9.4 repository: https://github.com/servo/core-foundation-rs @@ -14709,6 +14936,237 @@ third_party_libraries: DEALINGS IN THE SOFTWARE. - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: json + package_version: 0.12.4 + repository: https://github.com/maciejhirsz/json-rust + license: MIT/Apache-2.0 + licenses: + - license: MIT + text: | + Copyright (c) 2016 Maciej Hirsz + + The MIT License (MIT) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + - license: Apache-2.0 + text: |2 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 Maciej Hirsz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. - package_name: kernel32-sys package_version: 0.2.2 repository: https://github.com/retep998/winapi-rs @@ -14760,9 +15218,9 @@ third_party_libraries: IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - package_name: lazy_static - package_version: 1.4.0 + package_version: 1.5.0 repository: https://github.com/rust-lang-nursery/lazy-static.rs - license: MIT/Apache-2.0 + license: MIT OR Apache-2.0 licenses: - license: MIT text: | @@ -22468,9 +22926,9 @@ third_party_libraries: - package_name: ring package_version: 0.17.8 repository: https://github.com/briansmith/ring - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.17.8/LICENSE) + license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.17.8/LICENSE) licenses: - - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.17.8/LICENSE) + - license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.17.8/LICENSE) text: "Note that it is easy for this file to get out of sync with the licenses in the\nsource code files. It's recommended to compare the licenses in the source code\nwith what's mentioned here.\n\n*ring* is derived from BoringSSL, so the licensing situation in *ring* is\nsimilar to BoringSSL.\n\n*ring* uses an ISC-style license like BoringSSL for code in new files,\nincluding in particular all the Rust code:\n\n Copyright 2015-2016 Brian Smith.\n\n Permission to use, copy, modify, and/or distribute this software for any\n purpose with or without fee is hereby granted, provided that the above\n copyright notice and this permission notice appear in all copies.\n\n THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHORS DISCLAIM ALL WARRANTIES\n WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY\n SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n\nBoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL\nlicensing. Files that are completely new have a Google copyright and an ISC\nlicense. This license is reproduced at the bottom of this file.\n\nContributors to BoringSSL are required to follow the CLA rules for Chromium:\nhttps://cla.developers.google.com/clas\n\nFiles in third_party/ have their own licenses, as described therein. The MIT\nlicense, for third_party/fiat, which, unlike other third_party directories, is\ncompiled into non-test libraries, is included below.\n\nThe OpenSSL toolkit stays under a dual license, i.e. both the conditions of the\nOpenSSL License and the original SSLeay license apply to the toolkit. See below\nfor the actual license texts. Actually both licenses are BSD-style Open Source\nlicenses. In case of any license issues related to OpenSSL please contact\nopenssl-core@openssl.org.\n\nThe following are Google-internal bug numbers where explicit permission from\nsome authors is recorded for use of their work:\n 27287199\n 27287880\n 27287883\n\n OpenSSL License\n ---------------\n\n/* ====================================================================\n * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n *\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer. \n *\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in\n * the documentation and/or other materials provided with the\n * distribution.\n *\n * 3. All advertising materials mentioning features or use of this\n * software must display the following acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit. (http://www.openssl.org/)\"\n *\n * 4. The names \"OpenSSL Toolkit\" and \"OpenSSL Project\" must not be used to\n * endorse or promote products derived from this software without\n * prior written permission. For written permission, please contact\n * openssl-core@openssl.org.\n *\n * 5. Products derived from this software may not be called \"OpenSSL\"\n * nor may \"OpenSSL\" appear in their names without prior written\n * permission of the OpenSSL Project.\n *\n * 6. Redistributions of any form whatsoever must retain the following\n * acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit (http://www.openssl.org/)\"\n *\n * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY\n * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR\n * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\n * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED\n * OF THE POSSIBILITY OF SUCH DAMAGE.\n * ====================================================================\n *\n * This product includes cryptographic software written by Eric Young\n * (eay@cryptsoft.com). This product includes software written by Tim\n * Hudson (tjh@cryptsoft.com).\n *\n */\n\n Original SSLeay License\n -----------------------\n\n/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)\n * All rights reserved.\n *\n * This package is an SSL implementation written\n * by Eric Young (eay@cryptsoft.com).\n * The implementation was written so as to conform with Netscapes SSL.\n * \n * This library is free for commercial and non-commercial use as long as\n * the following conditions are aheared to. The following conditions\n * apply to all code found in this distribution, be it the RC4, RSA,\n * lhash, DES, etc., code; not just the SSL code. The SSL documentation\n * included with this distribution is covered by the same copyright terms\n * except that the holder is Tim Hudson (tjh@cryptsoft.com).\n * \n * Copyright remains Eric Young's, and as such any Copyright notices in\n * the code are not to be removed.\n * If this package is used in a product, Eric Young should be given attribution\n * as the author of the parts of the library used.\n * This can be in the form of a textual message at program startup or\n * in documentation (online or textual) provided with the package.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. All advertising materials mentioning features or use of this software\n * must display the following acknowledgement:\n * \"This product includes cryptographic software written by\n * Eric Young (eay@cryptsoft.com)\"\n * The word 'cryptographic' can be left out if the rouines from the library\n * being used are not cryptographic related :-).\n * 4. If you include any Windows specific code (or a derivative thereof) from \n * the apps directory (application code) you must include an acknowledgement:\n * \"This product includes software written by Tim Hudson (tjh@cryptsoft.com)\"\n * \n * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND\n * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n * SUCH DAMAGE.\n * \n * The licence and distribution terms for any publically available version or\n * derivative of this code cannot be changed. i.e. this code cannot simply be\n * copied and put under another distribution licence\n * [including the GNU Public Licence.]\n */\n\n\nISC license used for completely new code in BoringSSL:\n\n/* Copyright (c) 2015, Google Inc.\n *\n * Permission to use, copy, modify, and/or distribute this software for any\n * purpose with or without fee is hereby granted, provided that the above\n * copyright notice and this permission notice appear in all copies.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY\n * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */\n\n\nThe code in third_party/fiat carries the MIT license:\n\nCopyright (c) 2015-2016 the fiat-crypto authors (see\nhttps://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS).\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" - package_name: rmp package_version: 0.8.14 diff --git a/ddcommon-ffi/src/option.rs b/ddcommon-ffi/src/option.rs index 550467819..41ab5af46 100644 --- a/ddcommon-ffi/src/option.rs +++ b/ddcommon-ffi/src/option.rs @@ -12,6 +12,13 @@ impl Option { pub fn to_std(self) -> std::option::Option { self.into() } + + pub fn to_std_ref(&self) -> std::option::Option<&T> { + match self { + Option::Some(ref s) => Some(s), + Option::None => None, + } + } } impl From> for std::option::Option { diff --git a/ddcommon-ffi/src/slice.rs b/ddcommon-ffi/src/slice.rs index f2cd52098..b64a142ce 100644 --- a/ddcommon-ffi/src/slice.rs +++ b/ddcommon-ffi/src/slice.rs @@ -76,6 +76,13 @@ pub trait AsBytes<'a> { fn to_utf8_lossy(&self) -> Cow<'a, str> { String::from_utf8_lossy(self.as_bytes()) } + + #[inline] + /// # Safety + /// Must only be used when the underlying data was already confirmed to be utf8. + unsafe fn assume_utf8(&self) -> &'a str { + std::str::from_utf8_unchecked(self.as_bytes()) + } } impl<'a> AsBytes<'a> for Slice<'a, u8> { diff --git a/ddcommon-ffi/src/vec.rs b/ddcommon-ffi/src/vec.rs index 7d5fe359b..147bc5d39 100644 --- a/ddcommon-ffi/src/vec.rs +++ b/ddcommon-ffi/src/vec.rs @@ -8,6 +8,7 @@ use core::ops::Deref; use std::io::Write; use std::marker::PhantomData; use std::mem::ManuallyDrop; +use std::ptr::NonNull; /// Holds the raw parts of a Rust Vec; it should only be created from Rust, /// never from C. @@ -97,6 +98,15 @@ impl Vec { pub fn as_slice(&self) -> Slice { unsafe { Slice::from_raw_parts(self.ptr, self.len) } } + + pub const fn new() -> Self { + Vec { + ptr: NonNull::dangling().as_ptr(), + len: 0, + capacity: 0, + _marker: PhantomData, + } + } } impl Deref for Vec { @@ -109,7 +119,7 @@ impl Deref for Vec { impl Default for Vec { fn default() -> Self { - Self::from(alloc::vec::Vec::new()) + Self::new() } } diff --git a/ddcommon/src/rate_limiter.rs b/ddcommon/src/rate_limiter.rs index fee7480d1..2c983e9a5 100644 --- a/ddcommon/src/rate_limiter.rs +++ b/ddcommon/src/rate_limiter.rs @@ -10,6 +10,8 @@ pub trait Limiter { /// Returns the effective rate per interval. /// Note: The rate is only guaranteed to be accurate immediately after a call to inc(). fn rate(&self) -> f64; + /// Updates the rate and returns it + fn update_rate(&self) -> f64; } /// A thread-safe limiter built on Atomics. @@ -83,16 +85,14 @@ impl LocalLimiter { self.last_limit.store(0, Ordering::Relaxed); self.granularity = TIME_PER_SECOND * seconds as i64; } -} -impl Limiter for LocalLimiter { - fn inc(&self, limit: u32) -> bool { + fn update(&self, limit: u32, inc: i64) -> i64 { let now = now(); let last = self.last_update.swap(now, Ordering::SeqCst); // Make sure reducing the limit doesn't stall for a long time let clear_limit = limit.max(self.last_limit.load(Ordering::Relaxed)); let clear_counter = (now as i64 - last as i64) * (clear_limit as i64); - let subtract = clear_counter - self.granularity; + let subtract = clear_counter - inc; let mut previous_hits = self.hit_count.fetch_sub(subtract, Ordering::SeqCst); // Handle where the limiter goes below zero if previous_hits < subtract { @@ -100,6 +100,13 @@ impl Limiter for LocalLimiter { self.hit_count.fetch_add(add, Ordering::Acquire); previous_hits += add - clear_counter; } + previous_hits + } +} + +impl Limiter for LocalLimiter { + fn inc(&self, limit: u32) -> bool { + let previous_hits = self.update(limit, self.granularity); if previous_hits / self.granularity >= limit as i64 { self.hit_count .fetch_sub(self.granularity, Ordering::Acquire); @@ -119,6 +126,11 @@ impl Limiter for LocalLimiter { let hit_count = self.hit_count.load(Ordering::Relaxed); (hit_count as f64 / (last_limit as i64 * self.granularity) as f64).clamp(0., 1.) } + + fn update_rate(&self) -> f64 { + self.update(0, self.granularity); + self.rate() + } } #[cfg(test)] diff --git a/ipc/src/platform/mem_handle.rs b/ipc/src/platform/mem_handle.rs index 62490609b..85d408a4c 100644 --- a/ipc/src/platform/mem_handle.rs +++ b/ipc/src/platform/mem_handle.rs @@ -147,6 +147,12 @@ impl MappedMem { } } +impl AsRef<[u8]> for MappedMem { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + impl MappedMem { pub fn get_path(&self) -> &[u8] { self.mem.get_path() @@ -205,12 +211,6 @@ impl From for PlatformHandle { unsafe impl Sync for MappedMem where T: FileBackedHandle {} unsafe impl Send for MappedMem where T: FileBackedHandle {} -impl AsRef<[u8]> for MappedMem { - fn as_ref(&self) -> &[u8] { - self.as_slice() - } -} - #[cfg(feature = "tiny-bytes")] impl UnderlyingBytes for MappedMem {} diff --git a/ipc/src/rate_limiter.rs b/ipc/src/rate_limiter.rs index f05439f39..abefd4b15 100644 --- a/ipc/src/rate_limiter.rs +++ b/ipc/src/rate_limiter.rs @@ -1,8 +1,9 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::platform::{FileBackedHandle, MappedMem, NamedShmHandle}; +use crate::platform::{FileBackedHandle, MappedMem, MemoryHandle, NamedShmHandle}; use ddcommon::rate_limiter::{Limiter, LocalLimiter}; +use std::cell::UnsafeCell; use std::ffi::CString; use std::fmt::{Debug, Formatter}; use std::io; @@ -12,18 +13,32 @@ use std::sync::{Arc, RwLock}; #[repr(C)] #[derive(Default)] -struct ShmLimiterData<'a> { +struct ShmLimiterData<'a, Inner> { next_free: AtomicU32, // free list rc: AtomicI32, limiter: LocalLimiter, - _phantom: PhantomData<&'a ShmLimiterMemory>, + inner: UnsafeCell, + _phantom: PhantomData<&'a ShmLimiterMemory>, } -#[derive(Clone)] -pub struct ShmLimiterMemory(Arc>>); +pub struct ShmLimiterMemory { + mem: Arc>>, + last_size: AtomicU32, + _phantom: PhantomData, +} + +impl Clone for ShmLimiterMemory { + fn clone(&self) -> Self { + ShmLimiterMemory { + mem: self.mem.clone(), + last_size: AtomicU32::new(self.last_size.load(Ordering::Relaxed)), + _phantom: Default::default(), + } + } +} -impl ShmLimiterMemory { - const START_OFFSET: u32 = std::mem::align_of::() as u32; +impl ShmLimiterMemory { + const START_OFFSET: u32 = std::mem::align_of::>() as u32; pub fn create(path: CString) -> io::Result { // Clean leftover shm @@ -41,13 +56,17 @@ impl ShmLimiterMemory { } fn new(handle: MappedMem) -> Self { - Self(Arc::new(RwLock::new(handle))) + Self { + last_size: AtomicU32::new(handle.get_size() as u32), + mem: Arc::new(RwLock::new(handle)), + _phantom: Default::default(), + } } /// The start of the ShmLimiter memory has 4 bytes indicating an offset to the first free /// element in the free list. It is zero if there is no element on the free list. fn first_free_ref(&self) -> &AtomicU32 { - unsafe { &*self.0.read().unwrap().as_slice().as_ptr().cast() } + unsafe { &*self.mem.read().unwrap().as_slice().as_ptr().cast() } } fn next_free(&mut self) -> u32 { @@ -62,10 +81,10 @@ impl ShmLimiterMemory { .load(Ordering::Relaxed); // Not yet used memory will always be 0. The next free entry will then be just above. if target_next_free == 0 { - target_next_free = first_free + std::mem::size_of::() as u32; + target_next_free = first_free + std::mem::size_of::>() as u32; // target_next_free is the end of the current entry - but we need one more - self.0.write().unwrap().ensure_space( - target_next_free as usize + std::mem::size_of::(), + self.mem.write().unwrap().ensure_space( + target_next_free as usize + std::mem::size_of::>(), ); } match self.first_free_ref().compare_exchange( @@ -80,7 +99,11 @@ impl ShmLimiterMemory { } } - pub fn alloc(&mut self) -> ShmLimiter { + pub fn alloc(&mut self) -> ShmLimiter { + self.alloc_with_granularity(1) + } + + pub fn alloc_with_granularity(&mut self, seconds: u32) -> ShmLimiter { let reference = ShmLimiter { idx: self.next_free(), memory: self.clone(), @@ -89,18 +112,36 @@ impl ShmLimiterMemory { limiter.rc.store(1, Ordering::Relaxed); // SAFETY: we initialize the struct here unsafe { - (*(limiter as *const _ as *mut ShmLimiterData)) + (*(limiter as *const _ as *mut ShmLimiterData)) .limiter - .reset(1) + .reset(seconds) }; reference } - pub fn get(&self, idx: u32) -> Option { + fn ensure_index(&self, idx: u32) -> Option<()> { + let end = idx + std::mem::size_of::>() as u32; + if end > self.last_size.load(Ordering::Relaxed) { + let mut mem = self.mem.write().unwrap(); + let mut cur_size = mem.mem.get_size() as u32; + if cur_size < end { + mem.ensure_space(end as usize); + cur_size = mem.mem.get_size() as u32; + if cur_size < end { + return None; + } + } + self.last_size.store(cur_size, Ordering::Relaxed); + } + Some(()) + } + + pub fn get(&self, idx: u32) -> Option> { assert_eq!( - idx % std::mem::size_of::() as u32, + idx % std::mem::size_of::>() as u32, Self::START_OFFSET ); + self.ensure_index(idx)?; let reference = ShmLimiter { idx, memory: self.clone(), @@ -120,25 +161,49 @@ impl ShmLimiterMemory { } } } + + pub fn find(&self, cond: F) -> Option> + where + F: Fn(&Inner) -> bool, + { + let mut cur = Self::START_OFFSET; + let mem = self.mem.read().unwrap(); + loop { + self.ensure_index(cur)?; + let data: &ShmLimiterData = + unsafe { &*mem.as_slice().as_ptr().add(cur as usize).cast() }; + if data.next_free.load(Ordering::Relaxed) == 0 { + return None; + } + if data.rc.load(Ordering::Relaxed) > 0 && cond(unsafe { &*data.inner.get() }) { + if let Some(limiter) = self.get(cur) { + if cond(limiter.data()) { + return Some(limiter); + } + } + } + cur += std::mem::size_of::>() as u32; + } + } } -pub struct ShmLimiter { +pub struct ShmLimiter { idx: u32, - memory: ShmLimiterMemory, + memory: ShmLimiterMemory, } -impl Debug for ShmLimiter { +impl Debug for ShmLimiter { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { self.idx.fmt(f) } } -impl ShmLimiter { - fn limiter(&self) -> &ShmLimiterData { +impl ShmLimiter { + fn limiter(&self) -> &ShmLimiterData { unsafe { &*self .memory - .0 + .mem .read() .unwrap() .as_slice() @@ -148,12 +213,51 @@ impl ShmLimiter { } } + pub fn data(&self) -> &Inner { + unsafe { &*self.limiter().inner.get() } + } + pub fn index(&self) -> u32 { self.idx } + + /// # Safety + /// Callers MUST NOT do any other operations on this instance if dropping was successful. + pub unsafe fn drop_if_rc_1(&mut self) -> bool { + let limiter = self.limiter(); + + if limiter + .rc + .compare_exchange(1, 0, Ordering::SeqCst, Ordering::Relaxed) + .is_ok() + { + self.actual_free(limiter); + self.idx = 0; + true + } else { + false + } + } + + fn actual_free(&self, limiter: &ShmLimiterData) { + let next_free_ref = self.memory.first_free_ref(); + let mut next_free = next_free_ref.load(Ordering::Relaxed); + loop { + limiter.next_free.store(next_free, Ordering::Relaxed); + match next_free_ref.compare_exchange( + next_free, + self.idx, + Ordering::SeqCst, + Ordering::Relaxed, + ) { + Ok(_) => return, + Err(found) => next_free = found, + } + } + } } -impl Limiter for ShmLimiter { +impl Limiter for ShmLimiter { fn inc(&self, limit: u32) -> bool { self.limiter().limiter.inc(limit) } @@ -161,26 +265,21 @@ impl Limiter for ShmLimiter { fn rate(&self) -> f64 { self.limiter().limiter.rate() } + + fn update_rate(&self) -> f64 { + self.limiter().limiter.update_rate() + } } -impl Drop for ShmLimiter { +impl Drop for ShmLimiter { fn drop(&mut self) { + if self.idx == 0 { + return; + } + let limiter = self.limiter(); if limiter.rc.fetch_sub(1, Ordering::SeqCst) == 1 { - let next_free_ref = self.memory.first_free_ref(); - let mut next_free = next_free_ref.load(Ordering::Relaxed); - loop { - limiter.next_free.store(next_free, Ordering::Relaxed); - match next_free_ref.compare_exchange( - next_free, - self.idx, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - Ok(_) => return, - Err(found) => next_free = found, - } - } + self.actual_free(limiter); } } } @@ -207,6 +306,10 @@ impl Limiter for AnyLimiter { fn rate(&self) -> f64 { self.limiter().rate() } + + fn update_rate(&self) -> f64 { + self.limiter().update_rate() + } } #[cfg(test)] @@ -224,7 +327,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_limiters() { - let mut limiters = ShmLimiterMemory::create(path()).unwrap(); + let mut limiters = ShmLimiterMemory::<()>::create(path()).unwrap(); let limiter = limiters.alloc(); let limiter_idx = limiter.idx; // Two are allowed, then one more because a small amount of time passed since the first one @@ -243,7 +346,7 @@ mod tests { let limiter2 = limiters.alloc(); assert_eq!( limiter2.idx, - limiter_idx + std::mem::size_of::() as u32 + limiter_idx + std::mem::size_of::>() as u32 ); drop(limiter); @@ -253,7 +356,7 @@ mod tests { let limiter3 = limiters.alloc(); assert_eq!( limiter3.idx, - limiter2.idx + std::mem::size_of::() as u32 + limiter2.idx + std::mem::size_of::>() as u32 ); } } diff --git a/live-debugger-ffi/Cargo.toml b/live-debugger-ffi/Cargo.toml new file mode 100644 index 000000000..c6397fb04 --- /dev/null +++ b/live-debugger-ffi/Cargo.toml @@ -0,0 +1,29 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +[package] +name = "datadog-live-debugger-ffi" +version = "0.0.1" +edition = "2021" + +[lib] +crate-type = ["lib", "staticlib", "cdylib"] +bench = false + +[dependencies] +datadog-live-debugger = { path = "../live-debugger" } +ddcommon = { path = "../ddcommon" } +ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false } +percent-encoding = "2.1" +uuid = { version = "1.7.0", features = ["v4"] } +serde_json = "1.0" +tokio = "1.36.0" +tokio-util = { version = "0.7", features = ["rt"] } +log = "0.4.21" + +[features] +default = ["cbindgen"] +cbindgen = ["build_common/cbindgen", "ddcommon-ffi/cbindgen"] + +[build-dependencies] +build_common = { path = "../build-common" } diff --git a/live-debugger-ffi/build.rs b/live-debugger-ffi/build.rs new file mode 100644 index 000000000..23e13ebfa --- /dev/null +++ b/live-debugger-ffi/build.rs @@ -0,0 +1,10 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 +extern crate build_common; + +use build_common::generate_and_configure_header; + +fn main() { + let header_name = "live-debugger.h"; + generate_and_configure_header(header_name); +} diff --git a/live-debugger-ffi/cbindgen.toml b/live-debugger-ffi/cbindgen.toml new file mode 100644 index 000000000..2d29f5793 --- /dev/null +++ b/live-debugger-ffi/cbindgen.toml @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +language = "C" +tab_width = 2 +header = """// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +typedef struct ddog_DebuggerCapture ddog_DebuggerCapture; +typedef struct ddog_DebuggerValue ddog_DebuggerValue; +""" +include_guard = "DDOG_LIVE_DEBUGGER_H" +style = "both" + +no_includes = true +sys_includes = ["stdbool.h", "stddef.h", "stdint.h", "stdio.h"] +includes = ["common.h"] + +[export] +prefix = "ddog_" +renaming_overrides_prefixing = true + +[export.mangle] +rename_types = "PascalCase" + +[enum] +prefix_with_name = true +rename_variants = "ScreamingSnakeCase" + +[fn] +must_use = "DDOG_CHECK_RETURN" + +[parse] +parse_deps = true +include = ["datadog-live-debugger", "ddcommon-ffi"] diff --git a/live-debugger-ffi/src/data.rs b/live-debugger-ffi/src/data.rs new file mode 100644 index 000000000..71d4c0b0e --- /dev/null +++ b/live-debugger-ffi/src/data.rs @@ -0,0 +1,322 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +use datadog_live_debugger::debugger_defs::{ProbeMetadata, ProbeMetadataLocation, ProbeStatus}; +use datadog_live_debugger::{ + CaptureConfiguration, DslString, EvaluateAt, InBodyLocation, MetricKind, ProbeCondition, + ProbeValue, SpanProbeTarget, +}; +use ddcommon_ffi::slice::AsBytes; +use ddcommon_ffi::{CharSlice, Option}; +use std::borrow::Cow; + +#[repr(C)] +pub struct CharSliceVec<'a> { + pub strings: *const CharSlice<'a>, + pub string_count: usize, +} + +impl<'a> Drop for CharSliceVec<'a> { + fn drop(&mut self) { + unsafe { + Vec::from_raw_parts( + self.strings as *mut CharSlice, + self.string_count, + self.string_count, + ) + }; + } +} + +impl<'a> From<&'a Vec> for CharSliceVec<'a> { + fn from(from: &'a Vec) -> Self { + let char_slices: Vec> = from.iter().map(|s| s.as_str().into()).collect(); + let new = CharSliceVec { + strings: char_slices.as_ptr(), + string_count: char_slices.len(), + }; + std::mem::forget(char_slices); + new + } +} + +#[repr(C)] +pub struct MetricProbe<'a> { + pub kind: MetricKind, + pub name: CharSlice<'a>, + pub value: &'a ProbeValue, +} + +impl<'a> From<&'a datadog_live_debugger::MetricProbe> for MetricProbe<'a> { + fn from(from: &'a datadog_live_debugger::MetricProbe) -> Self { + MetricProbe { + kind: from.kind, + name: from.name.as_str().into(), + value: &from.value, + } + } +} + +#[repr(C)] +pub struct LogProbe<'a> { + pub segments: &'a DslString, + pub when: &'a ProbeCondition, + pub capture: &'a CaptureConfiguration, + pub capture_snapshot: bool, + pub sampling_snapshots_per_second: u32, +} + +impl<'a> From<&'a datadog_live_debugger::LogProbe> for LogProbe<'a> { + fn from(from: &'a datadog_live_debugger::LogProbe) -> Self { + LogProbe { + segments: &from.segments, + when: &from.when, + capture: &from.capture, + capture_snapshot: from.capture_snapshot, + sampling_snapshots_per_second: from.sampling_snapshots_per_second, + } + } +} + +#[repr(C)] +pub struct Tag<'a> { + pub name: CharSlice<'a>, + pub value: &'a DslString, +} + +#[repr(C)] +pub struct SpanProbeTag<'a> { + pub tag: Tag<'a>, + pub next_condition: bool, +} + +#[repr(C)] +pub struct SpanDecorationProbe<'a> { + pub target: SpanProbeTarget, + pub conditions: *const &'a ProbeCondition, + pub span_tags: *const SpanProbeTag<'a>, + pub span_tags_num: usize, +} + +impl<'a> From<&'a datadog_live_debugger::SpanDecorationProbe> for SpanDecorationProbe<'a> { + fn from(from: &'a datadog_live_debugger::SpanDecorationProbe) -> Self { + let mut tags = vec![]; + let mut conditions = vec![]; + for decoration in from.decorations.iter() { + let mut next_condition = true; + for (name, value) in decoration.tags.iter() { + tags.push(SpanProbeTag { + tag: Tag { + name: CharSlice::from(name.as_str()), + value, + }, + next_condition, + }); + next_condition = false; + } + conditions.push(&decoration.condition); + } + let new = SpanDecorationProbe { + target: from.target, + conditions: conditions.as_ptr(), + span_tags: tags.as_ptr(), + span_tags_num: tags.len(), + }; + std::mem::forget(conditions); + std::mem::forget(tags); + new + } +} + +#[no_mangle] +extern "C" fn drop_span_decoration_probe(_: SpanDecorationProbe) {} + +impl<'a> Drop for SpanDecorationProbe<'a> { + fn drop(&mut self) { + unsafe { + let tags = Vec::from_raw_parts( + self.span_tags as *mut SpanProbeTag, + self.span_tags_num, + self.span_tags_num, + ); + let num_conditions = tags.iter().filter(|p| p.next_condition).count(); + _ = Vec::from_raw_parts( + self.conditions as *mut &ProbeCondition, + num_conditions, + num_conditions, + ); + }; + } +} + +#[repr(C)] +pub enum ProbeType<'a> { + Metric(MetricProbe<'a>), + Log(LogProbe<'a>), + Span, + SpanDecoration(SpanDecorationProbe<'a>), +} + +impl<'a> From<&'a datadog_live_debugger::ProbeType> for ProbeType<'a> { + fn from(from: &'a datadog_live_debugger::ProbeType) -> Self { + match from { + datadog_live_debugger::ProbeType::Metric(metric) => ProbeType::Metric(metric.into()), + datadog_live_debugger::ProbeType::Log(log) => ProbeType::Log(log.into()), + datadog_live_debugger::ProbeType::Span(_) => ProbeType::Span, + datadog_live_debugger::ProbeType::SpanDecoration(span_decoration) => { + ProbeType::SpanDecoration(span_decoration.into()) + } + } + } +} + +#[repr(C)] +pub struct ProbeTarget<'a> { + pub type_name: CharSlice<'a>, + pub method_name: CharSlice<'a>, + pub source_file: CharSlice<'a>, + pub signature: Option>, // we need to distinguish empty signature and not present + pub lines: *const u32, + pub lines_count: u32, + pub in_body_location: InBodyLocation, +} + +impl<'a> From<&'a datadog_live_debugger::ProbeTarget> for ProbeTarget<'a> { + fn from(from: &'a datadog_live_debugger::ProbeTarget) -> Self { + ProbeTarget { + type_name: from + .type_name + .as_ref() + .map_or(CharSlice::empty(), |s| s.as_str().into()), + method_name: from + .method_name + .as_ref() + .map_or(CharSlice::empty(), |s| s.as_str().into()), + source_file: from + .source_file + .as_ref() + .map_or(CharSlice::empty(), |s| s.as_str().into()), + signature: from.signature.as_ref().map(|s| s.as_str().into()).into(), + lines: from.lines.as_ptr(), + lines_count: from.lines.len() as u32, + in_body_location: from.in_body_location, + } + } +} + +#[repr(C)] +pub struct Probe<'a> { + pub id: CharSlice<'a>, + pub version: u64, + pub language: CharSlice<'a>, + pub tags: CharSliceVec<'a>, + pub target: ProbeTarget<'a>, // "where" is rust keyword + pub evaluate_at: EvaluateAt, + pub probe: ProbeType<'a>, + pub diagnostic_msg: CharSlice<'a>, + pub status: ProbeStatus, + pub status_msg: CharSlice<'a>, + pub status_exception: CharSlice<'a>, + pub status_stacktrace: CharSlice<'a>, +} + +impl<'a> From<&'a datadog_live_debugger::Probe> for Probe<'a> { + fn from(from: &'a datadog_live_debugger::Probe) -> Self { + Probe { + id: from.id.as_str().into(), + version: from.version, + language: from + .language + .as_ref() + .map_or(CharSlice::empty(), |s| s.as_str().into()), + tags: (&from.tags).into(), + target: (&from.target).into(), + evaluate_at: from.evaluate_at, + probe: (&from.probe).into(), + status: ProbeStatus::Received, + diagnostic_msg: CharSlice::empty(), + status_msg: CharSlice::empty(), + status_exception: CharSlice::empty(), + status_stacktrace: CharSlice::empty(), + } + } +} + +impl<'a> From<&Probe<'a>> for ProbeMetadata<'a> { + fn from(val: &Probe<'a>) -> Self { + fn to_cow_option<'a>(s: &CharSlice<'a>) -> core::option::Option> { + if s.len() == 0 { + None + } else { + unsafe { Some(s.assume_utf8().into()) } + } + } + // SAFETY: These values are unmodified original rust strings. Just convert it back. + ProbeMetadata { + id: unsafe { val.id.assume_utf8() }.into(), + location: ProbeMetadataLocation { + method: to_cow_option(&val.target.method_name), + r#type: to_cow_option(&val.target.type_name), + }, + } + } +} + +#[repr(C)] +pub struct FilterList<'a> { + pub package_prefixes: CharSliceVec<'a>, + pub classes: CharSliceVec<'a>, +} + +impl<'a> From<&'a datadog_live_debugger::FilterList> for FilterList<'a> { + fn from(from: &'a datadog_live_debugger::FilterList) -> Self { + FilterList { + package_prefixes: (&from.package_prefixes).into(), + classes: (&from.classes).into(), + } + } +} + +#[repr(C)] +pub struct ServiceConfiguration<'a> { + pub id: CharSlice<'a>, + pub allow: FilterList<'a>, + pub deny: FilterList<'a>, + pub sampling_snapshots_per_second: u32, +} + +impl<'a> From<&'a datadog_live_debugger::ServiceConfiguration> for ServiceConfiguration<'a> { + fn from(from: &'a datadog_live_debugger::ServiceConfiguration) -> Self { + ServiceConfiguration { + id: from.id.as_str().into(), + allow: (&from.allow).into(), + deny: (&from.deny).into(), + sampling_snapshots_per_second: from.sampling_snapshots_per_second, + } + } +} + +#[repr(C)] +pub enum LiveDebuggingData<'a> { + None, + Probe(Probe<'a>), + ServiceConfiguration(ServiceConfiguration<'a>), +} + +impl<'a> From<&'a datadog_live_debugger::LiveDebuggingData> for LiveDebuggingData<'a> { + fn from(from: &'a datadog_live_debugger::LiveDebuggingData) -> Self { + match from { + datadog_live_debugger::LiveDebuggingData::Probe(probe) => { + LiveDebuggingData::Probe(probe.into()) + } + datadog_live_debugger::LiveDebuggingData::ServiceConfiguration(config) => { + LiveDebuggingData::ServiceConfiguration(config.into()) + } + } + } +} + +#[no_mangle] +pub extern "C" fn ddog_capture_defaults() -> CaptureConfiguration { + CaptureConfiguration::default() +} diff --git a/live-debugger-ffi/src/evaluator.rs b/live-debugger-ffi/src/evaluator.rs new file mode 100644 index 000000000..a55a3af7c --- /dev/null +++ b/live-debugger-ffi/src/evaluator.rs @@ -0,0 +1,325 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +use datadog_live_debugger::debugger_defs::SnapshotEvaluationError; +use datadog_live_debugger::{DslString, ProbeCondition, ProbeValue, ResultError, ResultValue}; +use ddcommon_ffi::slice::AsBytes; +use ddcommon_ffi::CharSlice; +use std::borrow::Cow; +use std::ffi::c_void; + +#[repr(C)] +pub enum IntermediateValue<'a> { + String(CharSlice<'a>), + Number(f64), + Bool(bool), + Null, + Referenced(&'a c_void), +} + +impl<'a> From<&'a datadog_live_debugger::IntermediateValue<'a, c_void>> for IntermediateValue<'a> { + fn from(value: &'a datadog_live_debugger::IntermediateValue<'a, c_void>) -> Self { + match value { + datadog_live_debugger::IntermediateValue::String(s) => { + IntermediateValue::String(s.as_ref().into()) + } + datadog_live_debugger::IntermediateValue::Number(n) => IntermediateValue::Number(*n), + datadog_live_debugger::IntermediateValue::Bool(b) => IntermediateValue::Bool(*b), + datadog_live_debugger::IntermediateValue::Null => IntermediateValue::Null, + datadog_live_debugger::IntermediateValue::Referenced(value) => { + IntermediateValue::Referenced(value) + } + } + } +} + +pub const EVALUATOR_RESULT_UNDEFINED: *const c_void = 0isize as *const c_void; +pub const EVALUATOR_RESULT_INVALID: *const c_void = -1isize as *const c_void; +pub const EVALUATOR_RESULT_REDACTED: *const c_void = -2isize as *const c_void; + +#[repr(C)] +pub struct VoidCollection { + pub count: isize, // set to < 0 on error + pub elements: *const c_void, + pub free: extern "C" fn(VoidCollection), +} + +#[repr(C)] +#[derive(Clone)] +pub struct Evaluator { + pub equals: + for<'a> extern "C" fn(&'a mut c_void, IntermediateValue<'a>, IntermediateValue<'a>) -> bool, + pub greater_than: + for<'a> extern "C" fn(&'a mut c_void, IntermediateValue<'a>, IntermediateValue<'a>) -> bool, + pub greater_or_equals: + for<'a> extern "C" fn(&'a mut c_void, IntermediateValue<'a>, IntermediateValue<'a>) -> bool, + pub fetch_identifier: + for<'a, 'b> extern "C" fn(&'a mut c_void, &CharSlice<'b>) -> *const c_void, /* special values: @duration, @return, @exception */ + pub fetch_index: for<'a, 'b> extern "C" fn( + &'a mut c_void, + &'a c_void, + IntermediateValue<'b>, + ) -> *const c_void, + pub fetch_nested: for<'a, 'b> extern "C" fn( + &'a mut c_void, + &'a c_void, + IntermediateValue<'b>, + ) -> *const c_void, + pub length: for<'a> extern "C" fn(&'a mut c_void, &'a c_void) -> usize, + pub try_enumerate: for<'a> extern "C" fn(&'a mut c_void, &'a c_void) -> VoidCollection, + pub stringify: for<'a> extern "C" fn(&'a mut c_void, &'a c_void) -> CharSlice<'static>, + pub get_string: for<'a> extern "C" fn(&'a mut c_void, &'a c_void) -> CharSlice<'static>, + pub convert_index: for<'a> extern "C" fn(&'a mut c_void, &'a c_void) -> isize, /* return < 0 on error */ + pub instanceof: for<'a> extern "C" fn(&'a mut c_void, &'a c_void, &CharSlice<'a>) -> bool, +} + +static mut FFI_EVALUATOR: Option = None; + +struct EvalCtx<'e> { + context: &'e mut c_void, + eval: &'static Evaluator, +} + +impl<'e> EvalCtx<'e> { + fn new(context: &'e mut c_void) -> Self { + EvalCtx { + context, + eval: unsafe { FFI_EVALUATOR.as_ref().unwrap() }, + } + } +} + +fn to_fetch_result<'e>(value: *const c_void) -> ResultValue<&'e c_void> { + match value { + EVALUATOR_RESULT_UNDEFINED => Err(ResultError::Undefined), + EVALUATOR_RESULT_INVALID => Err(ResultError::Invalid), + EVALUATOR_RESULT_REDACTED => Err(ResultError::Redacted), + _ => Ok(unsafe { &*value }), + } +} + +impl<'e> datadog_live_debugger::Evaluator<'e, c_void> for EvalCtx<'e> { + fn equals( + &mut self, + a: datadog_live_debugger::IntermediateValue<'e, c_void>, + b: datadog_live_debugger::IntermediateValue<'e, c_void>, + ) -> bool { + (self.eval.equals)(self.context, (&a).into(), (&b).into()) + } + + fn greater_than( + &mut self, + a: datadog_live_debugger::IntermediateValue<'e, c_void>, + b: datadog_live_debugger::IntermediateValue<'e, c_void>, + ) -> bool { + (self.eval.greater_than)(self.context, (&a).into(), (&b).into()) + } + + fn greater_or_equals( + &mut self, + a: datadog_live_debugger::IntermediateValue<'e, c_void>, + b: datadog_live_debugger::IntermediateValue<'e, c_void>, + ) -> bool { + (self.eval.greater_or_equals)(self.context, (&a).into(), (&b).into()) + } + + fn fetch_identifier(&mut self, identifier: &str) -> ResultValue<&'e c_void> { + to_fetch_result((self.eval.fetch_identifier)( + self.context, + &CharSlice::from(identifier), + )) + } + + fn fetch_index( + &mut self, + value: &'e c_void, + index: datadog_live_debugger::IntermediateValue<'e, c_void>, + ) -> ResultValue<&'e c_void> { + to_fetch_result((self.eval.fetch_index)( + self.context, + value, + (&index).into(), + )) + } + + fn fetch_nested( + &mut self, + value: &'e c_void, + member: datadog_live_debugger::IntermediateValue<'e, c_void>, + ) -> ResultValue<&'e c_void> { + to_fetch_result((self.eval.fetch_nested)( + self.context, + value, + (&member).into(), + )) + } + + fn length(&mut self, value: &'e c_void) -> usize { + (self.eval.length)(self.context, value) + } + + fn try_enumerate(&mut self, value: &'e c_void) -> ResultValue> { + let collection = (self.eval.try_enumerate)(self.context, value); + if collection.count < 0 { + Err(if collection.count == EVALUATOR_RESULT_REDACTED as isize { + ResultError::Redacted + } else { + ResultError::Invalid + }) + } else { + // We need to copy, Vec::from_raw_parts with only free in the allocator would be + // unstable... + let mut vec = Vec::with_capacity(collection.count as usize); + unsafe { + vec.extend_from_slice(std::slice::from_raw_parts( + collection.elements as *const &c_void, + collection.count as usize, + )) + }; + (collection.free)(collection); + Ok(vec) + } + } + + fn stringify(&mut self, value: &'e c_void) -> Cow<'e, str> { + (self.eval.stringify)(self.context, value).to_utf8_lossy() + } + + fn get_string(&mut self, value: &'e c_void) -> Cow<'e, str> { + (self.eval.get_string)(self.context, value).to_utf8_lossy() + } + + fn convert_index(&mut self, value: &'e c_void) -> ResultValue { + let index = (self.eval.convert_index)(self.context, value); + match index as *const c_void { + EVALUATOR_RESULT_INVALID => Err(ResultError::Invalid), + EVALUATOR_RESULT_REDACTED => Err(ResultError::Redacted), + _ => Ok(index as usize), + } + } + + fn instanceof(&mut self, value: &'e c_void, class: &'e str) -> bool { + (self.eval.instanceof)(self.context, value, &class.into()) + } +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_register_expr_evaluator(eval: &Evaluator) { + FFI_EVALUATOR = Some(eval.clone()); +} + +#[repr(C)] +pub enum ConditionEvaluationResult { + Success, + Failure, + Error(Box>), +} + +#[no_mangle] +pub extern "C" fn ddog_evaluate_condition( + condition: &ProbeCondition, + context: &mut c_void, +) -> ConditionEvaluationResult { + let mut ctx = EvalCtx::new(context); + match datadog_live_debugger::eval_condition(&mut ctx, condition) { + Ok(true) => ConditionEvaluationResult::Success, + Ok(false) => ConditionEvaluationResult::Failure, + Err(error) => ConditionEvaluationResult::Error(Box::new(vec![error])), + } +} + +pub fn ddog_evaluate_string<'a>( + condition: &'a DslString, + context: &'a mut c_void, + errors: &mut Option>>, +) -> Cow<'a, str> { + let mut ctx = EvalCtx::new(context); + let (result, new_errors) = datadog_live_debugger::eval_string(&mut ctx, condition); + let found_errors = if !new_errors.is_empty() { + Some(Box::new(new_errors)) + } else { + None + }; + std::mem::forget(std::mem::replace(errors, found_errors)); + result +} + +// This is unsafe, but we want to use it as function pointer... +#[no_mangle] +extern "C" fn ddog_drop_void_collection_string(void: VoidCollection) { + unsafe { + String::from_raw_parts( + void.elements as *mut u8, + void.count as usize, + void.count as usize, + ); + } +} + +fn into_void_collection_string(s: &dyn ToString) -> VoidCollection { + let string = s.to_string(); + let new = VoidCollection { + count: string.len() as isize, + elements: string.as_ptr() as *const c_void, + free: ddog_drop_void_collection_string as extern "C" fn(VoidCollection), + }; + std::mem::forget(string); + new +} + +#[no_mangle] +pub extern "C" fn ddog_evaluate_unmanaged_string( + segments: &DslString, + context: &mut c_void, + errors: &mut Option>>, +) -> VoidCollection { + into_void_collection_string(&ddog_evaluate_string(segments, context, errors)) +} + +pub struct InternalIntermediateValue<'a>(datadog_live_debugger::IntermediateValue<'a, c_void>); + +#[repr(C)] +pub enum ValueEvaluationResult<'a> { + Success(Box>), + Error(Box>), +} + +#[no_mangle] +pub extern "C" fn ddog_evaluate_value<'a>( + value: &'a ProbeValue, + context: &'a mut c_void, +) -> ValueEvaluationResult<'a> { + let mut ctx = EvalCtx::new(context); + match datadog_live_debugger::eval_value(&mut ctx, value) { + Ok(value) => ValueEvaluationResult::Success(Box::new(InternalIntermediateValue(value))), + Err(error) => ValueEvaluationResult::Error(Box::new(vec![error])), + } +} + +#[no_mangle] +pub extern "C" fn ddog_evaluated_value_get<'a>( + value: &'a InternalIntermediateValue<'a>, +) -> IntermediateValue<'a> { + (&value.0).into() +} + +#[no_mangle] +pub extern "C" fn ddog_evaluated_value_drop(_: Box) {} + +#[allow(clippy::boxed_local)] +pub fn ddog_evaluated_value_into_string<'a>( + value: Box>, + context: &'a mut c_void, +) -> Cow<'a, str> { + let mut ctx = EvalCtx::new(context); + datadog_live_debugger::eval_intermediate_to_string(&mut ctx, value.0) +} + +#[no_mangle] +pub extern "C" fn ddog_evaluated_value_into_unmanaged_string<'a>( + value: Box>, + context: &'a mut c_void, +) -> VoidCollection { + into_void_collection_string(&ddog_evaluated_value_into_string(value, context)) +} diff --git a/live-debugger-ffi/src/lib.rs b/live-debugger-ffi/src/lib.rs new file mode 100644 index 000000000..54bf2487a --- /dev/null +++ b/live-debugger-ffi/src/lib.rs @@ -0,0 +1,8 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +pub mod data; +pub mod evaluator; +pub mod parse; +pub mod send_data; +pub mod sender; diff --git a/live-debugger-ffi/src/parse.rs b/live-debugger-ffi/src/parse.rs new file mode 100644 index 000000000..ac8260358 --- /dev/null +++ b/live-debugger-ffi/src/parse.rs @@ -0,0 +1,40 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::data::LiveDebuggingData; +use ddcommon_ffi::slice::AsBytes; +use ddcommon_ffi::CharSlice; + +#[repr(C)] +pub struct LiveDebuggingParseResult { + pub data: LiveDebuggingData<'static>, + opaque_data: Option>, +} + +#[no_mangle] +pub extern "C" fn ddog_parse_live_debugger_json(json: CharSlice) -> LiveDebuggingParseResult { + if let Ok(parsed) = + datadog_live_debugger::parse_json(unsafe { std::str::from_utf8_unchecked(json.as_bytes()) }) + { + let parsed = Box::new(parsed); + LiveDebuggingParseResult { + // we have the box. Rust doesn't allow us to specify a self-referential struct, so + // pretend it's 'static + data: unsafe { + std::mem::transmute::<&_, &'static datadog_live_debugger::LiveDebuggingData>( + &*parsed, + ) + } + .into(), + opaque_data: Some(parsed), + } + } else { + LiveDebuggingParseResult { + data: LiveDebuggingData::None, + opaque_data: None, + } + } +} + +#[no_mangle] +pub extern "C" fn ddog_drop_live_debugger_parse_result(_: LiveDebuggingParseResult) {} diff --git a/live-debugger-ffi/src/send_data.rs b/live-debugger-ffi/src/send_data.rs new file mode 100644 index 000000000..d9e67b49e --- /dev/null +++ b/live-debugger-ffi/src/send_data.rs @@ -0,0 +1,455 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use ddcommon_ffi::CharSlice; +use std::borrow::Cow; +use std::collections::hash_map; +use std::mem::transmute; +// Alias to prevent cbindgen panic +use crate::data::Probe; +use datadog_live_debugger::debugger_defs::{ + Capture as DebuggerCaptureAlias, Capture, Captures, DebuggerData, DebuggerPayload, Diagnostics, + DiagnosticsError, Entry, Fields, ProbeMetadata, ProbeMetadataLocation, ProbeStatus, Snapshot, + SnapshotEvaluationError, Value as DebuggerValueAlias, +}; +use datadog_live_debugger::sender::generate_new_id; +use datadog_live_debugger::{ + add_redacted_name, add_redacted_type, is_redacted_name, is_redacted_type, +}; +use ddcommon_ffi::slice::AsBytes; + +#[repr(C)] +pub enum FieldType { + STATIC, + ARG, + LOCAL, +} + +#[repr(C)] +pub struct CaptureValue<'a> { + pub r#type: CharSlice<'a>, + pub value: CharSlice<'a>, + pub fields: Option>>, + pub elements: Vec>, + pub entries: Vec>, + pub is_null: bool, + pub truncated: bool, + pub not_captured_reason: CharSlice<'a>, + pub size: CharSlice<'a>, +} + +impl<'a> From> for DebuggerValueAlias<'a> { + fn from(val: CaptureValue<'a>) -> Self { + DebuggerValueAlias { + r#type: val.r#type.to_utf8_lossy(), + value: if val.value.len() == 0 { + None + } else { + Some(val.value.to_utf8_lossy()) + }, + fields: if let Some(boxed) = val.fields { + *boxed + } else { + Fields::default() + }, + elements: unsafe { + transmute::>, Vec>>(val.elements) + }, // SAFETY: is transparent + entries: val.entries, + is_null: val.is_null, + truncated: val.truncated, + not_captured_reason: if val.not_captured_reason.len() == 0 { + None + } else { + Some(val.not_captured_reason.to_utf8_lossy()) + }, + size: if val.size.len() == 0 { + None + } else { + Some(val.size.to_utf8_lossy()) + }, + } + } +} + +/// cbindgen:no-export +#[repr(transparent)] +pub struct DebuggerValue<'a>(DebuggerValueAlias<'a>); +/// cbindgen:no-export +#[repr(transparent)] +pub struct DebuggerCapture<'a>(DebuggerCaptureAlias<'a>); + +#[repr(C)] +pub struct ExceptionSnapshot<'a> { + pub data: *mut DebuggerPayload<'a>, + pub capture: *mut DebuggerCapture<'a>, +} + +#[no_mangle] +pub extern "C" fn ddog_create_exception_snapshot<'a>( + buffer: &mut Vec>, + service: CharSlice<'a>, + language: CharSlice<'a>, + id: CharSlice<'a>, + exception_id: CharSlice<'a>, + exception_hash: CharSlice<'a>, + frame_index: u32, + type_name: CharSlice<'a>, + method_name: CharSlice<'a>, + timestamp: u64, +) -> *mut DebuggerCapture<'a> { + let snapshot = DebuggerPayload { + service: service.to_utf8_lossy(), + ddsource: "dd_debugger", + timestamp, + message: None, + debugger: DebuggerData::Snapshot(Snapshot { + captures: Some(Captures { + r#return: Some(Capture::default()), + ..Default::default() + }), + probe: Some(ProbeMetadata { + id: Default::default(), + location: ProbeMetadataLocation { + method: if method_name.is_empty() { + None + } else { + Some(method_name.to_utf8_lossy()) + }, + r#type: if type_name.is_empty() { + None + } else { + Some(type_name.to_utf8_lossy()) + }, + }, + }), + language: language.to_utf8_lossy(), + id: id.to_utf8_lossy(), + exception_capture_id: Some(exception_id.to_utf8_lossy()), + exception_hash: Some(exception_hash.to_utf8_lossy()), + frame_index: Some(frame_index), + timestamp, + ..Default::default() + }), + }; + buffer.push(snapshot); + let DebuggerData::Snapshot(ref mut snapshot) = buffer.last_mut().unwrap().debugger else { + unreachable!(); + }; + unsafe { + transmute( + snapshot + .captures + .as_mut() + .unwrap() + .r#return + .as_mut() + .unwrap(), + ) + } +} + +#[no_mangle] +pub extern "C" fn ddog_create_log_probe_snapshot<'a>( + probe: &'a Probe, + message: Option<&CharSlice<'a>>, + service: CharSlice<'a>, + language: CharSlice<'a>, + timestamp: u64, +) -> Box> { + Box::new(DebuggerPayload { + service: service.to_utf8_lossy(), + ddsource: "dd_debugger", + timestamp, + message: message.map(|m| m.to_utf8_lossy()), + debugger: DebuggerData::Snapshot(Snapshot { + captures: Some(Captures { + ..Default::default() + }), + language: language.to_utf8_lossy(), + id: Cow::Owned(generate_new_id().as_hyphenated().to_string()), + probe: Some(probe.into()), + timestamp, + ..Default::default() + }), + }) +} + +#[no_mangle] +pub extern "C" fn ddog_update_payload_message<'a>( + payload: &mut DebuggerPayload<'a>, + message: CharSlice<'a>, +) { + payload.message = Some(message.to_utf8_lossy()); +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_snapshot_entry<'a>( + payload: &mut DebuggerPayload<'a>, +) -> *mut DebuggerCapture<'a> { + let DebuggerData::Snapshot(ref mut snapshot) = payload.debugger else { + unreachable!(); + }; + transmute( + snapshot + .captures + .as_mut() + .unwrap() + .entry + .insert(Capture::default()), + ) +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_snapshot_lines<'a>( + payload: &mut DebuggerPayload<'a>, + line: u32, +) -> *mut DebuggerCapture<'a> { + let DebuggerData::Snapshot(ref mut snapshot) = payload.debugger else { + unreachable!(); + }; + transmute( + match snapshot.captures.as_mut().unwrap().lines.entry(line) { + hash_map::Entry::Occupied(e) => e.into_mut(), + hash_map::Entry::Vacant(e) => e.insert(Capture::default()), + }, + ) +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_snapshot_exit<'a>( + payload: &mut DebuggerPayload<'a>, +) -> *mut DebuggerCapture<'a> { + let DebuggerData::Snapshot(ref mut snapshot) = payload.debugger else { + unreachable!(); + }; + transmute( + snapshot + .captures + .as_mut() + .unwrap() + .r#return + .insert(Capture::default()), + ) +} + +#[no_mangle] +pub extern "C" fn ddog_snapshot_redacted_name(name: CharSlice) -> bool { + is_redacted_name(name.as_bytes()) +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_snapshot_add_redacted_name(name: CharSlice) { + add_redacted_name(name.as_bytes()) +} + +#[no_mangle] +pub extern "C" fn ddog_snapshot_redacted_type(name: CharSlice) -> bool { + is_redacted_type(name.as_bytes()) +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_snapshot_add_redacted_type(name: CharSlice) { + add_redacted_type(name.as_bytes()) +} + +#[no_mangle] +#[allow(improper_ctypes_definitions)] // Vec has a fixed size, and we care only about that here +pub extern "C" fn ddog_snapshot_add_field<'a, 'b: 'a, 'c: 'a>( + capture: &mut DebuggerCapture<'a>, + r#type: FieldType, + name: CharSlice<'b>, + value: CaptureValue<'c>, +) { + let fields = match r#type { + FieldType::STATIC => &mut capture.0.static_fields, + FieldType::ARG => &mut capture.0.arguments, + FieldType::LOCAL => &mut capture.0.locals, + }; + fields.insert(name.to_utf8_lossy(), value.into()); +} + +#[no_mangle] +#[allow(improper_ctypes_definitions)] // Vec has a fixed size, and we care only about that here +pub extern "C" fn ddog_capture_value_add_element<'a, 'b: 'a>( + value: &mut CaptureValue<'a>, + element: CaptureValue<'b>, +) { + value.elements.push(DebuggerValue(element.into())); +} + +#[no_mangle] +#[allow(improper_ctypes_definitions)] // Vec has a fixed size, and we care only about that here +pub extern "C" fn ddog_capture_value_add_entry<'a, 'b: 'a, 'c: 'a>( + value: &mut CaptureValue<'a>, + key: CaptureValue<'b>, + element: CaptureValue<'c>, +) { + value.entries.push(Entry(key.into(), element.into())); +} + +#[no_mangle] +#[allow(improper_ctypes_definitions)] // Vec has a fixed size, and we care only about that here +pub extern "C" fn ddog_capture_value_add_field<'a, 'b: 'a, 'c: 'a>( + value: &mut CaptureValue<'a>, + key: CharSlice<'b>, + element: CaptureValue<'c>, +) { + let fields = match value.fields { + None => { + value.fields = Some(Box::default()); + value.fields.as_mut().unwrap() + } + Some(ref mut f) => f, + }; + fields.insert(key.to_utf8_lossy(), element.into()); +} + +#[no_mangle] +pub extern "C" fn ddog_snapshot_format_new_uuid(buf: &mut [u8; 36]) { + generate_new_id().as_hyphenated().encode_lower(buf); +} + +#[no_mangle] +#[allow(clippy::ptr_arg)] +pub extern "C" fn ddog_evaluation_error_first_msg(vec: &Vec) -> CharSlice { + CharSlice::from(vec[0].message.as_str()) +} + +#[no_mangle] +pub extern "C" fn ddog_evaluation_error_drop(_: Box>) {} + +#[no_mangle] +pub extern "C" fn ddog_evaluation_error_snapshot<'a>( + probe: &'a Probe, + service: CharSlice<'a>, + language: CharSlice<'a>, + errors: Box>, + timestamp: u64, +) -> Box> { + Box::new(DebuggerPayload { + service: service.to_utf8_lossy(), + ddsource: "dd_debugger", + timestamp, + message: Some(Cow::Owned(format!( + "Evaluation errors for probe id {}", + probe.id + ))), + debugger: DebuggerData::Snapshot(Snapshot { + language: language.to_utf8_lossy(), + id: Cow::Owned(generate_new_id().as_hyphenated().to_string()), + probe: Some(probe.into()), + timestamp, + evaluation_errors: *errors, + ..Default::default() + }), + }) +} + +pub fn serialize_debugger_payload(payload: &DebuggerPayload) -> String { + serde_json::to_string(payload).unwrap() +} + +#[no_mangle] +pub extern "C" fn ddog_serialize_debugger_payload( + payload: &DebuggerPayload, + callback: extern "C" fn(CharSlice), +) { + let payload = serialize_debugger_payload(payload); + callback(CharSlice::from(payload.as_str())) +} + +#[no_mangle] +pub extern "C" fn ddog_drop_debugger_payload(_: Box) {} + +pub fn ddog_debugger_diagnostics_create_unboxed<'a>( + probe: &'a Probe, + service: Cow<'a, str>, + runtime_id: Cow<'a, str>, + timestamp: u64, +) -> DebuggerPayload<'a> { + let mut diagnostics = Diagnostics { + probe_id: probe.id.to_utf8_lossy(), + probe_version: probe.version, + status: probe.status, + runtime_id, + ..Default::default() + }; + match probe.status { + ProbeStatus::Error => { + diagnostics.exception = Some(DiagnosticsError { + r#type: probe.status_exception.to_utf8_lossy(), + message: probe.status_msg.to_utf8_lossy(), + stacktrace: if probe.status_exception.len() > 0 { + Some(probe.status_exception.to_utf8_lossy()) + } else { + None + }, + }); + } + ProbeStatus::Warning => diagnostics.details = Some(probe.status_msg.to_utf8_lossy()), + _ => {} + } + DebuggerPayload { + service, + ddsource: "dd_debugger", + timestamp, + message: Some(if probe.diagnostic_msg.len() > 0 { + probe.diagnostic_msg.to_utf8_lossy() + } else { + Cow::Owned(match probe.status { + ProbeStatus::Received => { + format!("Received definition for probe {}", &diagnostics.probe_id) + } + ProbeStatus::Installed | ProbeStatus::Emitting => { + format!("Instrumented probe {}", &diagnostics.probe_id) + } + ProbeStatus::Blocked => { + format!("Instrumentation denied for probe {}", &diagnostics.probe_id) + } + ProbeStatus::Error => format!( + "Encountered error while instrumenting probe {}: {}", + &diagnostics.probe_id, + diagnostics.exception.as_ref().unwrap().message + ), + ProbeStatus::Warning => format!( + "Probe {} warning: {}", + &diagnostics.probe_id, + diagnostics.details.as_ref().unwrap() + ), + }) + }), + debugger: DebuggerData::Diagnostics(diagnostics), + } +} + +#[no_mangle] +pub extern "C" fn ddog_debugger_diagnostics_create<'a>( + probe: &'a Probe, + service: CharSlice<'a>, + runtime_id: CharSlice<'a>, + timestamp: u64, +) -> Box> { + Box::new(ddog_debugger_diagnostics_create_unboxed( + probe, + service.to_utf8_lossy(), + runtime_id.to_utf8_lossy(), + timestamp, + )) +} + +#[no_mangle] +pub extern "C" fn ddog_debugger_diagnostics_set_parent_id<'a>( + payload: &mut DebuggerPayload<'a>, + parent_id: CharSlice<'a>, +) { + let DebuggerData::Diagnostics(ref mut diagnostics) = payload.debugger else { + unreachable!(); + }; + diagnostics.parent_id = Some(parent_id.to_utf8_lossy()); +} diff --git a/live-debugger-ffi/src/sender.rs b/live-debugger-ffi/src/sender.rs new file mode 100644 index 000000000..2a7057ad0 --- /dev/null +++ b/live-debugger-ffi/src/sender.rs @@ -0,0 +1,170 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::send_data::serialize_debugger_payload; +use datadog_live_debugger::debugger_defs::DebuggerPayload; +use datadog_live_debugger::sender; +use datadog_live_debugger::sender::{generate_tags, Config, DebuggerType}; +use ddcommon::tag::Tag; +use ddcommon::Endpoint; +use ddcommon_ffi::slice::AsBytes; +use ddcommon_ffi::{CharSlice, MaybeError}; +use log::{debug, warn}; +use percent_encoding::{percent_encode, CONTROLS}; +use std::sync::Arc; +use std::thread::JoinHandle; +use tokio::sync::mpsc; +use tokio_util::task::TaskTracker; + +macro_rules! try_c { + ($failable:expr) => { + match $failable { + Ok(o) => o, + Err(e) => return MaybeError::Some(ddcommon_ffi::Error::from(format!("{:?}", e))), + } + }; +} + +#[repr(C)] +pub struct OwnedCharSlice { + slice: CharSlice<'static>, + free: extern "C" fn(CharSlice<'static>), +} + +unsafe impl Send for OwnedCharSlice {} + +impl Drop for OwnedCharSlice { + fn drop(&mut self) { + (self.free)(self.slice) + } +} + +enum SendData { + Raw(Vec, DebuggerType), + Wrapped(OwnedCharSlice, DebuggerType), +} + +async fn sender_routine(config: Arc, tags: String, mut receiver: mpsc::Receiver) { + let tags = Arc::new(tags); + let tracker = TaskTracker::new(); + loop { + let data = match receiver.recv().await { + None => break, + Some(data) => data, + }; + + let config = config.clone(); + let tags = tags.clone(); + tracker.spawn(async move { + let (debugger_type, data) = match data { + SendData::Raw(ref vec, r#type) => (r#type, vec.as_slice()), + SendData::Wrapped(ref wrapped, r#type) => (r#type, wrapped.slice.as_bytes()), + }; + + if let Err(e) = sender::send(data, &config, debugger_type, &tags).await { + warn!("Failed to send {debugger_type:?} debugger data: {e:?}"); + } else { + debug!( + "Successfully sent {} debugger data byte {debugger_type:?} payload", + data.len() + ); + } + }); + } + + tracker.wait().await; +} + +pub struct SenderHandle { + join: JoinHandle<()>, + channel: mpsc::Sender, +} + +#[no_mangle] +pub extern "C" fn ddog_live_debugger_build_tags( + debugger_version: CharSlice, + env: CharSlice, + version: CharSlice, + runtime_id: CharSlice, + global_tags: ddcommon_ffi::Vec, +) -> Box { + Box::new(generate_tags( + &debugger_version.to_utf8_lossy(), + &env.to_utf8_lossy(), + &version.to_utf8_lossy(), + &runtime_id.to_utf8_lossy(), + &mut global_tags.into_iter(), + )) +} + +#[no_mangle] +pub extern "C" fn ddog_live_debugger_tags_from_raw(tags: CharSlice) -> Box { + Box::new(percent_encode(tags.as_bytes(), CONTROLS).to_string()) +} + +#[no_mangle] +pub extern "C" fn ddog_live_debugger_spawn_sender( + endpoint: &Endpoint, + tags: Box, + handle: &mut *mut SenderHandle, +) -> MaybeError { + let runtime = try_c!(tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()); + + let (tx, mailbox) = mpsc::channel(5000); + let mut config = Config::default(); + try_c!(config.set_endpoint(endpoint.clone(), endpoint.clone())); + let config = Arc::new(config); + + *handle = Box::into_raw(Box::new(SenderHandle { + join: std::thread::spawn(move || { + runtime.block_on(sender_routine(config, *tags, mailbox)); + runtime.shutdown_background(); + }), + channel: tx, + })); + + MaybeError::None +} + +#[no_mangle] +pub extern "C" fn ddog_live_debugger_send_raw_data( + handle: &mut SenderHandle, + debugger_type: DebuggerType, + data: OwnedCharSlice, +) -> bool { + handle + .channel + .try_send(SendData::Wrapped(data, debugger_type)) + .is_ok() +} + +#[no_mangle] +pub extern "C" fn ddog_live_debugger_send_payload( + handle: &mut SenderHandle, + data: &DebuggerPayload, +) -> bool { + let debugger_type = DebuggerType::of_payload(data); + handle + .channel + .try_send(SendData::Raw( + serialize_debugger_payload(data).into_bytes(), + debugger_type, + )) + .is_err() +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_live_debugger_drop_sender(sender: *mut SenderHandle) { + drop(Box::from_raw(sender)); +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +pub unsafe extern "C" fn ddog_live_debugger_join_sender(sender: *mut SenderHandle) { + let sender = Box::from_raw(sender); + drop(sender.channel); + _ = sender.join.join(); +} diff --git a/live-debugger/Cargo.toml b/live-debugger/Cargo.toml new file mode 100644 index 000000000..4edd736a3 --- /dev/null +++ b/live-debugger/Cargo.toml @@ -0,0 +1,25 @@ +[package] +edition = "2021" +license = "Apache 2.0" +name = "datadog-live-debugger" +version = "0.0.1" + +[dependencies] +anyhow = "1.0" +ddcommon = { path = "../ddcommon" } +lazy_static = "1.4" +hyper = { version = "0.14", features = ["client"] } +regex = "1.9.3" +json = "0.12.4" +percent-encoding = "2.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sys-info = { version = "0.9.0" } +uuid = { version = "1.0", features = ["v4"] } +regex-automata = "0.4.5" +smallvec = "1.13.2" +constcat = "0.4.1" +tokio = "1.36.0" + +[lib] +bench = false diff --git a/live-debugger/src/debugger_defs.rs b/live-debugger/src/debugger_defs.rs new file mode 100644 index 000000000..4f15eb93f --- /dev/null +++ b/live-debugger/src/debugger_defs.rs @@ -0,0 +1,156 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::collections::HashMap; + +#[derive(Serialize, Deserialize)] +pub struct DebuggerPayload<'a> { + pub service: Cow<'a, str>, + pub ddsource: &'static str, + pub timestamp: u64, + pub debugger: DebuggerData<'a>, + pub message: Option>, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::large_enum_variant)] +pub enum DebuggerData<'a> { + Snapshot(Snapshot<'a>), + Diagnostics(Diagnostics<'a>), +} + +#[derive(Serialize, Deserialize)] +pub struct ProbeMetadataLocation<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + pub method: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct ProbeMetadata<'a> { + pub id: Cow<'a, str>, + pub location: ProbeMetadataLocation<'a>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SnapshotEvaluationError { + pub expr: String, + pub message: String, +} + +#[derive(Serialize, Deserialize)] +pub struct SnapshotStackFrame { + pub expr: String, + pub message: String, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Snapshot<'a> { + pub language: Cow<'a, str>, + pub id: Cow<'a, str>, + pub timestamp: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub exception_capture_id: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub exception_hash: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub frame_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub captures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub probe: Option>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub evaluation_errors: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub stack: Vec, +} + +#[derive(Default, Serialize, Deserialize)] +pub struct Captures<'a> { + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub lines: HashMap>, + #[serde(skip_serializing_if = "Option::is_none")] + pub entry: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#return: Option>, +} + +pub type Fields<'a> = HashMap, Value<'a>>; +#[derive(Default, Serialize, Deserialize)] +pub struct Capture<'a> { + #[serde(skip_serializing_if = "HashMap::is_empty")] + #[serde(rename = "staticFields")] + pub static_fields: Fields<'a>, + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub arguments: Fields<'a>, + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub locals: Fields<'a>, + #[serde(skip_serializing_if = "Option::is_none")] + pub throwable: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct Entry<'a>(pub Value<'a>, pub Value<'a>); + +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Value<'a> { + pub r#type: Cow<'a, str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option>, + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub fields: Fields<'a>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub elements: Vec>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entries: Vec>, + #[serde(skip_serializing_if = "<&bool as std::ops::Not>::not")] + pub is_null: bool, + #[serde(skip_serializing_if = "<&bool as std::ops::Not>::not")] + pub truncated: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub not_captured_reason: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option>, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Diagnostics<'a> { + pub probe_id: Cow<'a, str>, + pub runtime_id: Cow<'a, str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_id: Option>, + pub probe_version: u64, + pub status: ProbeStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub exception: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option>, +} + +#[derive(Serialize, Deserialize, Default, Copy, Clone)] +#[serde(rename_all = "UPPERCASE")] +#[repr(C)] +pub enum ProbeStatus { + #[default] + Received, + Installed, + Emitting, + Error, + Blocked, + Warning, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DiagnosticsError<'a> { + pub r#type: Cow<'a, str>, + pub message: Cow<'a, str>, + pub stacktrace: Option>, +} diff --git a/live-debugger/src/expr_defs.rs b/live-debugger/src/expr_defs.rs new file mode 100644 index 000000000..9a720f138 --- /dev/null +++ b/live-debugger/src/expr_defs.rs @@ -0,0 +1,252 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +use std::fmt::{Display, Formatter}; + +#[derive(Debug)] +pub enum CollectionSource { + Reference(Reference), + FilterOperator(Box<(CollectionSource, Condition)>), +} + +impl Display for CollectionSource { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + CollectionSource::Reference(r) => r.fmt(f), + CollectionSource::FilterOperator(b) => { + let (source, cond) = &**b; + write!(f, "filter({source}, {cond})") + } + } + } +} + +#[derive(Debug)] +pub enum Reference { + IteratorVariable, + Base(String), + Index(Box<(CollectionSource, Value)>), // i.e. foo[bar] + Nested(Box<(Reference, Value)>), // i.e. foo.bar +} + +impl Display for Reference { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Reference::IteratorVariable => f.write_str("@it"), + Reference::Base(s) => s.fmt(f), + Reference::Index(b) => { + let (source, index) = &**b; + write!(f, "{source}[{index}]") + } + Reference::Nested(b) => { + let (source, member) = &**b; + if let Value::String(StringSource::String(s)) = member { + write!(f, "{source}.{s}") + } else { + write!(f, "{source}.{member}") + } + } + } + } +} + +#[derive(Debug)] +pub enum BinaryComparison { + Equals, + NotEquals, + GreaterThan, + GreaterOrEquals, + LowerThan, + LowerOrEquals, +} + +impl Display for BinaryComparison { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + BinaryComparison::Equals => "==", + BinaryComparison::NotEquals => "!=", + BinaryComparison::GreaterThan => ">", + BinaryComparison::GreaterOrEquals => ">=", + BinaryComparison::LowerThan => "<", + BinaryComparison::LowerOrEquals => "<=", + }) + } +} + +#[derive(Debug)] +pub enum StringComparison { + StartsWith, + EndsWith, + Contains, + Matches, +} + +impl Display for StringComparison { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + StringComparison::StartsWith => "startsWith", + StringComparison::EndsWith => "endsWith", + StringComparison::Contains => "contains", + StringComparison::Matches => "matches", + }) + } +} + +#[derive(Debug)] +pub enum CollectionMatch { + All, + Any, +} + +impl Display for CollectionMatch { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + CollectionMatch::All => "all", + CollectionMatch::Any => "any", + }) + } +} + +#[derive(Debug)] +pub enum Condition { + Always, + Never, + Disjunction(Box<(Condition, Condition)>), + Conjunction(Box<(Condition, Condition)>), + Negation(Box), + StringComparison(StringComparison, StringSource, String), + BinaryComparison(Value, BinaryComparison, Value), + CollectionMatch(CollectionMatch, Reference, Box), + Instanceof(Reference, String), + IsDefinedReference(Reference), + IsEmptyReference(Reference), +} + +struct NonAssocBoolOp<'a>(&'a Condition, bool); + +impl<'a> Display for NonAssocBoolOp<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.1 { + write!(f, "({})", self.0) + } else { + self.0.fmt(f) + } + } +} + +impl Display for Condition { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Condition::Always => f.write_str("true"), + Condition::Never => f.write_str("false"), + Condition::Disjunction(b) => { + let (x, y) = &**b; + fn is_nonassoc(condition: &Condition) -> NonAssocBoolOp { + NonAssocBoolOp(condition, matches!(condition, Condition::Conjunction(_))) + } + write!(f, "{} || {}", is_nonassoc(x), is_nonassoc(y)) + } + Condition::Conjunction(b) => { + let (x, y) = &**b; + fn is_nonassoc(condition: &Condition) -> NonAssocBoolOp { + NonAssocBoolOp(condition, matches!(condition, Condition::Disjunction(_))) + } + write!(f, "{} && {}", is_nonassoc(x), is_nonassoc(y)) + } + Condition::Negation(b) => write!( + f, + "!{}", + NonAssocBoolOp( + b, + matches!( + **b, + Condition::Conjunction(_) + | Condition::Disjunction(_) + | Condition::BinaryComparison(..) + ) + ) + ), + Condition::StringComparison(cmp, s, v) => write!(f, "{cmp}({s}, {v})"), + Condition::BinaryComparison(x, cmp, y) => write!(f, "{x} {cmp} {y}"), + Condition::CollectionMatch(op, s, c) => write!(f, "{op}({s}, {})", **c), + Condition::IsDefinedReference(r) => write!(f, "isDefined({r})"), + Condition::IsEmptyReference(r) => write!(f, "isEmpty({r})"), + Condition::Instanceof(r, class) => write!(f, "instanceof({r}, {class})"), + } + } +} + +#[derive(Debug)] +pub enum NumberSource { + Number(f64), + CollectionSize(CollectionSource), + Reference(Reference), +} + +impl Display for NumberSource { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + NumberSource::Number(n) => n.fmt(f), + NumberSource::CollectionSize(s) => write!(f, "len({s})"), + NumberSource::Reference(r) => r.fmt(f), + } + } +} + +#[derive(Debug)] +pub enum StringSource { + String(String), + Substring(Box<(StringSource, NumberSource, NumberSource)>), + Null, + Reference(Reference), +} + +impl Display for StringSource { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + StringSource::String(s) => { + write!(f, "\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } + StringSource::Substring(b) => { + let (source, start, end) = &**b; + write!(f, "substring({source}, {start}, {end})") + } + StringSource::Null => f.write_str("null"), + StringSource::Reference(r) => r.fmt(f), + } + } +} + +#[derive(Debug)] +pub enum Value { + Bool(Box), + String(StringSource), + Number(NumberSource), +} + +impl Display for Value { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Value::Bool(b) => (**b).fmt(f), + Value::String(s) => s.fmt(f), + Value::Number(s) => s.fmt(f), + } + } +} + +#[derive(Debug)] +pub enum DslPart { + Ref(CollectionSource), + Value(Value), + String(String), +} + +impl Display for DslPart { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + DslPart::Ref(r) => write!(f, "{{{r}}}"), + DslPart::Value(v) => write!(f, "{{{v}}}"), + DslPart::String(s) => s.fmt(f), + } + } +} diff --git a/live-debugger/src/expr_eval.rs b/live-debugger/src/expr_eval.rs new file mode 100644 index 000000000..b584aea55 --- /dev/null +++ b/live-debugger/src/expr_eval.rs @@ -0,0 +1,1373 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +use crate::debugger_defs::SnapshotEvaluationError; +use crate::expr_defs::{ + BinaryComparison, CollectionMatch, CollectionSource, Condition, DslPart, NumberSource, + Reference, StringComparison, StringSource, Value, +}; +use regex::Regex; +use std::borrow::Cow; +use std::cmp::min; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +#[derive(Debug)] +pub struct DslString(pub(crate) Vec); +#[derive(Debug)] +pub struct ProbeValue(pub(crate) Value); +#[derive(Debug)] +pub struct ProbeCondition(pub(crate) Condition); + +impl Display for DslString { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for p in self.0.iter() { + p.fmt(f)?; + } + Ok(()) + } +} + +impl Display for ProbeValue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl Display for ProbeCondition { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +pub enum IntermediateValue<'a, I> { + String(Cow<'a, str>), + Number(f64), + Bool(bool), + Null, + Referenced(&'a I), +} + +impl<'a, I> Clone for IntermediateValue<'a, I> { + fn clone(&self) -> Self { + match self { + IntermediateValue::String(s) => IntermediateValue::String(s.clone()), + IntermediateValue::Number(n) => IntermediateValue::Number(*n), + IntermediateValue::Bool(b) => IntermediateValue::Bool(*b), + IntermediateValue::Null => IntermediateValue::Null, + IntermediateValue::Referenced(r) => IntermediateValue::Referenced(*r), + } + } +} + +pub enum ResultError { + Undefined, + Invalid, + Redacted, +} + +pub type ResultValue = Result; + +pub trait Evaluator<'e, I> { + fn equals(&mut self, a: IntermediateValue<'e, I>, b: IntermediateValue<'e, I>) -> bool; + fn greater_than(&mut self, a: IntermediateValue<'e, I>, b: IntermediateValue<'e, I>) -> bool; + fn greater_or_equals( + &mut self, + a: IntermediateValue<'e, I>, + b: IntermediateValue<'e, I>, + ) -> bool; + fn fetch_identifier(&mut self, identifier: &str) -> ResultValue<&'e I>; // special values: @duration, @return, @exception + fn fetch_index(&mut self, value: &'e I, index: IntermediateValue<'e, I>) -> ResultValue<&'e I>; + fn fetch_nested( + &mut self, + value: &'e I, + member: IntermediateValue<'e, I>, + ) -> ResultValue<&'e I>; + fn length(&mut self, value: &'e I) -> usize; + fn try_enumerate(&mut self, value: &'e I) -> ResultValue>; + fn stringify(&mut self, value: &'e I) -> Cow<'e, str>; // generic string representation + fn get_string(&mut self, value: &'e I) -> Cow<'e, str>; // log output-formatted string + fn convert_index(&mut self, value: &'e I) -> ResultValue; + fn instanceof(&mut self, value: &'e I, class: &'e str) -> bool; +} + +enum InvalidFetch<'a, I> { + NoIterator, + Identifier(ResultError, &'a String), + Index( + ResultError, + &'a CollectionSource, + &'a Value, + &'a I, + IntermediateValue<'a, I>, + ), + IndexEvaluated(&'a CollectionSource, &'a Value, Vec<&'a I>, usize), + Nested( + ResultError, + &'a Reference, + &'a Value, + &'a I, + IntermediateValue<'a, I>, + ), +} + +#[derive(Debug)] +struct EvErr(String); + +impl EvErr { + pub fn str(str: T) -> Self { + EvErr(str.to_string()) + } + + pub fn refed<'a, 'e, I, E: Evaluator<'e, I>>( + eval: &mut Eval<'a, 'e, I, E>, + reference: InvalidFetch<'e, I>, + ) -> Self { + fn fetch_mode(e: ResultError) -> &'static str { + match e { + ResultError::Undefined => "fetch", + ResultError::Invalid => "read invalid", + ResultError::Redacted => "read redacted", + } + } + EvErr(match reference { + InvalidFetch::NoIterator => "Attempted to use @it in non-iterator context".to_string(), + InvalidFetch::Identifier(e, reference) => { + format!("Could not {} variable {reference}", fetch_mode(e)) + } + InvalidFetch::Index(e, source, dim, base, index) => { + if matches!( + dim, + Value::String(StringSource::String(_)) | Value::Number(NumberSource::Number(_)) + ) { + format!( + "Could not {} index {dim} on {source} (evaluated to {})", + fetch_mode(e), + eval.iref_string(base) + ) + } else { + format!( + "Could not {} index {dim} (evaluated to {}) on {source} (evaluated to {})", + fetch_mode(e), + eval.get_string(index), + eval.iref_string(base) + ) + } + } + InvalidFetch::IndexEvaluated(source, dim, vec, index) => { + let first_entries: Vec<_> = vec + .into_iter() + .take(5) + .map(|i| eval.iref_string(i)) + .collect(); + if matches!( + dim, + Value::String(StringSource::String(_)) | Value::Number(NumberSource::Number(_)) + ) { + format!( + "Could not fetch index {dim} on {source} (evaluated to [{}])", + first_entries.join(", ") + ) + } else { + format!("Could not fetch index {dim} (evaluated to {index}) on {source} (evaluated to [{}])", first_entries.join(", ")) + } + } + InvalidFetch::Nested(e, source, prop, base, nested) => { + if matches!( + prop, + Value::String(StringSource::String(_)) | Value::Number(NumberSource::Number(_)) + ) { + format!( + "Could not {} property {prop} on {source} (evaluated to {})", + fetch_mode(e), + eval.iref_string(base) + ) + } else { + format!("Could not {} property {prop} (evaluated to {}) on {source} (evaluated to {})", fetch_mode(e), eval.get_string(nested), eval.iref_string(base)) + } + } + }) + } +} + +type EvalResult = Result; + +struct DefOrUndefRef<'a, I>(Result<&'a I, InvalidFetch<'a, I>>); + +impl<'e, I> DefOrUndefRef<'e, I> { + pub fn into_imm(self) -> InternalImm<'e, I> { + match self.0 { + Ok(referenced) => InternalImm::Def(IntermediateValue::Referenced(referenced)), + Err(reference) => InternalImm::Undef(reference), + } + } + + pub fn try_use<'a, E: Evaluator<'e, I>>( + self, + eval: &mut Eval<'a, 'e, I, E>, + ) -> EvalResult<&'e I> { + match self.0 { + Ok(referenced) => Ok(referenced), + Err(reference) => Err(EvErr::refed(eval, reference)), + } + } +} + +enum InternalImm<'a, I> { + Def(IntermediateValue<'a, I>), + Undef(InvalidFetch<'a, I>), +} + +impl<'e, I> InternalImm<'e, I> { + pub fn try_use<'a, E: Evaluator<'e, I>>( + self, + eval: &mut Eval<'a, 'e, I, E>, + ) -> EvalResult> { + match self { + InternalImm::Def(referenced) => Ok(referenced), + InternalImm::Undef(reference) => Err(EvErr::refed(eval, reference)), + } + } +} + +struct Eval<'a, 'e, I, E: Evaluator<'e, I>> { + eval: &'a mut E, + it: Option<&'e I>, +} + +impl<'a, 'e, I, E: Evaluator<'e, I>> Eval<'a, 'e, I, E> { + fn iref_string(&mut self, value: &'e I) -> Cow<'e, str> { + self.eval.get_string(value) + } + + fn value(&mut self, value: &'e Value) -> EvalResult> { + Ok(match value { + Value::Bool(condition) => { + InternalImm::Def(IntermediateValue::Bool(self.condition(condition)?)) + } + Value::String(s) => self.string_source(s)?, + Value::Number(n) => self.number_source(n)?, + }) + } + + fn number_source(&mut self, value: &'e NumberSource) -> EvalResult> { + Ok(match value { + NumberSource::Number(n) => InternalImm::Def(IntermediateValue::Number(*n)), + NumberSource::CollectionSize(collection) => { + InternalImm::Def(IntermediateValue::Number(match collection { + CollectionSource::Reference(reference) => { + let immediate = self.reference(reference)?.try_use(self)?; + self.eval.length(immediate) as f64 + } + CollectionSource::FilterOperator(_) => { + self.collection_source(collection)?.len() as f64 + } + })) + } + NumberSource::Reference(reference) => self.reference(reference)?.into_imm(), + }) + } + + fn convert_index(&mut self, value: IntermediateValue<'e, I>) -> EvalResult { + match value { + IntermediateValue::String(s) => usize::from_str(&s).map_err(EvErr::str), + IntermediateValue::Number(n) => Ok(n as usize), + IntermediateValue::Bool(_) => Err(EvErr::str("Cannot take index of boolean")), + IntermediateValue::Null => Ok(0), + IntermediateValue::Referenced(referenced) => { + self.eval.convert_index(referenced).map_err(|e| match e { + ResultError::Invalid => EvErr::str(format!( + "Cannot convert {} to an index", + self.iref_string(referenced) + )), + ResultError::Redacted => EvErr::str("Could not evaluate redacted value"), + _ => unreachable!("Invalid ResultError for convert_index"), + }) + } + } + } + + fn number_to_index(&mut self, value: &'e NumberSource) -> EvalResult { + let val = self.number_source(value)?.try_use(self)?; + self.convert_index(val) + .map_err(|e| EvErr::str(format!("{} (from {value})", e.0))) + } + + fn string_source(&mut self, value: &'e StringSource) -> EvalResult> { + Ok(match value { + StringSource::String(s) => { + InternalImm::Def(IntermediateValue::String(Cow::Borrowed(s.as_str()))) + } + StringSource::Substring(boxed) => { + let (string, start, end) = &**boxed; + let str = self.stringify(string)?; + let start = self.number_to_index(start)?; + let mut end = self.number_to_index(end)?; + if start > end || start > str.len() || (start == str.len() && start != end) { + return Err(EvErr::str(format!( + "[{start}..{end}] is out of bounds of {value} (string size: {})", + str.len() + ))); + } + end = min(end, str.len()); + InternalImm::Def(IntermediateValue::String(match str { + Cow::Owned(s) => Cow::Owned(s[start..end].to_string()), + Cow::Borrowed(s) => Cow::Borrowed(&s[start..end]), + })) + } + StringSource::Null => InternalImm::Def(IntermediateValue::Null), + StringSource::Reference(reference) => self.reference(reference)?.into_imm(), + }) + } + + fn reference_collection(&mut self, reference: &'e Reference) -> EvalResult> { + let immediate = self.reference(reference)?.try_use(self)?; + self.eval.try_enumerate(immediate).map_err(|e| match e { + ResultError::Invalid => EvErr::str(format!( + "Cannot enumerate non iterable type: {reference}; evaluating to: {}", + self.iref_string(immediate) + )), + ResultError::Redacted => { + EvErr::str(format!("Cannot enumerate redacted value {reference}")) + } + _ => unreachable!("Invalid ResultError for try_enumerate"), + }) + } + + fn reference(&mut self, reference: &'e Reference) -> EvalResult> { + Ok(DefOrUndefRef(match reference { + Reference::IteratorVariable => self.it.ok_or(InvalidFetch::NoIterator), + Reference::Base(ref identifier) => self + .eval + .fetch_identifier(identifier.as_str()) + .map_err(|e| InvalidFetch::Identifier(e, identifier)), + Reference::Index(ref boxed) => { + let (source, dimension) = &**boxed; + let dimension_val = self.value(dimension)?.try_use(self)?; + match source { + CollectionSource::FilterOperator(_) => { + let index = self + .convert_index(dimension_val) + .map_err(|e| EvErr::str(format!("{} (from {dimension}", e.0)))?; + let vec = self.collection_source(source)?; + if index < vec.len() { + Ok(vec[index]) + } else { + Err(InvalidFetch::IndexEvaluated(source, dimension, vec, index)) + } + } + CollectionSource::Reference(ref reference) => { + self.reference(reference)?.0.and_then(|reference_val| { + self.eval + .fetch_index(reference_val, dimension_val.clone()) + .map_err(|e| { + InvalidFetch::Index( + e, + source, + dimension, + reference_val, + dimension_val, + ) + }) + }) + } + } + } + Reference::Nested(ref boxed) => { + let (source, member) = &**boxed; + let member_val = self.value(member)?.try_use(self)?; + self.reference(source)?.0.and_then(|source_val| { + self.eval + .fetch_nested(source_val, member_val.clone()) + .map_err(|e| { + InvalidFetch::Nested(e, source, member, source_val, member_val) + }) + }) + } + })) + } + + fn collection_source(&mut self, collection: &'e CollectionSource) -> EvalResult> { + Ok(match collection { + CollectionSource::Reference(ref reference) => self.reference_collection(reference)?, + CollectionSource::FilterOperator(ref boxed) => { + let (source, condition) = &**boxed; + let mut values = vec![]; + let it = self.it; + for item in self.collection_source(source)? { + self.it = Some(item); + if self.condition(condition)? { + values.push(item); + } + } + self.it = it; + values + } + }) + } + + fn stringify_intermediate(&mut self, value: IntermediateValue<'e, I>) -> Cow<'e, str> { + match value { + IntermediateValue::String(s) => s, + IntermediateValue::Number(n) => Cow::Owned(n.to_string()), + IntermediateValue::Bool(b) => Cow::Owned(b.to_string()), + IntermediateValue::Null => Cow::Borrowed(""), + IntermediateValue::Referenced(referenced) => self.eval.stringify(referenced), + } + } + + fn get_string(&mut self, value: IntermediateValue<'e, I>) -> Cow<'e, str> { + if let IntermediateValue::Referenced(referenced) = value { + self.iref_string(referenced) + } else { + self.stringify_intermediate(value) + } + } + + fn stringify(&mut self, value: &'e StringSource) -> EvalResult> { + let value = self.string_source(value)?.try_use(self)?; + Ok(self.stringify_intermediate(value)) + } + + fn condition(&mut self, condition: &'e Condition) -> EvalResult { + Ok(match condition { + Condition::Always => true, + Condition::Never => false, + Condition::StringComparison(comparer, haystack, needle) => { + let haystack = self.stringify(haystack)?; + match comparer { + StringComparison::StartsWith => haystack.starts_with(needle), + StringComparison::EndsWith => haystack.ends_with(needle), + StringComparison::Contains => haystack.contains(needle), + StringComparison::Matches => { + return Regex::new(needle.as_str()) + .map_err(|e| EvErr::str(format!("{needle} is an invalid regex: {e}"))) + .map(|r| r.is_match(&haystack)) + } + } + } + Condition::BinaryComparison(a, comparer, b) => { + let (a, b) = (self.value(a)?.try_use(self)?, self.value(b)?.try_use(self)?); + match comparer { + BinaryComparison::Equals => self.eval.equals(a, b), + BinaryComparison::NotEquals => !self.eval.equals(a, b), + BinaryComparison::GreaterThan => self.eval.greater_than(a, b), + BinaryComparison::GreaterOrEquals => self.eval.greater_or_equals(a, b), + BinaryComparison::LowerThan => !self.eval.greater_or_equals(a, b), + BinaryComparison::LowerOrEquals => !self.eval.greater_than(a, b), + } + } + Condition::CollectionMatch(match_type, reference, condition) => { + let vec = self.reference_collection(reference)?; + let it = self.it; + let mut result; + match match_type { + CollectionMatch::All => { + result = true; + for v in vec { + self.it = Some(v); + if !self.condition(condition)? { + result = false; + break; + } + } + } + CollectionMatch::Any => { + result = false; + for v in vec { + self.it = Some(v); + if self.condition(condition)? { + result = true; + break; + } + } + } + } + self.it = it; + result + } + Condition::IsDefinedReference(reference) => self.reference(reference)?.0.is_ok(), + Condition::IsEmptyReference(reference) => { + let immediate = self.reference(reference)?.try_use(self)?; + self.eval.length(immediate) == 0 + } + Condition::Disjunction(boxed) => { + let (a, b) = &**boxed; + self.condition(a)? || self.condition(b)? + } + Condition::Conjunction(boxed) => { + let (a, b) = &**boxed; + self.condition(a)? && self.condition(b)? + } + Condition::Negation(boxed) => !self.condition(boxed)?, + Condition::Instanceof(reference, name) => { + let immediate = self.reference(reference)?.try_use(self)?; + self.eval.instanceof(immediate, name.as_str()) + } + }) + } +} + +pub fn eval_condition<'e, I: 'e, E: Evaluator<'e, I>>( + eval: &mut E, + condition: &'e ProbeCondition, +) -> Result { + Eval { eval, it: None } + .condition(&condition.0) + .map_err(|e| SnapshotEvaluationError { + expr: condition.to_string(), + message: e.0, + }) +} + +pub fn eval_string<'a, 'e, 'v, I: 'e, E: Evaluator<'e, I>>( + eval: &mut E, + dsl: &'v DslString, +) -> (Cow<'a, str>, Vec) +where + 'v: 'e, + 'e: 'a, +{ + let mut errors = vec![]; + let mut eval = Eval { eval, it: None }; + let mut map_error = |err: EvErr, expr: &dyn ToString| { + errors.push(SnapshotEvaluationError { + expr: expr.to_string(), + message: err.0, + }); + Cow::Borrowed("UNDEFINED") + }; + let mut vec = dsl + .0 + .iter() + .map(|p| match p { + DslPart::String(str) => Cow::Borrowed(str.as_str()), + DslPart::Value(val) => eval + .value(val) + .and_then(|value| { + let immediate = value.try_use(&mut eval)?; + Ok(eval.get_string(immediate)) + }) + .unwrap_or_else(|err| map_error(err, val)), + DslPart::Ref(reference) => match reference { + CollectionSource::Reference(reference) => { + eval.reference(reference).and_then(|referenced| { + let immediate = referenced.try_use(&mut eval)?; + Ok(eval.get_string(IntermediateValue::Referenced(immediate))) + }) + } + CollectionSource::FilterOperator(_) => { + eval.collection_source(reference).map(|vec| { + let mut strings = vec![]; + for referenced in vec { + strings + .push(eval.get_string(IntermediateValue::Referenced(referenced))); + } + Cow::Owned(format!("[{}]", strings.join(", "))) + }) + } + } + .unwrap_or_else(|err| map_error(err, reference)), + }) + .collect::>>(); + ( + if vec.len() == 1 { + vec.remove(0) + } else { + Cow::Owned(vec.join("")) + }, + errors, + ) +} + +pub fn eval_value<'e, 'v, I: 'e, E: Evaluator<'e, I>>( + eval: &mut E, + value: &'v ProbeValue, +) -> Result, SnapshotEvaluationError> +where + 'v: 'e, +{ + let mut eval = Eval { eval, it: None }; + eval.value(&value.0) + .and_then(|v| v.try_use(&mut eval)) + .map_err(|e| SnapshotEvaluationError { + expr: value.to_string(), + message: e.0, + }) +} + +pub fn eval_intermediate_to_string<'e, I, E: Evaluator<'e, I>>( + eval: &mut E, + value: IntermediateValue<'e, I>, +) -> Cow<'e, str> { + let mut eval = Eval { eval, it: None }; + eval.get_string(value) +} + +#[cfg(test)] +mod tests { + use crate::expr_defs::Value; + use crate::expr_defs::{ + BinaryComparison, CollectionMatch, CollectionSource, Condition, DslPart, NumberSource, + Reference, StringComparison, StringSource, + }; + use crate::{ + eval_condition, eval_intermediate_to_string, eval_string, eval_value, DslString, Evaluator, + IntermediateValue, ProbeCondition, ProbeValue, ResultError, ResultValue, + }; + use std::borrow::Cow; + use std::cmp::Ordering; + use std::collections::HashMap; + + struct EvalCtx<'e> { + variables: &'e HashMap, + } + + #[derive(Clone, PartialEq)] + struct OrdMap(HashMap); + + impl PartialOrd for OrdMap { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.len().partial_cmp(&other.0.len()) + } + } + + #[derive(Clone, PartialOrd, PartialEq)] + enum Val { + Null, + Num(f64), + Str(String), + Bool(bool), + Vec(Vec), + Obj(OrdMap), + } + + impl<'a> From> for Val { + fn from(value: IntermediateValue<'a, Val>) -> Self { + match value { + IntermediateValue::String(s) => Val::Str(s.to_string()), + IntermediateValue::Number(n) => Val::Num(n), + IntermediateValue::Bool(b) => Val::Bool(b), + IntermediateValue::Null => Val::Null, + IntermediateValue::Referenced(v) => v.clone(), + } + } + } + + impl<'e> Evaluator<'e, Val> for EvalCtx<'e> { + fn equals(&mut self, a: IntermediateValue<'e, Val>, b: IntermediateValue<'e, Val>) -> bool { + Val::from(a) == b.into() + } + + fn greater_than( + &mut self, + a: IntermediateValue<'e, Val>, + b: IntermediateValue<'e, Val>, + ) -> bool { + Val::from(a) > b.into() + } + + fn greater_or_equals( + &mut self, + a: IntermediateValue<'e, Val>, + b: IntermediateValue<'e, Val>, + ) -> bool { + Val::from(a) >= b.into() + } + + fn fetch_identifier(&mut self, identifier: &str) -> ResultValue<&'e Val> { + self.variables.get(identifier).ok_or(ResultError::Undefined) + } + + fn fetch_index( + &mut self, + value: &'e Val, + index: IntermediateValue<'e, Val>, + ) -> ResultValue<&'e Val> { + if let Val::Vec(vec) = value { + if let Val::Num(idx) = index.into() { + let idx = idx as usize; + if idx < vec.len() { + return Ok(&vec[idx]); + } + } + } + + Err(ResultError::Undefined) + } + + fn fetch_nested( + &mut self, + value: &'e Val, + member: IntermediateValue<'e, Val>, + ) -> ResultValue<&'e Val> { + if let Val::Obj(obj) = value { + if let Val::Str(str) = member.into() { + return obj.0.get(&str).ok_or(ResultError::Undefined); + } + } + + Err(ResultError::Undefined) + } + + fn length(&mut self, value: &'e Val) -> usize { + match value { + Val::Null => 0, + Val::Num(n) => n.to_string().len(), + Val::Str(s) => s.len(), + Val::Bool(_) => 0, + Val::Vec(v) => v.len(), + Val::Obj(o) => o.0.len(), + } + } + + fn try_enumerate(&mut self, value: &'e Val) -> ResultValue> { + match value { + Val::Vec(v) => Ok(v.iter().collect()), + Val::Obj(o) => Ok(o.0.values().collect()), + _ => Err(ResultError::Invalid), + } + } + + fn stringify(&mut self, value: &'e Val) -> Cow<'e, str> { + match value { + Val::Null => Cow::Borrowed(""), + Val::Num(n) => Cow::Owned(n.to_string()), + Val::Str(s) => Cow::Borrowed(s.as_str()), + Val::Bool(b) => Cow::Borrowed(if *b { "true" } else { "false" }), + Val::Vec(v) => Cow::Owned(format!("vec[{}]", v.len())), + Val::Obj(o) => Cow::Owned(format!("obj[{}]", o.0.len())), + } + } + + fn get_string(&mut self, value: &'e Val) -> Cow<'e, str> { + match value { + Val::Vec(v) => Cow::Owned(format!( + "vec{{{}}}", + v.iter() + .map(|e| self.get_string(e)) + .collect::>() + .join(", ") + )), + Val::Obj(o) => Cow::Owned(format!( + "obj{{{}}}", + o.0.iter() + .map(|(k, v)| format!("{k}: {}", self.get_string(v))) + .collect::>() + .join(", ") + )), + _ => self.stringify(value), + } + } + + fn convert_index(&mut self, value: &'e Val) -> ResultValue { + if let Val::Num(n) = value { + Ok(*n as usize) + } else { + Err(ResultError::Invalid) + } + } + + fn instanceof(&mut self, value: &'e Val, class: &str) -> bool { + if let Val::Obj(o) = value { + if let Some(Val::Str(s)) = o.0.get("class") { + return s == class; + } + } + false + } + } + + fn num(n: f64) -> Value { + Value::Number(NumberSource::Number(n)) + } + + fn string(s: &'static str) -> Value { + Value::String(StringSource::String(s.to_string())) + } + + fn numval(v: &'static str) -> Value { + Value::Number(NumberSource::Reference(Reference::Base(v.to_string()))) + } + + fn strvar(v: &'static str) -> StringSource { + StringSource::Reference(Reference::Base(v.to_string())) + } + + fn strval(v: &'static str) -> Value { + Value::String(strvar(v)) + } + + fn vecvar(v: &'static str) -> CollectionSource { + CollectionSource::Reference(Reference::Base(v.to_string())) + } + + fn it_ref() -> StringSource { + StringSource::Reference(Reference::IteratorVariable) + } + + macro_rules! assert_cond_err { + ($vars:expr, $expr:expr, $err:expr) => { + let cond = ProbeCondition($expr); + let mut ctx = EvalCtx { variables: &$vars }; + match eval_condition(&mut ctx, &cond) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!(e.message, $err), + } + }; + } + macro_rules! assert_cond_true { + ($vars:expr, $expr:expr) => { + let cond = ProbeCondition($expr); + let mut ctx = EvalCtx { variables: &$vars }; + assert!(eval_condition(&mut ctx, &cond).unwrap()); + }; + } + macro_rules! assert_cond_false { + ($vars:expr, $expr:expr) => { + let cond = ProbeCondition($expr); + let mut ctx = EvalCtx { variables: &$vars }; + assert!(!eval_condition(&mut ctx, &cond).unwrap()); + }; + } + + macro_rules! assert_val_err { + ($vars:expr, $expr:expr, $err:expr) => { + let val = ProbeValue($expr); + let mut ctx = EvalCtx { variables: &$vars }; + match eval_value(&mut ctx, &val) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!(e.message, $err), + } + }; + } + macro_rules! assert_val_eq { + ($vars:expr, $expr:expr, $eq:expr) => { + let val = ProbeValue($expr); + let mut ctx = EvalCtx { variables: &$vars }; + let value = eval_value(&mut ctx, &val).unwrap(); + assert_eq!(eval_intermediate_to_string(&mut ctx, value), $eq); + }; + } + + macro_rules! assert_dsl_eq { + ($vars:expr, $expr:expr, $eq:expr) => { + let dsl = &DslString($expr); + let mut ctx = EvalCtx { variables: &$vars }; + let (result, errors) = eval_string(&mut ctx, dsl); + assert_eq!(result, $eq); + assert_eq!(errors.len(), 0); + }; + } + + #[test] + fn test_eval() { + let vars = HashMap::from([ + ("var".to_string(), Val::Str("bar".to_string())), + ( + "vec".to_string(), + Val::Vec(vec![Val::Num(10.), Val::Num(11.), Val::Num(12.)]), + ), + ( + "vecvec".to_string(), + Val::Vec(vec![ + Val::Vec(vec![Val::Num(10.), Val::Num(11.)]), + Val::Vec(vec![Val::Num(12.)]), + ]), + ), + ("empty".to_string(), Val::Str("".to_string())), + ("emptyvec".to_string(), Val::Vec(vec![])), + ("null".to_string(), Val::Null), + ("zero".to_string(), Val::Num(0.)), + ("two".to_string(), Val::Num(2.)), + ( + "objA".to_string(), + Val::Obj(OrdMap(HashMap::from([( + "class".to_string(), + Val::Str("A".to_string()), + )]))), + ), + ( + "objB".to_string(), + Val::Obj(OrdMap(HashMap::from([( + "class".to_string(), + Val::Str("B".to_string()), + )]))), + ), + ]); + + assert_cond_true!(vars, Condition::Always); + assert_cond_false!(vars, Condition::Never); + + assert_cond_err!( + vars, + Condition::IsEmptyReference(Reference::Base("foo".to_string())), + "Could not fetch variable foo" + ); + assert_cond_err!( + vars, + Condition::IsEmptyReference(Reference::Nested(Box::new(( + Reference::Base("foo".to_string()), + string("bar") + )))), + "Could not fetch variable foo" + ); + assert_cond_err!( + vars, + Condition::IsEmptyReference(Reference::Nested(Box::new(( + Reference::Base("objA".to_string()), + string("foo") + )))), + "Could not fetch property \"foo\" on objA (evaluated to obj{class: A})" + ); + assert_cond_err!( + vars, + Condition::IsEmptyReference(Reference::Index(Box::new((vecvar("foo"), num(0.))))), + "Could not fetch variable foo" + ); + assert_cond_err!( + vars, + Condition::IsEmptyReference(Reference::Index(Box::new((vecvar("vec"), num(3.))))), + "Could not fetch index 3 on vec (evaluated to vec{10, 11, 12})" + ); + assert_cond_false!( + vars, + Condition::IsDefinedReference(Reference::Base("foo".to_string())) + ); + assert_cond_true!( + vars, + Condition::IsDefinedReference(Reference::Base("var".to_string())) + ); + assert_cond_true!( + vars, + Condition::IsDefinedReference(Reference::Index(Box::new((vecvar("vec"), num(0.))))) + ); + assert_cond_false!( + vars, + Condition::IsDefinedReference(Reference::Index(Box::new((vecvar("vec"), num(3.))))) + ); + assert_cond_false!( + vars, + Condition::IsDefinedReference(Reference::Index(Box::new((vecvar("foo"), num(0.))))) + ); + assert_cond_true!( + vars, + Condition::IsDefinedReference(Reference::Nested(Box::new(( + Reference::Base("objA".to_string()), + string("class") + )))) + ); + assert_cond_false!( + vars, + Condition::IsDefinedReference(Reference::Nested(Box::new(( + Reference::Base("objA".to_string()), + string("foo") + )))) + ); + + assert_cond_true!( + vars, + Condition::IsEmptyReference(Reference::Base("empty".to_string())) + ); + assert_cond_false!( + vars, + Condition::IsEmptyReference(Reference::Base("var".to_string())) + ); + + assert_cond_true!( + vars, + Condition::BinaryComparison( + Value::String(StringSource::Null), + BinaryComparison::Equals, + strval("null") + ) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(string("bar"), BinaryComparison::Equals, strval("var")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::Equals, numval("zero")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::Equals, numval("two")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(numval("zero"), BinaryComparison::Equals, num(0.)) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::NotEquals, numval("zero")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::GreaterThan, numval("zero")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::GreaterOrEquals, numval("zero")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::LowerThan, numval("zero")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::LowerOrEquals, numval("zero")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::GreaterThan, numval("two")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::GreaterOrEquals, numval("two")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::LowerThan, numval("two")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(0.), BinaryComparison::LowerOrEquals, numval("two")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(4.), BinaryComparison::GreaterThan, numval("two")) + ); + assert_cond_true!( + vars, + Condition::BinaryComparison(num(4.), BinaryComparison::GreaterOrEquals, numval("two")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(4.), BinaryComparison::LowerThan, numval("two")) + ); + assert_cond_false!( + vars, + Condition::BinaryComparison(num(4.), BinaryComparison::LowerOrEquals, numval("two")) + ); + + assert_cond_false!(vars, Condition::Negation(Box::new(Condition::Always))); + assert_cond_true!(vars, Condition::Negation(Box::new(Condition::Never))); + + assert_cond_true!( + vars, + Condition::Conjunction(Box::new((Condition::Always, Condition::Always))) + ); + assert_cond_false!( + vars, + Condition::Conjunction(Box::new((Condition::Never, Condition::Always))) + ); + assert_cond_false!( + vars, + Condition::Conjunction(Box::new((Condition::Always, Condition::Never))) + ); + assert_cond_false!( + vars, + Condition::Conjunction(Box::new((Condition::Never, Condition::Never))) + ); + + assert_cond_true!( + vars, + Condition::Disjunction(Box::new((Condition::Always, Condition::Always))) + ); + assert_cond_true!( + vars, + Condition::Disjunction(Box::new((Condition::Never, Condition::Always))) + ); + assert_cond_true!( + vars, + Condition::Disjunction(Box::new((Condition::Always, Condition::Never))) + ); + assert_cond_false!( + vars, + Condition::Disjunction(Box::new((Condition::Never, Condition::Never))) + ); + + assert_cond_true!( + vars, + Condition::StringComparison( + StringComparison::StartsWith, + StringSource::String("bar".to_string()), + "ba".to_string() + ) + ); + assert_cond_false!( + vars, + Condition::StringComparison( + StringComparison::StartsWith, + strvar("var"), + "ar".to_string() + ) + ); + assert_cond_false!( + vars, + Condition::StringComparison( + StringComparison::EndsWith, + strvar("var"), + "ba".to_string() + ) + ); + assert_cond_true!( + vars, + Condition::StringComparison( + StringComparison::EndsWith, + strvar("var"), + "ar".to_string() + ) + ); + assert_cond_true!( + vars, + Condition::StringComparison(StringComparison::Contains, strvar("var"), "a".to_string()) + ); + assert_cond_false!( + vars, + Condition::StringComparison(StringComparison::Contains, strvar("var"), "x".to_string()) + ); + assert_cond_true!( + vars, + Condition::StringComparison(StringComparison::Matches, strvar("var"), ".*".to_string()) + ); + assert_cond_false!( + vars, + Condition::StringComparison(StringComparison::Matches, strvar("var"), "o+".to_string()) + ); + + assert_cond_true!( + vars, + Condition::CollectionMatch( + CollectionMatch::Any, + Reference::Base("vec".to_string()), + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::Equals, + Value::String(it_ref()) + )) + ) + ); + assert_cond_false!( + vars, + Condition::CollectionMatch( + CollectionMatch::Any, + Reference::Base("vec".to_string()), + Box::new(Condition::BinaryComparison( + num(9.), + BinaryComparison::Equals, + Value::String(it_ref()) + )) + ) + ); + assert_cond_true!( + vars, + Condition::CollectionMatch( + CollectionMatch::All, + Reference::Base("vec".to_string()), + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::LowerOrEquals, + Value::String(it_ref()) + )) + ) + ); + assert_cond_false!( + vars, + Condition::CollectionMatch( + CollectionMatch::All, + Reference::Base("vec".to_string()), + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::GreaterOrEquals, + Value::String(it_ref()) + )) + ) + ); + + assert_cond_true!( + vars, + Condition::CollectionMatch( + CollectionMatch::Any, + Reference::Base("vecvec".to_string()), + Box::new(Condition::CollectionMatch( + CollectionMatch::Any, + Reference::IteratorVariable, + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::Equals, + Value::String(it_ref()) + )) + )) + ) + ); + assert_cond_false!( + vars, + Condition::CollectionMatch( + CollectionMatch::All, + Reference::Base("vecvec".to_string()), + Box::new(Condition::CollectionMatch( + CollectionMatch::Any, + Reference::IteratorVariable, + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::Equals, + Value::String(it_ref()) + )) + )) + ) + ); + + assert_cond_true!( + vars, + Condition::Instanceof(Reference::Base("objA".to_string()), "A".to_string()) + ); + assert_cond_false!( + vars, + Condition::Instanceof(Reference::Base("objA".to_string()), "B".to_string()) + ); + + assert_val_eq!(vars, string("foo"), "foo"); + assert_val_eq!(vars, strval("var"), "bar"); + assert_val_eq!(vars, strval("vec"), "vec{10, 11, 12}"); + assert_val_eq!(vars, strval("objA"), "obj{class: A}"); + + assert_val_eq!( + vars, + Value::String(StringSource::Substring(Box::new(( + StringSource::String("bar".to_string()), + NumberSource::Number(1.), + NumberSource::Number(2.) + )))), + "a" + ); + assert_val_eq!( + vars, + Value::String(StringSource::Substring(Box::new(( + StringSource::String("".to_string()), + NumberSource::Number(0.), + NumberSource::Number(0.) + )))), + "" + ); + assert_val_eq!( + vars, + Value::String(StringSource::Substring(Box::new(( + strvar("vec"), + NumberSource::Number(3.), + NumberSource::Number(6.) + )))), + "[3]" + ); + assert_val_err!( + vars, + Value::String(StringSource::Substring(Box::new(( + strvar("var"), + NumberSource::Number(1.), + NumberSource::Number(0.) + )))), + "[1..0] is out of bounds of substring(var, 1, 0) (string size: 3)" + ); + assert_val_err!( + vars, + Value::String(StringSource::Substring(Box::new(( + strvar("var"), + NumberSource::Number(10.), + NumberSource::Number(13.) + )))), + "[10..13] is out of bounds of substring(var, 10, 13) (string size: 3)" + ); + + assert_val_eq!( + vars, + Value::Number(NumberSource::CollectionSize(vecvar("vec"))), + "3" + ); + assert_val_eq!( + vars, + Value::Number(NumberSource::CollectionSize(CollectionSource::Reference( + Reference::Base("var".to_string()) + ))), + "3" + ); + assert_val_eq!( + vars, + Value::Number(NumberSource::CollectionSize(CollectionSource::Reference( + Reference::Base("null".to_string()) + ))), + "0" + ); + + assert_dsl_eq!(vars, vec![], ""); + assert_dsl_eq!(vars, vec![DslPart::String("test".to_string())], "test"); + assert_dsl_eq!(vars, vec![DslPart::Value(string("test"))], "test"); + assert_dsl_eq!(vars, vec![DslPart::Ref(vecvar("var"))], "bar"); + assert_dsl_eq!(vars, vec![DslPart::Ref(vecvar("vec"))], "vec{10, 11, 12}"); + assert_dsl_eq!( + vars, + vec![DslPart::Ref(CollectionSource::FilterOperator(Box::new(( + vecvar("vec"), + Condition::BinaryComparison( + num(10.), + BinaryComparison::LowerThan, + Value::String(it_ref()) + ) + ))))], + "[11, 12]" + ); + assert_dsl_eq!( + vars, + vec![DslPart::Ref(CollectionSource::FilterOperator(Box::new(( + vecvar("vecvec"), + Condition::CollectionMatch( + CollectionMatch::All, + Reference::IteratorVariable, + Box::new(Condition::BinaryComparison( + num(10.), + BinaryComparison::NotEquals, + Value::String(it_ref()) + )) + ) + ))))], + "[vec{12}]" + ); + assert_dsl_eq!( + vars, + vec![ + DslPart::String("a zero: ".to_string()), + DslPart::Ref(vecvar("zero")) + ], + "a zero: 0" + ); + + let dsl = &DslString(vec![ + DslPart::Value(Value::String(StringSource::Substring(Box::new(( + strvar("var"), + NumberSource::Reference(Reference::Base("var".to_string())), + NumberSource::Number(3.), + ))))), + DslPart::String(" - ".to_string()), + DslPart::Ref(CollectionSource::FilterOperator(Box::new(( + vecvar("var"), + Condition::Always, + )))), + DslPart::String(" - ".to_string()), + DslPart::Value(strval("var")), + ]); + let mut ctx = EvalCtx { variables: &vars }; + let (result, errors) = eval_string(&mut ctx, dsl); + assert_eq!(result, "UNDEFINED - UNDEFINED - bar"); + assert_eq!(errors.len(), 2); + assert_eq!( + errors[0].message, + "Cannot convert bar to an index (from var)" + ); + assert_eq!(errors[0].expr, "substring(var, var, 3)"); + assert_eq!( + errors[1].message, + "Cannot enumerate non iterable type: var; evaluating to: bar" + ); + assert_eq!(errors[1].expr, "filter(var, true)"); + } +} diff --git a/live-debugger/src/lib.rs b/live-debugger/src/lib.rs new file mode 100644 index 000000000..bf5970f64 --- /dev/null +++ b/live-debugger/src/lib.rs @@ -0,0 +1,16 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +mod expr_defs; +mod expr_eval; +mod parse_json; +mod probe_defs; + +pub mod debugger_defs; +mod redacted_names; +pub mod sender; + +pub use expr_eval::*; +pub use parse_json::parse as parse_json; +pub use probe_defs::*; +pub use redacted_names::*; diff --git a/live-debugger/src/parse_json.rs b/live-debugger/src/parse_json.rs new file mode 100644 index 000000000..832aedb91 --- /dev/null +++ b/live-debugger/src/parse_json.rs @@ -0,0 +1,845 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::expr_defs::{ + BinaryComparison, CollectionMatch, CollectionSource, Condition, DslPart, NumberSource, + Reference, StringComparison, StringSource, Value, +}; +use crate::{ + CaptureConfiguration, DslString, EvaluateAt, FilterList, InBodyLocation, LiveDebuggingData, + LogProbe, MetricKind, MetricProbe, Probe, ProbeCondition, ProbeTarget, ProbeType, ProbeValue, + ServiceConfiguration, SpanDecorationProbe, SpanProbe, SpanProbeDecoration, SpanProbeTarget, +}; +use anyhow::Context; +use serde::Deserialize; +use std::fmt::{Display, Formatter}; + +pub fn parse(json: &str) -> anyhow::Result { + let parsed: RawTopLevelItem = serde_json::from_str(json)?; + fn err(result: Result) -> anyhow::Result { + result.map_err(|(str, expr)| anyhow::format_err!("{str}: {expr}")) + } + Ok(match parsed.r#type { + ContentType::ServiceConfiguration => { + LiveDebuggingData::ServiceConfiguration(ServiceConfiguration { + id: parsed.id, + allow: parsed.allow.unwrap_or_default(), + deny: parsed.deny.unwrap_or_default(), + sampling_snapshots_per_second: parsed + .sampling + .map(|s| s.snapshots_per_second) + .unwrap_or(5000), + }) + } + probe_type => LiveDebuggingData::Probe({ + let mut probe = Probe { + id: parsed.id, + version: parsed.version.unwrap_or(0), + language: parsed.language, + tags: parsed.tags.unwrap_or_default(), + target: { + let target = parsed + .r#where + .ok_or_else(|| anyhow::format_err!("Missing where for Probe"))?; + ProbeTarget { + type_name: target.type_name, + method_name: target.method_name, + source_file: target.source_file, + signature: target.signature, + lines: { + let mut lines = vec![]; + for line in target.lines.unwrap_or(vec![]) { + lines.push(line.parse().with_context(|| { + format!("'{line}' is not a valid Probe line target") + })?); + } + lines + }, + in_body_location: target.in_body_location.unwrap_or(InBodyLocation::None), + } + }, + evaluate_at: parsed.evaluate_at.unwrap_or(EvaluateAt::Exit), + probe: match probe_type { + ContentType::MetricProbe => ProbeType::Metric(MetricProbe { + kind: parsed + .kind + .ok_or_else(|| anyhow::format_err!("Missing kind for MetricProbe"))?, + name: parsed + .metric_name + .ok_or_else(|| anyhow::format_err!("Missing name for MetricProbe"))?, + value: ProbeValue(err(parsed + .value + .unwrap_or(Expression { + json: RawExpr::Number(1f64), + }) + .json + .try_into())?), + }), + ContentType::LogProbe => ProbeType::Log(LogProbe { + segments: err(parsed + .segments + .ok_or_else(|| anyhow::format_err!("Missing segments for LogProbe"))? + .try_into())?, + when: ProbeCondition( + err(parsed.when.map(|expr| expr.json.try_into()).transpose())? + .unwrap_or(Condition::Always), + ), + capture: parsed.capture.unwrap_or_default(), + capture_snapshot: parsed.capture_snapshot.unwrap_or(false), + sampling_snapshots_per_second: parsed + .sampling + .map(|s| s.snapshots_per_second) + .unwrap_or(5000), + }), + ContentType::SpanProbe => ProbeType::Span(SpanProbe {}), + ContentType::SpanDecorationProbe => { + ProbeType::SpanDecoration(SpanDecorationProbe { + target: parsed.target_span.unwrap_or(SpanProbeTarget::Active), + decorations: { + let mut decorations = vec![]; + for decoration in parsed.decorations.ok_or_else(|| { + anyhow::format_err!( + "Missing decorations for SpanDecorationProbe" + ) + })? { + decorations.push(SpanProbeDecoration { + condition: ProbeCondition( + err(decoration + .when + .map(|expr| expr.json.try_into()) + .transpose())? + .unwrap_or(Condition::Always), + ), + tags: { + let mut tags = vec![]; + for tag in decoration.tags { + tags.push(( + tag.name, + err(tag.value.segments.try_into())?, + )); + } + tags + }, + }) + } + decorations + }, + }) + } + _ => unreachable!(), + }, + }; + // unconditional log probes always capture their entry context + if matches!( + probe.probe, + ProbeType::Log(LogProbe { + when: ProbeCondition(Condition::Always), + .. + }) + ) { + probe.evaluate_at = EvaluateAt::Entry; + } + probe + }), + }) +} + +#[derive(Deserialize, PartialEq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +enum ContentType { + MetricProbe, + LogProbe, + SpanProbe, + SpanDecorationProbe, + ServiceConfiguration, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawTopLevelItem { + r#type: ContentType, + id: String, + version: Option, + language: Option, + r#where: Option, + when: Option, + tags: Option>, + segments: Option>, + capture_snapshot: Option, + capture: Option, + kind: Option, + decorations: Option>, + metric_name: Option, + value: Option, + evaluate_at: Option, + allow: Option, + deny: Option, + sampling: Option, + target_span: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ServiceConfigurationSampling { + snapshots_per_second: u32, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ProbeWhere { + type_name: Option, + method_name: Option, + source_file: Option, + signature: Option, + lines: Option>, + in_body_location: Option, +} + +#[derive(Deserialize)] +struct Expression { + json: RawExpr, +} + +#[derive(Deserialize)] +struct RawSpanProbeDecoration { + when: Option, + tags: Vec, +} + +#[derive(Deserialize)] +struct RawSpanProbeDecorationTag { + name: String, + value: RawSegments, +} + +#[derive(Deserialize)] +struct RawSegments { + segments: Vec, +} + +#[derive(Deserialize)] +struct RawSegmentString { + str: String, +} + +#[derive(Deserialize)] +#[serde(untagged)] +#[serde(rename_all = "camelCase")] +enum RawSegment { + Str(RawSegmentString), + Expr(Expression), +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let result: Result = self.try_into()?; + result.or_else(|expr| Ok(CollectionSource::Reference(expr.try_into()?))) + } +} + +impl TryInto> for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result, Self::Error> { + Ok(Ok(match self { + RawExpr::Expr(Some(RawExprValue::Filter([source, cond]))) => { + CollectionSource::FilterOperator(Box::new(( + (*source).try_into()?, + (*cond).try_into()?, + ))) + } + expr => return Ok(Err(expr)), + })) + } +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + Ok(match self { + RawExpr::Expr(Some(RawExprValue::Ref(identifier))) => { + if identifier == "@it" { + Reference::IteratorVariable + } else { + Reference::Base(identifier) + } + } + RawExpr::Expr(Some(RawExprValue::Index([source, index]))) => { + Reference::Index(Box::new(((*source).try_into()?, (*index).try_into()?))) + } + RawExpr::Expr(Some(RawExprValue::Getmember([source, member]))) => { + Reference::Nested(Box::new(((*source).try_into()?, (*member).try_into()?))) + } + expr => return Err(("Found unexpected value for a reference", expr)), + }) + } +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let result: Result = self.try_into()?; + result.map_err(|expr| ("Found unexpected value for a condition", expr)) + } +} + +impl TryInto> for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result, Self::Error> { + Ok(Ok(match self { + RawExpr::Bool(true) => Condition::Always, + RawExpr::Bool(false) => Condition::Never, + RawExpr::Expr(None) => Condition::Never, + RawExpr::Expr(Some(RawExprValue::Or([a, b]))) => { + Condition::Disjunction(Box::new(((*a).try_into()?, (*b).try_into()?))) + } + RawExpr::Expr(Some(RawExprValue::And([a, b]))) => { + Condition::Conjunction(Box::new(((*a).try_into()?, (*b).try_into()?))) + } + RawExpr::Expr(Some(RawExprValue::Not(a))) => { + Condition::Negation(Box::new((*a).try_into()?)) + } + RawExpr::Expr(Some(RawExprValue::Eq([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::Equals, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::Ne([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::NotEquals, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::Gt([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::GreaterThan, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::Ge([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::GreaterOrEquals, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::Lt([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::LowerThan, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::Le([a, b]))) => Condition::BinaryComparison( + (*a).try_into()?, + BinaryComparison::LowerOrEquals, + (*b).try_into()?, + ), + RawExpr::Expr(Some(RawExprValue::StartsWith((source, value)))) => { + Condition::StringComparison( + StringComparison::StartsWith, + (*source).try_into()?, + value, + ) + } + RawExpr::Expr(Some(RawExprValue::EndsWith((source, value)))) => { + Condition::StringComparison( + StringComparison::EndsWith, + (*source).try_into()?, + value, + ) + } + RawExpr::Expr(Some(RawExprValue::Contains((source, value)))) => { + Condition::StringComparison( + StringComparison::Contains, + (*source).try_into()?, + value, + ) + } + RawExpr::Expr(Some(RawExprValue::Matches((source, value)))) => { + Condition::StringComparison(StringComparison::Matches, (*source).try_into()?, value) + } + RawExpr::Expr(Some(RawExprValue::Any([a, b]))) => Condition::CollectionMatch( + CollectionMatch::Any, + (*a).try_into()?, + Box::new((*b).try_into()?), + ), + RawExpr::Expr(Some(RawExprValue::All([a, b]))) => Condition::CollectionMatch( + CollectionMatch::All, + (*a).try_into()?, + Box::new((*b).try_into()?), + ), + RawExpr::Expr(Some(RawExprValue::Instanceof((source, name)))) => { + Condition::Instanceof((*source).try_into()?, name) + } + RawExpr::Expr(Some(RawExprValue::IsUndefined(source))) => Condition::Negation( + Box::new(Condition::IsDefinedReference((*source).try_into()?)), + ), + RawExpr::Expr(Some(RawExprValue::IsDefined(source))) => { + Condition::IsDefinedReference((*source).try_into()?) + } + RawExpr::Expr(Some(RawExprValue::IsEmpty(source))) => { + Condition::IsEmptyReference((*source).try_into()?) + } + expr => return Ok(Err(expr)), + })) + } +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let result: Result = self.try_into()?; + result.or_else(|expr| Ok(NumberSource::Reference(expr.try_into()?))) + } +} + +impl TryInto> for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result, Self::Error> { + Ok(Ok(match self { + RawExpr::Number(num) => NumberSource::Number(num), + RawExpr::Expr(Some(RawExprValue::Count(source))) + | RawExpr::Expr(Some(RawExprValue::Len(source))) => { + NumberSource::CollectionSize((*source).try_into()?) + } + expr => return Ok(Err(expr)), + })) + } +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let result: Result = self.try_into()?; + result.or_else(|expr| Ok(StringSource::Reference(expr.try_into()?))) + } +} + +impl TryInto> for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result, Self::Error> { + Ok(Ok(match self { + RawExpr::String(str) => StringSource::String(str), + RawExpr::Expr(None) => StringSource::Null, + RawExpr::Expr(Some(RawExprValue::Substring([source, start, end]))) => { + StringSource::Substring(Box::new(( + (*source).try_into()?, + (*start).try_into()?, + (*end).try_into()?, + ))) + } + expr => return Ok(Err(expr)), + })) + } +} + +impl TryInto for RawExpr { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let string: Result = self.try_into()?; + Ok(match string { + Ok(string) => Value::String(string), + Err(expr) => { + let num: Result = expr.try_into()?; + match num { + Ok(num) => Value::Number(num), + Err(expr) => { + let num: Result = expr.try_into()?; + match num { + Ok(num) => Value::Bool(Box::new(num)), + Err(expr) => Value::String(StringSource::Reference(expr.try_into()?)), + } + } + } + } + }) + } +} + +impl TryInto for Vec { + type Error = (&'static str, RawExpr); + + fn try_into(self) -> Result { + let mut dsl_parts = vec![]; + for segment in self { + dsl_parts.push(match segment { + RawSegment::Str(str) => DslPart::String(str.str), + RawSegment::Expr(expr) => match expr.json.try_into()? { + Ok(reference) => DslPart::Ref(reference), + Err(expr) => DslPart::Value(expr.try_into()?), + }, + }); + } + Ok(DslString(dsl_parts)) + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawExpr { + Bool(bool), + String(String), + Number(f64), + Expr(Option), +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +enum RawExprValue { + Ref(String), + Any([Box; 2]), + All([Box; 2]), + Or([Box; 2]), + And([Box; 2]), + Eq([Box; 2]), + Ne([Box; 2]), + Lt([Box; 2]), + Le([Box; 2]), + Gt([Box; 2]), + Ge([Box; 2]), + Contains((Box, String)), + Matches((Box, String)), + StartsWith((Box, String)), + EndsWith((Box, String)), + Filter([Box; 2]), + Getmember([Box; 2]), + Not(Box), + Count(Box), + IsEmpty(Box), + IsDefined(Box), + IsUndefined(Box), + Len(Box), + Instanceof((Box, String)), + Index([Box; 2]), + Substring([Box; 3]), +} + +impl Display for RawExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + RawExpr::Bool(true) => f.write_str("true"), + RawExpr::Bool(false) => f.write_str("false"), + RawExpr::String(s) => write!(f, "\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")), + RawExpr::Number(n) => n.fmt(f), + RawExpr::Expr(None) => f.write_str("null"), + RawExpr::Expr(Some(RawExprValue::Ref(id))) => id.fmt(f), + RawExpr::Expr(Some(RawExprValue::Any([a, b]))) => write!(f, "any({a}, {b})"), + RawExpr::Expr(Some(RawExprValue::All([a, b]))) => write!(f, "all({a}, {b})"), + RawExpr::Expr(Some(RawExprValue::Or([a, b]))) => write!(f, "{a} || {b}"), + RawExpr::Expr(Some(RawExprValue::And([a, b]))) => write!(f, "{a} && {b}"), + RawExpr::Expr(Some(RawExprValue::Eq([a, b]))) => write!(f, "{a} == {b}"), + RawExpr::Expr(Some(RawExprValue::Ne([a, b]))) => write!(f, "{a} != {b}"), + RawExpr::Expr(Some(RawExprValue::Lt([a, b]))) => write!(f, "{a} < {b}"), + RawExpr::Expr(Some(RawExprValue::Le([a, b]))) => write!(f, "{a} <= {b}"), + RawExpr::Expr(Some(RawExprValue::Gt([a, b]))) => write!(f, "{a} > {b}"), + RawExpr::Expr(Some(RawExprValue::Ge([a, b]))) => write!(f, "{a} >= {b}"), + RawExpr::Expr(Some(RawExprValue::Contains((src, str)))) => { + write!(f, "contains({src}, {str})") + } + RawExpr::Expr(Some(RawExprValue::Matches((src, str)))) => { + write!(f, "matches({src}, {str})") + } + RawExpr::Expr(Some(RawExprValue::StartsWith((src, str)))) => { + write!(f, "startsWith({src}, {str})") + } + RawExpr::Expr(Some(RawExprValue::EndsWith((src, str)))) => { + write!(f, "endsWith({src}, {str})") + } + RawExpr::Expr(Some(RawExprValue::Filter([a, b]))) => write!(f, "filter({a}, {b})"), + RawExpr::Expr(Some(RawExprValue::Getmember([a, b]))) => { + if let RawExpr::String(ref s) = **b { + write!(f, "{a}.{s}") + } else { + write!(f, "{a}.{b}") + } + } + RawExpr::Expr(Some(RawExprValue::Not(a))) => write!(f, "!{a}"), + RawExpr::Expr(Some(RawExprValue::Count(a))) => write!(f, "count({a})"), + RawExpr::Expr(Some(RawExprValue::IsEmpty(a))) => write!(f, "isEmpty({a})"), + RawExpr::Expr(Some(RawExprValue::IsDefined(a))) => write!(f, "isDefined({a})"), + RawExpr::Expr(Some(RawExprValue::IsUndefined(a))) => write!(f, "isUndefined({a})"), + RawExpr::Expr(Some(RawExprValue::Len(a))) => write!(f, "len({a})"), + RawExpr::Expr(Some(RawExprValue::Instanceof((src, class)))) => { + write!(f, "instanceof({src}, {class})") + } + RawExpr::Expr(Some(RawExprValue::Index([a, b]))) => write!(f, "{a}[{b}]"), + RawExpr::Expr(Some(RawExprValue::Substring([src, start, end]))) => { + write!(f, "substring({src}, {start}, {end})") + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + parse_json, CaptureConfiguration, EvaluateAt, LiveDebuggingData, LogProbe, MetricKind, + MetricProbe, Probe, ProbeType, SpanDecorationProbe, SpanProbeTarget, + }; + + #[test] + fn test_spandecoration_probe_deserialize() { + let json = r#" +{ + "id": "2142910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 42, + "type": "SPAN_DECORATION_PROBE", + "language": "java", + "tags": ["foo:bar", "baz:baaz"], + "where": { + "typeName": "VetController", + "methodName": "showVetList" + }, + "targetSpan": "ACTIVE", + "decorations": [{ + "when": { + "dsl": "field1 > 10", + "json": { + "gt": [{"ref": "field1"}, 10] + } + }, + "tags": [{ + "name": "transactions", + "value": { + "template": "{transactions.id}-{filter(transactions, startsWith(@it.status, 2))}", + "segments": [{ + "dsl": "transactions.id", + "json": { + "getmember": [{"ref": "transactions"}, "id"] + } + }, { + "str": "-" + }, { + "dsl": "filter(transactions, startsWith(@it[\"status\"], 2))", + "json": { + "filter": [{"ref": "transaction"}, { + "startsWith": [ + { + "index": [{"ref": "@it"}, "status"] + }, + "2" + ] + }] + } + }] + } + }] + }, { + "when": { + "dsl": "!(obj == null)", + "json": { + "not": {"eq": [{"ref": "obj"}, null]} + } + }, + "tags": [{ + "name": "value", + "value": { + "template": "{substring(arr[obj.key], 0, len(@return))}", + "segments": [{ + "dsl": "obj.value", + "json": { + "substring": [ + {"index": [{"ref": "arr"}, {"getmember": [{"ref": "obj"}, "key"]}]}, + 0, + {"len": {"ref": "@return"}} + ] + } + }] + } + }] + }] +} +"#; + + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + id, + version, + language, + tags, + target, + evaluate_at, + probe: + ProbeType::SpanDecoration(SpanDecorationProbe { + target: probe_target, + decorations, + }), + }) = parsed + { + assert_eq!(id, "2142910d-d2ff-4679-85cc-bfc317d74e8f"); + assert_eq!(version, 42); + assert_eq!(language, Some("java".to_string())); + assert_eq!(tags, vec!["foo:bar".to_string(), "baz:baaz".to_string()]); + assert_eq!(target.method_name, Some("showVetList".to_string())); + assert!(matches!(evaluate_at, EvaluateAt::Exit)); + assert!(matches!(probe_target, SpanProbeTarget::Active)); + assert_eq!(decorations[0].condition.to_string(), "field1 > 10"); + let (tag, expr) = &decorations[0].tags[0]; + assert_eq!(tag, "transactions"); + assert_eq!( + expr.to_string(), + r#"{transactions.id}-{filter(transaction, startsWith(@it["status"], 2))}"# + ); + assert_eq!(decorations[1].condition.to_string(), "!(obj == null)"); + let (tag, expr) = &decorations[1].tags[0]; + assert_eq!(tag, "value"); + assert_eq!( + expr.to_string(), + r#"{substring(arr[obj.key], 0, len(@return))}"# + ); + } else { + unreachable!(); + } + } + + #[test] + fn test_log_probe_deserialize() { + let json = r#" +{ + "id": "2142910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 42, + "type": "LOG_PROBE", + "language": "java", + "tags": ["foo:bar", "baz:baaz"], + "evaluateAt": "ENTRY", + "where": { + "typeName": "VetController", + "methodName": "showVetList" + }, + "template": "Id of transaction: {transactionId}", + "segments": [{ + "str": "Id of transaction: " + }, { + "dsl": "transactionId", + "json": {"ref": "transactionId"} + } + ], + "captureSnapshot": true, + "when": { + "dsl": "(@duration > 500 && (!(isDefined(myField)) && localVar1.field1.field2 != 15)) || (isEmpty(this.collectionField) || any(this.collectionField, { isEmpty(@it.name) }))", + "json": { + "or": [{ + "and": [{ + "gt": [{"ref": "@duration"}, 500] + }, { + "and": [{ + "not": { + "isDefined": {"ref": "myField"} + } + }, { + "ne": [ + { + "getmember": [ + { + "getmember": [{"ref": "localVar1"}, "field1"] + }, + "field2" + ] + }, + 15 + ] + }] + }] + }, { + "or": [{ + "isEmpty": {"ref": "this.collectionField"} + }, { + "any": [{ + "ref": "this.collectionField" + }, { + "isEmpty": { "ref": "@it.name" } + }] + }] + }] + } + }, + "capture": { + "maxReferenceDepth": 3, + "maxCollectionSize": 1, + "maxLength": 255, + "maxFieldCount": 20 + }, + "sampling": { + "snapshotsPerSecond": 10 + } +} +"#; + + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + evaluate_at, + probe: + ProbeType::Log(LogProbe { + segments, + when, + capture: + CaptureConfiguration { + max_reference_depth, + max_collection_size, + max_length, + max_field_count, + }, + capture_snapshot, + sampling_snapshots_per_second, + }), + .. + }) = parsed + { + assert!(matches!(evaluate_at, EvaluateAt::Entry)); + assert_eq!(segments.to_string(), "Id of transaction: {transactionId}"); + assert_eq!(when.to_string(), "(@duration > 500 && !isDefined(myField) && localVar1.field1.field2 != 15) || isEmpty(this.collectionField) || any(this.collectionField, isEmpty(@it.name))"); + assert_eq!(max_reference_depth, 3); + assert_eq!(max_collection_size, 1); + assert_eq!(max_length, 255); + assert_eq!(max_field_count, 20); + assert_eq!(sampling_snapshots_per_second, 10); + assert!(capture_snapshot); + } else { + unreachable!(); + } + } + + #[test] + fn test_metric_probe_deserialize() { + let json = r#" +{ + "id": "2142910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 42, + "type": "METRIC_PROBE", + "language": "java", + "tags": ["foo:bar", "baz:baaz"], + "where": { + "typeName": "VetController", + "methodName": "showVetList" + }, + "evaluateAt": "EXIT", + "metricName": "showVetList.callcount", + "kind": "COUNT", + "value": { + "dsl": "arg", + "json": {"ref": "arg"} + } +} +"#; + + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + evaluate_at, + probe: ProbeType::Metric(MetricProbe { kind, name, value }), + .. + }) = parsed + { + assert!(matches!(evaluate_at, EvaluateAt::Exit)); + assert!(matches!(kind, MetricKind::Count)); + assert_eq!(name, "showVetList.callcount"); + assert_eq!(value.to_string(), "arg"); + } else { + unreachable!(); + } + } +} diff --git a/live-debugger/src/probe_defs.rs b/live-debugger/src/probe_defs.rs new file mode 100644 index 000000000..0e15031c9 --- /dev/null +++ b/live-debugger/src/probe_defs.rs @@ -0,0 +1,158 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache +// License Version 2.0. This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. + +use crate::{DslString, ProbeCondition, ProbeValue}; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +#[repr(C)] +pub struct CaptureConfiguration { + #[serde(default = "default_max_reference_depth")] + pub max_reference_depth: u32, + #[serde(default = "default_max_collection_size")] + pub max_collection_size: u32, + #[serde(default = "default_max_length")] + pub max_length: u32, + #[serde(default = "default_max_field_count")] + pub max_field_count: u32, +} + +fn default_max_reference_depth() -> u32 { + 3 +} +fn default_max_collection_size() -> u32 { + 100 +} +fn default_max_length() -> u32 { + 255 +} +fn default_max_field_count() -> u32 { + 20 +} + +impl Default for CaptureConfiguration { + fn default() -> Self { + CaptureConfiguration { + max_reference_depth: default_max_reference_depth(), + max_collection_size: default_max_collection_size(), + max_length: default_max_length(), + max_field_count: default_max_field_count(), + } + } +} + +#[repr(C)] +#[derive(Deserialize, Copy, Clone, Debug)] +#[serde(rename_all = "UPPERCASE")] +pub enum MetricKind { + Count, + Gauge, + Histogram, + Distribution, +} + +#[derive(Debug)] +pub struct MetricProbe { + pub kind: MetricKind, + pub name: String, + pub value: ProbeValue, // May be Value::Null +} + +#[repr(C)] +#[derive(Deserialize, Copy, Clone, Debug)] +#[serde(rename_all = "UPPERCASE")] +pub enum SpanProbeTarget { + Active, + Root, +} + +#[derive(Debug)] +pub struct SpanProbeDecoration { + pub condition: ProbeCondition, + pub tags: Vec<(String, DslString)>, +} + +#[derive(Debug)] +pub struct LogProbe { + pub segments: DslString, + pub when: ProbeCondition, + pub capture: CaptureConfiguration, + pub capture_snapshot: bool, + pub sampling_snapshots_per_second: u32, +} + +#[derive(Debug)] +pub struct SpanProbe {} + +#[derive(Debug)] +pub struct SpanDecorationProbe { + pub target: SpanProbeTarget, + pub decorations: Vec, +} + +#[derive(Debug)] +pub enum ProbeType { + Metric(MetricProbe), + Log(LogProbe), + Span(SpanProbe), + SpanDecoration(SpanDecorationProbe), +} + +#[repr(C)] +#[derive(Deserialize, Copy, Clone, Debug)] +#[serde(rename_all = "UPPERCASE")] +pub enum InBodyLocation { + None, + Start, + End, +} + +#[derive(Debug)] +pub struct ProbeTarget { + pub type_name: Option, + pub method_name: Option, + pub source_file: Option, + pub signature: Option, + pub lines: Vec, + pub in_body_location: InBodyLocation, +} + +#[repr(C)] +#[derive(Deserialize, Copy, Clone, Debug)] +#[serde(rename_all = "UPPERCASE")] +pub enum EvaluateAt { + Entry, + Exit, +} + +#[derive(Debug)] +pub struct Probe { + pub id: String, + pub version: u64, + pub language: Option, + pub tags: Vec, + pub target: ProbeTarget, // "where" is rust keyword + pub evaluate_at: EvaluateAt, + pub probe: ProbeType, +} + +#[derive(Debug, Default, Deserialize)] +pub struct FilterList { + pub package_prefixes: Vec, + pub classes: Vec, +} + +#[derive(Debug)] +pub struct ServiceConfiguration { + pub id: String, + pub allow: FilterList, + pub deny: FilterList, + pub sampling_snapshots_per_second: u32, +} + +#[derive(Debug)] +pub enum LiveDebuggingData { + Probe(Probe), + ServiceConfiguration(ServiceConfiguration), +} diff --git a/live-debugger/src/redacted_names.rs b/live-debugger/src/redacted_names.rs new file mode 100644 index 000000000..dde9e8669 --- /dev/null +++ b/live-debugger/src/redacted_names.rs @@ -0,0 +1,209 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![allow(invalid_reference_casting)] + +use lazy_static::lazy_static; +use regex_automata::dfa::regex::Regex; +use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; + +lazy_static! { + static ref REDACTED_NAMES: HashSet<&'static [u8]> = HashSet::from([ + b"2fa" as &[u8], + b"accesstoken", + b"aiohttpsession", + b"apikey", + b"apisecret", + b"apisignature", + b"applicationkey", + b"auth", + b"authorization", + b"authtoken", + b"ccnumber", + b"certificatepin", + b"cipher", + b"clientid", + b"clientsecret", + b"connectionstring", + b"connectsid", + b"cookie", + b"credentials", + b"creditcard", + b"csrf", + b"csrftoken", + b"cvv", + b"databaseurl", + b"dburl", + b"encryptionkey", + b"encryptionkeyid", + b"env", + b"geolocation", + b"gpgkey", + b"ipaddress", + b"jti", + b"jwt", + b"licensekey", + b"masterkey", + b"mysqlpwd", + b"nonce", + b"oauth", + b"oauthtoken", + b"otp", + b"passhash", + b"passwd", + b"password", + b"passwordb", + b"pemfile", + b"pgpkey", + b"phpsessid", + b"pin", + b"pincode", + b"pkcs8", + b"privatekey", + b"publickey", + b"pwd", + b"recaptchakey", + b"refreshtoken", + b"routingnumber", + b"salt", + b"secret", + b"secretkey", + b"secrettoken", + b"securityanswer", + b"securitycode", + b"securityquestion", + b"serviceaccountcredentials", + b"session", + b"sessionid", + b"sessionkey", + b"setcookie", + b"signature", + b"signaturekey", + b"sshkey", + b"ssn", + b"symfony", + b"token", + b"transactionid", + b"twiliotoken", + b"usersession", + b"voterid", + b"xapikey", + b"xauthtoken", + b"xcsrftoken", + b"xforwardedfor", + b"xrealip", + b"xsrf", + b"xsrftoken", + b"customidentifier1", + b"customidentifier2", + ]); + static ref ADDED_REDACTED_NAMES: Vec> = vec![]; + static ref REDACTED_TYPES: HashSet<&'static [u8]> = HashSet::new(); + static ref ADDED_REDACTED_TYPES: Vec> = vec![]; + static ref REDACTED_WILDCARD_TYPES_PATTERN: String = "".to_string(); + static ref REDACTED_TYPES_REGEX: Regex = { + REDACTED_TYPES_INITIALIZED.store(true, Ordering::Relaxed); + Regex::new(&REDACTED_WILDCARD_TYPES_PATTERN).unwrap() + }; + static ref ASSUMED_SAFE_NAME_LEN: usize = { + REDACTED_NAMES_INITIALIZED.store(true, Ordering::Relaxed); + REDACTED_NAMES.iter().map(|n| n.len()).max().unwrap() + 5 + }; +} + +static REDACTED_NAMES_INITIALIZED: AtomicBool = AtomicBool::new(false); +static REDACTED_TYPES_INITIALIZED: AtomicBool = AtomicBool::new(false); + +/// # Safety +/// May only be called while not running yet - concurrent access to is_redacted_name is forbidden. +pub unsafe fn add_redacted_name>>(name: I) { + assert!(!REDACTED_NAMES_INITIALIZED.load(Ordering::Relaxed)); + // I really don't want to Mutex this often checked value. + // Hence, unsafe, and caller has to ensure safety. + // An UnsafeCell would be perfect, but it isn't Sync... + (*(&*ADDED_REDACTED_NAMES as *const Vec>).cast_mut()).push(name.into()); + (*(&*REDACTED_NAMES as *const HashSet<&'static [u8]>).cast_mut()) + .insert(&ADDED_REDACTED_NAMES[ADDED_REDACTED_NAMES.len() - 1]); +} + +/// # Safety +/// May only be called while not running yet - concurrent access to is_redacted_type is forbidden. +pub unsafe fn add_redacted_type>(name: I) { + assert!(!REDACTED_TYPES_INITIALIZED.load(Ordering::Relaxed)); + let name = name.as_ref(); + if name.ends_with(b"*") { + let regex_str = &mut *(&*REDACTED_WILDCARD_TYPES_PATTERN as *const String).cast_mut(); + if !regex_str.is_empty() { + regex_str.push('|') + } + let name = String::from_utf8_lossy(name); + regex_str.push_str(regex::escape(&name[..name.len() - 1]).as_str()); + regex_str.push_str(".*"); + } else { + (*(&*ADDED_REDACTED_TYPES as *const Vec>).cast_mut()).push(name.to_vec()); + (*(&*REDACTED_TYPES as *const HashSet<&'static [u8]>).cast_mut()) + .insert(&ADDED_REDACTED_TYPES[ADDED_REDACTED_TYPES.len() - 1]); + } +} + +pub fn is_redacted_name>(name: I) -> bool { + fn invalid_char(c: u8) -> bool { + c == b'_' || c == b'-' || c == b'$' || c == b'@' + } + let name = name.as_ref(); + if name.len() > *ASSUMED_SAFE_NAME_LEN { + return true; // short circuit for long names, assume them safe + } + let mut copy = smallvec::SmallVec::<[u8; 21]>::with_capacity(name.len()); + let mut i = 0; + while i < name.len() { + let mut c = name[i]; + if !invalid_char(c) { + if c.is_ascii_uppercase() { + c |= 0x20; // lowercase it + } + copy.push(c); + } + i += 1; + } + REDACTED_NAMES.contains(©[0..copy.len()]) +} + +pub fn is_redacted_type>(name: I) -> bool { + let name = name.as_ref(); + if REDACTED_TYPES.contains(name) { + true + } else if !REDACTED_WILDCARD_TYPES_PATTERN.is_empty() { + REDACTED_TYPES_REGEX.is_match(name) + } else { + false + } +} + +#[test] +#[cfg_attr(miri, ignore)] +fn test_redacted_name() { + unsafe { add_redacted_name("test") } + + assert!(is_redacted_name("test")); + assert!(is_redacted_name("te-st")); + assert!(is_redacted_name("CSRF")); + assert!(is_redacted_name("$XSRF")); + assert!(!is_redacted_name("foo")); + assert!(!is_redacted_name("@")); +} + +#[test] +#[cfg_attr(miri, ignore)] +fn test_redacted_type() { + unsafe { + add_redacted_type("other"); + add_redacted_type("type*"); + } + + assert!(is_redacted_type("other")); + assert!(is_redacted_type("type")); + assert!(is_redacted_type("type.foo")); + assert!(!is_redacted_type("typ")); +} diff --git a/live-debugger/src/sender.rs b/live-debugger/src/sender.rs new file mode 100644 index 000000000..2e80eb4e5 --- /dev/null +++ b/live-debugger/src/sender.rs @@ -0,0 +1,265 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::debugger_defs::{DebuggerData, DebuggerPayload}; +use constcat::concat; +use ddcommon::connector::Connector; +use ddcommon::tag::Tag; +use ddcommon::Endpoint; +use hyper::body::{Bytes, Sender}; +use hyper::client::ResponseFuture; +use hyper::http::uri::PathAndQuery; +use hyper::{Body, Client, Method, Response, Uri}; +use percent_encoding::{percent_encode, CONTROLS}; +use serde::{Deserialize, Serialize}; +use std::fmt::Display; +use std::hash::Hash; +use std::str::FromStr; +use tokio::task::JoinHandle; +use uuid::Uuid; + +pub const PROD_LOGS_INTAKE_SUBDOMAIN: &str = "http-intake.logs"; +pub const PROD_DIAGNOSTICS_INTAKE_SUBDOMAIN: &str = "debugger-intake"; + +const DIRECT_DEBUGGER_LOGS_URL_PATH: &str = "/api/v2/logs"; +const DIRECT_DEBUGGER_DIAGNOSTICS_URL_PATH: &str = "/api/v2/debugger"; +const AGENT_DEBUGGER_LOGS_URL_PATH: &str = "/debugger/v1/input"; +const AGENT_DEBUGGER_DIAGNOSTICS_URL_PATH: &str = "/debugger/v1/diagnostics"; + +#[derive(Clone, Default)] +pub struct Config { + pub logs_endpoint: Option, + pub diagnostics_endpoint: Option, +} + +impl Config { + pub fn set_endpoint( + &mut self, + mut logs_endpoint: Endpoint, + mut diagnostics_endpoint: Endpoint, + ) -> anyhow::Result<()> { + let mut logs_uri_parts = logs_endpoint.url.into_parts(); + let mut diagnostics_uri_parts = diagnostics_endpoint.url.into_parts(); + if logs_uri_parts.scheme.is_some() + && logs_uri_parts.scheme.as_ref().unwrap().as_str() != "file" + { + logs_uri_parts.path_and_query = Some(PathAndQuery::from_static( + if logs_endpoint.api_key.is_some() { + DIRECT_DEBUGGER_LOGS_URL_PATH + } else { + AGENT_DEBUGGER_LOGS_URL_PATH + }, + )); + diagnostics_uri_parts.path_and_query = Some(PathAndQuery::from_static( + if diagnostics_endpoint.api_key.is_some() { + DIRECT_DEBUGGER_DIAGNOSTICS_URL_PATH + } else { + AGENT_DEBUGGER_DIAGNOSTICS_URL_PATH + }, + )); + } + + logs_endpoint.url = Uri::from_parts(logs_uri_parts)?; + diagnostics_endpoint.url = Uri::from_parts(diagnostics_uri_parts)?; + self.logs_endpoint = Some(logs_endpoint); + self.diagnostics_endpoint = Some(diagnostics_endpoint); + Ok(()) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[repr(C)] +pub enum DebuggerType { + Diagnostics, + Logs, +} + +impl DebuggerType { + pub fn of_payload(payload: &DebuggerPayload) -> DebuggerType { + match payload.debugger { + DebuggerData::Snapshot(_) => DebuggerType::Logs, + DebuggerData::Diagnostics(_) => DebuggerType::Diagnostics, + } + } +} + +pub fn encode(data: Vec) -> Vec { + serde_json::to_vec(&data).unwrap() +} + +pub fn generate_tags( + debugger_version: &dyn Display, + env: &dyn Display, + version: &dyn Display, + runtime_id: &dyn Display, + custom_tags: &mut dyn Iterator, +) -> String { + let mut tags = format!( + "debugger_version:{debugger_version},env:{env},version:{version},runtime_id:{runtime_id}" + ); + if let Ok(hostname) = sys_info::hostname() { + tags.push_str(",host_name:"); + tags.push_str(hostname.as_str()); + } + for tag in custom_tags { + tags.push(','); + tags.push_str(tag.as_ref()); + } + percent_encode(tags.as_bytes(), CONTROLS).to_string() +} + +#[derive(Debug, Default)] +enum SenderFuture { + #[default] + Error, + Outstanding(ResponseFuture), + Submitted(JoinHandle>>), +} + +pub struct PayloadSender { + future: SenderFuture, + sender: Sender, + needs_boundary: bool, + payloads: u32, +} + +const BOUNDARY: &str = "------------------------44617461646f67"; +const BOUNDARY_LINE: &str = concat!("--", BOUNDARY, "\r\n"); + +impl PayloadSender { + pub fn new( + config: &Config, + debugger_type: DebuggerType, + percent_encoded_tags: &str, + ) -> anyhow::Result { + let endpoint = match debugger_type { + DebuggerType::Diagnostics => &config.diagnostics_endpoint, + DebuggerType::Logs => &config.logs_endpoint, + } + .as_ref() + .unwrap(); + + let mut url = endpoint.url.clone(); + let mut parts = url.into_parts(); + let query = format!( + "{}?ddtags={}", + parts.path_and_query.unwrap(), + percent_encoded_tags + ); + parts.path_and_query = Some(PathAndQuery::from_str(&query)?); + url = Uri::from_parts(parts)?; + + let mut req = endpoint + .into_request_builder(concat!("Tracer/", env!("CARGO_PKG_VERSION")))? + .method(Method::POST) + .uri(url); + + if endpoint.api_key.is_some() { + req = req.header("DD-EVP-ORIGIN", "agent-debugger"); + } + + let (sender, body) = Body::channel(); + + let needs_boundary = debugger_type == DebuggerType::Diagnostics; + let req = req.header( + "Content-type", + if needs_boundary { + concat!("multipart/form-data; boundary=", BOUNDARY) + } else { + "application/json" + }, + ); + + let future = Client::builder() + .build(Connector::default()) + .request(req.body(body)?); + Ok(PayloadSender { + future: SenderFuture::Outstanding(future), + sender, + needs_boundary, + payloads: 0, + }) + } + + pub async fn append(&mut self, data: &[u8]) -> anyhow::Result<()> { + let first = match std::mem::take(&mut self.future) { + SenderFuture::Outstanding(future) => { + if self.needs_boundary { + let header = concat!( + BOUNDARY_LINE, + "Content-Disposition: form-data; name=\"event\"; filename=\"event.json\"\r\n", + "Content-Type: application/json\r\n", + "\r\n", + ); + self.sender.send_data(header.into()).await?; + } + + self.future = SenderFuture::Submitted(tokio::spawn(future)); + true + } + future => { + self.future = future; + false + } + }; + + // Skip the [] of the Vec + let data = &data[..data.len() - 1]; + let mut data = data.to_vec(); + if !first { + data[0] = b','; + } + self.sender.send_data(Bytes::from(data)).await?; + + self.payloads += 1; + Ok(()) + } + + pub async fn finish(mut self) -> anyhow::Result { + if let SenderFuture::Submitted(future) = self.future { + // insert a trailing ] + if self.needs_boundary { + self.sender + .send_data(concat!("]\r\n", BOUNDARY_LINE).into()) + .await?; + } else { + self.sender.send_data(Bytes::from_static(b"]")).await?; + } + + drop(self.sender); + match future.await? { + Ok(response) => { + let status = response.status().as_u16(); + if status >= 400 { + let body_bytes = hyper::body::to_bytes(response.into_body()).await?; + let response_body = + String::from_utf8(body_bytes.to_vec()).unwrap_or_default(); + anyhow::bail!( + "Server did not accept debugger payload ({status}): {response_body}" + ); + } + Ok(self.payloads) + } + Err(e) => anyhow::bail!("Failed to send traces: {e}"), + } + } else { + Ok(0) + } + } +} + +pub async fn send( + payload: &[u8], + config: &Config, + debugger_type: DebuggerType, + percent_encoded_tags: &str, +) -> anyhow::Result<()> { + let mut batch = PayloadSender::new(config, debugger_type, percent_encoded_tags)?; + batch.append(payload).await?; + batch.finish().await?; + Ok(()) +} + +pub fn generate_new_id() -> Uuid { + Uuid::new_v4() +} diff --git a/remote-config/Cargo.toml b/remote-config/Cargo.toml index 004f7d8b4..ee34771ce 100644 --- a/remote-config/Cargo.toml +++ b/remote-config/Cargo.toml @@ -12,6 +12,7 @@ anyhow = { version = "1.0" } ddcommon = { path = "../ddcommon" } datadog-dynamic-configuration = { path = "../dynamic-configuration" } datadog-trace-protobuf = { path = "../trace-protobuf" } +datadog-live-debugger = { path = "../live-debugger" } hyper = { version = "0.14", features = ["client"], default-features = false } http = "0.2" base64 = "0.21.0" diff --git a/remote-config/src/fetch/fetcher.rs b/remote-config/src/fetch/fetcher.rs index d988096c1..8ce9b69c8 100644 --- a/remote-config/src/fetch/fetcher.rs +++ b/remote-config/src/fetch/fetcher.rs @@ -12,11 +12,14 @@ use datadog_trace_protobuf::remoteconfig::{ TargetFileHash, TargetFileMeta, }; use ddcommon::{connector, Endpoint}; +use http::uri::Scheme; use hyper::http::uri::PathAndQuery; use hyper::{Client, StatusCode}; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, Sha512}; use std::collections::{HashMap, HashSet}; use std::mem::transmute; +use std::ops::Add; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; use tracing::{debug, trace, warn}; @@ -78,6 +81,21 @@ pub struct ConfigFetcherState { pub expire_unused_files: bool, } +#[derive(Default, Serialize, Deserialize)] +pub struct ConfigFetcherStateStats { + pub active_files: u32, +} + +impl Add for ConfigFetcherStateStats { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + ConfigFetcherStateStats { + active_files: self.active_files + rhs.active_files, + } + } +} + pub struct ConfigFetcherFilesLock<'a, S> { inner: MutexGuard<'a, HashMap, StoredTargetFile>>, } @@ -145,6 +163,12 @@ impl ConfigFetcherState { } } } + + pub fn stats(&self) -> ConfigFetcherStateStats { + ConfigFetcherStateStats { + active_files: self.target_files_by_path.lock().unwrap().len() as u32, + } + } } pub struct ConfigFetcher { @@ -397,6 +421,7 @@ impl ConfigFetcher { } else { None }; + // If the file isn't there, it's not meant for us. if let Some(raw_file) = incoming_files.get(path) { if let Ok(decoded) = base64::engine::general_purpose::STANDARD.decode(raw_file) { let computed_hash = hasher(decoded.as_slice()); @@ -450,11 +475,6 @@ impl ConfigFetcher { String::from_utf8_lossy(raw_file) ) } - } else { - anyhow::bail!( - "Found changed config data for path {path}, but no file; existing files: {:?}", - incoming_files.keys().collect::>() - ) } } @@ -475,8 +495,16 @@ impl ConfigFetcher { } } -fn get_product_endpoint(_subdomain: &str, endpoint: &Endpoint) -> Endpoint { +fn get_product_endpoint(subdomain: &str, endpoint: &Endpoint) -> Endpoint { let mut parts = endpoint.url.clone().into_parts(); + if parts.authority.is_some() && parts.scheme.is_none() { + parts.scheme = Some(Scheme::HTTPS); + parts.authority = Some( + format!("{}.{}", subdomain, parts.authority.unwrap()) + .parse() + .unwrap(), + ); + } parts.path_and_query = Some(PathAndQuery::from_static("/v0.7/config")); Endpoint { url: hyper::Uri::from_parts(parts).unwrap(), diff --git a/remote-config/src/fetch/multitarget.rs b/remote-config/src/fetch/multitarget.rs index f87b134c8..4fa61e918 100644 --- a/remote-config/src/fetch/multitarget.rs +++ b/remote-config/src/fetch/multitarget.rs @@ -3,17 +3,19 @@ use crate::fetch::{ ConfigApplyState, ConfigFetcherState, ConfigInvariants, FileStorage, RefcountedFile, - RefcountingStorage, SharedFetcher, + RefcountingStorage, RefcountingStorageStats, SharedFetcher, }; use crate::Target; use futures_util::future::Shared; use futures_util::FutureExt; use manual_future::ManualFuture; +use serde::{Deserialize, Serialize}; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::default::Default; use std::fmt::Debug; use std::hash::Hash; +use std::ops::Add; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -52,6 +54,31 @@ where fetcher_semaphore: Semaphore, } +#[derive(Default, Serialize, Deserialize)] +pub struct MultiTargetStats { + known_runtimes: u32, + starting_fetchers: u32, + active_fetchers: u32, + inactive_fetchers: u32, + removing_fetchers: u32, + storage: RefcountingStorageStats, +} + +impl Add for MultiTargetStats { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + MultiTargetStats { + known_runtimes: self.known_runtimes + rhs.known_runtimes, + starting_fetchers: self.starting_fetchers + rhs.starting_fetchers, + active_fetchers: self.active_fetchers + rhs.active_fetchers, + inactive_fetchers: self.inactive_fetchers + rhs.inactive_fetchers, + removing_fetchers: self.removing_fetchers + rhs.removing_fetchers, + storage: self.storage + rhs.storage, + } + } +} + enum KnownTargetStatus { Pending, Alive, @@ -78,7 +105,7 @@ pub trait NotifyTarget: Sync + Send + Sized + Hash + Eq + Clone + Debug { } pub trait MultiTargetHandlers { - fn fetched(&self, target: &Arc, files: &[Arc]) -> bool; + fn fetched(&self, runtime_id: &Arc, target: &Arc, files: &[Arc]) -> bool; fn expired(&self, target: &Arc); @@ -134,7 +161,7 @@ where known_service.runtimes.remove(runtime_id); let mut status = known_service.status.lock().unwrap(); *status = match *status { - KnownTargetStatus::Pending => break 'drop_service, + KnownTargetStatus::Pending => KnownTargetStatus::Alive, // not really KnownTargetStatus::Alive => { KnownTargetStatus::RemoveAt(Instant::now() + Duration::from_secs(3666)) } @@ -142,6 +169,10 @@ where unreachable!() } }; + // We've marked it Alive so that the Pending check in start_fetcher() will fail + if matches!(*status, KnownTargetStatus::Alive) { + break 'drop_service; + } 0 } else { if *known_service.fetcher.runtime_id.lock().unwrap() == runtime_id { @@ -164,6 +195,7 @@ where }; break 'service_handling; } + trace!("Remove {target:?} from services map while in pending state"); services.remove(target); } @@ -366,10 +398,12 @@ where .run( this.storage.clone(), Box::new(move |files| { - let notify = inner_this - .storage - .storage - .fetched(&inner_fetcher.target, files); + let runtime_id = Arc::new(inner_fetcher.runtime_id.lock().unwrap().clone()); + let notify = inner_this.storage.storage.fetched( + &runtime_id, + &inner_fetcher.target, + files, + ); if notify { // notify_targets is Hash + Eq + Clone, allowing us to deduplicate. Also @@ -431,6 +465,10 @@ where { // scope lock before await + trace!( + "Remove {:?} from services map at fetcher end", + fetcher.target + ); let mut services = this.services.lock().unwrap(); services.remove(&fetcher.target); if services.is_empty() && this.pending_async_insertions.load(Ordering::Relaxed) == 0 @@ -458,6 +496,41 @@ where } } } + + pub fn active_runtimes(&self) -> usize { + self.runtimes.lock().unwrap().len() + } + + pub fn invariants(&self) -> &ConfigInvariants { + self.storage.invariants() + } + + pub fn stats(&self) -> MultiTargetStats { + let (starting_fetchers, active_fetchers, inactive_fetchers, removing_fetchers) = { + let services = self.services.lock().unwrap(); + let mut starting = 0; + let mut active = 0; + let mut inactive = 0; + let mut removing = 0; + for (_, known_target) in services.iter() { + match *known_target.status.lock().unwrap() { + KnownTargetStatus::Pending => starting += 1, + KnownTargetStatus::Alive => active += 1, + KnownTargetStatus::RemoveAt(_) => inactive += 1, + KnownTargetStatus::Removing(_) => removing += 1, + } + } + (starting, active, inactive, removing) + }; + MultiTargetStats { + known_runtimes: self.runtimes.lock().unwrap().len() as u32, + starting_fetchers, + active_fetchers, + inactive_fetchers, + removing_fetchers, + storage: self.storage.stats(), + } + } } #[cfg(test)] @@ -504,7 +577,12 @@ mod tests { } impl MultiTargetHandlers for MultiFileStorage { - fn fetched(&self, target: &Arc, files: &[Arc]) -> bool { + fn fetched( + &self, + _runtime_id: &Arc, + target: &Arc, + files: &[Arc], + ) -> bool { match self.recent_fetches.lock().unwrap().entry(target.clone()) { Entry::Occupied(_) => panic!("Double fetch without recent_fetches clear"), Entry::Vacant(e) => { diff --git a/remote-config/src/fetch/shared.rs b/remote-config/src/fetch/shared.rs index 8c5d266ac..dbffa466f 100644 --- a/remote-config/src/fetch/shared.rs +++ b/remote-config/src/fetch/shared.rs @@ -2,11 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 use crate::fetch::{ - ConfigApplyState, ConfigClientState, ConfigFetcher, ConfigFetcherState, ConfigInvariants, - FileStorage, + ConfigApplyState, ConfigClientState, ConfigFetcher, ConfigFetcherState, + ConfigFetcherStateStats, ConfigInvariants, FileStorage, }; use crate::{RemoteConfigPath, Target}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::ops::Add; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -138,6 +140,23 @@ where run_id: Arc, } +#[derive(Default, Serialize, Deserialize)] +pub struct RefcountingStorageStats { + pub inactive_files: u32, + pub fetcher: ConfigFetcherStateStats, +} + +impl Add for RefcountingStorageStats { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + RefcountingStorageStats { + inactive_files: self.inactive_files + rhs.inactive_files, + fetcher: self.fetcher + rhs.fetcher, + } + } +} + impl Clone for RefcountingStorage where S::StoredFile: RefcountedFile, @@ -191,6 +210,13 @@ where pub fn invariants(&self) -> &ConfigInvariants { &self.state.invariants } + + pub fn stats(&self) -> RefcountingStorageStats { + RefcountingStorageStats { + inactive_files: self.inactive.lock().unwrap().len() as u32, + fetcher: self.state.stats(), + } + } } impl FileStorage for RefcountingStorage diff --git a/remote-config/src/parse.rs b/remote-config/src/parse.rs index 37cbd4063..c2893867e 100644 --- a/remote-config/src/parse.rs +++ b/remote-config/src/parse.rs @@ -3,11 +3,12 @@ use crate::{RemoteConfigPath, RemoteConfigProduct, RemoteConfigSource}; use datadog_dynamic_configuration::data::DynamicConfigFile; +use datadog_live_debugger::LiveDebuggingData; #[derive(Debug)] pub enum RemoteConfigData { DynamicConfig(DynamicConfigFile), - LiveDebugger(()), + LiveDebugger(LiveDebuggingData), } impl RemoteConfigData { @@ -20,7 +21,8 @@ impl RemoteConfigData { RemoteConfigData::DynamicConfig(datadog_dynamic_configuration::parse_json(data)?) } RemoteConfigProduct::LiveDebugger => { - RemoteConfigData::LiveDebugger(/* placeholder */ ()) + let parsed = datadog_live_debugger::parse_json(&String::from_utf8_lossy(data))?; + RemoteConfigData::LiveDebugger(parsed) } }) } diff --git a/sidecar-ffi/Cargo.toml b/sidecar-ffi/Cargo.toml index b0fa91da5..1e30e4c1a 100644 --- a/sidecar-ffi/Cargo.toml +++ b/sidecar-ffi/Cargo.toml @@ -19,6 +19,7 @@ ddcommon = { path = "../ddcommon" } ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false } ddtelemetry-ffi = { path = "../ddtelemetry-ffi", default-features = false } datadog-remote-config = { path = "../remote-config" } +datadog-live-debugger = { path = "../live-debugger" } paste = "1" libc = "0.2" dogstatsd-client = { path = "../dogstatsd-client" } diff --git a/sidecar-ffi/cbindgen.toml b/sidecar-ffi/cbindgen.toml index dd0268147..75568489f 100644 --- a/sidecar-ffi/cbindgen.toml +++ b/sidecar-ffi/cbindgen.toml @@ -34,4 +34,4 @@ must_use = "DDOG_CHECK_RETURN" [parse] parse_deps = true -include = ["ddcommon", "ddtelemetry", "datadog-sidecar", "ddtelemetry-ffi", "ddcommon-ffi", "datadog-ipc", "datadog-remote-config"] +include = ["ddcommon", "ddtelemetry", "datadog-sidecar", "ddtelemetry-ffi", "ddcommon-ffi", "datadog-ipc", "datadog-live-debugger", "datadog-remote-config"] diff --git a/sidecar-ffi/src/lib.rs b/sidecar-ffi/src/lib.rs index 895a1b9fb..3953ac356 100644 --- a/sidecar-ffi/src/lib.rs +++ b/sidecar-ffi/src/lib.rs @@ -4,6 +4,7 @@ use datadog_ipc::platform::{ FileBackedHandle, MappedMem, NamedShmHandle, PlatformHandle, ShmHandle, }; +use datadog_live_debugger::debugger_defs::DebuggerPayload; use datadog_remote_config::fetch::ConfigInvariants; use datadog_remote_config::{RemoteConfigCapabilities, RemoteConfigProduct, Target}; use datadog_sidecar::agent_remote_config::{ @@ -487,6 +488,7 @@ pub unsafe extern "C" fn ddog_sidecar_session_set_config( language: ffi::CharSlice, tracer_version: ffi::CharSlice, flush_interval_milliseconds: u32, + remote_config_poll_interval_millis: u32, telemetry_heartbeat_interval_millis: u32, force_flush_size: usize, force_drop_size: usize, @@ -513,6 +515,9 @@ pub unsafe extern "C" fn ddog_sidecar_session_set_config( language: language.to_utf8_lossy().into(), tracer_version: tracer_version.to_utf8_lossy().into(), flush_interval: Duration::from_millis(flush_interval_milliseconds as u64), + remote_config_poll_interval: Duration::from_millis( + remote_config_poll_interval_millis as u64 + ), telemetry_heartbeat_interval: Duration::from_millis( telemetry_heartbeat_interval_millis as u64 ), @@ -620,6 +625,41 @@ pub unsafe extern "C" fn ddog_sidecar_send_trace_v04_bytes( MaybeError::None } +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +#[allow(improper_ctypes_definitions)] // DebuggerPayload is just a pointer, we hide its internals +pub unsafe extern "C" fn ddog_sidecar_send_debugger_data( + transport: &mut Box, + instance_id: &InstanceId, + queue_id: QueueId, + payloads: Vec, +) -> MaybeError { + if payloads.is_empty() { + return MaybeError::None; + } + + try_c!(blocking::send_debugger_data_shm_vec( + transport, + instance_id, + queue_id, + payloads, + )); + + MaybeError::None +} + +#[no_mangle] +#[allow(clippy::missing_safety_doc)] +#[allow(improper_ctypes_definitions)] // DebuggerPayload is just a pointer, we hide its internals +pub unsafe extern "C" fn ddog_sidecar_send_debugger_datum( + transport: &mut Box, + instance_id: &InstanceId, + queue_id: QueueId, + payload: Box, +) -> MaybeError { + ddog_sidecar_send_debugger_data(transport, instance_id, queue_id, vec![*payload]) +} + #[no_mangle] #[allow(clippy::missing_safety_doc)] pub unsafe extern "C" fn ddog_sidecar_set_remote_config_data( @@ -629,6 +669,7 @@ pub unsafe extern "C" fn ddog_sidecar_set_remote_config_data( service_name: ffi::CharSlice, env_name: ffi::CharSlice, app_version: ffi::CharSlice, + global_tags: &ddcommon_ffi::Vec, ) -> MaybeError { try_c!(blocking::set_remote_config_data( transport, @@ -637,6 +678,7 @@ pub unsafe extern "C" fn ddog_sidecar_set_remote_config_data( service_name.to_utf8_lossy().into(), env_name.to_utf8_lossy().into(), app_version.to_utf8_lossy().into(), + global_tags.to_vec(), )); MaybeError::None diff --git a/sidecar-ffi/tests/sidecar.rs b/sidecar-ffi/tests/sidecar.rs index 1ca319b4b..a857118a3 100644 --- a/sidecar-ffi/tests/sidecar.rs +++ b/sidecar-ffi/tests/sidecar.rs @@ -95,6 +95,7 @@ fn test_ddog_sidecar_register_app() { "".into(), 1000, 1000000, + 1, 10000000, 10000000, "".into(), @@ -147,6 +148,7 @@ fn test_ddog_sidecar_register_app() { "".into(), 1000, 1000000, + 1, 10000000, 10000000, "".into(), diff --git a/sidecar/Cargo.toml b/sidecar/Cargo.toml index d60525e77..bb63ce30d 100644 --- a/sidecar/Cargo.toml +++ b/sidecar/Cargo.toml @@ -25,6 +25,7 @@ datadog-trace-protobuf = { path = "../trace-protobuf" } datadog-trace-utils = { path = "../trace-utils" } datadog-trace-normalization = { path = "../trace-normalization" } datadog-remote-config = { path = "../remote-config" } +datadog-live-debugger = { path = "../live-debugger" } datadog-crashtracker = { path = "../crashtracker" } dogstatsd-client = { path = "../dogstatsd-client" } tinybytes = { path = "../tinybytes" } @@ -97,6 +98,7 @@ sendfd = { version = "0.4", features = ["tokio"] } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["securitybaseapi", "sddl"] } +windows-sys = { version = "0.52.0", features = ["Win32_System_SystemInformation"] } [target.'cfg(windows_seh_wrapper)'.dependencies] microseh = "0.1.1" diff --git a/sidecar/src/config.rs b/sidecar/src/config.rs index 2ca4616a5..aeeea7668 100644 --- a/sidecar/src/config.rs +++ b/sidecar/src/config.rs @@ -12,6 +12,8 @@ const ENV_SIDECAR_IPC_MODE: &str = "_DD_DEBUG_SIDECAR_IPC_MODE"; const SIDECAR_IPC_MODE_SHARED: &str = "shared"; const SIDECAR_IPC_MODE_PER_PROCESS: &str = "instance_per_process"; +const ENV_SIDECAR_LOG_LEVEL: &str = "_DD_DEBUG_SIDECAR_LOG_LEVEL"; + const ENV_SIDECAR_LOG_METHOD: &str = "_DD_DEBUG_SIDECAR_LOG_METHOD"; const SIDECAR_LOG_METHOD_DISABLED: &str = "disabled"; const SIDECAR_LOG_METHOD_STDOUT: &str = "stdout"; @@ -80,6 +82,7 @@ impl std::fmt::Display for LogMethod { pub struct Config { pub ipc_mode: IpcMode, pub log_method: LogMethod, + pub log_level: String, pub idle_linger_time: Duration, pub self_telemetry: bool, pub library_dependencies: Vec, @@ -185,6 +188,10 @@ impl FromEnv { } } + pub fn log_level() -> String { + std::env::var(ENV_SIDECAR_LOG_LEVEL).unwrap_or_default() + } + fn idle_linger_time() -> Duration { std::env::var(ENV_IDLE_LINGER_TIME_SECS) .unwrap_or_default() @@ -205,6 +212,7 @@ impl FromEnv { Config { ipc_mode: Self::ipc_mode(), log_method: Self::log_method(), + log_level: Self::log_level(), idle_linger_time: Self::idle_linger_time(), self_telemetry: Self::self_telemetry(), library_dependencies: vec![], diff --git a/sidecar/src/entry.rs b/sidecar/src/entry.rs index 6349c0cf2..fd473e5b4 100644 --- a/sidecar/src/entry.rs +++ b/sidecar/src/entry.rs @@ -27,6 +27,7 @@ use crate::setup::{self, IpcClient, IpcServer, Liaison}; use crate::config::{self, Config}; use crate::self_telemetry::self_telemetry; +use crate::tracer::SHM_LIMITER; use crate::watchdog::Watchdog; use crate::{ddog_daemon_entry_point, setup_daemon_process}; @@ -79,6 +80,9 @@ where .await; }); + // Init. Early, before we start listening. + drop(SHM_LIMITER.lock()); + let server = SidecarServer::default(); let (shutdown_complete_tx, shutdown_complete_rx) = mpsc::channel::<()>(1); diff --git a/sidecar/src/lib.rs b/sidecar/src/lib.rs index 62349a761..bcb404921 100644 --- a/sidecar/src/lib.rs +++ b/sidecar/src/lib.rs @@ -11,7 +11,7 @@ pub mod one_way_shared_memory; mod self_telemetry; pub mod setup; pub mod shm_remote_config; -mod tracer; +pub mod tracer; mod watchdog; pub use entry::*; @@ -22,6 +22,7 @@ mod unix; pub use unix::*; pub mod service; +mod tokio_util; #[cfg(windows)] mod windows; diff --git a/sidecar/src/log.rs b/sidecar/src/log.rs index 449a847de..5fa4ee72d 100644 --- a/sidecar/src/log.rs +++ b/sidecar/src/log.rs @@ -381,7 +381,11 @@ pub(crate) fn enable_logging() -> anyhow::Result<()> { MULTI_LOG_FILTER.add(env); // this also immediately drops it, but will retain it for few // seconds during startup } - MULTI_LOG_WRITER.add(config::Config::get().log_method); // same than MULTI_LOG_FILTER + let config = config::Config::get(); + if !config.log_level.is_empty() { + MULTI_LOG_FILTER.add(config.log_level.clone()); + } + MULTI_LOG_WRITER.add(config.log_method); // same than MULTI_LOG_FILTER LogTracer::init()?; diff --git a/sidecar/src/service/blocking.rs b/sidecar/src/service/blocking.rs index cd48d87be..85f7ffc3d 100644 --- a/sidecar/src/service/blocking.rs +++ b/sidecar/src/service/blocking.rs @@ -5,9 +5,12 @@ use super::{ InstanceId, QueueId, RuntimeMetadata, SerializedTracerHeaderTags, SessionConfig, SidecarAction, SidecarInterfaceRequest, SidecarInterfaceResponse, }; -use datadog_ipc::platform::{Channel, ShmHandle}; +use datadog_ipc::platform::{Channel, FileBackedHandle, ShmHandle}; use datadog_ipc::transport::blocking::BlockingTransport; +use datadog_live_debugger::sender::DebuggerType; +use ddcommon::tag::Tag; use dogstatsd_client::DogStatsDActionOwned; +use serde::Serialize; use std::sync::Mutex; use std::{ borrow::Cow, @@ -271,6 +274,100 @@ pub fn send_trace_v04_shm( }) } +/// Sends raw data from shared memory to the debugger endpoint. +/// +/// # Arguments +/// +/// * `transport` - The transport used for communication. +/// * `instance_id` - The ID of the instance. +/// * `queue_id` - The unique identifier for the trace context. +/// * `handle` - The handle to the shared memory. +/// * `debugger_type` - Whether it's log or diagnostic data. +/// +/// # Returns +/// +/// An `io::Result<()>` indicating the result of the operation. +pub fn send_debugger_data_shm( + transport: &mut SidecarTransport, + instance_id: &InstanceId, + queue_id: QueueId, + handle: ShmHandle, + debugger_type: DebuggerType, +) -> io::Result<()> { + transport.send(SidecarInterfaceRequest::SendDebuggerDataShm { + instance_id: instance_id.clone(), + queue_id, + handle, + debugger_type, + }) +} + +/// Sends a collection of debugger payloads to the debugger endpoint. +/// +/// # Arguments +/// +/// * `transport` - The transport used for communication. +/// * `instance_id` - The ID of the instance. +/// * `queue_id` - The unique identifier for the trace context. +/// * `payloads` - The payloads to be sent +/// +/// # Returns +/// +/// An `anyhow::Result<()>` indicating the result of the operation. +pub fn send_debugger_data_shm_vec( + transport: &mut SidecarTransport, + instance_id: &InstanceId, + queue_id: QueueId, + payloads: Vec, +) -> anyhow::Result<()> { + if payloads.is_empty() { + return Ok(()); + } + let debugger_type = DebuggerType::of_payload(&payloads[0]); + + struct SizeCount(usize); + + impl io::Write for SizeCount { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0 += buf.len(); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let mut size_serializer = serde_json::Serializer::new(SizeCount(0)); + payloads.serialize(&mut size_serializer).unwrap(); + + let mut mapped = ShmHandle::new(size_serializer.into_inner().0)?.map()?; + let mut serializer = serde_json::Serializer::new(mapped.as_slice_mut()); + payloads.serialize(&mut serializer).unwrap(); + + Ok(send_debugger_data_shm( + transport, + instance_id, + queue_id, + mapped.into(), + debugger_type, + )?) +} + +/// Acquire an exception hash rate limiter +/// +/// # Arguments +/// * `exception_hash` - the ID +/// * `granularity` - how much time needs to pass between two exceptions +pub fn acquire_exception_hash_rate_limiter( + transport: &mut SidecarTransport, + exception_hash: u64, + granularity: Duration, +) -> io::Result<()> { + transport.send(SidecarInterfaceRequest::AcquireExceptionHashRateLimiter { + exception_hash, + granularity, + }) +} + /// Sets the state of the current remote config operation. /// The queue id is shared with telemetry and the associated data will be freed upon a /// `Lifecycle::Stop` event. @@ -294,6 +391,7 @@ pub fn set_remote_config_data( service_name: String, env_name: String, app_version: String, + global_tags: Vec, ) -> io::Result<()> { transport.send(SidecarInterfaceRequest::SetRemoteConfigData { instance_id: instance_id.clone(), @@ -301,6 +399,7 @@ pub fn set_remote_config_data( service_name, env_name, app_version, + global_tags, }) } diff --git a/sidecar/src/service/exception_hash_rate_limiter.rs b/sidecar/src/service/exception_hash_rate_limiter.rs new file mode 100644 index 000000000..1948d508a --- /dev/null +++ b/sidecar/src/service/exception_hash_rate_limiter.rs @@ -0,0 +1,113 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::primary_sidecar_identifier; +use datadog_ipc::rate_limiter::{ShmLimiter, ShmLimiterMemory}; +use ddcommon::rate_limiter::Limiter; +use lazy_static::lazy_static; +use std::ffi::CString; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::Duration; + +lazy_static! { + pub(crate) static ref EXCEPTION_HASH_LIMITER: Mutex = + Mutex::new(ManagedExceptionHashRateLimiter::create().unwrap()); +} + +pub(crate) struct ManagedExceptionHashRateLimiter { + limiter: ExceptionHashRateLimiter, + active: Vec, + _drop: tokio::sync::oneshot::Sender<()>, +} + +impl ManagedExceptionHashRateLimiter { + fn create() -> io::Result { + let (send, recv) = tokio::sync::oneshot::channel::<()>(); + + tokio::spawn(async move { + async fn do_loop() { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + let mut this = EXCEPTION_HASH_LIMITER.lock().unwrap(); + this.active.retain_mut(|limiter| { + // This technically could discard + limiter.shm.update_rate() > 0. || !unsafe { limiter.shm.drop_if_rc_1() } + }); + } + } + + tokio::select! { + _ = do_loop() => {} + _ = recv => { } + } + }); + + Ok(ManagedExceptionHashRateLimiter { + limiter: ExceptionHashRateLimiter::create()?, + active: vec![], + _drop: send, + }) + } + + pub fn add(&mut self, hash: u64, granularity: Duration) { + let limiter = self.limiter.add(hash, granularity); + self.active.push(limiter); + } +} + +pub struct ExceptionHashRateLimiter { + mem: ShmLimiterMemory, +} + +struct EntryData { + pub hash: AtomicU64, +} + +pub struct HashLimiter { + shm: ShmLimiter, +} + +impl HashLimiter { + pub fn inc(&self) -> bool { + self.shm.inc(1) + } +} + +fn path() -> CString { + CString::new(format!("/ddexhlimit-{}", primary_sidecar_identifier())).unwrap() +} + +impl ExceptionHashRateLimiter { + pub fn create() -> io::Result { + Ok(ExceptionHashRateLimiter { + mem: ShmLimiterMemory::create(path())?, + }) + } + + pub fn open() -> io::Result { + Ok(ExceptionHashRateLimiter { + mem: ShmLimiterMemory::open(&path())?, + }) + } + + fn add(&mut self, hash: u64, granularity: Duration) -> HashLimiter { + let allocated = self + .mem + .alloc_with_granularity(granularity.as_secs() as u32); + let data = allocated.data(); + data.hash.store(hash, Ordering::Relaxed); + allocated.inc(1); + HashLimiter { shm: allocated } + } + + pub fn find(&self, hash: u64) -> Option { + Some(HashLimiter { + shm: self + .mem + .find(|data| data.hash.load(Ordering::Relaxed) == hash)?, + }) + } +} diff --git a/sidecar/src/service/mod.rs b/sidecar/src/service/mod.rs index 8100ea29b..4760f601b 100644 --- a/sidecar/src/service/mod.rs +++ b/sidecar/src/service/mod.rs @@ -28,6 +28,7 @@ use session_info::SessionInfo; use sidecar_interface::{SidecarInterface, SidecarInterfaceRequest, SidecarInterfaceResponse}; pub mod blocking; +pub mod exception_hash_rate_limiter; mod instance_id; mod queue_id; mod remote_configs; @@ -48,6 +49,7 @@ pub struct SessionConfig { pub language: String, pub tracer_version: String, pub flush_interval: Duration, + pub remote_config_poll_interval: Duration, pub telemetry_heartbeat_interval: Duration, pub force_flush_size: usize, pub force_drop_size: usize, diff --git a/sidecar/src/service/remote_configs.rs b/sidecar/src/service/remote_configs.rs index e2d3499e9..0a0f2e785 100644 --- a/sidecar/src/service/remote_configs.rs +++ b/sidecar/src/service/remote_configs.rs @@ -2,10 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use crate::shm_remote_config::{ShmRemoteConfigs, ShmRemoteConfigsGuard}; -use datadog_remote_config::fetch::{ConfigInvariants, NotifyTarget}; +use datadog_remote_config::fetch::{ConfigInvariants, MultiTargetStats, NotifyTarget}; use std::collections::hash_map::Entry; use std::fmt::Debug; use std::sync::{Arc, Mutex}; +use std::time::Duration; use zwohash::HashMap; #[cfg(windows)] @@ -93,9 +94,11 @@ pub struct RemoteConfigs( pub type RemoteConfigsGuard = ShmRemoteConfigsGuard; impl RemoteConfigs { + #[allow(clippy::too_many_arguments)] pub fn add_runtime( &self, invariants: ConfigInvariants, + poll_interval: Duration, runtime_id: String, notify_target: RemoteConfigNotifyTarget, env: String, @@ -112,6 +115,7 @@ impl RemoteConfigs { Box::new(move || { this.lock().unwrap().remove(&invariants); }), + poll_interval, )) } } @@ -123,4 +127,13 @@ impl RemoteConfigs { rc.shutdown(); } } + + pub fn stats(&self) -> MultiTargetStats { + self.0 + .lock() + .unwrap() + .values() + .map(|rc| rc.stats()) + .fold(MultiTargetStats::default(), |a, b| a + b) + } } diff --git a/sidecar/src/service/runtime_info.rs b/sidecar/src/service/runtime_info.rs index 4a4817845..6a30dd812 100644 --- a/sidecar/src/service/runtime_info.rs +++ b/sidecar/src/service/runtime_info.rs @@ -6,12 +6,16 @@ use crate::service::{ telemetry::{AppInstance, AppOrQueue}, InstanceId, QueueId, }; +use datadog_live_debugger::sender::{generate_tags, PayloadSender}; +use ddcommon::tag::Tag; use futures::{ future::{self, join_all, Shared}, FutureExt, }; use manual_future::{ManualFuture, ManualFutureCompleter}; +use simd_json::prelude::ArrayTrait; use std::collections::HashMap; +use std::fmt::Display; use std::sync::{Arc, Mutex, MutexGuard}; use tracing::{debug, info}; @@ -31,7 +35,6 @@ pub(crate) struct SharedAppManualFut { pub(crate) struct RuntimeInfo { pub(crate) apps: Arc>, applications: Arc>>, - #[cfg(feature = "tracing")] pub(crate) instance_id: InstanceId, } @@ -41,10 +44,17 @@ pub(crate) struct RuntimeInfo { /// Each app is represented by a shared future that may contain an `Option`. /// Each action is represented by an `AppOrQueue` enum. Combining apps and actions are necessary /// because service and env names are not known until later in the initialization process. +/// Similarly, each application has its own global tags. #[derive(Default)] pub(crate) struct ActiveApplication { pub app_or_actions: AppOrQueue, pub remote_config_guard: Option, + pub env: Option, + pub app_version: Option, + pub global_tags: Vec, + pub live_debugger_tag_cache: Option>, + pub debugger_logs_payload_sender: Arc>>, + pub debugger_diagnostics_payload_sender: Arc>>, } impl RuntimeInfo { @@ -80,7 +90,6 @@ impl RuntimeInfo { /// Shuts down the runtime. /// This involves shutting down all the instances in the runtime. pub(crate) async fn shutdown(self) { - #[cfg(feature = "tracing")] info!( "Shutting down runtime_id {} for session {}", self.instance_id.runtime_id, self.instance_id.session_id @@ -105,7 +114,6 @@ impl RuntimeInfo { .collect(); future::join_all(instances_shutting_down).await; - #[cfg(feature = "tracing")] debug!( "Successfully shut down runtime_id {} for session {}", self.instance_id.runtime_id, self.instance_id.session_id @@ -134,4 +142,58 @@ impl RuntimeInfo { } } +impl ActiveApplication { + /// Sets the cached debugger tags if not set and returns them. + /// + /// # Arguments + /// + /// * `env` - The environment of the current application. + /// * `app_version` - The version of the current application. + /// * `global_tags` - The global tags of the current application. + pub fn set_metadata(&mut self, env: String, app_version: String, global_tags: Vec) { + self.env = Some(env); + self.app_version = Some(app_version); + self.global_tags = global_tags; + self.live_debugger_tag_cache = None; + } + + /// Sets the cached debugger tags if not set and returns them. + /// + /// # Arguments + /// + /// * `debugger_version` - The version of the live debugger to report. + /// * `queue_id` - The unique identifier for the trace context. + /// + /// # Returns + /// + /// * `Arc` - A percent encoded string to be passed to + /// datadog_live_debugger::sender::send. + /// * `bool` - Whether new tags were set and a new sender needs to be started. + pub fn get_debugger_tags( + &mut self, + debugger_version: &dyn Display, + runtime_id: &str, + ) -> (Arc, bool) { + if let Some(ref cached) = self.live_debugger_tag_cache { + return (cached.clone(), false); + } + if let Some(env) = &self.env { + if let Some(version) = &self.app_version { + let tags = Arc::new(generate_tags( + debugger_version, + env, + version, + &runtime_id, + &mut self.global_tags.iter(), + )); + self.live_debugger_tag_cache = Some(tags.clone()); + return (tags, true); + } + } + let tags = Arc::new(format!("debugger_version:{debugger_version}")); + self.live_debugger_tag_cache = Some(tags.clone()); + (tags, true) + } +} + // TODO: APM-1079 - Add unit tests for RuntimeInfo diff --git a/sidecar/src/service/session_info.rs b/sidecar/src/service/session_info.rs index f060b1207..b57b5a932 100644 --- a/sidecar/src/service/session_info.rs +++ b/sidecar/src/service/session_info.rs @@ -2,19 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 use std::sync::atomic::AtomicI32; +use std::time::Duration; use std::{ collections::HashMap, sync::{Arc, Mutex, MutexGuard}, }; -use datadog_remote_config::fetch::ConfigInvariants; use futures::future; -use tracing::{enabled, info, Level}; + +use datadog_live_debugger::sender::{DebuggerType, PayloadSender}; +use datadog_remote_config::fetch::ConfigInvariants; +use tracing::log::warn; +use tracing::{debug, error, info, trace}; use crate::log::{MultiEnvFilterGuard, MultiWriterGuard}; -use crate::tracer; +use crate::{spawn_map_err, tracer}; -use crate::service::{InstanceId, RuntimeInfo}; +use crate::service::{InstanceId, QueueId, RuntimeInfo}; /// `SessionInfo` holds information about a session. /// /// It contains a list of runtimes, session configuration, tracer configuration, and log guards. @@ -23,15 +27,16 @@ use crate::service::{InstanceId, RuntimeInfo}; pub(crate) struct SessionInfo { runtimes: Arc>>, pub(crate) session_config: Arc>>, + debugger_config: Arc>, tracer_config: Arc>, dogstatsd: Arc>>, remote_config_invariants: Arc>>, + pub(crate) remote_config_interval: Arc>, #[cfg(windows)] pub(crate) remote_config_notify_function: Arc>, pub(crate) log_guard: Arc, MultiWriterGuard<'static>)>>>, - #[cfg(feature = "tracing")] pub(crate) session_id: String, pub(crate) pid: Arc, } @@ -41,9 +46,11 @@ impl Clone for SessionInfo { SessionInfo { runtimes: self.runtimes.clone(), session_config: self.session_config.clone(), + debugger_config: self.debugger_config.clone(), tracer_config: self.tracer_config.clone(), dogstatsd: self.dogstatsd.clone(), remote_config_invariants: self.remote_config_invariants.clone(), + remote_config_interval: self.remote_config_interval.clone(), #[cfg(windows)] remote_config_notify_function: self.remote_config_notify_function.clone(), log_guard: self.log_guard.clone(), @@ -69,18 +76,15 @@ impl SessionInfo { Some(runtime) => runtime.clone(), None => { let mut runtime = RuntimeInfo::default(); + runtime.instance_id = InstanceId { + session_id: self.session_id.clone(), + runtime_id: runtime_id.clone(), + }; runtimes.insert(runtime_id.clone(), runtime.clone()); - #[cfg(feature = "tracing")] - if enabled!(Level::INFO) { - runtime.instance_id = InstanceId { - session_id: self.session_id.clone(), - runtime_id: runtime_id.clone(), - }; - info!( - "Registering runtime_id {} for session {}", - runtime_id, self.session_id - ); - } + info!( + "Registering runtime_id {} for session {}", + runtime_id, self.session_id + ); runtime } } @@ -179,6 +183,17 @@ impl SessionInfo { f(&mut self.get_dogstatsd()); } + pub fn get_debugger_config(&self) -> MutexGuard { + self.debugger_config.lock().unwrap() + } + + pub fn modify_debugger_config(&self, mut f: F) + where + F: FnMut(&mut datadog_live_debugger::sender::Config), + { + f(&mut self.get_debugger_config()); + } + pub fn set_remote_config_invariants(&self, invariants: ConfigInvariants) { *self.remote_config_invariants.lock().unwrap() = Some(invariants); } @@ -186,6 +201,102 @@ impl SessionInfo { pub fn get_remote_config_invariants(&self) -> MutexGuard> { self.remote_config_invariants.lock().unwrap() } + + pub fn send_debugger_data + Sync + Send + 'static>( + &self, + debugger_type: DebuggerType, + runtime_id: &str, + queue_id: QueueId, + payload: R, + ) { + async fn do_send( + config: Arc>, + debugger_type: DebuggerType, + new_tags: bool, + tags: Arc, + guard: Arc>>, + payload: &[u8], + ) -> anyhow::Result<()> { + async fn finish_sender(debugger_type: DebuggerType, sender: PayloadSender) { + match sender.finish().await { + Ok(payloads) => debug!("Successfully sent {payloads} payloads to live debugger {debugger_type:?} endpoint"), + Err(e) => error!("Error sending to live debugger endpoint: {e:?}"), + } + } + + let mut sender = guard.lock().await; + if new_tags { + if let Some(sender) = sender.take() { + spawn_map_err!(finish_sender(debugger_type, sender), |e| { + error!("Error sending to live debugger {debugger_type:?} endpoint: {e:?}"); + }); + } + } + if sender.is_none() { + let config = &*config.lock().unwrap(); + *sender = Some(PayloadSender::new(config, debugger_type, tags.as_str())?); + let guard = guard.clone(); + spawn_map_err!( + async move { + tokio::time::sleep(Duration::from_millis(500)).await; + if let Some(sender) = guard.lock().await.take() { + finish_sender(debugger_type, sender).await; + } + }, + |e| error!("Error sending to live debugger {debugger_type:?} endpoint: {e:?}") + ); + } + trace!( + "Submitting live debugger {debugger_type:?} payload {:?}", + String::from_utf8_lossy(payload) + ); + sender.as_mut().unwrap().append(payload).await + } + + async fn send + Sync + Send>( + config: Arc>, + debugger_type: DebuggerType, + new_tags: bool, + tags: Arc, + guard: Arc>>, + payload: R, + ) { + let payload = payload.as_ref(); + if let Err(e) = do_send(config, debugger_type, new_tags, tags, guard, payload).await { + error!("Error sending to live debugger {debugger_type:?} endpoint: {e:?}"); + debug!("Attempted to send the following payload: {:?}", payload); + } + } + + let invariants = self.get_remote_config_invariants(); + let version = invariants + .as_ref() + .map(|i| i.tracer_version.as_str()) + .unwrap_or("0.0.0"); + if let Some(runtime) = self.lock_runtimes().get(runtime_id) { + if let Some(app) = runtime.lock_applications().get_mut(&queue_id) { + let (tags, new_tags) = app.get_debugger_tags(&version, runtime_id); + let sender = match debugger_type { + DebuggerType::Diagnostics => app.debugger_diagnostics_payload_sender.clone(), + DebuggerType::Logs => app.debugger_logs_payload_sender.clone(), + }; + let config = self.debugger_config.clone(); + spawn_map_err!( + send(config, debugger_type, new_tags, tags, sender, payload), + |e| { + error!("Error sending to live debugger {debugger_type:?} endpoint: {e:?}"); + } + ); + } else { + warn!("Did not find queue_id {queue_id:?} for runtime id {runtime_id} of session id {} - skipping live debugger data", self.session_id); + } + } else { + warn!( + "Did not find runtime {runtime_id} for session id {} - skipping live debugger data", + self.session_id + ); + } + } } #[cfg(test)] diff --git a/sidecar/src/service/sidecar_interface.rs b/sidecar/src/service/sidecar_interface.rs index dc198d67d..f03f4fe5e 100644 --- a/sidecar/src/service/sidecar_interface.rs +++ b/sidecar/src/service/sidecar_interface.rs @@ -1,6 +1,8 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#![allow(clippy::too_many_arguments)] + use crate::service::{ InstanceId, QueueId, RequestIdentification, RequestIdentifier, RuntimeMetadata, SerializedTracerHeaderTags, SessionConfig, SidecarAction, @@ -8,7 +10,10 @@ use crate::service::{ use anyhow::Result; use datadog_ipc::platform::ShmHandle; use datadog_ipc::tarpc; +use datadog_live_debugger::sender::DebuggerType; +use ddcommon::tag::Tag; use dogstatsd_client::DogStatsDActionOwned; +use std::time::Duration; // This is a bit weird, but depending on the OS we're interested in different things... // and the macro expansion is not going to be happy with #[cfg()] instructions inside them. @@ -111,6 +116,27 @@ pub trait SidecarInterface { headers: SerializedTracerHeaderTags, ); + /// Transfers raw data to a live-debugger endpoint. + /// + /// # Arguments + /// * `instance_id` - The ID of the instance. + /// * `queue_id` - The unique identifier for the trace context. + /// * `handle` - The data to send. + /// * `debugger_type` - Whether it's log or diagnostic data. + async fn send_debugger_data_shm( + instance_id: InstanceId, + queue_id: QueueId, + #[SerializedHandle] handle: ShmHandle, + debugger_type: DebuggerType, + ); + + /// Acquire an exception hash rate limiter + /// + /// # Arguments + /// * `exception_hash` - the ID + /// * `granularity` - how much time needs to pass between two exceptions + async fn acquire_exception_hash_rate_limiter(exception_hash: u64, granularity: Duration); + /// Sets contextual data for the remote config client. /// /// # Arguments @@ -119,12 +145,14 @@ pub trait SidecarInterface { /// * `service_name` - The name of the service. /// * `env_name` - The name of the environment. /// * `app_version` - The application version. + /// * `global_tags` - Global tags which need to be propagated. async fn set_remote_config_data( instance_id: InstanceId, queue_id: QueueId, service_name: String, env_name: String, app_version: String, + global_tags: Vec, ); /// Sends DogStatsD actions. diff --git a/sidecar/src/service/sidecar_server.rs b/sidecar/src/service/sidecar_server.rs index 88cd38275..b8fe99e4d 100644 --- a/sidecar/src/service/sidecar_server.rs +++ b/sidecar/src/service/sidecar_server.rs @@ -31,21 +31,25 @@ use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; -use tracing::{debug, enabled, error, info, warn, Level}; +use std::time::Duration; +use tracing::{debug, error, info, warn}; use futures::FutureExt; use serde::{Deserialize, Serialize}; use tokio::task::{JoinError, JoinHandle}; use crate::config::get_product_endpoint; +use crate::service::exception_hash_rate_limiter::EXCEPTION_HASH_LIMITER; use crate::service::remote_configs::{RemoteConfigNotifyTarget, RemoteConfigs}; use crate::service::runtime_info::ActiveApplication; use crate::service::telemetry::enqueued_telemetry_stats::EnqueuedTelemetryStats; use crate::service::tracing::trace_flusher::TraceFlusherStats; use datadog_ipc::platform::FileBackedHandle; use datadog_ipc::tarpc::server::{Channel, InFlightRequest}; -use datadog_remote_config::fetch::ConfigInvariants; +use datadog_live_debugger::sender::DebuggerType; +use datadog_remote_config::fetch::{ConfigInvariants, MultiTargetStats}; use datadog_trace_utils::tracer_header_tags::TracerHeaderTags; +use ddcommon::tag::Tag; use dogstatsd_client::{new_flusher, DogStatsDActionOwned}; use tinybytes; @@ -66,6 +70,7 @@ struct SidecarStats { enqueued_apps: u32, enqueued_telemetry_data: EnqueuedTelemetryStats, remote_config_clients: u32, + remote_configs: MultiTargetStats, telemetry_metrics_contexts: u32, telemetry_worker: TelemetryWorkerStats, telemetry_worker_errors: u32, @@ -137,7 +142,7 @@ impl SidecarServer { let mut executor = datadog_ipc::sequential::execute_sequential( server.requests(), self.clone().serve(), - 100, + 500, ); let (tx, rx) = tokio::sync::mpsc::channel::<_>(100); let tx = executor.swap_sender(tx); @@ -216,11 +221,8 @@ impl SidecarServer { Some(session) => session.clone(), None => { let mut session = SessionInfo::default(); - #[cfg(feature = "tracing")] - if enabled!(Level::INFO) { - session.session_id.clone_from(session_id); - info!("Initializing new session: {}", session_id); - } + session.session_id.clone_from(session_id); + info!("Initializing new session: {}", session_id); sessions.insert(session_id.clone(), session.clone()); session } @@ -385,6 +387,7 @@ impl SidecarServer { .sum::() }) .sum(), + remote_configs: self.remote_configs.stats(), telemetry_metrics_contexts: sessions .values() .map(|s| { @@ -646,6 +649,8 @@ impl SidecarInterface for SidecarServer { remote_config_notify_function: crate::service::remote_configs::RemoteConfigNotifyFunction, config: SessionConfig, ) -> Self::SetSessionConfigFut { + debug!("Set session config for {session_id} to {config:?}"); + let session = self.get_session(&session_id); #[cfg(unix)] { @@ -673,6 +678,17 @@ impl SidecarInterface for SidecarServer { let d = new_flusher(config.dogstatsd_endpoint.clone()).ok(); *dogstatsd = d; }); + session.modify_debugger_config(|cfg| { + let logs_endpoint = get_product_endpoint( + datadog_live_debugger::sender::PROD_LOGS_INTAKE_SUBDOMAIN, + &config.endpoint, + ); + let diagnostics_endpoint = get_product_endpoint( + datadog_live_debugger::sender::PROD_DIAGNOSTICS_INTAKE_SUBDOMAIN, + &config.endpoint, + ); + cfg.set_endpoint(logs_endpoint, diagnostics_endpoint).ok(); + }); session.set_remote_config_invariants(ConfigInvariants { language: config.language, tracer_version: config.tracer_version, @@ -680,6 +696,7 @@ impl SidecarInterface for SidecarServer { products: config.remote_config_products, capabilities: config.remote_config_capabilities, }); + *session.remote_config_interval.lock().unwrap() = config.remote_config_poll_interval; self.trace_flusher .interval_ms .store(config.flush_interval.as_millis() as u64, Ordering::Relaxed); @@ -793,6 +810,48 @@ impl SidecarInterface for SidecarServer { no_response() } + type SendDebuggerDataShmFut = NoResponse; + + fn send_debugger_data_shm( + self, + _: Context, + instance_id: InstanceId, + queue_id: QueueId, + handle: ShmHandle, + debugger_type: DebuggerType, + ) -> Self::SendDebuggerDataShmFut { + let session = self.get_session(&instance_id.session_id); + match handle.map() { + Ok(mapped) => { + session.send_debugger_data( + debugger_type, + &instance_id.runtime_id, + queue_id, + mapped, + ); + } + Err(e) => error!("Failed mapping shared debugger data memory: {}", e), + } + + no_response() + } + + type AcquireExceptionHashRateLimiterFut = NoResponse; + + fn acquire_exception_hash_rate_limiter( + self, + _: Context, + exception_hash: u64, + granularity: Duration, + ) -> Self::AcquireExceptionHashRateLimiterFut { + EXCEPTION_HASH_LIMITER + .lock() + .unwrap() + .add(exception_hash, granularity); + + no_response() + } + type SetRemoteConfigDataFut = NoResponse; fn set_remote_config_data( @@ -803,7 +862,10 @@ impl SidecarInterface for SidecarServer { service_name: String, env_name: String, app_version: String, + global_tags: Vec, ) -> Self::SetRemoteConfigDataFut { + debug!("Registered remote config metadata: instance {instance_id:?}, queue_id: {queue_id:?}, service: {service_name}, env: {env_name}, version: {app_version}"); + let session = self.get_session(&instance_id.session_id); #[cfg(windows)] let notify_target = if let Some(handle) = self.process_handle { @@ -819,24 +881,24 @@ impl SidecarInterface for SidecarServer { pid: session.pid.load(Ordering::Relaxed), }; let runtime_info = session.get_runtime(&instance_id.runtime_id); - runtime_info - .lock_applications() - .entry(queue_id) - .or_default() - .remote_config_guard = Some( + let mut applications = runtime_info.lock_applications(); + let app = applications.entry(queue_id).or_default(); + app.remote_config_guard = Some( self.remote_configs.add_runtime( session .get_remote_config_invariants() .as_ref() .expect("Expecting remote config invariants to be set early") .clone(), + *session.remote_config_interval.lock().unwrap(), instance_id.runtime_id, notify_target, - env_name, + env_name.clone(), service_name, - app_version, + app_version.clone(), ), ); + app.set_metadata(env_name, app_version, global_tags); no_response() } diff --git a/sidecar/src/shm_remote_config.rs b/sidecar/src/shm_remote_config.rs index 62f51468d..6d570f643 100644 --- a/sidecar/src/shm_remote_config.rs +++ b/sidecar/src/shm_remote_config.rs @@ -5,14 +5,16 @@ use crate::one_way_shared_memory::{ open_named_shm, OneWayShmReader, OneWayShmWriter, ReaderOpener, }; use crate::primary_sidecar_identifier; +use crate::tracer::SHM_LIMITER; use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; use datadog_ipc::platform::{FileBackedHandle, MappedMem, NamedShmHandle}; +use datadog_ipc::rate_limiter::ShmLimiter; use datadog_remote_config::fetch::{ ConfigInvariants, FileRefcountData, FileStorage, MultiTargetFetcher, MultiTargetHandlers, - NotifyTarget, RefcountedFile, + MultiTargetStats, NotifyTarget, RefcountedFile, }; -use datadog_remote_config::{RemoteConfigPath, RemoteConfigValue, Target}; +use datadog_remote_config::{RemoteConfigPath, RemoteConfigProduct, RemoteConfigValue, Target}; use priority_queue::PriorityQueue; use sha2::{Digest, Sha224}; use std::cmp::Reverse; @@ -24,6 +26,8 @@ use std::hash::{Hash, Hasher}; use std::io; #[cfg(windows)] use std::io::Write; +use std::str::FromStr; +use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::time::Instant; @@ -87,6 +91,7 @@ struct ConfigFileStorage { struct StoredShmFile { handle: Mutex, + limiter: Option, refcount: FileRefcountData, } @@ -107,6 +112,11 @@ impl FileStorage for ConfigFileStorage { ) -> anyhow::Result> { Ok(Arc::new(StoredShmFile { handle: Mutex::new(store_shm(version, &path, file)?), + limiter: if path.product == RemoteConfigProduct::LiveDebugger { + Some(SHM_LIMITER.lock().unwrap().alloc()) + } else { + None + }, refcount: FileRefcountData::new(version, path), })) } @@ -140,7 +150,7 @@ fn store_shm( let len = len + 4; let mut handle = NamedShmHandle::create(CString::new(name)?, len)?.map()?; - #[cfg_attr(not(windows), allow(unused_mut))] + #[allow(unused_mut)] let mut target_slice = handle.as_slice_mut(); #[cfg(windows)] { @@ -152,7 +162,12 @@ fn store_shm( } impl MultiTargetHandlers for ConfigFileStorage { - fn fetched(&self, target: &Arc, files: &[Arc]) -> bool { + fn fetched( + &self, + runtime_id: &Arc, + target: &Arc, + files: &[Arc], + ) -> bool { let mut writers = self.writers.lock().unwrap(); let writer = match writers.entry(target.clone()) { Entry::Occupied(e) => e.into_mut(), @@ -166,14 +181,18 @@ impl MultiTargetHandlers for ConfigFileStorage { }), }; - let len = files - .iter() - .map(|f| f.handle.lock().unwrap().get_path().len() + 2) - .sum(); - let mut serialized = Vec::with_capacity(len); + let mut serialized = vec![]; + serialized.extend_from_slice(runtime_id.as_bytes()); + serialized.push(b'\n'); for file in files.iter() { serialized.extend_from_slice(file.handle.lock().unwrap().get_path()); serialized.push(b':'); + if let Some(ref limiter) = file.limiter { + serialized.extend_from_slice(limiter.index().to_string().as_bytes()); + } else { + serialized.push(b'0'); + } + serialized.push(b':'); serialized.extend_from_slice( BASE64_URL_SAFE_NO_PAD .encode(file.refcount.path.to_string()) @@ -225,6 +244,17 @@ impl Drop for ShmRemoteConfigsGuard { self.remote_configs .0 .delete_runtime(&self.runtime_id, &self.target); + if self + .remote_configs + .0 + .invariants() + .endpoint + .test_token + .is_some() + && self.remote_configs.0.active_runtimes() == 0 + { + self.remote_configs.shutdown() + } } } @@ -240,13 +270,21 @@ pub struct ShmRemoteConfigs( // pertaining to that env refcounting RemoteConfigIdentifier tuples by their unique runtime_id impl ShmRemoteConfigs { - pub fn new(invariants: ConfigInvariants, on_dead: Box) -> Self { + pub fn new( + invariants: ConfigInvariants, + on_dead: Box, + interval: Duration, + ) -> Self { let storage = ConfigFileStorage { invariants: invariants.clone(), writers: Default::default(), on_dead: Arc::new(Mutex::new(Some(on_dead))), }; - ShmRemoteConfigs(MultiTargetFetcher::new(storage, invariants)) + let fetcher = MultiTargetFetcher::new(storage, invariants); + fetcher + .remote_config_interval + .store(interval.as_nanos() as u64, Ordering::Relaxed); + ShmRemoteConfigs(fetcher) } pub fn is_dead(&self) -> bool { @@ -278,16 +316,23 @@ impl ShmRemoteConfigs { pub fn shutdown(&self) { self.0.shutdown(); } + + pub fn stats(&self) -> MultiTargetStats { + self.0.stats() + } } -fn read_config(path: &str) -> anyhow::Result { - if let [shm_path, rc_path] = &path.split(':').collect::>()[..] { +fn read_config(path: &str) -> anyhow::Result<(RemoteConfigValue, u32)> { + if let [shm_path, limiter, rc_path] = &path.split(':').collect::>()[..] { let mapped = NamedShmHandle::open(&CString::new(*shm_path)?)?.map()?; let rc_path = String::from_utf8(BASE64_URL_SAFE_NO_PAD.decode(rc_path)?)?; let data = mapped.as_slice(); #[cfg(windows)] let data = &data[4..(4 + u32::from_ne_bytes((&data[0..4]).try_into()?) as usize)]; - RemoteConfigValue::try_parse(&rc_path, data) + Ok(( + RemoteConfigValue::try_parse(&rc_path, data)?, + u32::from_str(limiter)?, + )) } else { anyhow::bail!( "could not read config; {} does not have exactly one colon", @@ -311,12 +356,16 @@ pub struct RemoteConfigManager { active_configs: HashMap, last_read_configs: Vec, check_configs: Vec, + pub current_runtime_id: String, } #[derive(Debug)] pub enum RemoteConfigUpdate { None, - Add(RemoteConfigValue), + Add { + value: RemoteConfigValue, + limiter_index: u32, + }, Remove(RemoteConfigPath), } @@ -331,6 +380,7 @@ impl RemoteConfigManager { active_configs: Default::default(), last_read_configs: Default::default(), check_configs: vec![], + current_runtime_id: "".to_string(), } } @@ -346,9 +396,18 @@ impl RemoteConfigManager { if changed { 'fetch_new: { let mut configs = vec![]; + let mut runtime_id: &[u8] = b""; if !data.is_empty() { let mut i = 0; - let mut start = 0; + while i < data.len() { + if data[i] == b'\n' { + break; + } + i += 1; + } + runtime_id = &data[0..i]; + i += 1; + let mut start = i; while i < data.len() { if data[i] == b'\n' { match std::str::from_utf8(&data[start..i]) { @@ -363,6 +422,13 @@ impl RemoteConfigManager { i += 1; } } + match std::str::from_utf8(runtime_id) { + Ok(s) => self.current_runtime_id = s.to_string(), + Err(e) => { + warn!("Failed reading received configurations {e:?}"); + break 'fetch_new; + } + } self.last_read_configs = configs; self.check_configs = self.active_configs.keys().cloned().collect(); } @@ -371,6 +437,8 @@ impl RemoteConfigManager { if *instant < Instant::now() - Duration::from_secs(3666) { let (target, _) = self.unexpired_targets.pop().unwrap(); self.encountered_targets.remove(&target); + } else { + break; } } } @@ -388,7 +456,7 @@ impl RemoteConfigManager { while let Some(config) = self.last_read_configs.pop() { if let Entry::Vacant(entry) = self.active_configs.entry(config) { match read_config(entry.key()) { - Ok(parsed) => { + Ok((parsed, limiter_index)) => { trace!("Adding remote config file {}: {:?}", entry.key(), parsed); entry.insert(RemoteConfigPath { source: parsed.source, @@ -396,7 +464,10 @@ impl RemoteConfigManager { config_id: parsed.config_id.clone(), name: parsed.name.clone(), }); - return RemoteConfigUpdate::Add(parsed); + return RemoteConfigUpdate::Add { + value: parsed, + limiter_index, + }; } Err(e) => warn!( "Failed reading remote config file {}; skipping: {:?}", @@ -456,6 +527,23 @@ impl RemoteConfigManager { self.check_configs.clear(); self.active_configs.clear(); } + + /// Can be used to fast-remove configs temporarily. Will be re-applied on next fetch_update(). + pub fn unload_configs(&mut self, configs: &[RemoteConfigProduct]) { + self.active_configs.retain(|key, path| { + if configs.contains(&path.product) { + // self.check_configs should generally be empty here, but be safe + if let Some(pos) = self.check_configs.iter().position(|x| x == key) { + self.check_configs.swap_remove(pos); + } + // And re-apply it on next read + self.last_read_configs.push(key.clone()); + false + } else { + true + } + }); + } } #[cfg(test)] @@ -522,6 +610,7 @@ mod tests { Box::new(|| { tokio::spawn(on_dead_completer.complete(())); }), + Duration::from_millis(10), ); let mut manager = RemoteConfigManager::new(server.dummy_invariants()); @@ -554,11 +643,11 @@ mod tests { receiver.recv().await; - if let RemoteConfigUpdate::Add(update) = manager.fetch_update() { - assert_eq!(update.config_id, PATH_FIRST.config_id); - assert_eq!(update.source, PATH_FIRST.source); - assert_eq!(update.name, PATH_FIRST.name); - if let RemoteConfigData::DynamicConfig(data) = update.data { + if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { + assert_eq!(value.config_id, PATH_FIRST.config_id); + assert_eq!(value.source, PATH_FIRST.source); + assert_eq!(value.name, PATH_FIRST.name); + if let RemoteConfigData::DynamicConfig(data) = value.data { assert!(matches!( >::from(data.lib_config)[0], Configs::TracingEnabled(true) @@ -573,6 +662,17 @@ mod tests { // just one update assert!(matches!(manager.fetch_update(), RemoteConfigUpdate::None)); + manager.unload_configs(&[PATH_FIRST.product]); + + if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { + assert_eq!(value.config_id, PATH_FIRST.config_id); + } else { + unreachable!(); + } + + // just one update + assert!(matches!(manager.fetch_update(), RemoteConfigUpdate::None)); + { let mut files = server.files.lock().unwrap(); files.insert( @@ -604,14 +704,14 @@ mod tests { } // then the adds - let was_second = if let RemoteConfigUpdate::Add(update) = manager.fetch_update() { - update.config_id == PATH_SECOND.config_id + let was_second = if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { + value.config_id == PATH_SECOND.config_id } else { unreachable!(); }; - if let RemoteConfigUpdate::Add(update) = manager.fetch_update() { + if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { assert_eq!( - &update.config_id, + &value.config_id, if was_second { &PATH_FIRST.config_id } else { @@ -637,9 +737,9 @@ mod tests { manager.track_target(&DUMMY_TARGET); // If we re-track it's added again immediately - if let RemoteConfigUpdate::Add(update) = manager.fetch_update() { + if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { assert_eq!( - &update.config_id, + &value.config_id, if was_second { &PATH_SECOND.config_id } else { diff --git a/sidecar/src/tokio_util.rs b/sidecar/src/tokio_util.rs new file mode 100644 index 000000000..d8136f59b --- /dev/null +++ b/sidecar/src/tokio_util.rs @@ -0,0 +1,13 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#[macro_export] +macro_rules! spawn_map_err { + ($fut:expr, $err:expr) => { + tokio::spawn(async move { + if let Err(e) = tokio::spawn($fut).await { + ($err)(e); + } + }) + }; +} diff --git a/sidecar/src/tracer.rs b/sidecar/src/tracer.rs index 8af193688..56e473ec9 100644 --- a/sidecar/src/tracer.rs +++ b/sidecar/src/tracer.rs @@ -1,10 +1,15 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use crate::primary_sidecar_identifier; +use datadog_ipc::rate_limiter::ShmLimiterMemory; use datadog_trace_utils::config_utils::trace_intake_url_prefixed; use ddcommon::Endpoint; use http::uri::PathAndQuery; +use lazy_static::lazy_static; +use std::ffi::CString; use std::str::FromStr; +use std::sync::Mutex; #[derive(Default)] pub struct Config { @@ -27,3 +32,12 @@ impl Config { Ok(()) } } + +pub fn shm_limiter_path() -> CString { + CString::new(format!("/ddlimiters-{}", primary_sidecar_identifier())).unwrap() +} + +lazy_static! { + pub static ref SHM_LIMITER: Mutex = + Mutex::new(ShmLimiterMemory::create(shm_limiter_path()).unwrap()); +} diff --git a/sidecar/src/watchdog.rs b/sidecar/src/watchdog.rs index 4b8f73a6a..cbc537a15 100644 --- a/sidecar/src/watchdog.rs +++ b/sidecar/src/watchdog.rs @@ -36,7 +36,7 @@ impl WatchdogHandle { impl Watchdog { pub fn from_receiver(shutdown_receiver: Receiver<()>) -> Self { Watchdog { - interval: tokio::time::interval(Duration::from_secs(5)), + interval: tokio::time::interval(Duration::from_secs(10)), max_memory_usage_bytes: 1024 * 1024 * 1024, // 1 GB shutdown_receiver, } diff --git a/spawn_worker/src/unix/spawn.rs b/spawn_worker/src/unix/spawn.rs index d30ddc999..32ea2fd77 100644 --- a/spawn_worker/src/unix/spawn.rs +++ b/spawn_worker/src/unix/spawn.rs @@ -77,6 +77,7 @@ fn write_to_tmp_file(data: &[u8]) -> anyhow::Result { use std::fs::File; +#[cfg(target_os = "linux")] use std::ffi::CStr; use std::io; use std::ops::RangeInclusive; diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 01b4ae2d9..e9ce10fd4 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -80,6 +80,8 @@ COPY "ddsketch/Cargo.toml" "ddsketch/" COPY "dogstatsd/Cargo.toml" "dogstatsd/" COPY "dogstatsd-client/Cargo.toml" "dogstatsd-client/" COPY "dynamic-configuration/Cargo.toml" "dynamic-configuration/" +COPY "live-debugger/Cargo.toml" "live-debugger/" +COPY "live-debugger-ffi/Cargo.toml" "live-debugger-ffi/" COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "profiling-replayer/Cargo.toml" "profiling-replayer/"