From 98e895deb87f94cee1ee41b1c6d61abf2f7af5db Mon Sep 17 00:00:00 2001 From: Bruno Paiva Lima da Silva Date: Wed, 16 Nov 2022 09:28:28 -0800 Subject: [PATCH] Telemetry work --- docs/design-document.md | 290 ++++++++++++++++++ docs/images/telemetry-app.png | Bin 0 -> 4607 bytes docs/images/telemetry-execution.png | Bin 0 -> 14614 bytes docs/images/telemetry-pluginlevel.png | Bin 0 -> 18853 bytes docs/metrics-document.md | 173 +++++++++++ examples/AWSDriverExample/build.gradle.kts | 5 + .../java/software/amazon/MetricsExample.java | 115 +++++++ wrapper/build.gradle.kts | 4 + .../amazon/jdbc/ConnectionPluginManager.java | 82 ++++- .../software/amazon/jdbc/PluginService.java | 4 + .../amazon/jdbc/PluginServiceImpl.java | 6 + .../amazon/jdbc/PropertyDefinition.java | 35 +++ .../AuroraHostListConnectionPlugin.java | 7 +- ...AuroraHostListConnectionPluginFactory.java | 2 +- .../plugin/DataCacheConnectionPlugin.java | 23 +- .../DataCacheConnectionPluginFactory.java | 2 +- .../jdbc/plugin/IamAuthConnectionPlugin.java | 11 + .../IamAuthConnectionPluginFactory.java | 2 +- .../efm/HostMonitoringConnectionPlugin.java | 9 + .../plugin/efm/MonitorConnectionContext.java | 12 +- .../amazon/jdbc/plugin/efm/MonitorImpl.java | 26 ++ .../jdbc/plugin/efm/MonitorServiceImpl.java | 14 +- .../plugin/staledns/AuroraStaleDnsHelper.java | 10 + .../plugin/staledns/AuroraStaleDnsPlugin.java | 1 - .../amazon/jdbc/util/WrapperUtils.java | 41 ++- .../telemetry/DefaultTelemetryFactory.java | 60 ++++ .../util/telemetry/NullTelemetryContext.java | 43 +++ .../util/telemetry/NullTelemetryCounter.java | 39 +++ .../util/telemetry/NullTelemetryGauge.java | 34 ++ .../util/telemetry/OpenTelemetryContext.java | 79 +++++ .../util/telemetry/OpenTelemetryCounter.java | 51 +++ .../util/telemetry/OpenTelemetryFactory.java | 60 ++++ .../util/telemetry/OpenTelemetryGauge.java | 43 +++ .../util/telemetry/OpenTelemetryMetrics.java | 4 + .../jdbc/util/telemetry/TelemetryContext.java | 27 ++ .../jdbc/util/telemetry/TelemetryCounter.java | 25 ++ .../jdbc/util/telemetry/TelemetryFactory.java | 27 ++ .../jdbc/util/telemetry/TelemetryGauge.java | 21 ++ .../util/telemetry/XRayTelemetryContext.java | 88 ++++++ .../util/telemetry/XRayTelemetryFactory.java | 25 ++ .../jdbc/wrapper/ConnectionWrapper.java | 15 +- .../jdbc/ConnectionPluginManagerTests.java | 35 ++- .../plugin/DataCacheConnectionPluginTest.java | 7 +- 43 files changed, 1522 insertions(+), 35 deletions(-) create mode 100644 docs/design-document.md create mode 100644 docs/images/telemetry-app.png create mode 100644 docs/images/telemetry-execution.png create mode 100644 docs/images/telemetry-pluginlevel.png create mode 100644 docs/metrics-document.md create mode 100644 examples/AWSDriverExample/src/main/java/software/amazon/MetricsExample.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/DefaultTelemetryFactory.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryContext.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryCounter.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryGauge.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryContext.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryCounter.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryFactory.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryGauge.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryMetrics.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryContext.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryCounter.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryFactory.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryGauge.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryContext.java create mode 100644 wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryFactory.java diff --git a/docs/design-document.md b/docs/design-document.md new file mode 100644 index 000000000..2263ac84a --- /dev/null +++ b/docs/design-document.md @@ -0,0 +1,290 @@ +# Design Document Template for the RDS Atlas Team + +In our specific case, where we will build and add features to multiple drivers, the title of any design document should be clear on: +- which feature is the document about +- which environment/application the feature is being applied to + +A good format for a clear title is: `[Feature short name] for [Driver name]` + +Examples: +``` +- Connection failover for the AWS JDBC MySQL Driver +- Read-Write splitting for the AWS JDBC Wrapper +- IAM authentication for the AWS ODBC MySQL Driver +- Automated timesheets for Workday +``` + +# Introduction + +Every document begins with an introduction. In the introduction, the writer should go all the way back to initial state of the service/application and explain why the new feature is important and required. + +In 2 (max 3) paragraphs, the writer should quickly introduce the service/application, and briefly explain how it works ***before*** introducing the new feature. If the new feature will use a new technology or will interact with a 3rd party tool or service, those should also be briefly introduced here. + +Another thing to mention in the introduction is to briefly mention the reasons why the new feature is required. Whether it is a mandatory feature, or something that has been requested by the users multiple times, it is worth adding a couple of lines to help the reader to understand the importance and the impact of the new feature. + +The Introduction section can often be merged with the problem section. There is actually no rule on whether to have them merged or not. If the introduction gets a bit longer, as it has to introduce and describe multiple services or applications, it can be worth splitting those sections to add a quick break in the read. When the introduction is shorter, one can simply state the problem in the end of it and move on to the goals. + +# Glossary + +The glossary is a section to explain or clarify the meaning of any acronyms and or technical terms that are used in the rest of the document. It is not a mandatory section but recommended if the document rely on some specific technical terms that may not be well known to some users/readers. + +Example: +``` +multi-AZ: application/service hosted in multiple Availability zones +IoT: Internet of Things +``` + +## Acronyms + +It is very common that a feature/component/strategy defined in the document has a long name often referred by an acronym. If the document refers to that acronym multiple times, it is recommended to use the component full name the first time it appears in the document, followed by the acronym in parenthesis. This allows the reader to search back for its name/meaning if needed. + +Acronyms are often introduced either in the Introduction section, or inside a Solution (Proposed or Discarded) subsection. + +Examples: +``` +Earlier this year, we have introduced Enhanced Failover Monitoring (EFM), a mechanism that +allows the driver to quickly identify a cluster failover and therefore trigger a driver failover. + +... (later in the doc) + +One of the advantages of EFM is that there is only one monitoring thread per node in +the cluster, no matter how many different connections are open to this node. + +``` + +# Problem + +The problem section is a paragraph, usually a one-liner, where the writer clearly defines the problem the document is addressing. + +When Introduction and Problem sections are merged, the Introduction should finish with one line where it clearly states the problem the document is trying to address. + +Ex: +``` +This documents addresses the issue of our team not being able to organize +team lunches by proposing the introduction of a team lunch schedule. +``` + +# Goal(s) + +The Goal (or Goals) section highlights the achievements that the writer expects to obtain by executing/implementing the solution that will be proposed in the document. + +A goal should be an atomic achievement that is clearly described, and also easily verifiable. People that will read the document will judge the quality of the proposed solution according to the effectiveness it will have to achieve the goals listed in this section. + +When goals involve measurements, it is recommended to include the numeric values desired when listing the project goals. Unclear or ambiguous value will very often lead to questions from the readers. + +Example: +``` +- Reduce the service response time - NOT GOOD (What is the expected response time?) +- Reduce significantly the service response time - UNCLEAR (What does significantly mean?) +- Reduce the service response time below 200ms - GOOD (Goal is clear) +``` + +When a project has multiple goals, it is common to list them one by one, either in priority order, either in execution order, in a list. + +Examples: +``` +(Priority) +In order to succesfully decrease the service response time, our goals are to: +- Identify and fix the memory leaks in the codebase +- Replace the database driver to a newer version +- Upgrade the disk space and memory of our cloud instances +``` + +``` +(Execution order) +In order to succesfully write a document, our goals are to: +- Write the introduction +- Write the core part of the document +- Insert an inspiring little doodling in the middle +- Do a barrel roll +- Write the conclusion +``` + +# Non-Goals + +The Non-Goals section is not the opposite of the Goals section, but more like another way of limiting the scope of the document. + +Non-Goals are basically goals that are not a priority or will not be addressed in the scope of this specific document. Non-goals are very often things that would be nice to have achieved though. + +Example: +``` +Non-Goals for this design document are: +- Not increasing the server response time +- Add a public API to interact with the new feature +- Reduce the amount of support tickets associated to the feature +``` + +The Non-Goals section is not mandatory in a document (it is not even very frequent), but it is handy to prevent future questions that a reader might raise when reading the proposed solution, such as `"Does your solution does something to address/prevent X?"` or `"Does your solution does something to reduce response/connection time?"` + +# Requirements/Constraints + +Requirements and/or Constraints are details that were defined prior to the design phase and that have an impact on the choice of proposed solution. + +Common cases of sources for those requirement/constraints are: +- Licence compatibility +- Available/Unavailable resources +- Decisions made by managers/PMs + +Examples: +``` +For this project, a requirement is to design a solution that addresses X without +adding any dependency to any 3rd party library using the GPL licence. +``` +``` +For this project, a requirement is to keep the software image size below +650MB as it will be shipped to our partners in CDs. +``` +``` +For this project, it was requested by the company board that languages like +Python and Ruby were avoided due to recent security issues found in the SSL +libraries for those languages. +``` + +Requirements often have a role on choosing which approach to use when addressing the problem, and even more often on defining which solutions to discard. When the requirements were not design choices made by the writer, it is important to clearly state those ***before*** starting describing the proposed and discarded solutions, so the reader will have the opportunity to validate the solutions against the requirements and constraints. + +# Solutions + +Here's the beginning of the critical section of the document. + +Everything written prior to this point had purpose of stating the problem this solution is addressing and providing a way for the reader to verify that your solution satisfies the problem. + +## No conclusion + +The solutions section should close the document. As it drives the reader into a more technical scope, it is not recommended to add any kind of text after this section. Conclusions are often either redundant or not required. + +Example: +``` +SOLUTION SECTION + +In order to ensure that the waiting line does not goes over 10 people, we will immediately +schedule an appointment for any person that accesses the website for the following day. + +CONCLUSION + +In conclusion, we believe that the solution is the most appropriate as it addresses +the issue of not letting the waiting line to go over 10 people at any time. +``` + +The conclusion in the example above is redundant, as it not only paraphrases what was probably already mentioned in the Goals section, but also repeats something that was previously mentioned when describing the solution. + +***As a rule of thumb, the writer should always let the solution speak for itself.*** + +## Presentation order + +There is no correct order between the presentation of discarded and proposed solutions. There is however situations where the writer will want to prefer to have one section or the other written first. + +As the document is gonna be read from the top to the bottom, it is important to start the solutions section with the solution that the writer want to be discussed more. + +If the goal of the document is to present a solution that was already defined as the one that better addresses the problem, and the writer is more interested in validation/feedback instead of being questioned about the choice of solution, the writer should definitely present the Proposed Solution section before the discarded solutions. In this situation, the reader will read the entire solution before going to check if there were any alternatives studied or discarded choices. + +In the other hand, if the document goal is to explain why some specific solutions previously proposed do not fully address the problem discarded, or do not respect the requirements and constrains of the project, one should perhaps start the solution section by describing the solutions that were studied and should not be adopted. In this case, the reader will have an idea of all the discarded solutions and their reasons before reaching the final solution proposed in the document. + +## Proposed Solution + +The goal of this subsection is to: +- Detail your solution so the reader has a clear idea of the changes being introduced +- Describe how your changes impact the system and where those changes will be made +- Explain how your solution satisfies the goals and requirements/constraints presented in the document +- Convince the audience that the solution is indeed the better suited solution for the problem + +### Diagrams + +When presenting a design, very often the writer will require diagrams to explain a solution. That is because very often visual elements are extremely helpful for people to understand concepts or integrations that would perhaps be complicated to describe with words. + +The most common diagrams one might need in a design document are: +- A global architecture diagram +- A sequence diagram +- A class/entity diagram + +When presenting a solution with multiple diagrams, it is recommended to start from the global architecture diagram, with is less specific and shows the system as a whole. Once the writer has explained how the feature will be implemented in the larger scope, it is then safe to proceed with the other diagrams that go more in detail in the implementation of the solution. + +[Still in progress] + +## Discarded Solutions + +The discarded solutions section should list all the solutions that were potential solutions to address the problem but were discarded for some reason. + +In this section, the writer should create a subsection for each discarded solution that will be presented. For each of those entries, it is recommended to give an name to the solution that summarizes or what made the approach unique. + +Example: +``` +Discarded Solutions +- Solution A: using an external cache +- Solution B: using a distributed hashmap +``` + +Inside a solution subsection, the write should describe the solution just as if it was the proposed one. That is in case some reader still prefers that solution over the proposed one. + +However, unlike a proposed solution, the writer should also remember to clearly state the reasons the solution was discarded. + +There are multiple reasons for a solution to get discarded: +- The solution does not address the problem or achieve the goals previously stated\ + The writer should highlight the elements (or the lack of) that show that the solution does not satisfy the goals the document wants to achieve.\ + Examples:\ + `This solution was discarded as while it would be effective for our premium users, it would not reach potential new customers currently in a free plan as they would not have access to this feature. The goal of the project was to make the new feature accessible to all our users independently of location or enrollment status.` + +- The solution does not respect some of the requirements or constraints previously specified\ + Just like the previous case, the writer should clearly identify the pieces of the solution that violate the constraints for the project. + Examples:\ + `This solution was discarded as it would add a round-trip connection to an external microservice that would increase the server response time above 200ms, which does not satisfy a requirement for this project.` + +- The solution is valid (or could work) but costs more in resources/time/money\ + In this case, a bit of argumentation added to some quantitative information about it will be required. If the matter is time, then it is expected that all valid solutions should have had a rough estimation provided. In case of money/resources, the difference or gap between sections should also be mentioned.\ + Those cases generally trigger discussions and questions from the audience/readers.\ + Examples:\ + `This solution is less preferable as its estimation of completion time would be around 6 months, instead of 3 months for other solutions.`\ + `Creating a new microservice instead of extending an existing one will increase the amount of devices in our cloud fleet and therefore the size of the infrastructure to be monitored by SRE/DevOps team.` + +- The solution is valid but not the one preferred by the writer\ + There are many valid reasons to prefer one solution to another. Strong argumentation will be necessary to convince the reader that it is the correct choice, as it is very likely to be questioned in a feedback session.\ + If a solution comes with positive aspects but also brings up some negative ones, it is also a good idea to introduce a `Pros/Cons` subsection after the solution is detailed. + +# Appendix + +Appendix is a part at the end of the document where the writer can add anything that might have value to the document/discussion but will make the document length too large/bulky. + +It is often used for large diagrams, code snippets, or the integrality of performance tests results. + +Every piece of information added to the appendix should be given an identifying reference (often numbers or letters) and a name. + +Example: +``` +Appendix A - Recipy for a nice spaghetti bolognaise + +Recipy: +Boil water +Add pasta +Wait 8 minutes +Add bolognaise sauce +Put sauce on top of pasta (not too much) +Add meatballs (optional) +Serve it hot +``` + +``` +Appendix B - Very nice house + + /\ + /\ //\\ + /\ //\\///\\\ /\ + //\\ ///\////\\\\ /\ //\\ + /\ / ^ \/^ ^/^ ^ ^ \/^ \/ ^ \ + / ^\ /\ / ^ / ^/ ^ ^ ^ ^\ ^/ ^^ \ + /^ \ / ^\/ ^ ^ ^ / ^ ^ ^ \/ ^ ^ \ * + / ^ ^ \/^ ^\ ^ ^ ^ ^ ^ ^ ____ ^ ^ \ /|\ + / ^ ^ ^ \ ^ _\___________________| |_____^ ^ \ /||o\ + / ^^ ^ ^ ^\ /______________________________\ ^ ^ \ /|o|||\ + / ^ ^^ ^ ^ /________________________________\ ^ /|||||o|\ + /^ ^ ^ ^^ ^ ||___|___||||||||||||___|__||| /||o||||||\ + / ^ ^ ^ ^ ||___|___||||||||||||___|__||| | | +/ ^ ^ ^ ^ ^ ^ ||||||||||||||||||||||||||||||oooooooooo| |ooooooo +ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo +``` + +It is important to add a reference in the document where having a look at a specific appendix might explain or make the situation clearer. + +Example: +``` +The choice of font in the recipy book was driven by the fact that cooking recipies +are often a long set of sequential operations, but often very simple (cf. Appendix A). +``` diff --git a/docs/images/telemetry-app.png b/docs/images/telemetry-app.png new file mode 100644 index 0000000000000000000000000000000000000000..9b982c065b450ed3a63234db6f9a6530171eabd8 GIT binary patch literal 4607 zcmb_gdpy)z*OyevQ%1U-qNv14jky{liMcXkj1eb?X|yae}4M|{K@m@ zEuUv%Vlv;?hY)CDV%h@aQ1jW~S()wK=cm+4>$~{kvJfM0aw(tsB%U9 z2sGaD@o@r{Ac`sm-brXA28whAYA32s5Y5j4fd_J&I932&WC1T$HnjvVN|Z@}h6IOU zp}_P%KDTm(+(hwznV5=1Am>I;N1;LmDoIEUHyM>2fs&DeG;w}D)94j~#3ZpyY6upJ zgaZFli{6ZJAqbxc68SA-V2b!#jyA$n2^>svuo%ts69i&HCNeSK*tv|my*ixH^up)%vS z1S}oF7E(DJ5rQEh60tA|AqmG-pomN%Q?nJ%(7>q_VjzVf1HMT<6c%1e4B^t0QBgd; zjD;486%tXflE@E1$HD{P!EEmYI+(~3E6`CvQ7~dKM+iQp)PSvlVg;MRP7wL25{cMY z4ka#@%pk``#WUqRxi?)Jjg51~Bs(B*@c}Y7mN%IcfQV&K!8Z%Rr%M!4cpzCJh(ZYA zd^Z)IFW~Ul6wy>EptdH<<9X2>JQI`<4(S)92niG=k-UXck}`^-z{5yNW`YJuWJV#7 zu8DEXKuR)7kpzoLCiwD2SV3}-3{RubiDYjQCm;l&3KY@;{1Rg@K0#4@NeodCJY_V1 z9EWGSaojMdLdK8L@B6Ug;E{^ z4;LT=oM<$U!<4~suw+t_D*;O-xglh}p!RSW5|1AOC$rUL0-ni5qL_FzQ;lFG#Uo)L zF*qcb5W>fhV-SpxKuGf^5Eg1%=>u9sfqf(A3A+a zNkw8>-;8VSWthw`H8Y3UdF?VuE1xAyA1E)0^Vwxk+F6*lK+OyjW|*R9r4O8rbae!? zr(6;MI{YTg4d#0QoYQKIA8c}}+8diJURnm5R zfvtPc_6J2`sT3Mkr$Lv5*SY6TqJ96Pe0ubwJQ_XE<8S`WqLk;QVY>F~ zNxBE8kd?8SwwD{fcHiDJ_WEgY8;_Vm>sY>6;`j~Ytwk09gjHKpu~@a?6AS<6<>2Dw z`I#}=@M>vTZ^BQ5%NjooH+;QN5?Yte?>-E^YORcY{&=+i zQESHy<>-M?oO6p{mfLU8;#=^YZegIsbY11S!)K>{oK<>hG9;_QhGjNxu@rjFTzA(#Q{V9Q%bR^~Vjljr?EZ+Xy#`(; zI$7Im;e-7$bIyWub=xOWKZIP7TNZD^>4E(U-j(o1%GgBL&ivec53P9wpDjB+#o^G~Cb!IWep8lSX#cH7H$a=6 z2;{Tv718ve`I%>z7T1rCj_R{Iva8H07N*m4-LvrWxeetsU9Y|`N9JkKA|NK0=k>AY zd!~L?EqGLjli!TMX|8yZMVZ$NtJsTO#!o!$;U=Bn@Lny<&cN@07uB-{X;6vFp2moh z?Lpd=aY;V}E1&*Sv$+4Y$5)Ue$CA|?@{+F>Q5J>*gr&9FYp!Oko_*-qE;i#()lauq zw9m%N(PyEH3K~8Q-T1QpL-(PKr_T<){tVoww}T3rU`dMWTCEDG1GbKY;TPp=c8ax& zJMwpa!S0(_=)W(8u`Ky>Z~DLs<(b-jdk1>9<@O)m@?~HrSG&c$trhiLM*9r7VjPP z*X|KZi)tFHU=HhM@u1C}zVd|~Z(a7+846^NX(djM{>WVAjd>vq&>ZCbayxH8<+8g! z2P0cDxW9Dv`^TSbFjge(MtgUh{%*$7QOd`ntbx0lDSvKxiM^56#aC+!Zm{X{Tu2>w zc0g=BH3j-L2Bu-xdX+%J>h_m;99lzhD!yX{Da6!FUdz~<=g-Kig1^?;I9%(hNm#Q= z{IcGIySg*uQPacJ(5g6HS2w#~XOoPVKM9r2``5nPRocCIzn8PMnY>G$O3lv~4?LOQ za{igho$PUYZEA4-`d`NG-!nJRrw)>RDkta7-KL>);f1LKjX3>p$4$NyI8L@YS zeAVF_A4Iy(>PcIU{PBEv+4G$2U69T2ru~*E6}~SpwW;UV45Zz+d1u#{B9X{rmMyke zguT4Y%`K!xQ%j7h`#3Lc_>zm>Rz^+OLP)_4dDRn&!P(UfX6XZhgyoC84Vq>L z5X1k6nKYY%{=Pm9VBE0Fq2EE7ajdiR^XbSsunC-5>(&`c$FA8MINsw(yJti@IM2Lg za55*?(ACcFF~!N}&`%#VX0vTEq~O&!=KBWB0ex~o9?HSmuxr_e+R=e#S%W<>$2Sx+vH#f zLPU!c3fIkTDH+MhHloe|sFyxEecFhc15k(lg~1xjW+^}o%|oNVqh1eCH^~x>sG$IL zm`7~PchtcEbynA(#$<&7)HhVmo_t4rc^9N(eZxJY+gc|(f^y`_rBg;^!64(osmF|N z7yKK_dV$n;a|)wxr`qAG%?&d(%U%7j@YSBE*<{Qzt@QSf6uM_ ztsFk(@1Ua{;c!3S8Hi)E^1xl=j`SvUTg{PYC85vR7nc{DMEIi#vd){=6E-W>f`q4y zfYKc~6GsGM{jI-8`&;uiZ+~1!KK%FsE$8vi_J$ zzkaF~s6b27wN=LIz$)EP#|?GEUS&<@(9-FBJKMgm>MyJ!F@DG~NDWg>(1Fq<*4nw0>f^i;k-yC=U1`{j$r)H&{7uS#QuLF_3!avD z?YDM(oaM}V-+!kKyV65-B)m?O^@>Pou9xcq55Pg|?y&3@oB@5@Qquq87Xh`?m9wd5 z;>Fj&SJl@Af4;1UzTRm*VD--NX+cjS=&-j?=76Z;PuQ-RVQOuyJ!|9KY$YwkXcP)p z{kuDs>Ty@kj3djvu}lqlZbwwn|?mK1<%%=5APE)5&nTDM95Shl#JSHJKv zmFA!#XO}%&k$ke-6AqhcW#<*S7nG;-*styzOwH;KdR5*nco4urQZ|vV!{cEDmTW8( zG(vVY`Cl1G~AgOD~*91>O_Cxi}4%_Ha4jk(CTr&EVamDmfg(p~ua_tOKk1zZgnMX&MkavO>ft-)b3=n{yY0D`zLxTYOt_2LertN zonQ6Jl8Z^^EtM{_Hi7H5@t*JJ6M5CK_FtiV+G%iVX|NpQxrRUr!e$zo!Yl^FguWeT z>^NRtTE%?h-5*l)qaES7S%s0FOlH`(p$!aY=XLhNE=ZX5gTC+f(nZe=lmgD;|J8Z3 kWea-Zzr3eiTJmuft6RDH?)=Og@JGkQm*`I@#YOJ^U&7uIxBvhE literal 0 HcmV?d00001 diff --git a/docs/images/telemetry-execution.png b/docs/images/telemetry-execution.png new file mode 100644 index 0000000000000000000000000000000000000000..8cee23cf2a0759ad3bb3affdb9a8c14159a232f8 GIT binary patch literal 14614 zcmdtJcT|&06fcS$D=LZ<5j-lW6iEVvB1rF{*U$_QNPr}S1VS+&iXhTdr1!2uKx!m3 zm8SGi6$lYgKp-H}lzKnTIeNTz-TT*F>%F&LmMeUjnQ!meduH~`?>E~pLw(Johj1QOW7|y!p6l561K*2FCYivWT?AuI6}FEr}Q(XS6$#?Gi-w zpQlS;2^5Y%yaZ9Z1O_8KJj9*Q2p3NTh9Hhb5`iM19)oj2JEM_K|Fi*1fFWWMU@;KH z6sUzLgFwKSl!UmH3>f}TdxQ%T`>#MLaS1@cWeYUc2}dFTkNT#-7f1pqhNu8fK!vpA zKaW1rl2?EdWe*Q`qy^GJ7Y%6Fkb*$PA<{tcWnC>(nBFC@D)8)%c0&SxG?9*OxSdy2 zQJy#qP@(~nkQE0+|E<#m;f(M^{~=<>5+o7f@{bkv#zqJk7kvka3(VBZ3#ToktPON; zr;Rt#lYquye;F$d5eM||R1keVkpGlAA-&O#fI`50b-^)4zRf2)IcM^p6lJxQDZuG>oJHGXr|8W@-QhMngr?Oji{o z3o`J;8|c9dEild&AQH;bS5wl(R2dDHb<+1Xa?^M4(sgyxa?&Ems#~I@)PdJ^pwgy9 zLvt-HEEwr&BB7}(tBf>sF~uTvH6*2gdQYq?(n8Y6Ro}?l6G4LbVn}8V+Q#Y<`X~vo ziyPo$6&D{(eFFm%Gy!fzboJ2n^T4kZzO~wnU z;^=^v_5~UHTA_$eB(Rkq6cFg9=P652SC=x>HBz>el5vJ&WK6tC1aAj#pwh__%|GahkdW6Bme@l#!&ghBEM?5z0(QTN$US zp(pFBZ-Um-RKu8}ebw~c9E_}#&5d=CniyqIO|Xl;i-nsBL|xg-rj5YCHI0 z2oCxvcS$J?cRy8xH%#4ISp{RF0{6lAN}G6VYZKsl1Z6dZj~>F?MMXu$z*j=o7!5<~ zI?4du(sFVHzRghxH3>KlW#NK0)UdEIq6YjrIKW5k^iRO-)ZP12qy1groy)m7BYJxt9& zUY1xmUegtZ@X^9bVsN^qhElqkNRyq(;OIiscLd=HFh37J7fYC|xrw=y7Xjys^ztwP zmac}Rl@uK0r;n6^`dPw&-@Ya)SWgpW14$`oCm9`eA1K67TMMv4#Y-8jsfE(>0NNO+ zc@uD27SfiIvOXwnsFRnGg@lTMj*psyvZb4NiZJfP4ITn#Tjd=8hRM%B4pt%=FS)w4U~p~ zv7wumj3LQK%3R-D3k$<*?aT#ZLkUAT*2UXkXMIU)dFvSasTp`7Eqs7FoEhE=B<*S7 z0o5UaFz!}JQwY#V%0t>gOB$u_siU=HgEWjNOEUD*f;mCGaB5(rA;?6>jA#hA($Mn7 z8JnVzx;TP~fd*d7Ov_S6+6ao$(j_?Jae#QFtE`!;xwb0=sb(ytB?-tfgCeC2WgwOg z5LX%Cw~C4rP^hfD6IFKp?YQ|@MEW~C0l)tZ^$_*zHYZSQY=UgsYRaZQcC6IHeW%TM zR-ugZy@$ZzfBj``XqEHGTuH3tj<`3S%Ty`O^rB#xa+~01F5;s%$0fj5_Z$+vTz#dyt-<=y-Du+xl)}dJD9kd3YjDgc%GkZpY>gtuV0SkqyJ7<{&_hlO89oz+>i5YY=4w#02=r%HLv`Z&$gT2fQ^%}pvuPfM_J^_ zS87n52-f+3Y}=`kOJftFjfd_y{I?Ojmu`n)pKuSW{?fUNop>EEtA%41+wXqtVwcnb zG+YK%hVDpt6~-w%QfO5+@+xQ#W8>$d^(XwtSG%YlV=7eF{;Sl$?Eq@PrXSUz>Chk* z#|#-#@SaUb204b{+`2QA73nGUo9j@Ruq2&Q&Y3iosTwNH&msT!xco1#=*EfS9h)QY^4_x^sv$@4oWlA`^@&xA z%Y7ycsX?3Z<|zIdY3#_<+VadKq+v3J5M^EF963zIr4PSA++`NSh2Sk!Ytsef;c9Qz zcPnHfl0_x2dv9-kY6@q}ekdcayA2k3HH31Iprbz7)mv*bB8ez|$2Y+Tw_b9^9I5_f zw%;ZlmMr~~48@JkN{k)x-Z7;D7$`T0!{L9-0+%dKbS&~rWot&$yk-Wf)Wf-cem|Vp zW0b97cBEAAg`%@32fy5BSj9@REFsUj$}@Lc;aQ5~%m8KX#}d&179b*0V7_VwO=o*dlUQ{BF8g*rA4fz(i6KJ`T*a zJFQD6b{~HmaXX?E+zqBakF){E`)UJd~ zGj^qn6kry7>5H{fX4@@${Na^K(W3sNEr4?8L2&QYpX_ZCv#fJh+fWxdG0WY~MP+Z* z9J2`Amn9DZ6Lv2!VKv|C37=;R5@o+^Qcalb#1>~9j-*5(q?%g=Tcqe~wpP=?_zf0@ znQxXUnsbUR7KO5#DLM$j7p8V;C>(k34wDp;3xA}{x8pEi>5#Ifmw0v`Mr&e5@ZB7SGd9oraYuu5{aNR7RFl3GWsC#zPcWypQorYR3ruoL@>l zcg!^Lz5@44m#(ys(axYStk2~adv-zquvj~kcB0P16G1P9hLWWAblwPfd;*me1e6)F z0l{MTeI<6FAF}K>v)VE*)b6x3e(YY{#f;b zx5J!|{IQgkGL!)o`-4hk|BFX#fbS!I$Aq2mAgKi`ctHtiV8Q_2&O9i652`O(-e=KH2A=R4vksRhZweM$~=jH&;-#UlM}`d9FtNv<@JH z=wAjB)VayJ`WYh(#}gV%PtMDv`Goo}gJ(Exv5_G9l(pyXFLUR@CD!7Sm8$$(HpJ@d z5~e)eBEmG7kAb$UmcexD$lg|l`~>dja+6xTwWFToT3m`!Rdvf>fI=a|xlvGHTk$m^ z+6b>>7M=PTXsf`5_+R?K%^6vDN!W01u7o`sS9S17HL0EXeI`6ioZ6g#S`w2D@+3tA z-E4zpjMO@A#o62I5v!z{)v|H#fVTdkZ>FrVKsWgpsT*q!S#1=M5byu1CJm3OrOV*o zbRIm?84VXCF2`gZn$kpnGy!h4`f1r=;_(!*bKe zr;E$0r=J&$geoqbhay`~L5Wk{e%=)SWYMLat!c89yuN~MNtN?@enUVSGWXm|uy0|C zp3P_uINSEV#BtN+h3}&QY_?-I_7y$Ex3fPdp|l(7euiR@H`1FFa-uv7SF`nuG1d@T z-H{A+2}nCg?t6h~7>VG4Y}8aNT2)lYV2aS9&8)tp_)LSgwVE797uI49Za8@go^^5a z9G}R+GrTT)Y&)Jn2_vKL^JBGBG;5ZV;26^9w~t!?D&`yuKFB#T-Ct1q2%D4Y2lGF- zX_ma8J<=Y};{xd$Q@5=x0qpl1*B>f6k$rJf3fFNv(YO2kRM_F0VseMMM4xg`TK1aD zI90G)%QtJ-%%(VpPXOjY>c0V@_<^RQ%|Q;nnI408k#G&#qZ0a|_3M+a21%&$t?hDu zcyHlEaVLH)coNbPlPXVg4p^}6P#c7Y4>zezjxFjSvQ!ri9O3?0oTe|&lB9aG;KdZ= zHP)fYn#E2+yq({(ZCGMKxH#?UumdVDKX&$}i10$;DyG@#-iIS?r>Wl6K_0RlWOYpKx_}1<%F*x zbjr3+tjd~KHCgAb`*gd!X{h_)6W$V^`gFV`zre3;H(XvWWX>-k^TfRG;zSj2q8Mm@ z+ccidLoSFcxu~!*$FQyn&z`+)86K%WhrUjXD94|2{(`Z4E{61s6+7Bu)AQq`TVzwL zh=q6C+hFF*5ABwccDD-r>R|m=zl(oS{gxkT(3a#?{jhBDPC_H8@cYXjohr;qhr3}PJaJT=Cc0tdccMcsTT(I-Olk^Z|1`-O};J%X5rCp`16&* z)SOx0r}31{F}S0mpq<+?If18Gis@B!Kjxg(&l=+-)X}H1gbAPVlXlAYJ_Ua{tId;h z@uOApaGGl|U0p^zy5ydDj7a<5mT>!acp!Dw*hih;`Jrkr1l28g<7+v$t3%S2W5QZ* zMcp;SZ;7;&ao!(_f7fIAsDv6`*xemUrsMMs(&vuGE216ezl}AVR?TNUK0+wlUuXmj zOEugN#l(*{|2%vY)T=EM8Tu8CG#``Eflxuh9f-E4xgB4=#n~s=nkDL0j3*rAXi1IS z!*_Rq_4Zqo@?$%E-i2`4w0m{SPFcO>>CzZD)w;A<>0RiH2=RAn1-Gt)=rK2xVrqN|bA|25wREywWC_a6(P&PiqlQ3MfUoj*4Z#wTZ#nNt^4d!a{1ZBl(M7Hl0RY zaQCEOw=79*k;(gAl*%->b0J8oHx%wx+ToUJ;?BoEILAJ~y9e(ec{trdhm+s=>6^7A zNzT;DZ&GfquCV86TeWoU6z^fl&aI~AiXqm3#Q5nX=8pkW8;;Y1tM@`^a9Pdl7VAp) z#AJCA_KU0hTsXh(e(-g46G^$J+0N(W;6cH!;+kcb_#>ySF&k#R4f^J-$FC+|Za9&g zmQ$^Q;D2emIp~<>RmyQKLIvjJ*MmhTnfL3Jb0v5L@{tx~UZ`7>jy|y}iK&PnS+*kd z7B7E&tHLao`{1FNi@sKtwNJV*+B9%HoaLuip2(Caqg1}BzK2d~spxOME`1+yc1W0_ z(3B<9sKf&mp*+wvDorU+Kxd7wt%6bfe8qkh$6E9U09VfViY!9gD~IrJs9H3$%nN7hffK;;ammO zscUfv`2|XWMs)r?W7daY3NmYapAOFqJkh}MjLYr_TO&Q9l!B}6XG{A=Rbg89D^otr zI5YNZ(v4Rzx9%n0cLvkJbnVTEmLP}9aCb%+NHzC}p>as=l+aM=n)Vlo1=XZ6n&JZaIUW>Ek-I(}bRtD@bb z=aqiJi6Usj$kf88rt+jPefcYm>%k#qseoyImO?7X%V2^0^Z;7dTMPd++9>{1R4j13 zukc4V>n==L_85i?&Fyc@6S=PMQvgBZ*_lg@j-Xl65ayLPh(2BB6U?!cdx`VYqHa%? z8}hjJY=h`cW9iE&{&MD09mya%n#FV1f4+7wTEPEYrfA|+_pK-Dfd6+~P>sxz(jQ+7 z@PDZ&mdw}n;fKS8&rTW&Vw{wdVL9Jb#ZKW@)b;I($qzTPo?{Zg~CD=4AJ4f zGEw4HpHFVbo%a8swk`rOrU=PN@9wdWSgzI2@jK?wn+L0lvX#3xbdg_Jw6+9)CVEeD z_8Pe@PK;qEW;T&DWw`D$3P6-G$Px2&Ub|PpLU3m27sEAw8&#gIbFOv;DHpC>I5()( z%(|p-aR@C%XXR8h>n5berUB1rN8y=89L1}Kz6^CQrYB9k=<>rhjq02pRBP!vg2EJ; zcb;f@RL+%8+13A8e?is~gNl4+2E>`zhG31_MaG<+G3VyChINHpJK%=u)V5*S#|FE` zOv=WRPkz!fd}-^$ccn1BXA6EGSM`Mp_e5m{E)kQqK^=PcQb_a`#|uWl0BKAt*E#CU z4N7`kn0XD_KmnLp$tBcTi#UT&s^?>U;~U!1FYCl--G01C7&iddP~MwhB&3>a z&dUeNe-R=0b@M6W;GK$G&pjgiI`aCpxD*LRZL64^c8znSV&JUx_4S+guUhqTJDMI6 zeqg^EGNZ|}j&m!+xYbK%GQ*_D-5r@*SZbi7Ef7=7x4i~!CuhEU75>;YAhRCZXVca*fTegkqL<;=oPYtI-Npn0e159+jmfqPLg5#CYugK7DK zs*kR{^)cuGEfJ+F2ao{yk7b@#0*(w@sjbMS1_T!;AMJuh)&QZ(=TlnqT`HlEHsZFh z>Z9E52yzEC=8z{vtH*lxuqWUr z>_tWAxWnwzxwa3(E4j~X0>8z_R_==yR6Pj8!scc6d}ys7jnB_Zm>M8<66OoZ=Y-@8 zDh~BXC$`>Ue`LZvmCX76j%0Y%;7=qH7@wYbXom&EeKt<>+;Sv?YUmY=fL zvn_pah%@!{yWP}=i{dY2c!ERfc$7pvIH#;{;=$}k$&lNYABDQ9uRb{)=M|FMb6*{I zN85&Y4<61zwjQ@ws-;t>4pndotW`D_Px)u#=%-!96!nO18$%8_A-wh5KH|fvVM$u$ z`xE40mAt&WYRo)(MN`z&Fc(;g|Dq5LO=Oi_hT!i2!rXYbarvdtimvT%^T zAs2xqi~uDm>=Y;CNQLxpsP>5S9Vu?{HEH(34rxuMG-u&!4?0!(CZF`RC?=?Nin_3cN>7l822d)*PGF2D_ePs7@!T6r$I@ zs5&o%>u_W@6_Z;a;;}?IY%+`uuEU7f+6Fmh&AjOwd|3E>QDF9L&9_g9b&~ZuKhx@>i+;>-6puVE?&daqaSa{i@#X`g9$G0&oz^uj|A>2yY0h&Fy;g!5( ztS5;{rZVS~+=}p|QD2u!eBzGRaiiC%#IHTr4-1kFg$c+))}2iF4Fh>rId&uTDBr}j zl$^}3{!3GI9Svq3U{Lye!{>J~oBQHyPuT!S4Ey@dKE?{}OQ(8ob{hZhS*{em7Ys0( zLry79z{XLpYx@I|>tD zm$m!c+R)G=qMmaID9*w@0T`b=Yu=eD;9v>pU1gnJTjs!TFl#BTl6NH z&e+@h>{yHSKD&HLbdr7jIb0tY2<_!=E_3ckdR{S*nJ5(_=eIPaV_V)N<GH zSrvN39(*1w+52(WvGMp5wg8<3r3w6abwK~>7emnsFx?w*GRdd+(rkVrqHU{1XsFf7 zVFJt?Yl>EAzaNIhHnFU4s1~~t@nBrnPgrTrH5@TtfX3mM(Z`!7(0hR$)D))w{mKX+b@ zfOKk+7u781hqYuwb&jc+P=@T9M9TQ$2*FbHR$5;F&HO}EzkWWy@UxT)ojJn=GtX;F z#aXPg=2Sb15bYc%zZ(xV-@U4RZM4>3#A`TVipxc``WH!S$iAD*C2sRRx6nB|Jx9-_ zg6cRDxVJHh`_Wj;i4yk4sq$?&& zy?IJk5*!m*dizkuU*aMxCrcjBFv8iJI}p`L4tp5jYDC;U0Q12U*^Bo z4?u^j06X9rmHl`AV69QY0N3tq9B%nr1>4?3%D}G3&cnX@_co@KkqS`#!`fmF_x{+( z_69u&6IYY`J@I#{f~3!PU)Xj)B)<^U`%m*O_P&csvYIe8_g`fB-)-l1Sbg?CoBcn{ z>VJeS0rYJ(m#Ag;$0GM*zlQ-Zr1Rnn6#cKk{2l5ZQf*NDhl2#W&jPzYVdVYcKRk37 zxNDHNyhZy%$p2Nda;L2A@A367F~gNr(Auvs#_^$5+^umw#_PKW7%}M^POC3M6$pdABJ9UT%Koc=Nw}K0AtQ1eeKCQzyp4i+JZY zVj5loF>S4k58PTzXVh=`gOIalmV?YLZ5(TTGqPHt>5~&8i$80fODg-E&KVI}G^h z#^j9~Qs1uK8as>~d5NC`Mf>XE?Z6)m0(13)r1YQd zE+d!XZSLI1fIR@KwqLH#*E1EqKZ;(8F-_8G(Nvf=ngcG#<^XGp<$;UWR6B4Hvm8?& zuvs&a48?gjgd8Pwk>?1tTWj;>z4`=U{S*(kfx-phT`UpD<;U?g?pE<9+1O6fc5(q4 z8NQRLv8?T_)q0Dl>}@JOdvm^eUUBul=JhwpyDFaHSGxkYdQ&}HuP6#d%5LU$?cum? z7-m$`JPwX9B2VQ+zvR(fI32wfJ%fzlJj~SPg;JyjB?J>GGiY`~=2nZlXPLyyZbuadcGv${W0}LTBrH!b;kVK;rHO1I3~Jd@>~F05+vW9VjSgvN>k#}87B zu))e?efsvsyHntc{98QeJ}7QefBuo2luC|pZV!!xi%#CoG`=O zT7d?dT6Z;+WDstoudxHu90RWZ(R6Lap>7s=jo`mBi*Hq$Ii&NU8Q`^P)cQ@<+*gA@ zKk|tZv*Hs!degT)d)=ak+kHMdmjH4ciG+?$X+)o6o!j_$z(OcfD64Mg_M7q4D7$8Y z#<#Fs9LSj2n9{s4wD~Rc2GM{#ZNd=P{-H_wnj23uxb=xM;PpU)&romLgaO--{qW31 z{3?S~t$cTXmHpEk0c`fkUT>(=f zXCR9(VCCYw#mhm<7L2L%U&R(r&IfM$<0|d8zQp0lYAN~36uK8;kJdh zfLTpa4L#=NzdE-p>`iuBvsWzgT{lyr7kM_>nkV#%4hZ}+y2;h&Z*nU(Ib_17Hy_OwTL(xgL*g4~GJ1 zv+u{SwaezKz=~z^xnOSie7m^{Or|~o10j~eyl?b+Pki0yPWk!k`7w9&xiq8BE(kyB zHMqe@S8~eQnVz&~Ci%4DfaQX^$l_aHL>LZF(1urnB6a!O9G>vZM2cQ!l_o}AO9jR)-3vnR zx-|i*y+S4DQlbh<1rq&vq!T?ObE!37#1vyXgg>awWEq`#Z4X59k$Y$CrY1`nQavM& z1Ar6@k9hlls`~qqs(J0@6ZMKT?#RjFxg)4faqG(VE@bCSEp&ITRkobl*38+0z4uOD zl*7&Jmg)OR!I29L^ek7(R{h>SVr6ND2_{91&8{9 ztBIL}+_{Yz{nE}4>FXntBZRcYv&9wV+bmq*45!J}!$#G`e*Fa_i)pLZsea4TF%ugm zY2Qx;KfSA@yFE)v2#b@YXN86p-Sy+ZX3@0z4f_`l!pikaY0Jg$Z?4KzIg)04W_^o7x4Q?%y;luHb|Cnfz(aiF-8Sl@yE z%B?3n-8gWATv|k8?CrPg?0+QpOVfh<}cw%mD=`d(I;5RiR^ji z%R4RVnCl7EBT5fFk9A(^4?xI<&K(Z`I=OPDNlw@e6F1nSSwTk}%uPgHs&-0{TJ2I? zcV7*+-+1Fz9DaqI?`1uSqU7tiPH``!V!C3`ZU)D?Jaf4rE9~jPg0TEG3*@d?SRds)>?V&r@*1()HpD8% zIL3Crh1D@29Y-Ab!0DA!VZ|$ZuYNPd#atFLsxV4756H-=+FB{mKXw(`eNfrijwIdw zv_%L${Co=%VZ@`KDmnW}e;5U#gJOLv*1AM}SBcik(<1zeKvXriyjqYMo_MC_<32?v zbY`h2G8_ht9h-{F0t?W>Gpdi$#GY2>ga!=mdtA!zhce_Sxg1^?Q?n(W8p|-3wmSUM z^-FGbd(Bo^s3EdkYVHlW0|gkHh1uHp8DMjLA8A+-s2|ccKhE6lAoh@zKXeytePAE<;4}@x5SQ7%6 ze=qZ}+_{mjDi-_UlinRIhAx+6i7tn1tq1$zC=KR&wStj_?o8Eyy6M-5eu}PUqD^Wb z5d7p^$)=^sGbM^TC{(Q}PtM%b^!1`;gGI}k!a1<@hw+E!RcVp?QfIDAK26##T?#tV<4~%HKvS8GKE)=Rj5Q0;kpk5gbDJDs}PdI+`jTxPko%zN6o!vYPYm z>Z<4!?7W}r)=)ChDJRye53=8hcTwPsLN0DgVT@fHIIHtZh$l3D;o>3xN;LVGy$c1h zL+TX#s$c9e0rLv4}DCEgKcXsmOE4t3gnK1W1Xh% zc9a~Rr&YO-fvdb8KES)i1$*Q^uFQ1z<)TRG5$UJioj)gpKW2Ei@8TMlR>WBqW#-uo zuBX~dVh_1xgYVSIDv24!0GUW(k1Q_heX4oOm2JBu7IMs2J*dw2k(fsHl=Tt*?+k*cA$H1WTUH)X;E-;=78dQ25ueIua`45e57LK!vXTJW`Xlx>P+XS6UJmPjA`*3>X>eb#(rm^e*^r?KT?C!0f1vJ!K5c#JY?DtTAqZfH>5J}%+A z-QCbV_xE9UbK2g-mmZ4LPC5O~q4I8BQ{Sb6r`=GI$4%}B$l=d)&5bf~<`=_19224? z(42Tr$HegPo^F*3}j(r=7^ar4jw>F}SQV0A{ba`TEXibO^iWd5^81lhwo6YO(lA zbouJR4BWiiLzlf%$-<0L93R=={_gb)O3dqDR|1B^xx_rlG~nZRj3bj zMw_c0`VZh49tVK$9d>2z-#j4OZk8&LQ>F+%apf0t0_3hGte^UInC`Qi&Vob&>q?178!g zS@WZ{!xa1aQHo{Bn`PE8HFw-%ShgS_?cc;Fif#3PKoOid&C+|SS-T!QTtN$jv^WX; z2HxF=g7Wmrk6qp~Y@J>M8w9wzjeS9DGM=B`K4-WK1C)>7$5%n8H&4}zpX(7nIZc0Q zrT}Nur*VJyFWJvYThN4)pRcw1Y!gqnZMC=Ak>`m6KvyilKtAa?RveI;TvKBau}5f-UG#s$WWa-c!f=Jzz2Aw$n8$BbVWw;3jizl_DcGGcfvjI@M)Ql zkt_f*zhI)1?<+CM*C>X3U=p|%T(FbUdm}^`b)4iw`0g0Qpqmi8{vZGI3x5cb0e{9V_fGi}c=NFxP)|(?fJ}KS7A!N?7 zhmk|p|K`YsuXDLRlq{sA_q_RZWc`X`VYt9JnuqnWrg_bAXvq7)4#ys#Cu(0e0we)< z#J>$*58W>IndnHKj1#vV@_eRWl@F@D`!BVVk$+GUNHIu4wdu5$Uzri%%#WZ{#=HxM zJw=8f2b|8vxF(JBegx7f^s14j`+D@Ukea$8&I=WyWFj;K zZbcu6h%~V*Ix&}Ry$mcK{la&cJ~ffDD>2+jmJlcBEU{rmL|Enbz2P@RUD#pe3NKuS zFU6*qLg?rgBY*Fb6Y;bJgaD8OC{LVvc!SuZng(#K3r#9-wOft|WE47Q3B!{5FsNXT zYeEWwJbaTcE((dRqn&S;ar|D=Z|^Ei0J#-etq;=z*1k$X>e0B=`}G`<6!e#~fG|qJ zR5{e+{l2@%EEadBr8XJJ*^}Sw<6z6Xqs!vsrz)2C!0%tu>KI%qj5Phmp zljrvZ(GcQQut&P=)(2#Km9BWz#ge?L4@pzbFrrOV7B^*Os8!a~jekH*;6}j6s%}d} z2O6M&i^7gYbH)=IFG+A^gV)i;!@B4xJy+0$${4e zc8a31nL@O*EA7(1BIDA2AhHm9{Ikp|GuD_Ahk2ZDt}oBTuMONQhpN0)doHPz%jw4s z7ym}f3S`1Xq;T>bL){;Ue&TE1%Yj3sS7qJS*yzazMk z*zTi1TI130{`1E(mNH2%38ufopj-?PoA=VJSBC5|E$)cZ_x-96G(-V=g-V;M$1k%0 sQR+ZOuz1$h#=U=RDgWPYwtavzsi$4N>3Rk5A064W)%DeiRBqn+A8-IlfB*mh literal 0 HcmV?d00001 diff --git a/docs/images/telemetry-pluginlevel.png b/docs/images/telemetry-pluginlevel.png new file mode 100644 index 0000000000000000000000000000000000000000..df3045f572d82f3c123867a3fbd8da5e5ff95ff1 GIT binary patch literal 18853 zcmeJFXHb;Q^9GC}MnpiOAQF@ypoAsoyuh;LoRKAm-31m{GGZc&2#A7YNs=Ta9{-=H7d{XQrp8ud90u)6!5Txp?&=0RaICOhr+L zfPiokeExOe9QgS<&8Z6fa|)xQEKg9>acz!(;M|@E)W`$rZRhA>OTZ?e@aGd7Kc52{ zg24nH5i05=~$H$*@WJQa|G z2!J1a0z3lzVkUpSZ)IYq~1FoW{uExf%06x1oI@^MOlx?k@(fC&s9Nf`J z(4qw46XOA0|CXt1WoPB?c%lpZNNhc`Zrh}0UjV9|FVaVtL>jw8(S|&YakE|9||V&?~H-3IR2d@Ze6s!wy3rUN>NPB z*vk$=2pPB|tX(l!s1~1h1=a2C9CWenCYst%B^`NRs3%edf|2vE z)rM&ZqlCQm9Mxck_M&iaH+y?49gMo6iK-l232iLoiuK?}i;AghxB`8w1YMofVTuT- zk~&yZC4FZ{bt?gFV?92MgOR8_)ETt7YxroW2neAZ@ajV~OqA5EG5Y$7qHaRM0gB( zVD?TL9wtB^cY9lm7Q`51g*6uR5mJY_$Sb3SAbg&#&U$hPV^>8RBWni@6FXxAbsw}B z6z=0@%_pw}krM?y3PI2sM((Z}7oFb?{f)#0dgXfT0E8 zKH#Cdl8~XHJl4il8R-Jk6NGCz2%G3SA(Ra~#GHj8+RlnDFn6dS!WwI>FYK@l6GvURftHP$qJxVN*2~rfu4|y= z<81Hf;{jE65rNpFk-jeWNCN{6H+gjvLAbAux4o~Ao1TrXHB8sU)=*v_JP>dZ)xmgK!(5CsoYmAZo}xCQ7y(-^S5deo)(z#323kQJ?L{!|0%(Mn zy}ONpAPUCk1rxB9Hw2HsA8UReQHX)3AVoT1#eA$Rg|WtuCB4Z4%o{gdRiy}6(ws$J8wfRTSJJeuqevGR><1O5sOgrwUcwy z)sfTGw^B8*5mk3KP_Ywnw0H6rHSkfvz|@?y^gI-tfJ%;71&EHliX#T&rVR$AX`|s` z#HY+JtdH?UD2Q4q_`+<(3~k-Kv>+k^);jjeqQ?AA+CEll+Gu4Bn2&?4qPm=qfu6gz zl^6=VE(p={vhh*$LOQ5v8|t8}1z^_vHg<}h>L?__Sp;rninfi}nJp z82tLr8~=9A;P=1BoB-7Gl-6ki0%ii3qMV+$#ZofKH){QA+?7Q3+$X_y+KM)Esc|u$ zgj{(K8Vc^8rFlZfcJWo(_bg`1t2V(IXK3lNPGvD(U!q*Rcq>hmkuC2w2TS8e);Co5 z%j1qWO=l$cabs3NUlyhOR$kPvR2OaE!>$N9ODzQU7i^k#N(!KJ6hcTD+-)CgB#X!I zC$2CPoVp-OKzR8s!5L7UlWO~emv;&{h)!ywN5tH!+XA!2tNnvg z)caRjlB7@R!eCiW62w~RfHER(q#>X-Ny_i7X`Oe;@LOb(X$j(?X_agG!bmkH!*_dW zc!pi$ifIku=#Yp-zqkT6Gx z87E;7?=3W`X!D%wZc%=%DYdCr=`p8M>A7G~={}_zWKsRGYUqU+CcD^gvNcM&jk~LG zy}!_OMh3Umc55__Q`i6S^D}PD;r=#OWZb{3cH$BJ#Td5=i2F~K@uET230^(vX_1&1&7F5r6K=ooAicZ}EjRcbyQEVYfOG?cfbe^{=AE?t4%##^P9 z1hCMhOL?b&LCGQrDMrFW;jFi?{{02UsKsw3(uoBttX^zY)IL0H8GX5JoUf(tyt~`f zyi@11K2q-3!!_+6lE<0VTqP}1f);*pcOaX&+|X_3Ul^Y~{hTJC@-278t2kPb2{io!t#|Oq5VFV{ zt!0qbH-v|6DTB7fw^3v#d>b~3Lp^2XNA3P# z@9qhWWN9^ZxZ6#AK{hQNC_&PqZjHC2mzpwzZ>$tT49Ol58b{tfZ%V%8&{5T3*y}$~ zWKNA~jb|!qa&N65`r4GJ8Ucau_-&-+GS(Cb+h2ope0+qTF$SQB=ZGjVe#Z3AG&;Pk zmulfE4Y{ElNjrh6TU9zP8PniLjfnC#cz$b&>8i|(^Gym*&J=ky`=&Baap`5ax)Y`ykK4FAxH z`7UkoDaIW9w$8=Rh*Yl5O{cn4IJ%^wB?>03T{%d=p4tBB2-r4*U zzGCPu0iurNrP`Gf&a4y$Jb*gx4)uSoLYDrrY-D9{?A)J?_S;#?3IhjFHg~1q-hZCL z;WQ8-?8 zhfiT~0R=Riah!xZBD%}(Xn%jtfs(KQSGY9GdIXKE+%n4jR?B)n3t>z3Pj_aTQ~l#c z&)rQq!h*;rt35H|k8aZ4o-0@3{x_62M{l=iReo4Ddpk!z46ldt8*v#Z{cjZz38cm47NUDrw|4tNQA> zJW|cW%}#gQ6LMGeIsu3*o_6>oIx9yrn&&!CwZ#WAI`&@oOW?I!=BZue$8Po+)|Oa* zsA&k;7k=$C6fC=Zq1XB_{?QY!3l)al)vOmw`_~udo+~+Lvj)I^jO(>jMZN?l;d=v; zM+_2P9y_;T%0neCqHnHt9Rr&p-vWD-`4{fY)7y8u&U%kG1XzTUT@!tCeSPKq{qv&* z#$^`?M;`3H6vNEZsN8ue%e-=CzU0W{j^Ie-%@^=+Na5yz?_OS4A}7r9=}u)4V&q2c z`gA*{CtDp8bi6-4T39(YFN_+Q)c^;Zh|ii$rOP1yxJW?uLzavEXAVQjS;Q4v^|#aH z?jKZx^KHG$n&CADoWTt|+$xkeH8y5A712Md;a_wgIee> z!&*0xll?^HiOO?X-KrL0uH zr8#zgdu7|VdHO-_bDnJD8gQ!X$*3m^enOWwRdamtau8{cv$=M26>Z*9b?)I~PTg$p zo7=!%_>70KCLZON)t>XcC-&slc$P>WzQ8EuOGQH&-<45r^XcimT*J|R4G4s>P$crl z16TN9U%p|n{7+@irIG3$f>-RV9ch6usO5+15#^g9WKij^) zceZ1`uhG8Frq)O;S-fvJbG^{)W$!cA`)r?>U)?^lrol{AR&&%OrZ!OU*sAGKbrw8f zuUdJ)w0f825>*^A|F?rm=YgWempN}e0qH z9%GNsi9`msOK49N`bcqoU6*Em!$jyvk^5oK`Kr8^iJd%0>*sd1?%O>%npn(ChbQQB z(*#-i%~M4!Y1 zF6+<``rJ%gaBq4<@=(Rq8<~erB&G`JuCvGfkbi$Ic_3VBox=QQBZ%;dsueM&B|b_K zb`JJ}RcQpWVvq*A_QZ_Ca-R>=$Uf&T)aYROT_xJw3z%k@{-=HFyN8?XTG5*W%~gbF zg_$0{|Kg4byD^q{KThl`E-1*}(JrgmTf!-u)%r$QkS_HkKmQlHXe9^vm;$+sVaAE{ z-pGljkSnNojy*FM`zz-DE@>RTW~!0SF&CIzxwc$&z3IiA5~MCrt7j6SSBAu{Hixf` z`>oVNuY7ui7(p4E>V2VUQH{7Av&x!4Vtf5-jO<3RN&tO!I<$DweLducTKe<1&z^?5 z!9PcI2uzscekDfJ(FO|k zWt2Kx*y&C_qZ6~LQytfGE@>!y>`~NEme+FKy2f!yJ!hI3YfUeK*@^~6TWN(pkY!lQ>UmWe|`{CIQyGgyg6Q4Ms%54YUaPSx8x0r8N?efvU zIX{ji;PFb(?kax60CXW6x{`D=VpC%87dxUZzKC#7N0V^VC`f_CYQPnQS)&SMQhJvs zr^dRotF1ayq%7{;^l~PN)=K5JvhjHW6%~Z3E zw=P*P-9D=iq$SD(1zuN05^LnOWWF>%+}phMQz!3`kBe$7(IDyS?4IFNBNfF$nK13M z^}=E02~)DypXih$Z&pik5C?mo$+#in&!~W%@L&j;ySnwH%4xv7z}2GXfIRtToQeDU zvsbDx9b!36>%8KYoiF1)(~-^4aX*_F))U0d(LFUqRksW>y!%Sy=Pm7WQ(SYdI@aOx zyWG4&;0pGA`nq+(r3zpj-gRzDd0T8x6bo9isOtUD0}XSWeYzTD&)*U;wLMM}U`$@W zeOwnW_}`i-r@kZUQCh7172Jua#AKp%wOKj*Rd$Bc5Zs-){q5KwR!fvMgBCZ;}s<|f|&le)~srFOsCE#J6 zlwNi?OE|IV%*mx1OU&|VV#Ef`$|2lI<##_AnitQWQEh#&v#Jr1CPjd%^hILOXx*?5 z0R%&KpOLvQ>ciMI+i~ULE`Pm()UL`~&t)P;S8uV``6Q{{GCQ@Ao`OG`lv=9VNoZKh zRj?jB+IRS_PCcGZH8r0a%cf8lK?wnuE6KWwc0x#o1;@3M1_PfbVPH)JP{M7(78<`X zfw9}@uc#EFH%HThe(^|6eEeja!fk%z(;Y|0%O0eJmpS0?mao3nD#2ZJL2FL%y7E6^ zz8p=kfR@zo;f`;{g|625im}y#!Nw)Ny80*PlO3kKbBJ8J{sjNHpOKi_FTv)GUP0vl zqRHkP@Al}4by37@tGo!wN|eTjUqG>Bn!L0BzS2M5&v+Vk5J8JxD$y#z!p^~4+Bp6P z=Wl~xv4n5xtro@cjaJOUw8te)exq*37?Xb?HeNjB{}uoLH2LV;AQq;eCH)rz<53O! zWl@h=tpbBv;j2a!S{ME;$nTX^Q3DW#mhR^NSR3#nGXuaL9-=zL|3C&lAg2Hf$4#YH zqJP0T{sol~K#wpy{x9OjOKS$GA}q#@@Z`(g*8!^&!cy^nq(A`FK-j$eX99Rg6YK$C zO3%c95a73z|G!c(x0V(}sI~NB?ypQsZNlh9UGG3o8y1>Oc`bgMu>10S!ejYw#BIL4 zJbEjIKIfgjq4|{8%GeSOkJ+p^cKO;&T!t+vqvR~RX3An~B;6#QsfbN3&C^VkM)l@u z^Be^AwTjJrX|VnLjKQG2%5740OQvw^)*$0vwa%dXI2r`*Bv%D0`0fgZr z&0&=FGW%OgNmYp^oth)eH~%fw6K6utE0YvhFO@rn;@uREEkiL?2^R*gUpJ5XR_dz& zVYH9cE*jsYc{jiP@QTGO4Y!G2N$2L<7SUG|_C%{hlY=jPKU@V$@@sHxrg5rGfbNh5?+pF{zOPpPk1hFuwZ$T1UdXfdY* zxPi_Wf{ssXmunU-lCR}{kGTuNdb{)#ofz#3`w-~(W-au)ro@sU+WGkCkeE=@lL%Jv ze&4C^=+deglh03bkvs1Q#vfujr7B%-LWT3xs^&P^>N2sqZJQXN+Hz6t@|tkO`@aN` zWX~*Z=;Puiv4a__zQf2$2CqP8b-Imnf46F|AW^S=B>Y05C2ovZh_aDAPW>yVP5>bJ zElA}-&)vy9bxEpfe|V0r(Fj0B5VImNv zZvN<2oH4df)_>DCuQv?h?yVnYGoJ!CXL@xtxf8QXk;B5gdYOk$ReI7f6))cS_NMAM za#^WHohg6@>qqMZWosmxs318Ld?}+MLvY*keV?~8?K@M(gv)Hdy|D@zbU;3T-hb|$ z=pqQ^_l*t>8TO9H>ivkxcO4|GY$UHvkmAZnl6TwA1-*0#dVa&!>(QZJtzAx*4qz9H z6%xZ6c!eAEQloBe>K-O&hDGLxNoimC<4e*u@zW4Ou1=IW^*Nzd3^ToBwxlz6Im^8@ zfydnBa+%liuZgDmCzpCCRQpT|0wUfjJv8UYl_%#u{iOkb6DsP-(w-Z`S&uXYTe6%lDd6O;bCpL~_UBSUxy!ZDM!G|6|~27N0hfp62s7+1bj52&JCFk)aA_ zVtL`A$HLlPSotO8;nI>DfMTn8V*hf6D8sDLt8tZTkA-UTZ7_#WgU$UjJ~|FJ1r#KARTZV^XZ6-}hIo0e)njb7*gyxC!B=Jqi90{oY;R96TUcRcGE%IwGA zW(;_^7Bdi%R0C7ZRnT7%Q@2Annfe>Do>5Uwc6}`GC~>EloVZ`u2jXBde$(3|Ey9#m zZCbrS;b%-YWNJz-8mmSoI4O@mL`W9!)C zjCU~I3&e-29?H6UHt@Wct9*rQYILO^>!9CbL)~g6$dehHCpsinHkfW5!pLH7v<{;w zvOfGdbAnAS^g<&OC{(QyWuG^ufdDq$U>TWbNI!HxapLs7^w~;Z))e^|CiVNF;bPFl zrfZ9DG@!NIl}0oAZFIc-#c$ClHLrePia`6sG=6TpTCB7@%b^nQhkLE!xohAw@z< zhP#m6|0w-gk7nz>*2c_z3k5->d&@3aM}^3_UrW4^QW7BlY-#n zqG^BJVA6FaJ7dw;G@?G4)gKeuK{8%rMIY>r*5w@F9lTBN`JA?#>S|x|PbAm%1RZUc z3f(VE6Q)WThe9B9P5iZV!wpRRG_@a@_eRw1J+s?!k0>f+mh)LMSLmYmWI9Z(8{aO< zbpm!2>z(^(|bC-;_av1 zDWYZ{S;|NMJMsZ@a(VK-Uu(o+mS(f}WQez5RL&~R*Z-(YhfCm0w-)WW_-_*MuabiN zgT2V%qQ|LYl8o@}JRxqrbPo+JFq3ZfA9;2h&W5$GSrvzN7XrDlN# z+e&9IdxFX&AVeyBPjoJFn@Pr|NF41KN#xG{6DLEGgQ_9<@3;-2oYH#Lh);yzGJYLgc+UJKahLIFU4nrM6| zgbn?_!fz4+q6Zr-y_YLuTvKJD%?OWV!hC@&1CX}(E(e~#v;56E+Ee+Oo1*>*DTBCP zpO529#bHOSNXzqelVz%ztaUwU&kG*gLC^-Z89k4YmDZ`No~6~yQQeVcf*h^IEXXls z2G2N&^)CXWOO@k{!=Xk9|GwAy%VnvD1%9*XAtRH~GRwCTXmK0e@X_6wR3==BMlvEn zy)4a(t6kiocJ-$f>ADj6odTiojbrrr>7}M?mX*=PeLa$UwSe72fly{l+cP?H*!4+U zs+6CXTwJPaBhgr>9ihvCRu=TXr3?0APkXugnHM)A(`=0N4PT9*ri*SA);{`vpR5M& zTt+>5)Rz{?_D)2uUzN@zZVAV8Om+gSzMK)keQG{SRp??&j72uL-pUb15*R?4EGb}SWdLDgFJ)4 z98KiOJi~b)!t2-nnZS>%U}Ws0_q0z;)c?;XFSxgPCt!PYCR;se`UN=pI|J5YQ$31( z+MN*sBPG@$`-dw*D`T5Q*cR!VCcn@@`$@HU8wG*afnldl2WG@xX5V-qM>F;HLT?y8 z2@P^_yuLA}^}ZGK!Yn%n zZ@My5K<^=Fb?@{WTPIz3-LSuwfLnZv9$!v5KA5Quqr6!Yt0`UG7xPu-&-&fn@4z-p z)__Ry92w){(Qao@4L!Q93k0h*8SI>uMcHASG0Kc?oxc5LBhR=^N8pxw-QV_Zy&kve z*;$<&eR5r4B%xu0Z5zlO+FTwTRtFrZ)OK~Jde2X_-uOh?@p8x5m9OuDPmfwhOHG3? zQJgo9m5;69vCNV8_Rm<&z4w{H8z55i37Jz)9d!Ew`A~H;#k(;z(_MUH&EskQzgmF$ z=KPK>w_a`K>JLmoBgZ&EO&00qXdE4bARpu0U*MLy*PC;3bMD&VH4q{%1L_m+=QfaS zN)@(*M&nSVu5m!oI=W_#9or!`Qw3&w@Tkr{#iw;?9EZ~i*_4~=GprzI1i6B>(=h*| z)##uq=i>VQWbBxah3nDI2N|!RqxIBLW$CpTwM*fwN}=~l1jobX$d#{6>~k(7tX(#J zydQf%I_hh+2dsI+7m2ae$AmLr%>C$a>EqqbTC#vyg=Oku?}t6mpu^39+C1GHngfc* z33yI^Oj&$;HZ$7mJn6Mxp0k+2+)fv@x#Mep>^1l{W!2K#;X3946OO3zJMRg3O{ke~ zSJ1$9M?p9qc-T|(+S5q6N=dcMp*}9KYwuQlc7^VY^x;O&*+S}+Oy7;#{=mY?J@ZrW zvo@P7h3pK}^Fn=W0XY?h3n)Mn$_{TgY)j!20pmsGKY`US;--(MMR0ClhCQ#CduXSV z8E@%R4A0+aWeW#!>HOI7(RM9w;A%wR`5cbv!|m}Ei?yc-Rk(vG-eu2XzvL@5Xusj% zgANTG+-!5UtLbnfN5(VfrF(dEpelj$ci5ez?+LDb@3haDO^xl3ZPPUG4B(DDx4WVP zdyk9wuU>#ZyS>^GN7*U0ZMGd>?B7C0CARfNpwnXrVv^*ywdi&;+2g6-)Gf<7-_7|c zg94+8^Tl3v;akVE$}(%7qyFpd%YDUx6~i^2`N!tk0ZStmo39P4#}AjWsJ1xHz_dZ~ z@KwN7R|9@_V8w3T|6`ElPU98vH*sXgdjq&Vlw)Rk#43I>zF(Oy>mZj{9VjxKY&^(^ zeb)@yi^aX^6rH@Zk}uM*8kI6oJAR0&v}!sRbbaWY+R^fGW%2k!GA8Mf%6yW+^D;Yf zGFyvgeZ?g4>@2$~Uh}AC5!|kNP=oa3U#D;L;vKNBxt6u%(b~}u(pwOwCwt8|r6*oN zhGz@@fG$=T;&YNe3#abENlnvW4QjDu2>lIP^Eb8@sa7+8cj~^ zkqIvAh4imrGSR`T+YgwIeuXUC^7_5FGTg0=+v_VH0jMQFR{sXUds-5<;V{UFeSd^uV=H~!4>hzG39&l)9G?0s8R9b7Y~-e zs!7WbjBR9#__rEfoN2GdFId)+V9yg$@jwdcOSR9OqLZmo*H zWmq%NuvKBfm*UxPyc`{Lkhy=fjaymjBj}AjQ`h!LqCqS2_{SC8uloTY0j17likbf6 zR%X}!x{Y-AR+~=7HVC3tYx^U38@_2$rH{w|U|CMQTrSbCG;2D(XK?=bQLDpguOY`A zAB)$z^mk#g>mSD{D^J*0 zfTO7IzIpKsmwTaX9HAlJ$RuVHXH9NTzg zdJXJ@#Q3g8aeQ3EtpHnunHT3bhhr{!Src{Vf_bew9-zeI=$fPKLIfeA1!OFWFBWkM zwGg@zkJJdUMd-M5+;yJUW{e4;kiUU6^kJb_xnyWnYJ)-nXK4w58>S%PX z7M9XbdFbhx$!VRHq_&B-(-pqkZ##D-UfVZ(**gSb=DToHW;d+U^$D%vn+*_M^PXwN zC#m*}q8{$C(A&fcTINex5AZ>Fx?bm&X1r>HJk6X^%PQkQ6snYoX-~1Hc$g~4!+&VZ zs8lU@MPADoO*yo9= z_}wWPle|X`1`*?U@@cJ2o)>TD%n~-{Xs4(|CS4flDf4AWc~EsUQfJZiAx4u> z?cDALy@A4SV9{?1eQraFaI$$=!G+}Ezv0DS+Vb7RnaASu9LL)Z z8wtacj3v$9c^|9BmI}6oFh3IrB-Ri1NI(A$(V1~)jDpq|B+AT!3VS_VM_DHOIm9M+ znMAu7`RK}1H0Q{F?@yalkb&6>(Cd(&id^6=U%1}k)T3K)WE8WS3#n6Q8$Zoap7JTp z(;ue!8w@bZ;Cr3SM?4c=LUOezimlLfZr?%hf2=%y!}uHZtikSXilhb#3<}Z8$GlKFuNkYyB$aq)be>f!!$YyWs>wZW+$-QJDe*Uk{ zz!;v`0<3b|@BHuyIAoFt;84~meC9-3DL!FfyK;yAUzY$W!a@3?vfOX(+KIMj__C`- z2A2~BnhuUY#4|765}eja--{AccdZ-!?;1ln=CTkJ&J zZ7?$XL5XiC3S7T-0}^ z77w!69T1;mYdZBp zoAs&PS>sOMXA5KX`&QF3pzr0+mPe(O_h)2|cTqo@!e&OPa;7{(K5h*$5sO` z{SQHWJ%}8u6YH-qy=haO`}U>f=BQ@i8s*KDPRaF1`@^yAh5-h*W^&J!q@rRxaTD?n<3Q|U$p_!7pmH@1 zygNC9FB<|NZmtEeFH+MtJ3-y(KL4HuWxir$TYhBQJmxU4M_r^EB)2~hC*skU6aWuJ zApTzl`qn1N1U1yoW<-pvr)*1ZCygJpjt1^8;e%<72t@!04tB*^e>P$CLmJAqzF1Un z){1-1>-8p;>PB<`s5uA4?UmZKU&St0&+IkiTXIC4j1kBSH7Iz zTL#KxgWw?5Z*;+8U}7W)Am3%M0CoFcL!^INtcDxzMD=YBSgrtye_k(|yWGH1wDsq} z>GP!CL7x*^d;FQ-wvZ089KD8Hs~^U=GD`bbt=N~VZ(4@zQ<@2LUVf+chNg!c_oQ^c;!TOg%F_7&%Va zuA6zYt2KatJW7NMiOgTUAB(K7om*`b@vjAn!^&+Q?as)75bM2c@E2 z@hJ~{x93ikoTn&0P^V<+FF>R zb{-!@TiS>qhkoLEd?u>#WXFi-fceZhlB++jTp7%T>LV;>9IZILI0!euUg{>H<$EW! z^Zh;wl#^Nb*KZc0Jwf$1hH2;B`4K!2+TY2yJZ$5oHTik`D@11JLf(yW6aPT4sI`Xm z^ZLtJ+0YxCemYf9gf8&y$X6DP5JvbVDLG{RvIO|1x-cx$3BzkxCITSKB zKk&K;5_LU{KdPlTV9EK&Bq6oDB!Njt^JDZ?-L*+PtmC~6Y(LuXdhssUkLE|HsAex< zPgq%xdQ}y>iLXL*772;;p7pOF=NGO5V2&HNvbfA^em-%TIfXB+v%8-Jbm^>b4>F9wV~O>bbHdwew|Yv7fw{QfWzr_*DX2Lu2*OE zB3`k3QBHV}1-Ao^&({{*S0}0|OigTrtnv%G z&8fP&qTA+nCYg?Bq2^@ zOVycsRkO4> zWGU;LCXy}ygrzE10=H{JlJlXS-LGH|QTYLo-fT9m$V_XsC8w?dJJp#_G`prhR7J%5+o{E2^g@o1vLU&E8ru*v)XuylHzBn~GZ&Qi@~GYIg?+ zIzTJE_n_;Gv(V61#el`|#E|&$!Fqr!bClSQI@&iD(1D&IyQY{I(?ZCXp0hP3H6D3n z<)PJKZswoE5ebN@z(ZeLI=SCj@$ydznp8Vq8MF?M06zX!Spup&gQQ;9O>1VgIGiqA zPzxqE54srlvn$B>D*Riw5r`97*E-Cc{{m>$79MEw(ZP&NZDsuuQhMmN+bajK=MRy* zPXyD)0MQwEL!Z^CaoKNtKjq4#l<&qba@BalW+Q}I*X=A9mB^r-CZ;X&uVo$LpCh=Y zmQ)uV_@9aDoq>Bg@f-Yv-_w+b-+;%~+~Az3-cutkq#;huQF*PHkg@$tpx{CY4S!e= z)QWYnoED}Cf5~fH+MFD?Hz!?M{Lt*4CenzGd#4ndbPAe7mDfj@#WLdBs~h=Q8lvg$ za6MuVrdHn7#f^rpQG4p?X%VN~m|V*`Q_)=2dRN$lqG0+o(nc*ZfBDzQmAda|Q>Gpe zQToq^wtb3Tdw3>*qm?Z(!&)t(u8rf0dF3Z+bKP?FBzcBs_n7I~ayN?=@9(9ZHJg;p zFhS-sMpeF1-!-?^yA-=oNh&rUNLB*3H^(#?mbP)%lpbroKrvo(B(!o2(EQqfrH3@tzRo44z#S`@1n2f0Jar_Z zFmx>+1s5wRSK5^y$h^mD!XjAg#L(3`F5Hq>T=pHOYR%HVS*c^{pNa%a?y{e=+z)eb z=!Q=|$8`Gcc`xkLE#s^%rS1fnP>4ir+*0z9ir!K&0j69a(pi!Gq)Uk zkbAcO!AoJLaQR!}MQexxDpt3*rXApB4dy%(M`PB9|V(2Jf}73>2W&H83Ct3 z+ZD@3MmA>jD!VF&kZle3U5wLM+;7OB0O0)i>C$KbG(q8XVunq zuVUU1UZ^nu%Si_3v1}OW+#cF+I#}4d?7T`mqkw!0hxA^3)!rbil*Q;OBHgdN3*U0k z2n}CN0Q~BUUzS;JF}=9>`N{nQ5cACpVDpT|oMM9?q$%n?6?yG&U5nK`#xLy1Jp^j8 z_|_$_G`tS}X0E>|LxLtxv&E5!>r;LZ+cVo*ni1BJjFB4R3<=IDQb14!XE?@_hKWa! z)KX^7TvrAogNdPnS5|-8dlU-u#-z8 zG1P*)XfT@y%BSyeSGi+RwL2o%$By^jgQTi#r%tTeIpZ~@y|>@TH(uD~DIFxSpa{RsK?Zv-=T(h-ihF-@YYzi;p3pB<$E<(?*=V|SI)&xj%U|0 zgS2g{z-0I|ZKKWCYs*ni)W^c%9+;u`IB*c=8myu-RYY+Vz%eWm}#dW2+NWEXrGltfS;to~#CTw{l~Zs=;HMDEx$gswNq6|?XG*FI&r z(!YwXClaX)JQy}R+C(uN_b0gq&MQaGd@%GZiTjksV#c#`PVrt10bK842KeyzWyLo!4Ieuw`^ zo#0jobQMjS{Oq00K@x!hX~d85YvP3(>@2MdXM}nC`00MO==KeU1?5jZODOw=wvg0M zPTr{$NBM{mA5XXFZdOu4EP`5Jpn^X!_blfPiXNB>G^!ci4H8&NNsMa339)?fIJ+a# zv+S7b7P|WAM~8*A+PT4XSR&2DxS*q5G0#DgRO%~7@+0+FyRG>~9C%cATRu`G!e42~ zmj7cY%e>9Gqe9AWk$Mq90zX2R?PI9^7iDv zJit(&j-}Ftm*DVP225*5sL=Bm;Zp}m!d)(XAwgwR(dC6!hw4cVfeQ-wz5JKA`Hiff zV1RdiDRFuIL|SwW4%XCi+)C^O=I;Q29J4I@GfaQ@uAeUfMxP;dDc{9s)iRZ$+`*NOg|8deF|G{n0vD9MAv*bgudO zXoOkgaPuG@!~XmfJNg`AZ3ZsOIZ;gZ{c?ImOSTkyXY`HL{eQPdVLkM0tUVNy^8?}3 zw_YCgJ$n7TGj=b};9NyYYOh7x-?3hW0=k2vo4Z?s)5`9i_q)5N&O#bI=DOvZ#ijJV-P2f6gzFodAa?Pa zcIdQexgTH>&XSwJeqN!R6t|!L@}dD55vTm{C&&qpfcrH5FWzX1Z&yt|UdJ+QWY&^| z@7$6f1=kO-3x$<5H^#cW9+5FEizv=l#@B;<-^gh)$?u)N!S_nT{zdHV<{$w0QGiBZ zSiTdtR_yXjR^(`EZ?Mo*1Rb)vGEi(efeK#*bUAv0^nO4;xFOP*-O(C)<1m(8eJzO1 zK&_yl^^x1#T7GW#ur89{go}d!Ffw6`ci|{Edw*kR<412X2oEGjf!r(1Bn%qXnoFB- zp?8~o?AdK&ccU6bj>INk&t-hbR4+c>{A%cpN&^uzVq@U|2=(i#zVrTG=^QV(zWP==w;?T@etLO0WC5twitVyBIV~x>Lf8V4&uW6U*;b_-&nJfn1?^lrr2Uz zEPGHiWQ+@}FP4zpsO~7s^)6YZQ0h>?<)+TB_vY!=?~o2BB)|Qnlf2hJyACisl`s)L zMOp8+y>&z?A_$P=!gn~)Cb)>z|1wc`q2m4SUsG}d1rYh^{jD=`pl*uhEDu|2W(Gv9 zI5gaiGOg>Q?<^tZB)D?2z`T|*C%}HMNtks}ew8k~;p>r3Mz3N?*eEc)D{HM0qufv4 zUD7pe3@vNMZHgqiBVrElH-t`E7BFneaJ3dnALfgzUHJSKTspe;wj@mH9k}Y?`{A9i zEQ?Ykhl^lb8Dl?n_^SJtc$&<0swIO5LHHZJXLI9dc@r3WTN60VfA8VoYvkZy|7dK? zSD|y{$_X)5rG=NrQ%-K(v7>-622|QxbSMiP@BqGi+~L$*GPd`{rqzPAekp)XdPT%}n$ zHX{d@QRiGTFK_iIR*Xi0X@U2I@eg=&OfNXR*4(-4n0Ma3%(ec`=|65KjRLqAHhKT? zVA%?Uq$DrQMlz30DJKl z9C06lOTELET|yBM5X#~IUjSoNY-`rv0>6*=vK7X;clOCW8gY-kORa&O=t9V zoIr|Vq-jcNpL6l`a!JhoAt3O0cC(-GF^R(x!@JoURjS=nZKD-2J6g`q1&TOa^R8`z z@NUr&aJv|K8Xl)>u7bc59rk9~|t++K3A4!{IugvLFY2aQPy9 zwW|0+WP`nA;zf$>LI};NO7;6LWK19XOx+`^8404s21BLE($?AfH>|jTT5J|S5C`Zn?n<+Q;Qw7BfI&4Bi{$Uz`~Luh>Z9ub literal 0 HcmV?d00001 diff --git a/docs/metrics-document.md b/docs/metrics-document.md new file mode 100644 index 000000000..7a2227736 --- /dev/null +++ b/docs/metrics-document.md @@ -0,0 +1,173 @@ +# Adding observability in the JDBC wrapper via metrics + +# Introduction + +AWS Aurora are databases provided by AWS that are hosted in the AWS cloud. Currently available in Aurora are instances of MySQL and PostgreSQL databases. In order to enable the access from an application to an Aurora database, users need to set up a driver that has the ability to connect and interact with an Aurora database instance set up remotely. Existing drivers for MySQL and PostgreSQL work fine with Aurora instances, but do not take advantage of any of the additional features provided by the Aurora databases. + +Recently, AWS has created the AWS JDBC Wrapper driver. The AWS JDBC Wrapper is a driver application that is not a driver in itself, but more of a driver enhancer - an application that set up alongside a database driver, enables some Aurora-specific features to the database driver. Examples of those features are the ability of driver failover (link to doc) and integration with IAM and AWS Secrets Manager (link). The structure of the AWS JDBC Wrapper is organized in plugins, where each new wrapper feature is isolated and independent from other features. That allows users to select which wrapper features/plugins they require in their application workflow. + +# Problem + +Through the driver, applications interact with a remote database by sending requests. Those requests trigger a chain of execution where every one of the plugins enabled by the user is activated and executed, until the request reaches the database and returns. + +In its current form, the AWS JDBC Wrapper is much like a black box: during the chain of execution, there is actually no possibility to measure the individual performance of each plugin throughout the execution of a query. Which means that in the event of loss of performance while using the Wrapper, the troubleshooting process is manual, requiring users to dive deep inside application logs and manually inspecting the behavior of each plugin enabled. + +This project aims to add internal observability of the Wrapper performance at plugin level. + +# Goals + +The goals of the project are the following: + +- Instrument the wrapper code in order to obtain metrics and traces at plugin level +- Define observability interfaces that make the code agnostic to a specific observability library or tool +- Add plugin-specific metrics to existing plugins +- Implement connectors/exporters to visualize the generated observability data + +# Proposed solution + +## Definitions + +As previously mentioned in the Goals section, one important thing we aim is while adding observability to the driver, to not attach our codebase to any vendor specific or specific existing metrics library. + +This is not only to avoid adding a new external dependency to the wrapper, but mostly due to the fact that very often, applications that use the wrapper to interact with a database already have their own observability and/or monitoring implementations. + +When that is the case, we would like ideally to have wrapper specific metrics simply added to the already existing users application monitoring. In order to achieve that, we needed to find some standard observability method that would fit most users workflow. For this project, we have decided to follow OpenTelemetry (link) notations and definitions for our monitoring entities. + +OpenTelemetry aims to define vendor and language agnostic specs on how to monitor applications. + +The entities we introduce in this project are the following: Metrics and Traces. + +### Traces + +We define a trace as a defined sequence of an application execution, specifically defined by its start and its end. A trace can contain the entire application execution, or a single atomic operation. Traces can be related either by hierarchy either with symbolic links. + +Collecting libraries will see the traces boundaries and measure the execution time for a particular trace. For a better understanding on the application behavior, different attributes can be attached to a trace, such as a success status and/or status code. + +In our project, traces are represented as `TelemetryContext` objects. A `TelemetryContext` object will be created/opened at every plugin invocation or execution. + +`TelemetryContext` objects are similar to what OpenTelemetry define as *spans*, and also to the concept of *trace* in AWS X-Ray, the trace visualization mechanism featured in AWS Cloudwatch. + +### Metrics + +Metrics will also be added to our observability suite for the numeric data that will be collected throughout the application execution. + +For metrics, we follow the same standard defined by OpenTelemetry, which consists of: +- Counters +- Gauge +- Histogram + +Gauges will be used for situations where the numeric data is varies, but not incrementally throughout the execution of the application. An example of that would be a hit/miss ratio for a cache. Further information on those 3 entities, and to which kind of data they suit better can be found in the OpenTelemetry documentation (link). + +### Warning + +It is important to state that despite the fact that the entities and objects that we define here are either similar, or either can be mapped to OpenTelemetry concepts, we are still creating our observability layer independently of any available library or suite. Using the OpenTelemetry libraries for JAVA require to write the mapping (in this case, an interface instantiation) from our definitions to the OpenTelemetry objects. + +## Plugin-level tracing + +Having defined our objects and entities for tracing, our solution will be first to open a telemetry context for every wrapper execution. In that way, the monitoring will be able to show traces for every different operation executed by the wrapper. Every operation, such as `createStatement()` and `executeQuery()` will now be individually traced. + +In order to achieve that, a `TelemetryContext` object will be opened inside every call to `executeWithPlugins()` in the `WrapperUtils` class. The object will be created before the execution and closed right after, tracing its execution. + +Then, once those traces are created, we will also open a new telemetry context for each plugin execution inside the wrapper execution. + +
+ +The plugin execution chain is created in the `ConnectionPluginManager` class, where execution lambdas are generated and invoke the `execute()` method for every plugin subscribed to a given execution. + +Inside the function that generates that plugin execution chain, we will encapsulate those lambdas with `TelemetryContext` objects that will then trace every plugin execution individually. + +
+ +The image above displays the different telemetry contexts that are opened inside the plugin execution chain, allowing plugin specific tracing. + +## Plugin-specific metrics + +In addition to plugin-level tracing, we will also introduce specific metrics related to performance for each available plugin. The list of the available plugins that will have metrics added is the following: + +- Data cache plugin +- Failover plugin +- IAM authentication plugin +- AWS Secrets manager plugin +- EFM (Enhanced failure monitoring) plugin + +### Data cache plugin + +Metrics: +(Metric name | metric type | Unit (if applicable) | Dimensions | Description) + +- Execution counter\ + Type: counter\ + Counts the amount of times the plugin has been executed + +- Put counter\ + Type: counter\ + Counts the amount of times any object has been put into the cache + +- Get counter\ + Type: counter\ + Counts the amount of times any object has been looked up in the cache + +- Hit counter\ + Type: counter\ + Counts the amounts of time an object was looked up in the cache and retrieved + +- Hit counter per query\ + Type: counter\ + Dimension: query\ + Counts the amounts of time a given query was looked up in the cache and retrieved + +- Miss counter\ + Type: counter\ + Counts the amounts of time an object was looked up in the cache and not found + +- Miss counter per query\ + Type: counter\ + Dimension: query + Counts the amounts of time each query was looked up in the cache and not found + +- Hit/Miss ratio\ + Type: percentage + In a given frequency, computes the percentage of hits / the amount of lookups + +- Hit/Miss ratio\ + Type: gauge\ + Dimension: query\ + In a given frequency, computes the percentage of hits / the amount of lookups for each query + +- Cache clear counter\ + Type: counter\ + Counts the amount of times the data cache has been cleared + +### Failover plugin + +Metrics: +(Metric name | metric type | Unit (if applicable) | Dimensions | Description) + +- Execution counter\ + Type: counter\ + Counts the amount of times the plugin has been executed + +- Failover trigger counter\ + Type: counter\ + Counts the amount of times failover has been triggered + +- Writer failover counter\ + Type; counter\ + Counts the amount of time failover was triggered and the driver has reconnected to a writer instance + +- Writer failover counter\ + Type; counter\ + Counts the amount of time failover was triggered and the driver has reconnected to a writer instance + +- Reader failover counter\ + Type; counter\ + Counts the amount of time failover was triggered and the driver has reconnected to a reader instance + +### EFM Plugin + +Metrics: +(Metric name | metric type | Unit (if applicable) | Dimensions | Description) + +- Execution counter\ + Type: counter\ + Counts the amount of times the plugin has been executed diff --git a/examples/AWSDriverExample/build.gradle.kts b/examples/AWSDriverExample/build.gradle.kts index e60e85d8a..8dd3b6330 100644 --- a/examples/AWSDriverExample/build.gradle.kts +++ b/examples/AWSDriverExample/build.gradle.kts @@ -19,4 +19,9 @@ dependencies { implementation("software.amazon.awssdk:rds:2.17.289") implementation("software.amazon.awssdk:secretsmanager:2.17.285") implementation(project(":aws-advanced-jdbc-wrapper")) + implementation("io.dropwizard.metrics:metrics-core:4.2.13") + implementation("io.opentelemetry:opentelemetry-api:1.20.1") + implementation("io.opentelemetry:opentelemetry-sdk:1.20.1") + implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.20.1") + implementation("com.amazonaws:aws-xray-recorder-sdk-core:2.13.0") } diff --git a/examples/AWSDriverExample/src/main/java/software/amazon/MetricsExample.java b/examples/AWSDriverExample/src/main/java/software/amazon/MetricsExample.java new file mode 100644 index 000000000..66285464c --- /dev/null +++ b/examples/AWSDriverExample/src/main/java/software/amazon/MetricsExample.java @@ -0,0 +1,115 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon; + +import com.amazonaws.xray.AWSXRay; +import com.amazonaws.xray.AWSXRayRecorderBuilder; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import software.amazon.jdbc.PropertyDefinition; + +public class MetricsExample { + + // User configures connection properties here + public static final String POSTGRESQL_CONNECTION_STRING = + "jdbc:aws-wrapper:postgresql://atlas-postgres.cluster-czygpppufgy4.us-east-2.rds.amazonaws.com:5432/postgres"; + private static final String USERNAME = "pgadmin"; + private static final String PASSWORD = "my_password_2020"; + + private static final String SQL_DBLIST = "SELECT datname FROM pg_database;"; + private static final String SQL_SLEEP = "select pg_sleep(120);"; + private static final String SQL_TABLELIST = "select * from information_schema.tables where table_schema='public';"; + + private final OpenTelemetry openTelemetry; + private final OtlpGrpcSpanExporter spanExporter; + private final OtlpGrpcMetricExporter metricExporter; + private final SdkTracerProvider tracerProvider; + private final SdkMeterProvider meterProvider; + + public MetricsExample() { + spanExporter = OtlpGrpcSpanExporter.builder().setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")).build(); + metricExporter = OtlpGrpcMetricExporter.builder().setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")).build(); + + tracerProvider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)).build(); + meterProvider = SdkMeterProvider.builder() + .registerMetricReader(PeriodicMetricReader.builder(metricExporter).setInterval(15, TimeUnit.SECONDS).build()) + .build(); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setMeterProvider(meterProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal(); + } + + public void doWork(Properties properties) throws SQLException { + try (final Connection conn = DriverManager.getConnection(POSTGRESQL_CONNECTION_STRING, properties); + final Statement statement = conn.createStatement(); + final ResultSet rs = statement.executeQuery(SQL_SLEEP)) { + System.out.println(Util.getResult(rs)); + } + } + + public static void main(String[] args) throws SQLException { + final MetricsExample example = new MetricsExample(); + + AWSXRayRecorderBuilder builder = AWSXRayRecorderBuilder.standard(); + AWSXRay.setGlobalRecorder(builder.build()); + + final Properties properties = new Properties(); + properties.setProperty(PropertyDefinition.PLUGINS.name, "dataCache, efm, failover"); + properties.setProperty(PropertyDefinition.USER.name, USERNAME); + properties.setProperty(PropertyDefinition.PASSWORD.name, PASSWORD); + + properties.setProperty(PropertyDefinition.ENABLE_TELEMETRY.name, String.valueOf(true)); + // Traces: Available values are XRAY, OTLP and NONE + properties.setProperty(PropertyDefinition.TELEMETRY_TRACES_BACKEND.name, "XRAY"); + // Metrics: Available values are OTLP and NONE + properties.setProperty(PropertyDefinition.TELEMETRY_METRICS_BACKEND.name, "NONE"); + + System.out.println("-- starting metrics e2e test"); + + System.out.println("-- env vars"); + System.out.println("AWS_REGION: " + System.getenv("AWS_REGION")); + System.out.println("OTEL_METRICS_EXPORTER: " + System.getenv("OTEL_METRICS_EXPORTER")); + System.out.println("OTEL_TRACES_EXPORTER: " + System.getenv("OTEL_TRACES_EXPORTER")); + System.out.println("OTEL_LOGS_EXPORTER: " + System.getenv("OTEL_LOGS_EXPORTER")); + System.out.println("OTEL_EXPORTER_OTLP_ENDPOINT: " + System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")); + System.out.println("OTEL_RESOURCE_ATTRIBUTES: " + System.getenv("OTEL_RESOURCE_ATTRIBUTES")); + + System.out.println("-- running application"); + + AWSXRay.beginSegment("application"); + example.doWork(properties); + AWSXRay.endSegment(); + } +} diff --git a/wrapper/build.gradle.kts b/wrapper/build.gradle.kts index fcb98233d..f6bc2c8e3 100644 --- a/wrapper/build.gradle.kts +++ b/wrapper/build.gradle.kts @@ -36,6 +36,10 @@ dependencies { compileOnly("org.postgresql:postgresql:42.5.0") compileOnly("org.mariadb.jdbc:mariadb-java-client:3.1.0") compileOnly("org.osgi:org.osgi.core:4.3.0") + compileOnly("io.opentelemetry:opentelemetry-api:1.20.1") + compileOnly("io.opentelemetry:opentelemetry-exporter-otlp:1.20.1") + compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.20.1-alpha") + compileOnly("com.amazonaws:aws-xray-recorder-sdk-core:2.13.0") testImplementation("org.junit.platform:junit-platform-commons:1.9.0") testImplementation("org.junit.platform:junit-platform-engine:1.9.0") diff --git a/wrapper/src/main/java/software/amazon/jdbc/ConnectionPluginManager.java b/wrapper/src/main/java/software/amazon/jdbc/ConnectionPluginManager.java index c72cae28b..4239b7094 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/ConnectionPluginManager.java +++ b/wrapper/src/main/java/software/amazon/jdbc/ConnectionPluginManager.java @@ -30,17 +30,28 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import software.amazon.jdbc.cleanup.CanReleaseResources; +import software.amazon.jdbc.plugin.AuroraConnectionTrackerPlugin; import software.amazon.jdbc.plugin.AuroraConnectionTrackerPluginFactory; +import software.amazon.jdbc.plugin.AuroraHostListConnectionPlugin; import software.amazon.jdbc.plugin.AuroraHostListConnectionPluginFactory; +import software.amazon.jdbc.plugin.AwsSecretsManagerConnectionPlugin; import software.amazon.jdbc.plugin.AwsSecretsManagerConnectionPluginFactory; +import software.amazon.jdbc.plugin.DataCacheConnectionPlugin; import software.amazon.jdbc.plugin.DataCacheConnectionPluginFactory; import software.amazon.jdbc.plugin.DefaultConnectionPlugin; +import software.amazon.jdbc.plugin.ExecutionTimeConnectionPlugin; import software.amazon.jdbc.plugin.ExecutionTimeConnectionPluginFactory; +import software.amazon.jdbc.plugin.IamAuthConnectionPlugin; import software.amazon.jdbc.plugin.IamAuthConnectionPluginFactory; +import software.amazon.jdbc.plugin.LogQueryConnectionPlugin; import software.amazon.jdbc.plugin.LogQueryConnectionPluginFactory; +import software.amazon.jdbc.plugin.efm.HostMonitoringConnectionPlugin; import software.amazon.jdbc.plugin.efm.HostMonitoringConnectionPluginFactory; +import software.amazon.jdbc.plugin.failover.FailoverConnectionPlugin; import software.amazon.jdbc.plugin.failover.FailoverConnectionPluginFactory; +import software.amazon.jdbc.plugin.readwritesplitting.ReadWriteSplittingPlugin; import software.amazon.jdbc.plugin.readwritesplitting.ReadWriteSplittingPluginFactory; +import software.amazon.jdbc.plugin.staledns.AuroraStaleDnsPlugin; import software.amazon.jdbc.plugin.staledns.AuroraStaleDnsPluginFactory; import software.amazon.jdbc.profile.DriverConfigurationProfiles; import software.amazon.jdbc.util.Messages; @@ -48,6 +59,8 @@ import software.amazon.jdbc.util.SqlState; import software.amazon.jdbc.util.StringUtils; import software.amazon.jdbc.util.WrapperUtils; +import software.amazon.jdbc.util.telemetry.TelemetryContext; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; import software.amazon.jdbc.wrapper.ConnectionWrapper; /** @@ -75,6 +88,24 @@ public class ConnectionPluginManager implements CanReleaseResources { } }; + protected static final Map, String> pluginNameByClass = + new HashMap, String>() { + { + put(ExecutionTimeConnectionPlugin.class, "executionTime"); + put(AuroraConnectionTrackerPlugin.class, "auroraConnectionTracker"); + put(AuroraHostListConnectionPlugin.class, "auroraHostList"); + put(LogQueryConnectionPlugin.class, "logQuery"); + put(DataCacheConnectionPlugin.class, "dataCache"); + put(HostMonitoringConnectionPlugin.class, "efm"); + put(FailoverConnectionPlugin.class, "failover"); + put(IamAuthConnectionPlugin.class, "iam"); + put(AwsSecretsManagerConnectionPlugin.class, "awsSecretsManager"); + put(AuroraStaleDnsPlugin.class, "auroraStaleDns"); + put(ReadWriteSplittingPlugin.class, "readWriteSplitting"); + put(DefaultConnectionPlugin.class, "driver"); + } + }; + protected static final String DEFAULT_PLUGINS = ""; private static final Logger LOGGER = Logger.getLogger(ConnectionPluginManager.class.getName()); @@ -91,13 +122,17 @@ public class ConnectionPluginManager implements CanReleaseResources { protected final ConnectionProvider connectionProvider; protected final ConnectionWrapper connectionWrapper; protected PluginService pluginService; + protected final TelemetryFactory telemetryFactory; @SuppressWarnings("rawtypes") protected final Map pluginChainFuncMap = new HashMap<>(); - public ConnectionPluginManager(ConnectionProvider connectionProvider, ConnectionWrapper connectionWrapper) { + public ConnectionPluginManager(ConnectionProvider connectionProvider, + ConnectionWrapper connectionWrapper, + TelemetryFactory telemetryFactory) { this.connectionProvider = connectionProvider; this.connectionWrapper = connectionWrapper; + this.telemetryFactory = telemetryFactory; } /** @@ -108,8 +143,9 @@ public ConnectionPluginManager(ConnectionProvider connectionProvider, Connection Properties props, ArrayList plugins, ConnectionWrapper connectionWrapper, - PluginService pluginService) { - this(connectionProvider, props, plugins, connectionWrapper); + PluginService pluginService, + TelemetryFactory telemetryFactory) { + this(connectionProvider, props, plugins, connectionWrapper, telemetryFactory); this.pluginService = pluginService; } @@ -120,11 +156,13 @@ public ConnectionPluginManager(ConnectionProvider connectionProvider, Connection ConnectionProvider connectionProvider, Properties props, ArrayList plugins, - ConnectionWrapper connectionWrapper) { + ConnectionWrapper connectionWrapper, + TelemetryFactory telemetryFactory) { this.connectionProvider = connectionProvider; this.props = props; this.plugins = plugins; this.connectionWrapper = connectionWrapper; + this.telemetryFactory = telemetryFactory; } public void lock() { @@ -251,6 +289,31 @@ protected T executeWithSubscribedPlugins( return pluginChainFunc.call(pluginPipeline, jdbcMethodFunc); } + + protected T executeWithTelemetry( + final @NonNull JdbcCallable execution, + final @NonNull String methodName, + final @NonNull String pluginName) throws E { + TelemetryContext context = null; + try { + try { + final String contextName = methodName + " - " + pluginName; + context = telemetryFactory.openTelemetryContext(contextName); + } catch (Exception e) { + throw new RuntimeException(e); + } + return execution.call(); + } finally { + if (context != null) { + try { + context.close(); + } catch (Exception e) { + // Ignore if an exception is thrown + } + } + } + } + @Nullable protected PluginChainJdbcCallable makePluginChainFunc( final @NonNull String methodName) { @@ -259,6 +322,7 @@ protected PluginChainJdbcCallable makePluginChain for (int i = this.plugins.size() - 1; i >= 0; i--) { final ConnectionPlugin plugin = this.plugins.get(i); + final String pluginName = pluginNameByClass.getOrDefault(plugin.getClass(), plugin.getClass().getSimpleName()); Set pluginSubscribedMethods = plugin.getSubscribedMethods(); boolean isSubscribed = pluginSubscribedMethods.contains(ALL_METHODS) @@ -266,11 +330,13 @@ protected PluginChainJdbcCallable makePluginChain if (isSubscribed) { if (pluginChainFunc == null) { - pluginChainFunc = (pipelineFunc, jdbcFunc) -> pipelineFunc.call(plugin, jdbcFunc); + pluginChainFunc = (pipelineFunc, jdbcFunc) -> + executeWithTelemetry(() -> pipelineFunc.call(plugin, jdbcFunc), methodName, pluginName); } else { final PluginChainJdbcCallable finalPluginChainFunc = pluginChainFunc; pluginChainFunc = (pipelineFunc, jdbcFunc) -> - pipelineFunc.call(plugin, () -> finalPluginChainFunc.call(pipelineFunc, jdbcFunc)); + executeWithTelemetry(() -> pipelineFunc.call(plugin, () -> finalPluginChainFunc.call(pipelineFunc, jdbcFunc)), + methodName, pluginName); } } } @@ -306,6 +372,10 @@ public ConnectionWrapper getConnectionWrapper() { return this.connectionWrapper; } + public TelemetryFactory getTelemetryFactory() { + return this.telemetryFactory; + } + public T execute( final Class resultType, final Class exceptionClass, diff --git a/wrapper/src/main/java/software/amazon/jdbc/PluginService.java b/wrapper/src/main/java/software/amazon/jdbc/PluginService.java index 593eba2a4..f6dca0052 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/PluginService.java +++ b/wrapper/src/main/java/software/amazon/jdbc/PluginService.java @@ -25,6 +25,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import software.amazon.jdbc.exceptions.ExceptionHandler; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; /** * Interface for retrieving the current active {@link Connection} and its {@link HostSpec}. @@ -67,4 +68,7 @@ EnumSet setCurrentConnection( void forceRefreshHostList(Connection connection) throws SQLException; Connection connect(HostSpec hostSpec, Properties props) throws SQLException; + + TelemetryFactory getTelemetryFactory(); + } diff --git a/wrapper/src/main/java/software/amazon/jdbc/PluginServiceImpl.java b/wrapper/src/main/java/software/amazon/jdbc/PluginServiceImpl.java index dc2e3ae87..167df7767 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/PluginServiceImpl.java +++ b/wrapper/src/main/java/software/amazon/jdbc/PluginServiceImpl.java @@ -35,6 +35,7 @@ import software.amazon.jdbc.hostlistprovider.StaticHostListProvider; import software.amazon.jdbc.util.ExpiringCache; import software.amazon.jdbc.util.Messages; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; public class PluginServiceImpl implements PluginService, CanReleaseResources, HostListProviderService, PluginManagerService { @@ -447,4 +448,9 @@ public boolean isLoginException(Throwable throwable) { public boolean isLoginException(String sqlState) { return this.exceptionManager.isLoginException(this.driverProtocol, sqlState); } + + public TelemetryFactory getTelemetryFactory() { + return this.pluginManager.getTelemetryFactory(); + } + } diff --git a/wrapper/src/main/java/software/amazon/jdbc/PropertyDefinition.java b/wrapper/src/main/java/software/amazon/jdbc/PropertyDefinition.java index 0b142ac47..626d6f5be 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/PropertyDefinition.java +++ b/wrapper/src/main/java/software/amazon/jdbc/PropertyDefinition.java @@ -61,6 +61,41 @@ public class PropertyDefinition { new AwsWrapperProperty( "database", null, "Driver database name"); + public static final AwsWrapperProperty ENABLE_TELEMETRY = + new AwsWrapperProperty( + "enableTelemetry", "false", + "Enables telemetry and observability of the driver"); + + public static final AwsWrapperProperty TELEMETRY_TOPLEVEL_TRACING = + new AwsWrapperProperty( + "telemetryToplevelTracing", + null, + "Method to handle top level traces when application does not have monitoring set up", + false, + new String[] { + "IGNORE", "WRAP", "SUBMIT" + }); + + public static final AwsWrapperProperty TELEMETRY_TRACES_BACKEND = + new AwsWrapperProperty( + "telemetryTracesBackend", + null, + "Method to export telemetry traces of the driver", + false, + new String[] { + "XRAY", "OTLP", "NONE" + }); + + public static final AwsWrapperProperty TELEMETRY_METRICS_BACKEND = + new AwsWrapperProperty( + "telemetryMetricsBackend", + null, + "Method to export telemetry metrics of the driver", + false, + new String[] { + "OTLP", "NONE" + }); + private static final Map PROPS_BY_NAME = new HashMap<>(); diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPlugin.java index c1734aec2..7f79b8394 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPlugin.java @@ -27,16 +27,13 @@ import software.amazon.jdbc.PluginService; import software.amazon.jdbc.hostlistprovider.AuroraHostListProvider; import software.amazon.jdbc.util.Messages; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryGauge; public class AuroraHostListConnectionPlugin extends AbstractConnectionPlugin { private static final Set subscribedMethods = Collections.unmodifiableSet(new HashSet<>( Collections.singletonList("initHostProvider"))); - private final PluginService pluginService; - - public AuroraHostListConnectionPlugin(PluginService pluginService, Properties properties) { - this.pluginService = pluginService; - } @Override public Set getSubscribedMethods() { diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPluginFactory.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPluginFactory.java index 13be8bd10..068fc21f5 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPluginFactory.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/AuroraHostListConnectionPluginFactory.java @@ -25,6 +25,6 @@ public class AuroraHostListConnectionPluginFactory implements ConnectionPluginFa @Override public ConnectionPlugin getInstance(PluginService pluginService, Properties props) { - return new AuroraHostListConnectionPlugin(pluginService, props); + return new AuroraHostListConnectionPlugin(); } } diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPlugin.java index 874805378..6245da3d0 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPlugin.java @@ -48,8 +48,12 @@ import java.util.logging.Logger; import software.amazon.jdbc.AwsWrapperProperty; import software.amazon.jdbc.JdbcCallable; +import software.amazon.jdbc.PluginService; import software.amazon.jdbc.util.Messages; import software.amazon.jdbc.util.StringUtils; +import software.amazon.jdbc.util.telemetry.TelemetryCounter; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryGauge; public class DataCacheConnectionPlugin extends AbstractConnectionPlugin { @@ -68,8 +72,21 @@ public class DataCacheConnectionPlugin extends AbstractConnectionPlugin { protected final String dataCacheTriggerCondition; - public DataCacheConnectionPlugin(Properties props) { + private final TelemetryFactory telemetryFactory; + private final TelemetryCounter hitCounter; + private final TelemetryCounter missCounter; + private final TelemetryCounter totalCallsCounter; + private final TelemetryGauge cacheSizeGauge; + + + public DataCacheConnectionPlugin(final PluginService pluginService, Properties props) { + this.telemetryFactory = pluginService.getTelemetryFactory(); this.dataCacheTriggerCondition = DATA_CACHE_TRIGGER_CONDITION.getString(props); + + this.hitCounter = telemetryFactory.createCounter("dataCache.cache.hit"); + this.missCounter = telemetryFactory.createCounter("dataCache.cache.miss"); + this.totalCallsCounter = telemetryFactory.createCounter("dataCache.cache.totalCalls"); + this.cacheSizeGauge = telemetryFactory.createGauge("dataCache.cache.size", dataCache.size()); } public static void clearCache() { @@ -89,6 +106,8 @@ public T execute(Class resultClass, Class excepti return jdbcMethodFunc.call(); } + totalCallsCounter.inc(); + ResultSet result; boolean needToCache = false; String sql = getQuery(jdbcMethodArgs); @@ -97,11 +116,13 @@ public T execute(Class resultClass, Class excepti result = dataCache.get(sql); if (result == null) { needToCache = true; + missCounter.inc(); LOGGER.finest( () -> Messages.get( "DataCacheConnectionPlugin.queryResultsCached", new Object[]{methodName, sql})); } else { + hitCounter.inc(); try { result.beforeFirst(); } catch (SQLException ex) { diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginFactory.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginFactory.java index 43642a17b..4375657ae 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginFactory.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginFactory.java @@ -25,6 +25,6 @@ public class DataCacheConnectionPluginFactory implements ConnectionPluginFactory @Override public ConnectionPlugin getInstance(PluginService pluginService, Properties props) { - return new DataCacheConnectionPlugin(props); + return new DataCacheConnectionPlugin(pluginService, props); } } diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPlugin.java index 83eec2021..210d9e14c 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPlugin.java @@ -32,11 +32,14 @@ import software.amazon.jdbc.AwsWrapperProperty; import software.amazon.jdbc.HostSpec; import software.amazon.jdbc.JdbcCallable; +import software.amazon.jdbc.PluginService; import software.amazon.jdbc.PropertyDefinition; import software.amazon.jdbc.authentication.AwsCredentialsManager; import software.amazon.jdbc.util.Messages; import software.amazon.jdbc.util.RdsUtils; import software.amazon.jdbc.util.StringUtils; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryGauge; public class IamAuthConnectionPlugin extends AbstractConnectionPlugin { @@ -64,6 +67,14 @@ public class IamAuthConnectionPlugin extends AbstractConnectionPlugin { protected final RdsUtils rdsUtils = new RdsUtils(); + private final TelemetryFactory telemetryFactory; + private final TelemetryGauge cacheSizeGauge; + + public IamAuthConnectionPlugin(final PluginService pluginService) { + this.telemetryFactory = pluginService.getTelemetryFactory(); + this.cacheSizeGauge = telemetryFactory.createGauge("iam.tokenCache.size", tokenCache.size()); + } + @Override public Set getSubscribedMethods() { return new HashSet<>(Collections.singletonList("connect")); diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPluginFactory.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPluginFactory.java index 69b3ba2dc..a1437701e 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPluginFactory.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/IamAuthConnectionPluginFactory.java @@ -24,6 +24,6 @@ public class IamAuthConnectionPluginFactory implements ConnectionPluginFactory { @Override public ConnectionPlugin getInstance(PluginService pluginService, Properties props) { - return new IamAuthConnectionPlugin(); + return new IamAuthConnectionPlugin(pluginService); } } diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/HostMonitoringConnectionPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/HostMonitoringConnectionPlugin.java index bf11097d1..b0f05f2e8 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/HostMonitoringConnectionPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/HostMonitoringConnectionPlugin.java @@ -41,6 +41,8 @@ import software.amazon.jdbc.cleanup.CanReleaseResources; import software.amazon.jdbc.plugin.AbstractConnectionPlugin; import software.amazon.jdbc.util.Messages; +import software.amazon.jdbc.util.telemetry.TelemetryCounter; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; /** * Monitor the server while the connection is executing methods for more sophisticated failure @@ -90,9 +92,13 @@ public class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin protected @NonNull Properties properties; private final @NonNull Supplier monitorServiceSupplier; private final @NonNull PluginService pluginService; + private final @NonNull TelemetryFactory telemetryFactory; + private final @NonNull Set nodeKeys = ConcurrentHashMap.newKeySet(); // Shared with monitor thread private MonitorService monitorService; + private final TelemetryCounter invalidatedConnectionCounter; + /** * Initialize the node monitoring plugin. * @@ -116,8 +122,11 @@ public HostMonitoringConnectionPlugin( } this.pluginService = pluginService; + this.telemetryFactory = pluginService.getTelemetryFactory(); this.properties = properties; this.monitorServiceSupplier = monitorServiceSupplier; + + this.invalidatedConnectionCounter = telemetryFactory.createCounter("efm.connection.invalidated"); } @Override diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorConnectionContext.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorConnectionContext.java index f3b40d4ef..31065da8d 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorConnectionContext.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorConnectionContext.java @@ -25,7 +25,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; + +import software.amazon.jdbc.PluginService; import software.amazon.jdbc.util.Messages; +import software.amazon.jdbc.util.telemetry.TelemetryCounter; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryGauge; /** * Monitoring context for each connection. This contains each connection's criteria for whether a @@ -35,6 +40,8 @@ public class MonitorConnectionContext { private static final Logger LOGGER = Logger.getLogger(MonitorConnectionContext.class.getName()); + private final TelemetryCounter abortedConnectionsCounter; + private final long failureDetectionIntervalMillis; private final long failureDetectionTimeMillis; private final long failureDetectionCount; @@ -64,13 +71,15 @@ public MonitorConnectionContext( Set hostAliases, long failureDetectionTimeMillis, long failureDetectionIntervalMillis, - long failureDetectionCount) { + long failureDetectionCount, + TelemetryCounter abortedConnectionsCounter) { this.connectionToAbort = connectionToAbort; // Variable is never written, so it does not need to be thread-safe this.hostAliases = new HashSet<>(hostAliases); this.failureDetectionTimeMillis = failureDetectionTimeMillis; this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; this.failureDetectionCount = failureDetectionCount; + this.abortedConnectionsCounter = abortedConnectionsCounter; } void setStartMonitorTimeNano(long startMonitorTimeNano) { @@ -139,6 +148,7 @@ synchronized void abortConnection() { } try { + abortedConnectionsCounter.inc(); this.connectionToAbort.close(); } catch (SQLException sqlEx) { // ignore diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorImpl.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorImpl.java index 18456d9e2..5bcf3cd80 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorImpl.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorImpl.java @@ -25,11 +25,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; + import org.checkerframework.checker.nullness.qual.NonNull; + import software.amazon.jdbc.HostSpec; import software.amazon.jdbc.PluginService; import software.amazon.jdbc.util.Messages; import software.amazon.jdbc.util.PropertyUtils; +import software.amazon.jdbc.util.telemetry.TelemetryContext; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryGauge; /** * This class uses a background thread to monitor a particular server with one or more active {@link @@ -57,6 +62,7 @@ static class ConnectionStatus { private final long monitorDisposalTimeMillis; private final Queue contexts = new ConcurrentLinkedQueue<>(); private final PluginService pluginService; + private final TelemetryFactory telemetryFactory; private final Properties properties; private final HostSpec hostSpec; private final AtomicLong contextLastUsedTimestampNano = new AtomicLong(); @@ -66,6 +72,9 @@ static class ConnectionStatus { private final AtomicBoolean isConnectionCheckIntervalInitialized = new AtomicBoolean(false); private Connection monitoringConn = null; + private final TelemetryGauge contextsSizeGauge; + private TelemetryContext telemetryContext; + /** * Store the monitoring configuration for a connection. * @@ -87,12 +96,14 @@ public MonitorImpl( long monitorDisposalTimeMillis, @NonNull MonitorService monitorService) { this.pluginService = pluginService; + this.telemetryFactory = pluginService.getTelemetryFactory(); this.hostSpec = hostSpec; this.properties = properties; this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; this.monitorService = monitorService; this.contextLastUsedTimestampNano.set(this.getCurrentTimeNano()); + this.contextsSizeGauge = telemetryFactory.createGauge("efm.contextsQueue.size", contexts.size()); } @Override @@ -134,6 +145,8 @@ public void clearContexts() { @Override public void run() { + this.telemetryContext = telemetryFactory.openTelemetryContext("monitoring thread"); + telemetryContext.setAttribute("url", hostSpec.getUrl()); try { this.stopped.set(false); while (true) { @@ -171,6 +184,13 @@ public void run() { // ignore } } + if (telemetryContext != null) { + try { + this.telemetryContext.close(); + } catch (Exception ex) { + // ignore + } + } this.stopped.set(true); } } @@ -202,7 +222,13 @@ ConnectionStatus checkConnectionStatus(final long shortestFailureDetectionInterv }); startNano = this.getCurrentTimeNano(); + TelemetryContext connectContext = telemetryFactory.openTelemetryContext("efm connect"); this.monitoringConn = this.pluginService.connect(this.hostSpec, monitoringConnProperties); + try { + connectContext.close(); + } catch (Exception ex) { + // ignore + } return new ConnectionStatus(true, this.getCurrentTimeNano() - startNano); } diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorServiceImpl.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorServiceImpl.java index 8d8e7e8c1..eef798df2 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorServiceImpl.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/efm/MonitorServiceImpl.java @@ -27,6 +27,8 @@ import software.amazon.jdbc.HostSpec; import software.amazon.jdbc.PluginService; import software.amazon.jdbc.util.Messages; +import software.amazon.jdbc.util.telemetry.TelemetryCounter; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; /** * This class handles the creation and clean up of monitoring threads to servers with one or more @@ -45,9 +47,13 @@ public class MonitorServiceImpl implements MonitorService { private MonitorThreadContainer threadContainer; final MonitorInitializer monitorInitializer; + final PluginService pluginService; + final TelemetryFactory telemetryFactory; + final TelemetryCounter abortedConnectionsCounter; public MonitorServiceImpl(final @NonNull PluginService pluginService) { this( + pluginService, (hostSpec, properties, monitorService) -> new MonitorImpl( pluginService, @@ -65,9 +71,12 @@ public MonitorServiceImpl(final @NonNull PluginService pluginService) { } MonitorServiceImpl( + PluginService pluginService, MonitorInitializer monitorInitializer, ExecutorServiceInitializer executorServiceInitializer) { - + this.pluginService = pluginService; + this.telemetryFactory = pluginService.getTelemetryFactory(); + this.abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); this.monitorInitializer = monitorInitializer; this.threadContainer = MonitorThreadContainer.getInstance(executorServiceInitializer); } @@ -98,7 +107,8 @@ public MonitorConnectionContext startMonitoring( nodeKeys, failureDetectionTimeMillis, failureDetectionIntervalMillis, - failureDetectionCount); + failureDetectionCount, + abortedConnectionsCounter); monitor.startMonitoring(context); diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsHelper.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsHelper.java index 03e013140..e2f40c11b 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsHelper.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsHelper.java @@ -36,12 +36,18 @@ import software.amazon.jdbc.util.RdsUtils; import software.amazon.jdbc.util.SqlState; import software.amazon.jdbc.util.Utils; +import software.amazon.jdbc.util.telemetry.TelemetryCounter; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; public class AuroraStaleDnsHelper { private static final Logger LOGGER = Logger.getLogger(AuroraStaleDnsHelper.class.getName()); private final PluginService pluginService; + private final TelemetryFactory telemetryFactory; + + private final TelemetryCounter staleDNSDetectedCounter; + private final RdsUtils rdsUtils = new RdsUtils(); private HostSpec writerHostSpec = null; @@ -54,6 +60,9 @@ public class AuroraStaleDnsHelper { public AuroraStaleDnsHelper(PluginService pluginService) { this.pluginService = pluginService; + this.telemetryFactory = pluginService.getTelemetryFactory(); + + this.staleDNSDetectedCounter = telemetryFactory.createCounter("staleDNS.stale.detected"); } public Connection getVerifiedConnection( @@ -134,6 +143,7 @@ public Connection getVerifiedConnection( LOGGER.fine(() -> Messages.get("AuroraStaleDnsHelper.staleDnsDetected", new Object[]{this.writerHostSpec})); + staleDNSDetectedCounter.inc(); Connection writerConn = this.pluginService.connect(this.writerHostSpec, props); diff --git a/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsPlugin.java b/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsPlugin.java index 5518a9fdc..941c20505 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsPlugin.java +++ b/wrapper/src/main/java/software/amazon/jdbc/plugin/staledns/AuroraStaleDnsPlugin.java @@ -18,7 +18,6 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/WrapperUtils.java b/wrapper/src/main/java/software/amazon/jdbc/util/WrapperUtils.java index f576f7460..d07c4d7ae 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/util/WrapperUtils.java +++ b/wrapper/src/main/java/software/amazon/jdbc/util/WrapperUtils.java @@ -55,6 +55,8 @@ import software.amazon.jdbc.ConnectionPluginManager; import software.amazon.jdbc.JdbcCallable; import software.amazon.jdbc.JdbcRunnable; +import software.amazon.jdbc.util.telemetry.TelemetryContext; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; import software.amazon.jdbc.wrapper.ArrayWrapper; import software.amazon.jdbc.wrapper.BlobWrapper; import software.amazon.jdbc.wrapper.CallableStatementWrapper; @@ -181,11 +183,20 @@ public static T executeWithPlugins( final Object... jdbcMethodArgs) { pluginManager.lock(); + TelemetryFactory telemetryFactory = pluginManager.getTelemetryFactory(); + TelemetryContext context = null; try { + try { + context = telemetryFactory.openTelemetryContext(methodName); + } catch (Exception e) { + throw new RuntimeException(e); + } final Object[] argsCopy = jdbcMethodArgs == null ? null : Arrays.copyOf(jdbcMethodArgs, jdbcMethodArgs.length); + context.setAttribute("methodName", methodName); + final T result = pluginManager.execute( resultClass, @@ -195,14 +206,23 @@ public static T executeWithPlugins( jdbcMethodFunc, argsCopy); + context.setSuccess(true); + try { return wrapWithProxyIfNeeded(resultClass, result, pluginManager); } catch (final InstantiationException e) { + context.setSuccess(false); throw new RuntimeException(e); } - } finally { pluginManager.unlock(); + if (context != null) { + try { + context.close(); + } catch (Exception e) { + // Ignore if an exception is thrown + } + } } } @@ -217,23 +237,42 @@ public static T executeWithPlugins( throws E { pluginManager.lock(); + TelemetryFactory telemetryFactory = pluginManager.getTelemetryFactory(); + TelemetryContext context = null; try { + try { + context = telemetryFactory.openTelemetryContext(methodName); + } catch (Exception e) { + throw new RuntimeException(e); + } final Object[] argsCopy = jdbcMethodArgs == null ? null : Arrays.copyOf(jdbcMethodArgs, jdbcMethodArgs.length); + context.setAttribute("methodName", methodName); + final T result = pluginManager.execute( resultClass, exceptionClass, methodInvokeOn, methodName, jdbcMethodFunc, argsCopy); + context.setSuccess(true); + try { return wrapWithProxyIfNeeded(resultClass, result, pluginManager); } catch (final InstantiationException e) { + context.setSuccess(false); throw new RuntimeException(e); } } finally { pluginManager.unlock(); + if (context != null) { + try { + context.close(); + } catch (Exception e) { + // Ignore if an exception is thrown + } + } } } diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/DefaultTelemetryFactory.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/DefaultTelemetryFactory.java new file mode 100644 index 000000000..9caef19ad --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/DefaultTelemetryFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import java.util.Properties; +import software.amazon.jdbc.PropertyDefinition; + +public class DefaultTelemetryFactory implements TelemetryFactory { + private final Properties properties; + + public DefaultTelemetryFactory(Properties properties) { + this.properties = properties; + } + + @Override + public TelemetryContext openTelemetryContext(String name) { + if (PropertyDefinition.ENABLE_TELEMETRY.getBoolean(properties)) { + if ("otlp".equalsIgnoreCase(PropertyDefinition.TELEMETRY_TRACES_BACKEND.getString(properties))) { + return OpenTelemetryFactory.openTelemetryContext(name); + } else if ("xray".equalsIgnoreCase(PropertyDefinition.TELEMETRY_TRACES_BACKEND.getString(properties))) { + return XRayTelemetryFactory.openTelemetryContext(name); + } + } + return new NullTelemetryContext(name); + } + + @Override + public TelemetryCounter createCounter(String name) { + if (PropertyDefinition.ENABLE_TELEMETRY.getBoolean(properties)) { + if ("otlp".equalsIgnoreCase(PropertyDefinition.TELEMETRY_METRICS_BACKEND.getString(properties))) { + return OpenTelemetryFactory.createCounter(name); + } + } + return new NullTelemetryCounter(name); + } + + @Override + public TelemetryGauge createGauge(String name, long measure) { + if (PropertyDefinition.ENABLE_TELEMETRY.getBoolean(properties)) { + if ("otlp".equalsIgnoreCase(PropertyDefinition.TELEMETRY_METRICS_BACKEND.getString(properties))) { + return OpenTelemetryFactory.createGauge(name, measure); + } + } + return new NullTelemetryGauge(name, measure); + } +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryContext.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryContext.java new file mode 100644 index 000000000..f50cd9d0d --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryContext.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public class NullTelemetryContext implements TelemetryContext { + + private String name; + + public NullTelemetryContext(String name) { + this.name = name; + } + + @Override + public void setSuccess(boolean success) { + } + + @Override + public void setAttribute(String key, String value) { + } + + @Override + public String getName() { + return name; + } + + @Override + public void close() { + } +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryCounter.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryCounter.java new file mode 100644 index 000000000..f1a247daa --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryCounter.java @@ -0,0 +1,39 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public class NullTelemetryCounter implements TelemetryCounter { + + private final String name; + + public NullTelemetryCounter(String name) { + this.name = name; + } + + @Override + public void add(long value) { + } + + @Override + public void inc() { + } + + public String getName() { + return name; + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryGauge.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryGauge.java new file mode 100644 index 000000000..de499edb7 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/NullTelemetryGauge.java @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public class NullTelemetryGauge implements TelemetryGauge { + + private final String name; + + private final long measure; + + public NullTelemetryGauge(String name, long measure) { + this.name = name; + this.measure = measure; + } + + public String getName() { + return name; + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryContext.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryContext.java new file mode 100644 index 000000000..110fcdbd9 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryContext.java @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; + +import java.time.Instant; + +public class OpenTelemetryContext implements TelemetryContext { + + private Span span; + + private Scope scope; + private String name; + + private final Tracer tracer; + + public OpenTelemetryContext(Tracer tracer, String name) { + this.name = name; + this.tracer = tracer; + + span = this.tracer.spanBuilder(name).setStartTimestamp(Instant.now()).startSpan(); + + scope = span.makeCurrent(); + } + + @Override + public void setSuccess(boolean success) { + if (success) { + span.setStatus(StatusCode.OK); + } else { + span.setStatus(StatusCode.ERROR); + } + } + + @Override + public void setAttribute(String key, String value) { + span.setAttribute(key, value); + } + + @Override + public String getName() { + return name; + } + + @Override + public String toString() { + return span.toString(); + } + + @Override + public void close() { + if (span != null) { + span.end(); + span = null; + } + if (scope != null) { + scope.close(); + scope = null; + } + } +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryCounter.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryCounter.java new file mode 100644 index 000000000..d575934b7 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryCounter.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; + +public class OpenTelemetryCounter implements TelemetryCounter { + + private final LongCounter counter; + + private final String name; + + private final Meter meter; + + OpenTelemetryCounter(Meter meter, String name) { + this.name = name; + this.meter = meter; + + counter = this.meter.counterBuilder(name).build(); + } + + @Override + public void add(long value) { + counter.add(value); + } + + @Override + public void inc() { + this.add(1); + } + + public String getName() { + return name; + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryFactory.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryFactory.java new file mode 100644 index 000000000..b6c328301 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Tracer; + +public class OpenTelemetryFactory { + + private static final String INSTRUMENTATION_NAME = TelemetryFactory.class.getName(); + + private static OpenTelemetry openTelemetry; + private static Tracer tracer; + private static Meter meter; + + private static final Object mutex = new Object(); + + private static OpenTelemetry getOpenTelemetry() { + if (openTelemetry == null) { + synchronized (mutex) { + if (openTelemetry == null) { + openTelemetry = GlobalOpenTelemetry.get(); + } + } + } + return openTelemetry; + } + + public static TelemetryContext openTelemetryContext(String name) { + tracer = getOpenTelemetry().getTracer(INSTRUMENTATION_NAME); + return new OpenTelemetryContext(tracer, name); + } + + public static TelemetryCounter createCounter(String name) { + meter = getOpenTelemetry().getMeter(INSTRUMENTATION_NAME); + return new OpenTelemetryCounter(meter, name); + } + + public static TelemetryGauge createGauge(String name, long measure) { + meter = getOpenTelemetry().getMeter(INSTRUMENTATION_NAME); + return new OpenTelemetryGauge(meter, name, measure); + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryGauge.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryGauge.java new file mode 100644 index 000000000..323050b2a --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryGauge.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableLongGauge; + +public class OpenTelemetryGauge implements TelemetryGauge { + + private final ObservableLongGauge gauge; + + private final String name; + + private final Meter meter; + + OpenTelemetryGauge(Meter meter, String name, long value) { + this.name = name; + this.meter = meter; + + this.gauge = this.meter.gaugeBuilder(name) + .ofLongs() + .buildWithCallback(entry -> entry.record(value)); + } + + public String getName() { + return name; + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryMetrics.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryMetrics.java new file mode 100644 index 000000000..d39a262e3 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/OpenTelemetryMetrics.java @@ -0,0 +1,4 @@ +package software.amazon.jdbc.util.telemetry; + +public class OpenTelemetryMetrics { +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryContext.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryContext.java new file mode 100644 index 000000000..e23118c15 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryContext.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public interface TelemetryContext extends AutoCloseable { + + void setSuccess(boolean success); + + void setAttribute(String key, String value); + + String getName(); + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryCounter.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryCounter.java new file mode 100644 index 000000000..95b2247f8 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryCounter.java @@ -0,0 +1,25 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public interface TelemetryCounter { + + void add(long value); + + void inc(); + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryFactory.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryFactory.java new file mode 100644 index 000000000..8ae11ec9d --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public interface TelemetryFactory { + + TelemetryContext openTelemetryContext(String name); + + TelemetryCounter createCounter(String name); + + TelemetryGauge createGauge(String name, long measure); + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryGauge.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryGauge.java new file mode 100644 index 000000000..2a255c614 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/TelemetryGauge.java @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public interface TelemetryGauge { + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryContext.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryContext.java new file mode 100644 index 000000000..4ff0bb426 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryContext.java @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +import com.amazonaws.xray.AWSXRay; +import com.amazonaws.xray.entities.Segment; +import com.amazonaws.xray.entities.Subsegment; + +public class XRayTelemetryContext implements TelemetryContext { + + private boolean isSubsegment; + + private Segment segment; + private Subsegment subsegment; + + private String name; + + public XRayTelemetryContext(String name) { + this.name = name; + isSubsegment = AWSXRay.getTraceEntity() != null; + if (isSubsegment) { + subsegment = AWSXRay.beginSubsegment(name); + } else { + segment = AWSXRay.beginSegment(name); + } + } + + @Override + public void setSuccess(boolean success) { + if (isSubsegment) { + subsegment.setError(!success); + } else { + segment.setError(!success); + } + } + + @Override + public void setAttribute(String key, String value) { + if (isSubsegment) { + subsegment.putAnnotation(key, value); + } else { + segment.putAnnotation(key, value); + } + } + + @Override + public String getName() { + return name; + } + + @Override + public String toString() { + if (isSubsegment) { + return subsegment.getId(); + } else { + return segment.getId(); + } + } + + @Override + public void close() { + if (isSubsegment) { + if (subsegment != null) { + subsegment.close(); + subsegment = null; + } + } else { + if (segment != null) { + segment.close(); + segment = null; + } + } + } +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryFactory.java b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryFactory.java new file mode 100644 index 000000000..f8234e408 --- /dev/null +++ b/wrapper/src/main/java/software/amazon/jdbc/util/telemetry/XRayTelemetryFactory.java @@ -0,0 +1,25 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.jdbc.util.telemetry; + +public class XRayTelemetryFactory { + + public static TelemetryContext openTelemetryContext(String name) { + return new XRayTelemetryContext(name); + } + +} diff --git a/wrapper/src/main/java/software/amazon/jdbc/wrapper/ConnectionWrapper.java b/wrapper/src/main/java/software/amazon/jdbc/wrapper/ConnectionWrapper.java index 437908b0a..cb64227ae 100644 --- a/wrapper/src/main/java/software/amazon/jdbc/wrapper/ConnectionWrapper.java +++ b/wrapper/src/main/java/software/amazon/jdbc/wrapper/ConnectionWrapper.java @@ -51,12 +51,15 @@ import software.amazon.jdbc.util.SqlState; import software.amazon.jdbc.util.StringUtils; import software.amazon.jdbc.util.WrapperUtils; +import software.amazon.jdbc.util.telemetry.DefaultTelemetryFactory; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; public class ConnectionWrapper implements Connection, CanReleaseResources { private static final Logger LOGGER = Logger.getLogger(ConnectionWrapper.class.getName()); protected ConnectionPluginManager pluginManager; + protected TelemetryFactory telemetryFactory; protected PluginService pluginService; protected HostListProviderService hostListProviderService; @@ -79,10 +82,12 @@ public ConnectionWrapper( this.originalUrl = url; this.targetDriverProtocol = getProtocol(url); - final ConnectionPluginManager pluginManager = new ConnectionPluginManager(connectionProvider, this); + final TelemetryFactory telemetryFactory = new DefaultTelemetryFactory(props); + final ConnectionPluginManager pluginManager = + new ConnectionPluginManager(connectionProvider, this, telemetryFactory); final PluginServiceImpl pluginService = new PluginServiceImpl(pluginManager, props, url, this.targetDriverProtocol); - init(props, pluginManager, pluginService, pluginService, pluginService); + init(props, pluginManager, telemetryFactory, pluginService, pluginService, pluginService); if (PropertyDefinition.LOG_UNCLOSED_CONNECTIONS.getBoolean(props)) { this.openConnectionStacktrace = new Throwable(Messages.get("ConnectionWrapper.unclosedConnectionInstantiated")); @@ -93,6 +98,7 @@ public ConnectionWrapper( @NonNull final Properties props, @NonNull final String url, @NonNull final ConnectionPluginManager connectionPluginManager, + @NonNull final TelemetryFactory telemetryFactory, @NonNull final PluginService pluginService, @NonNull final HostListProviderService hostListProviderService, @NonNull final PluginManagerService pluginManagerService) @@ -102,16 +108,19 @@ public ConnectionWrapper( throw new IllegalArgumentException("url"); } - init(props, connectionPluginManager, pluginService, hostListProviderService, pluginManagerService); + init(props, + connectionPluginManager, telemetryFactory, pluginService, hostListProviderService, pluginManagerService); } protected void init( final Properties props, final ConnectionPluginManager connectionPluginManager, + final TelemetryFactory telemetryFactory, final PluginService pluginService, final HostListProviderService hostListProviderService, final PluginManagerService pluginManagerService) throws SQLException { this.pluginManager = connectionPluginManager; + this.telemetryFactory = telemetryFactory; this.pluginService = pluginService; this.hostListProviderService = hostListProviderService; this.pluginManagerService = pluginManagerService; diff --git a/wrapper/src/test/java/software/amazon/jdbc/ConnectionPluginManagerTests.java b/wrapper/src/test/java/software/amazon/jdbc/ConnectionPluginManagerTests.java index 0218a729a..21f5a07a7 100644 --- a/wrapper/src/test/java/software/amazon/jdbc/ConnectionPluginManagerTests.java +++ b/wrapper/src/test/java/software/amazon/jdbc/ConnectionPluginManagerTests.java @@ -41,6 +41,7 @@ import software.amazon.jdbc.mock.TestPluginThree; import software.amazon.jdbc.mock.TestPluginThrowException; import software.amazon.jdbc.mock.TestPluginTwo; +import software.amazon.jdbc.util.telemetry.TelemetryFactory; import software.amazon.jdbc.wrapper.ConnectionWrapper; public class ConnectionPluginManagerTests { @@ -75,10 +76,12 @@ public void testExecuteJdbcCallA() throws Exception { final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); + final Object[] testArgs = new Object[] {10, "arg2", 3.33}; final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Object result = target.execute( @@ -120,10 +123,12 @@ public void testExecuteJdbcCallB() throws Exception { final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); + final Object[] testArgs = new Object[] {10, "arg2", 3.33}; final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Object result = target.execute( @@ -163,10 +168,12 @@ public void testExecuteJdbcCallC() throws Exception { final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); + final Object[] testArgs = new Object[] {10, "arg2", 3.33}; final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Object result = target.execute( @@ -203,8 +210,9 @@ public void testConnect() throws Exception { final Properties testProperties = new Properties(); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Connection conn = target.connect("any", new HostSpec("anyHost"), testProperties, true); @@ -230,8 +238,9 @@ public void testConnectWithSQLExceptionBefore() { final Properties testProperties = new Properties(); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); assertThrows( SQLException.class, @@ -256,8 +265,9 @@ public void testConnectWithSQLExceptionAfter() { final Properties testProperties = new Properties(); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); assertThrows( SQLException.class, @@ -285,8 +295,9 @@ public void testConnectWithUnexpectedExceptionBefore() { final Properties testProperties = new Properties(); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Exception ex = assertThrows( @@ -312,8 +323,9 @@ public void testConnectWithUnexpectedExceptionAfter() { final Properties testProperties = new Properties(); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final ConnectionPluginManager target = - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory); final Exception ex = assertThrows( @@ -344,10 +356,12 @@ public void testExecuteCachedJdbcCallA() throws Exception { final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); + final Object[] testArgs = new Object[] {10, "arg2", 3.33}; final ConnectionPluginManager target = Mockito.spy( - new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper)); + new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, mockTelemetryFactory)); Object result = target.execute( @@ -418,6 +432,7 @@ public void testExecuteAgainstOldConnection() throws Exception { final PluginService mockPluginService = mock(PluginService.class); final ConnectionProvider mockConnectionProvider = mock(ConnectionProvider.class); final ConnectionWrapper mockConnectionWrapper = mock(ConnectionWrapper.class); + final TelemetryFactory mockTelemetryFactory = mock(TelemetryFactory.class); final Connection mockOldConnection = mock(Connection.class); final Connection mockCurrentConnection = mock(Connection.class); final Statement mockOldStatement = mock(Statement.class); @@ -429,7 +444,7 @@ public void testExecuteAgainstOldConnection() throws Exception { final ConnectionPluginManager target = new ConnectionPluginManager(mockConnectionProvider, testProperties, testPlugins, mockConnectionWrapper, - mockPluginService); + mockPluginService, mockTelemetryFactory); assertThrows(SQLException.class, () -> target.execute(String.class, Exception.class, mockOldConnection, "testJdbcCall_A", () -> "result", null)); diff --git a/wrapper/src/test/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginTest.java b/wrapper/src/test/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginTest.java index b6aae500a..4d36c594b 100644 --- a/wrapper/src/test/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginTest.java +++ b/wrapper/src/test/java/software/amazon/jdbc/plugin/DataCacheConnectionPluginTest.java @@ -31,6 +31,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; import software.amazon.jdbc.JdbcCallable; +import software.amazon.jdbc.PluginService; class DataCacheConnectionPluginTest { @@ -38,6 +39,8 @@ class DataCacheConnectionPluginTest { private AutoCloseable closeable; + @Mock + PluginService mockPluginService; @Mock ResultSet mockResult1; @Mock @@ -78,7 +81,7 @@ void cleanUp() throws Exception { void test_execute_withEmptyCache() throws SQLException { final String methodName = "Statement.executeQuery"; - final DataCacheConnectionPlugin plugin = new DataCacheConnectionPlugin(props); + final DataCacheConnectionPlugin plugin = new DataCacheConnectionPlugin(mockPluginService, props); final ResultSet rs = plugin.execute( ResultSet.class, @@ -95,7 +98,7 @@ void test_execute_withEmptyCache() throws SQLException { void test_execute_withCache() throws Exception { final String methodName = "Statement.executeQuery"; - final DataCacheConnectionPlugin plugin = new DataCacheConnectionPlugin(props); + final DataCacheConnectionPlugin plugin = new DataCacheConnectionPlugin(mockPluginService, props); when(mockCallable.call()).thenReturn(mockResult1, mockResult2);