From 560bfc44b23cdc7ec252440fe2624fbc37be9755 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Tue, 4 Apr 2023 16:42:41 -0700 Subject: [PATCH] update schema folder (#1490) * update schema folder - add aws_alb log mapping template / samples - add cloud log mapping template/ samples - add http log mapping template/ samples - add container log mapping template/ samples - add communication.mapping log mapping template / samples - add logs/metrics/traces README docs - add gauge / sum / histogram / metrics mapping template / samples - add services log mapping template/ samples - add traceGroup log mapping template/ samples Signed-off-by: YANGDB * add missing content Signed-off-by: YANGDB --------- Signed-off-by: YANGDB --- docs/img/correlation.png | Bin 0 -> 21763 bytes docs/schema/observability/README.md | 57 ++ docs/schema/observability/logs/README.md | 46 +- .../logs/sample/aws/aws_elb-log.json | 66 ++ .../observability/logs/sample/cloud.json | 5 + .../logs/sample/communication.json | 9 + .../observability/logs/sample/container.json | 6 + .../observability/logs/sample/http/http.json | 18 + .../sample/{ => http}/http_client-log.json | 0 .../sample/{ => http}/http_server-log.json | 0 .../traces/samples/load_samples.md | 41 +- .../traces/samples/serviceA.json | 11 + .../traces/samples/serviceB.json | 11 + .../traces/samples/serviceC.json | 11 + .../observability/traces/samples/traceA.json | 4 + .../observability/traces/samples/traceB.json | 2 + .../observability/traces/samples/traceC.json | 2 + .../traces/samples/traceGroups.json | 8 + docs/schema/system/README.md | 16 +- .../samples/integrations-fields-list.json | 578 ++++++++++++++++++ src/main/resources/schema/README.md | 136 +++++ .../resources/schema/observability/README.md | 163 +++++ .../schema/observability/catalog.json | 39 +- .../schema/observability/logs/README.md | 234 +++++++ .../schema/observability/logs/Usage.md | 117 ++++ .../schema/observability/logs/aws_alb.mapping | 147 +++++ .../schema/observability/logs/aws_alb.schema | 259 ++++++++ .../schema/observability/logs/cloud.mapping | 76 +++ .../observability/logs/communication.mapping | 6 + .../observability/logs/communication.schema | 121 ++-- .../observability/logs/container.mapping | 67 ++ .../schema/observability/logs/http.mapping | 6 + .../schema/observability/logs/http.schema | 12 +- .../schema/observability/logs/logs.mapping | 31 +- .../schema/observability/logs/logs.schema | 8 +- .../schema/observability/metrics/README.md | 105 ++++ .../observability/metrics/metrics.mapping | 31 +- .../observability/metrics/metrics.schema | 9 +- .../schema/observability/traces/README.md | 153 +++++ .../schema/observability/traces/Usage.md | 113 ++++ .../observability/traces/services.mapping | 121 ++-- .../observability/traces/services.schema | 45 ++ .../observability/traces/traceGroups.mapping | 31 + .../observability/traces/traceGroups.schema | 33 + .../observability/traces/traces.mapping | 25 +- .../schema/observability/traces/traces.schema | 16 +- src/main/resources/schema/security/README.md | 9 + src/main/resources/schema/system/README.md | 57 ++ .../system/integration-fields-list.schema | 66 ++ 49 files changed, 2963 insertions(+), 164 deletions(-) create mode 100644 docs/img/correlation.png create mode 100644 docs/schema/observability/logs/sample/aws/aws_elb-log.json create mode 100644 docs/schema/observability/logs/sample/cloud.json create mode 100644 docs/schema/observability/logs/sample/communication.json create mode 100644 docs/schema/observability/logs/sample/container.json create mode 100644 docs/schema/observability/logs/sample/http/http.json rename docs/schema/observability/logs/sample/{ => http}/http_client-log.json (100%) rename docs/schema/observability/logs/sample/{ => http}/http_server-log.json (100%) create mode 100644 docs/schema/observability/traces/samples/serviceA.json create mode 100644 docs/schema/observability/traces/samples/serviceB.json create mode 100644 docs/schema/observability/traces/samples/serviceC.json create mode 100644 docs/schema/observability/traces/samples/traceGroups.json create mode 100644 docs/schema/system/samples/integrations-fields-list.json create mode 100644 src/main/resources/schema/README.md create mode 100644 src/main/resources/schema/observability/README.md create mode 100644 src/main/resources/schema/observability/logs/README.md create mode 100644 src/main/resources/schema/observability/logs/Usage.md create mode 100644 src/main/resources/schema/observability/logs/aws_alb.mapping create mode 100644 src/main/resources/schema/observability/logs/aws_alb.schema create mode 100644 src/main/resources/schema/observability/logs/cloud.mapping create mode 100644 src/main/resources/schema/observability/logs/container.mapping create mode 100644 src/main/resources/schema/observability/metrics/README.md create mode 100644 src/main/resources/schema/observability/traces/README.md create mode 100644 src/main/resources/schema/observability/traces/Usage.md create mode 100644 src/main/resources/schema/observability/traces/services.schema create mode 100644 src/main/resources/schema/observability/traces/traceGroups.mapping create mode 100644 src/main/resources/schema/observability/traces/traceGroups.schema create mode 100644 src/main/resources/schema/security/README.md create mode 100644 src/main/resources/schema/system/README.md create mode 100644 src/main/resources/schema/system/integration-fields-list.schema diff --git a/docs/img/correlation.png b/docs/img/correlation.png new file mode 100644 index 0000000000000000000000000000000000000000..2d01ddc7db76f460fa3745831d4dbd039821a40f GIT binary patch literal 21763 zcmcG$1yo#H7bb{H2pR$j?hX|Q!QGwU?(PJ43Bf{e_uwG}cMtCF?(SAN%!TB=|L>Vz zYr5C;u&Rn%$1ZoDeYSp^;Eyt*$S-hSKtMnsi;D@#LqI^`LO?*iM1TWYtneJ5fiG%i zf`T8#1qF#d+SwSJSsFn=P(?aM@PnY`F#TlCO-qDGig;YtglNU)+wwncK^ z8;De_85s+ad3{0Fz`gU2QoVv)2>K#KM_w*!&VM#Pl}Z0e$NDfn`pnuI1#MsXEo}fq zjO9>|QJ#Ut-nHtqXplCl2%6l7_?*{9k#tvS2TF~CeEoBbrF+ze8d&MOvjoLLI%lGk z-uG4$1-L~bXMP+Rj9t)B7^P)fj4kl(ekl0D9#qqo$N9?wexTj74b6ZW0lW!1B}y-{ zxEJ(wGp#wRMPNt2nx>%=mAH0-UE+Rz{+H!q`cNElSoz*}`E|U0`0w`HTCk*4kGEx$ zFat5(%*9%{^8ULHT+w2}?hoF4 zxSzz%3NkEg@^BpC?A~gytyz2rX=j&ATb1_T*C1<(yr75;468rbwI%_yRyI-D=zKQ)6$h@OWySa6f5N`E90w6QZHVqtj4@Q#G%1rZSumz|+8hrE!; z-|4_NZW2=m2U`vh$l2MM!TCLdjhzXIk)53#^o|L{#6%B_ptpCmcF=R7x3(w!v&cW| z2pQQM*qPZnnAunpJ*}&!Z{z5|O+xZ?qkn(?{GCQFX8*a9wf*160v-_b)B|EU~7zt2>% zH?kA7u>x-C!1JG-`8)Z)JO57P0zEzXe^KHOnV(w$I`h2X0{y#ZJTG2kypn`~;DZns z;#YKmJV=F4!0ea|2rn^=qtk{mnwUS2Rl27I|~Pisj2t7u5n{Zj&-}|8WZmWcX!{5 z-Wic!bK-Nkd<j5PMZZ_DR z{0ww*^`=_T6!)s7+ZWZYb(s*HOYwGUW$#+Y?u8*=sfn>wF`-6`fcX2m67(4i;KiGH z@lO-q8z^ek(_x40VdAql9biM@g3n~<5qG148m42`NWCe+VD{kk%^py^KY?p%-Y_L^ zP8{vixBG5d>|!_5`#)Qd;8O?W5WV!<5M z2F>Rc__5ZW8B%*n0b^r!&tGSo6wDhXOd8)G#ivzejn0H`R%++O1x9=Am~ql%U0_Sg z4qZyh&dKE;aOO-FI81V$J@=&En`-YVnY*&>JWXR#_Z~Sv_xL}zp!}_5sSR9*1T5yh z(NEKOuf2?en3$NF%e@N@({#^s`|$rJz-3^n?I7c|KNEkj3WN5=FvS)j6%6Gm(q=}j zt!9mhAfjBP8y(MN)IPTe(!XE`u1*8zF-<28Yc^a|@Wy6f z!N#|1$lfZvwUjnxSah3B@BJ9^S#q^7?>_%!6$LZ44flbp(GTGk@{`H22SqrIS4kny zD~x9@N2%^He*O^{l>Y`pWEd$eq1!(M6Z@yQm$g*)Q0Ui7S7ER9JeaHNUu1ShnG1yT z<&m5$TqOTe5Aun6KD`wGQqK-?c&6K^h4e3ZiG1IneJ@9c{$cqadc^}4cL3oap}cJ+6uJZdvhl zu_p#&FV<|>HLFkYmLL(0(s_qE;JUG@Bs&*+akplGNiO}r#WmZO?J%^s=SRD)?j15O!pD zhJauqtOVLPlJ_6}+jGW0pnWs4+IdtEFH7_0U0Uifj2p7{T5@Q|!Nc*3qPb6vjX&j! zgD|cfdVc+gt6w0b22*{%B77OMZ)9C~Qy9pDM#8;#@79PiyTyXk{>#!e5hUzOXvtNo z;ChMVNPX}0(8RS<&)kv;h1N~$y?u5@{>Ful=W$rD-hczf8I!pBD$n0_G<+Dd;FO=J z_%0)T={Crn<&=?+%m1IF_nQ-7zeP2<=zkiB2muan1++sh*x7SzYnJ`n$wO0l-qqU%=bcP8UMGC}P3(=w?d z7=-ck8p=F3+obslc`{8!r^hR8n+l^VpEE@>#99j`6yJ?FXND4Ko<9wmv zdn1Sip;RgzcH|YBjry?Oa=TURPnWjA*xbP?TWz8L_xe+wfp7aX$Q7W!YpN)i$G@>KCS01IQHgZ zuWvX*aKroYHhl;BzTsk0!&ljOFooJ`uIj@SqWi%;e2mK<(fKhOA$wU;>J`s4IxFOn zj;O>9>~cK8jX<)K#{3Nk1-#%eCOukfZ{m7;P`$DA_)tCXb$h_%4z_&DsQ;w_U{0n_ zt3oZLk+elM0_NjCh##-Ic)vmWbm7`Eh4I{&)`Dj(jK+V+$v1hpx*k+c^{>6+61WDo z!|3vG;WOy*_NZA$+hqlI&EtG;YKV=cFCK)AC!5S7tzKbDTmp;eX~NGbTkC7&gZU)JKC6JJX7WWM^8-%Lk z_Dp%6+~d_+_(1$k4`sULscQb!RP%!o9y+G3(n^Z`AfwI=Fz1CP?d~uNUi8@@C#u>^ zy>sy|o#sV07+aR*mqdC-db{KKOm^#H(SeQA^^69cFB9MCeV%t-^3271{El+7{i^F= zwlW%?&|{b_qT8Sn=0dFGBK`3$Pp!plwj#e2o={Pi`y%F3jCJFMa@N-V5~lj&cW$y+ zinY^lz$}o{lVc=Vb|erQ8*__YK{IdcwRKZJd@>E>=bR-9Rd#+kZ#M3-4;k>(A_yhJ zs~5``r=bfKf{IMhM<8gm(Pd8j-8W)YUJy_&Bi*FAA4UTFmUzRFT%`PbJzlIz4|_#QW6-kx(4in*#Zqub)Id22Nm; zZQp_CY>XY(*!$j&X5U*7U<_hYlxasd4tnh?4SAu2N5}ipg(Yh6Qp?9XK~h*$o?H#SJe z`+njV!;fnx5QaOC@jw!}O~XlmrpHM~uJnYTQ2ox%jxWjf!*Ls`$k8C@5p%e*)8V|C zGAGaVdbCd(fy3^&xOKycNO@GIkxL(}43e?{_-M)IW|Bk9BVjgrL7Tp-a_LhSzOO~Q zd5Fg2PuV}3=p!?rM8rxywWh&0NVx5nb|r~YT&*%_)m5m8Uk!Ob7!XMQQY#m!bKIkF ziCK*sWLtZA=8xLNuR^%*X3JM%_t~k$7c<+KFUif&kMlv1gvx*Rtf)Z67x_poyE9t`|yctRB zxhf2NIO&Lri_6Wek!ZfXnAL7Lm}~N^cm3iE()#8f2G1f_q**)K0Q(taPK2E`QxgTQa2!9+b)2Z(rz5RCnfv*+8d^5K)PNq!0E7ONBZq zTTj_uIPZmvYEY%*=ZJ(B7`%+vUKSa%K0gBU`5RwQ0UrMr#&X zIzO%aBx+!Lc;bYud~v*uhL~J%@|7K|%^1m7hPG@FWB68F8y+#j2Lyv=dzcfieVS8f z8rpER8uHnNDJVZWHkMqY+WJFsF?u;%a#f-jk=hd<#zn3>0&Wpz0tD1HF$6x;qMj^< zHngY@S9|;jk+VXs6r-Fq`aRZ(dC7~tmhpqcAyGJR==A z@^cNP{!2Km7CSB*o^1gsT|6#*NQ};CZE(r$C%JDjPcjLGFgy27#atwjN4}J{;2_eH zcayrkgW3))U7gc(7!fw+#njFi>h6B$b{7b<%1^X}E>t>iHFvmTULX<^k{cMEvFBmA zY>COCeS7gUkjGT#-A;q843<|5OoF5cls1pfu>!=^YFAnY5~yKVXu0un(OvqBMnha^ z#GR_Y%lU{XY#3w;l4aT9NLHX)Xc`4^YT9Ue*vO2!M3Pajn^IzUh3U%_*wh=;k<__Nz#% z9w!gh(Uz*D&Nm&G@9~P8_{){300eJC8A|fLc(Dq(mjzCvyiz`2d~k1eerHRk`1U z=aO-BW~)QXl@4SLYW`gWE>fSjtHfNC@MT~zyk4Sjt)M1zRpN!yALk(!SMC|_c7$N} z<>#fD69g|yzazVILdV0c@N++jQIKK0jHn9y?e;^kRYT(nnPgy1yDi;odhN~BSgv%k z4+=rE$HVnj+M)gV%Ef~}gM!_wVOOj}nl?{w+aVD)eLBh~fdUW725F0M9LSA2E$aJ) zwI*WxE#EuNFBe@e5Ti8Cb=pUL+cM?sPW(|QA-rvw_r>u81H7B_b8gwc`*qQQSxp>J z5w!)YDeqEzD1LS7cEAl{zCvvG1I$U6kH&<62P+%^(R6;I9Hh~Dbg8QYPmr$r*jazn zWI+&5#M23jt*HyPTb9H}cSElubi2YQ2KdaD2(1D&{QYsVkgXKji1YDsi(VG=TJ@I_ zUYJ{vVboRo-*7^Q9RYG|psQ=h{kh*w81Ahg#$v7d$@YGi!P%pXaJ_vFE(%{&bik3L zEwhY=5&JHNu{~c~eqFf^2@RLf_0k0<1F`wnH&&$*_H)4c{1E9@`Ur{3FZ%6`9!t!s z9MU313oenZCW-K4hs{f5LZ11C9Vk(#a1OLBXwPXCdJos}zQAQUsza`Gk zg9p`1)+A7*qb2sFI;k9vFtnu8QZKsTc-emoL7kBN{;qvOGT;SR+h1WIzbth9gan&y zYa@TaOQt(PfDV@gw3g1X%Lgp}t7SZ*8&IZID|v!&RuW6Ocul+D z($~eU`W?b^V{W|-uY5>nOU$@uqDxg+Z2GSq4;!lGxdy+N%W-wqPV10A#0M{#ji8?D zqqakR%v5}Bb{u|WG#1rV=Do!#q)&=wmi8O6<%Mq%3!dwGr0@IEC|{7BiGXYNcJoGW z?0feQDO>)1q)Z@|tUB_Bh0H4Uy^-)_Pnkwr*9JAei3Bl^)>PcR)95;<66Gj~UTi`S z!o8xGO9c-zf*g)LYH|XB)3j&T?n`}-Y2K8Z_t+gMc2n>nnHLp_FkBclfbBc*cn&xM zeYzsv?>Rk8t1UIyWtc7>RwGF!zS2Opzijy_#s7-nwO$7q`cam`&@NS88vfml4SzA~ z{B^vRb5n|oy1VO9D9CVF>9RM$7q#a!U0HJKOeW?PM>VpudMU$JuShAu!>avB&`r=% z`WwJ+nTZM9S`T{oQ4FGB3VtD)MmvFn8Xv<{35(LYsRP&ZaG)a=9JP@HE`6Wl@biLq zzPZfsvU7@VYejp{TZ?*hc^$UQq;F2Xbkjeo(k9l?)w?GEmb>35uGp(?-16Ed8?;ZZ zgo_;6|E z3ATESM59lY1J`>)*BcQ(F6F{$>%{Lx9Q%zHIPT16DoF*+C-aH(V`)|GFF_CdvCaq6dE`1+GxHF zdZTGYw!6b~Fsf{$H-Y67{T$0t418yQMCx3K_PI)t<@#mOnyc{F-+*x<>_(pI`L4weIl4WS|*h zABYN+U+Jzs*do9lQOYNwVrfK>yxcBvQBj8n}fL8C}VspE0wS8*^cppIu>;lXxkk$BPSE?=x?wX;}X@HVT( z;6}V--0t+}hS*)LUV%gNzC-z=2!5#S;)8P2t$uKdlMH3Kom}2xZ;Cb^2e}tL0~vw{D^M=BJScaWnzcDe^|(I~=%e1Iyug`=3y|LJ!9MZ^ z+1zb;+OyM#ThURbPVzH3ZQgMl9cr;-3_>W2Fdxj->m1e`@0WQlXq0$LmuAocf_5g9#MH{68ZAwt_Mfc$N{wqPnPfwB zF@&Pj>zZAD^HMa1jxy0b4sLMav=7Z*^I&^LzRUsHY<#1d51*iisCLFkL83-*vAIQX z^}-&{Waene_7Yti>Cnq=X<6n4W3|B^!06W=pN&U19w0O7S&%Pt#TUl9nd>{0!1R&@>kElw@X@U3^XXZ9Y4c zdEHkbQKexu@iY3CB2xLcURmD3q@$^NvvIVVUQ4npPJ{H`DtVUY&=%kyp34CvISzM+$@f`X3P$UiG|B1 zOfs(u-9jOdy_45PG1F`)cdF1HUVOuCN*vrmZ@2xs?zh3~Z9uBf=JZagIQG@n3Zfip z>3P00>1p|9bhmFL%~UJO~1j53lfcbWM)EaNVky=b~%O4Aesi+m>PD}E5IyX^!Z(R(nk2yDe=(-4( zDL;(1oT>2&`J}2&Fg6`yqK49I(>Ix5P@m=dxouLGmneTsuA1Mf#T;H9*`m0+ZX&s# z@nNKfW;ekUmSDuxT>pUGcT7_?EMPX0u!kG>I-}G6D;{J+=;y|_(*?e4&{?A43>e9R zkafld@Y%g`tbL30M{2j!+V64aQ3w}JJdY2ezgB`um1%m_+^J_J3Rz^;uhWVQ$}R5g zfyB~8o$4*;;>!{Ck~Ix#mBR8$*)_HnxY6DV8)liF7VhSRdx9Jq=E{`l-Q4h>qD$6; z&gFYSQS8mc7^9F#QJ?upmPDr=)QTEz?+P?OOw4N5wt4IjdY>|W=E6hbdeO3U9q$xN zGYL0BYrSoKaAZntK~G6`_M3k%G}_4+1gCE{Z)ZYy2KS!m%{D@t?Nr3%K z%^qRT;GRw}4g{)oR`r=5J#D=s;H)GQfTn;=SQCjxL=AfMZ@!MFVC{2;bXmfoGi{v z7R)InQ=LFrd?@?sM7Pc!R*gMnoo*iQMRzMaM^gN{*l@k~Em(dAl+)IH+KdW@$jl)b z6d_p@mBhX`P6xjp5>szGQOwa zVi`}G3Z!4K`Q>97XqbDEEB46WxUnc|Pe6S*TdMJJt2n5ghfUx;wtvBPe-{noEoLi& zwwBcTUz1Qe|i1nT*oJiM25_S+tSwy|kiX=K6$2f&#+@iP2vhIFy zPa3Ri9LL3ue`z`*DXNN%^X8Qn+?fSPq{f8RI$?hZW?@%OIRUH}{uMd4AlTnbC-Y3w z;r4+urzmAQzhSSY?zI=Ew@%vm!fdys+SiMqEGcu^y}}6Dev3S}Lv-?9Ra_BHwZSfw zp85M|tSyZ=aqi9rnh>SvdBs-s6H^=IkIO!)A#J`tZJl5-^&Os~3eEbegHf(*(Jqex zq^}xGNK>$`QYXO}X3kQ9wlu^WA>Qfk4}a-3STGC(bj4~sCmlEu*xp42=r)gPUbARZ9@}}t^ql+y^nCgmVX$?;Tv^y zJjBt`a_K0%l~zL;nd!MG*tK_$q?Nl)vAn?p1-aEA;cLGlWo_hyU#MbMTux}w*!w!!D+;!N zH@b4pEBYS(a#rcpgUeN)r%^vHD_KLT6Leo}5fSoF*4Q;1$4KMKa74u&7nm1#KmdDG z4!;`+bMpx&KtHFw_3Ds>YC?0d4?W^2u_24kh?3GK3tozf|4&oxN<*#$XFbA1vs}Y& zv2GWiLWnr_Zr_%fpMgdzWH{N&9}qu-1ahY_d<3h_x^P@?yB>Q_r9*Pu=L+5svs87@ zxIw+@wpkb4B6kgK*ik|~a1R|0Mm=`X5!K$eFVp9_Mp$=#=r9_B!wW5Ruj*DixDm@z zFWfP0H+yKwfVQ!Pe`F6a?#wrF68yWBwJ`dJmdhiE%FG-eI613T2s%3m}qPR2u0D;fewe^OjuU->3*;!eL*uQzX7ZD-aie!b>% zqWd>7?CXLNR9o&V{dN8&&JZ#35_V_Ck49L`2Xoawf51mYg!XwrD-fAd{+sT!d`fpB zt5p4)?mTAo^$ST=7IAv&Ke1~}uOgJ+wO zoE4IHt8wWJG;)Mm3?{#PxjNzd#KIoJXiDofDcvb+9naw} zk@BOt>)XtHn29YV-fctU7g9j7v2pOvCXX<%J!R7+n)fyC##&9KVgGy38K;gu0e{7B zvT2_qm{VNHK{GKXVX=L?iAx}OyxGwHor}wD-gM;Zx)7dpbZpE_XV`T84z;h(g{s+f zZPStGwh;a{iv06-o}}@~gKfV;s&~i68TtHu3pXyqD?j1C`@jeF00drE2qmljJ%`Hz zKr#{u6D#r`l*5%M(BMrsgYgeFeUN;!omm!>&(bOirWP-bstRoE`Q<%An3$LhZ_al* zupx?5{zT>}VC3~j&7nTH&8p?~z#}cPVl_Uj?lB!*RIksdy_aeNvS)~fW-EzMlxq*S zNAbb0D3i2Kpy6IpaT!d%f4&{xD7x59Y$%gaUw33>7_)bNtbC3`;wb+$&z25(xlST| ze9EH2gX(?Y-RT&}?l!c#Z|7hz0&?n35^HeC)8x*tnkR-JPup;pB0PyCF%f>?%a!ZN zke$}hL$1gAZ+F9~Fs)pVjsWc+7x9Zk7RrUye45 zl35hUMt8@E6i^$z8?+l?$%herw^|IBJTAo^_d8RkD;k8|RrGV~KMz$T&n5?qc3&wN zO$}->UEL7p&Yd`i#@Ai!^*-KTe#)0CuahEl_9nAmRo-%7VtW5xdMQZoo$d1Pnwgfc zzvl+t7ZV0)G<&53U6(?>w?I3)3M1oi@Kf7*>moLef;E9#xdYEVCORz}4PH>9Y=f@9 z$v{-KNh3WzhhqdSoeA{LA!3`+T0rRO;l6W@a;iF$evVvouUhR**jL$7a7<3a1StZA_v7rSB`G_mV#{)F8H zgbg(GvlS5|f?WxOWJt{NV(`<+jBe+qGi630Gv#u*Qg01aX2^tkJ;c6YB|4(V{G^~zAT zlVHbIz`gv)U?;q<-yjA%-Ag=R{+*%(o}9qVWu11@%{r~wWLbX7#JyAxR|Iun8On?s z-MnDlm~XXx05Yy}jYz5UKXwV4tG&pmR?#i&PWY6*f9BcS)Q$eNL8h;sMMnlKp<#^% zDN-hjZXh2cDdd$!_&jxF6P1%HAkaJg42LykNe@mP2kqdM8Qb z#>8~&VVh0Q$n_4P4P_dnAW1to|SoLq)uHNJuMq^p=ENBJN$d?0I`e zRzSC7HTZop{~jw>FB{PhL7fG6puxd1u2VW3>pZ?2QKoCT#jg;#l)d=etu%Hp09#`G z2^8YH3SFcxxKNJUr~ z9rycFL#`6H^IKU{w<&kss6gTcyZM6c@oVSZS{m}@C;fj>j6Sk}<#z>BR>RVUoY~>H z?z-@#ot9GpsNB)7A3OhpK7v69biZ7Y1MG~qt;~x;)|Bn$55n~*(*m~wqV0R0GF}<`F^FVaI`0UJ&w2~e z$~LzoAK1D=bY7$&nb^^m@0m7AxToH$?h(5?)QBis-iTxUdXVx`zkSUbI?La&{U({e z6u4cLO2TVoWbC3fj3SMvgq_W5doYu6e}XLy|HjCy!p4))B0psR7hG{aI_9LvaXZp5 zSulQTI||~q5;dxL$lTeveuisSdghJ0#SXfeEI#$*loJkEG)mB;Qa(=;*#kU}v}gS1 zf6yBLZwuZhh@uGx;;@%F-lKfxt&T}cV=yx_vsUtDaw!6Xp(J=c{j zr6T|t70?h?Z1CoD**7AbFKvbmb^%P+k?GHA?Jb?{dBI?n!S(ny(55?!UENA^kAx4d zP_(0=tJX7X5dtVJmbkZX-`as^O!^KN>aB9PUW!5#kU6llj_9#JTOK|b-^(+|=v)Ea z`<#u;E=&mg*J^?#;xk#ZaZ#P9IcmCf;U^iwO!ZlqQLT3>YyeP6BeJLbYfpHSCePcts`K?|S&5^?My)+n zv1cKU<_$PObemu$WPGS1YbwN}(=I~qnEJ!IzFS1t;r3lvW#xoFx*sDQI4>68_6u-0 z3^f-%VnpUxqiHp|RldYw7V~<%cYeZY(WzH-cLU%uiMc8(Y0`3&p*RIuZYA@HpX4e! zo|TrOU16K_aZjqd!fb3E$h$J35aNZTaym-@qe90@Jg)7= zHUXFw5xsgv#9nbT0o(c${6%Wn@cR+)#7eoc=?qySp-islBS3iVcT=d)h6?5(>M;fH z$8PC2ti|adwvao!nU$cs&#tBnAkD9^M>6n zd){rYP#)09Pulr<6pzQFhU<2}oXK{X&%jM@TIG#qe}~%})D1|r3;|fPvE0x|d=7^Q zQ1jg=^Lqfv)AF+&xF%68t=Y~Dd8zYDfKIjK_c!*GmuzHPDfVGKZH#KoUX8j(uy}92 zAz`#;8sxwX22h;eAJ!k?zzcWn0GZnN6Hd2^yitnxVjMrto`BYGi@hq-P459vHa95c z+cPXTMN6IF39_3ri!RgojO!g_pbvlQl0QJ8TziJ~n#9L-J$25=eg_eQw4*tEWU%utV3)bx#)X@6~R>*Dw#rv z_B?n=Z{Hiku%+7dQlDN->4Yp0aPy|DUj;<9+(&eug@gUW-by?}pjlkKeFSVtp3E?$ zE`N8H7&N0Mgy|)b;PKz0gq~z;UE%SvJlE}XS^5)V3INkdvAGNkC4G?ex>|uhGertp zUTkz9AXMhGZF-OIzF+ElPxwQlRNr60-}R3rBS6Myl7+THsWGK$HeGHat6Ti#{${dJ zgR)S)k|~+H+G>tsi0kZ|?hyPafPSO{b`i6IqaJ`R?Eomd8P>5v4OUlxSq;}ZLvyu$ zx#)%mzUL?00&ryVaSU2C(?Pv|+^&Bj8<^sf=T?|+mYO`>?U4bbCd$l|4{|a*3f{Yd z*V0MMjJSNw_h5~3P?l^u&t?OF^Ag`0OohEQNn)|05e!865U%k(oK^MMz-3zTD@>Qj z7y$mXp6z6xX8nx(1MD`D^#`X)g$G6LKSu8^4F~0tX5-9qT+rx}`Wlz3v>M9h#`JT! zPi!D?2F2R~kO$uDwy#qGfUX!?l|f6aj=k~^7L&}$QlGQ|(CIpGK$&R?TwcE&M7U|k zr3&YL03J{T$fq0jM3LuOP5U%OW;QMu#$2KP5fJ?VPAiCg6>y!cUpm&Cv%OHw8UCX3 z?I7QJp^liAge@;)BSlDZ8{4Ml-5=+ZjKFuka=X1e6pNecL7WL{w_{iYG+8gYmxMahP9 zB(VV>){@TzaA=KrQNa?^eR;C!hw9c-)tn4qJ9kbhkV5XaV6I&{(ey#Dkz5VeZ1BSQ z2zF*_n(Jnwuf@h65%0qaz}X~M*o$>lZ~SXp&~2D8!El7%f7_T(N5D`~eD;Ji&_prS zp^9m+j5zH@9qa=nf&gcEtjobv0YMVjoY3KZwnqAv?>+MqupO1#*5hm_@lIeSgKT7z zP=#f_D>#R2S>pGD#=D{ybbojiff#rQ7{tGEA%FmPWD%L+GQ&UMAcH6Qot&y%$Ctr1 zEnY=cKyArw)ttVKXm};}Kw$KBrR+ibY4rPyqt>b+vuJMF+kIxcFvq6tngH|lIF}7p zy;c|i_3UsM%piK5>dhbx`?X;fn~k^?^1wbY>#U=Zr8?%$PHj58dq{@#D&ewNZ2em6hvA>WSFlyihvb*Yawj8$~eHQk`U~cC-1^P_W zqq<`UPX-DRxdAeo)D57zhde{Ysv2iiQjT`-E42cW%?^eWLe}3N+e%Iio?LW4b z7COw{j)8$7sk`1Cskf)f|7=1mKcUMxvE!nAXd#C?cE5Krw8;dxlRP=GuId0`LGKDk-GP=J9X$LGV9`tMO= z0t}tV>ykK_&j`m>F+_O%PmJh82}KPyO$@RNxGXwJv&_zpTm`C;bbx9k07?p+isG(; zFrkv6(BQZc9`Z=q78voa57iXs-L%asZGrAddD=Ij{+=Dz56Jiv^BBczl3um_`=aY6 zd6D-0x2!RU60M)aTLw1%hd3-?5YLqYhcHZ{y6I!?1&z{BVz_eA1x;J((XUI%p_!u1 zHu2`02`9yXJ=wlz)71P{CjWva^akbCov$%dBRN*k zm^Un&4z#dOCInk_R@!+S2-$_5wfX}d;~A_tZ@ggedK_8bR>+j=krF8KYJ>Lk4-`fQ zHu+Jv*hmMt=h~y3*{yz2T42Lg27!-}^9t>pG;Enp37gAVS;MWec%!9G!u4Ta{FSAB zHd8y1DF1UY6pFn-*&krzn{}$7r@m1=88MdUswgx+JlN+xj()_RPW5nfAuIHE6Mjx1 zL28<$zYA||6ZRL5L-A&=H1ns*;$zC15EAtziTdoeoov+UlAG&Pz(w~TchedD(Gj#F z*1S64v*+%26+uOp#nFD;vR+O&wO!Af-IwD1jVOt;%>`2cB1{B)h&+0CPAxxhL&$<3 ziYCNJ4Y6g8T#s`&^LEc@L9|VO6!p0b$kzpZ8E(WSl0pCeIW~`j=dFQW&Q;rQWqSuP zQ~!X=UUO2gUiVHL8Q8m062R_JMXtUK=^#Mn;?Se zI7MWbk!>>`9#=nzx@QV$zsQ-irc~3w#zxdIBYV>NHBXEs;%~HB`08*(>|3zUr|!Y| z#5bM8FP;pnzvVKW3jgZ2hF8G&)Yl*WE%`bHSWblJ`w8Y>kDCbc2u2{1S7D>IA1B~-bX)cbtWuu?L}KzE9o)|=wOmNf@4b~@M^mM9*o}wZzV+Q^W#KQoz zLGSmgg`u>)$A7gEpcjoXFOekrEaY_o3!qKLuM1Ij`FN#SXbDHc#lVc{c`YA6h5EfB zHbb~8vkOh}nz53vRbt(7;P<}X4I~_>G%_KKJ3-~9rI)GKWN3PjNxV8fW4bQImRg81 zqTM*vIAq*fp5G{V5W^v8KzrH*F-$0G99$^4r+j0lk%>w2qn5}mVeVfAylU)MwWVwV z>MQm%r(BeEqBz%@2D4k0ZQWN5Ejr1u4RR%rn_Rvn>#8uNv*G5|T70x@N||yFx&vQZ zFnOQIC*HJ&oSu)@8(sHSGTH6EvfB_gN$jh;KBOMaQQ&la>w47Kg^Uc;oeQzCT*fUR_nMN!bY_l=YaLgr32yjIS6wdmobc1VZt~~I zlhHOuNXA`5r#Gt!UW!mz@E&M@T?|xEg4JuB+9=E>Dk48Ik9QhqFqh&5A297L$ZjN> zkyfb6?Am^#yS1w9;bpvL*m zyN#=WLAvUr@Q;Nl7^kmR(n4*^{WgTZ^7R1;i-k^J*xUCJ$Na}@#k3n=oC>q`ukFg5 z?sI=%Sk9IW;H`~FR?@_h-3Qi|T{R|-HaWiE@xF!WDI-W^{KKyQB#q=#5G-e^6uj668Z5A*sSpc5w-@i=fH`M3ZcB!VPFSa%-^K=mv~#OTKMttIrLzADcXv}lS6 zc3z}oMNKDZqu?T`zubS1E1)1R(c9=ePj<<}CWw?Q{X80&X-ImBF)5WIZEXsxxxJRpy(#W|fDzn0BZc^zCsKbf@Jkk2e(svIqKNWvcStSwT+cN96)SVC59ZvXdL~cD$ zX6165v3yILyU6zWG+pi6MdiQ@CHAL$t_fq=yNi3d-M9@J9>rxiTY>~*+v=d>hiXQO z2Y5grU%fWUgMAOt?EbP>gk)AS`AOUo%^+az#D@NN6(NvF=sn%&8@ST6ANaszHOtJt z5u=a|R4@X?b?tmU-~ExWC5p8hrLKjswa9e563_M@fl8M2^AfyT_Co?1%y5% zyIs~Id!|bbo{9$TCe=)O?yk=S9aJM!VV=V63qx#bxnX!j&Zsq=Fb@)RNM?Y>fj}f< z1wyG6QbKrbw=-Ai9zbHADe>RLJm*3EPHu8GkheJn%BlLELPLJ=sBx@I9%F0}HjNi? z4acoWW%gS#PlbLP-`)>KfDh*S0QZ*xVIH8uaZABMDkCGPv<(w}8p>#vv<^YuEjJThG#YcFPIb ztuVst(g>+EjDbIPkB!Xj;0mw$KoL` zcF!O$Pn74JSF9lX<(H8#dCemjwNowqfP*l7*ZBekiP-oT`4`#hp)B(Gy(5f!I$f;l z@J}|BQ4JXQVB@C$|Epjhdj}!&sR~fjbt9I^ep6~^x-{z(8FR0GfpQ@fx_C0P8401s zv2V}OY^CH!c*-2PtQ1xZTa>ujjJz!sprDHlw>zFL@hku;Y^y(!^u2LE!gdS4?|VNE zmtBZoR#b+0=NhO77GLTdkqDz3HN4`IL@VQOq>JL@jc7RPO%_d{)DqndFRLcXJ{@dD zmDnYq@I_NP$DWhbh3cB>+whdG(@OfmAAo5>QMBDsBX;^TEUKD#Eo5+`?itt2kR>W-zPBJteZL zm{7Z6k+0V5nc`-^8i{~6v+Od{bde`U^XsXU&R1XfyI9xE8)nmp^Mg5+$zmNFu-=Vj&PK=hFY5+6_3FF%*Z^Cm&XBC=Q^ zFU-}_Pn78@c4B;7QHfU3^_@X%b`?j?M=RYeb{HBZ8!}w)+j&+zHFzhn9;TMFvFgql zv?l@8?#mbH#PSy10H{+dr)IO{ zh(AfubjInTc4hqXY(k%IG%JnyIWkxcp8m63GGC zk=Tp~Hr8uJ}4Z`<1BGZw|=%LaRdTQp~% zvjs)Z|0_ag(I;lam~>sIiPSDIS!`wyB6opzUsO(`Cw1NUd01*ucF34V4gsCh_63QxJ ziGT_gSrRr8OdteMR%Hrt`JMmy$NK)@EtDKQIe_v)SWa}@9=^9X)TJ?IR_T78C#%{2!nrvvRg{FHD_v{p6eh1wfc|MLpD zQSpcwE+fz)Bkkw837=S_5v)RAx>HHm=?e{)CX>R6w31g1Ef22^?h6|23!Lv<9ZJ0Z zsT*$ry7B5tJ2F0adrckfeaWXt|2%^rZeC_Ju3AWr!o7R@ssd=NJhpyN=iVurTinLg z%xzbTJW+^g-W1mAIg5Nn!tp5HalHEAE3N&5K3FBSP=D$6@plbbBlxFk9*2^KIryG5 zLe)3ISLkqQg5H!w!-gVvsc(_VkeQ-AymQ`)KMdIn?Ax+UZ9k@MsXWh zlTF>7AqNZs{xlG6#!LtoVW#5HfsYSP7~x=tJuL^8_R2~}Sd zU-S2(M`ZLH&DC?+igL$Lne>RzG}8ezo~gy(UBf7+e&shpj%nfy1v47aEpJv^Z5Ypx z#jb&`+ydzw_>}?dLYk`SChUB7Zkl~$_z?l6vKJ$hi;G442bW4km-c9I%i}2^87qbO z&F~h0%H$5(!)qs}qsv4)%QfKs8^d+meIZjJESik?M7yj*Au^{Z=1ElKAKKGcj?j< zJrR*;?9DFW=oS~Y5-2*&U|L%A{fI>05BjWpC%AqqDlwCx3J~;TdA@4SJj2}Br1~17 zp}t$5#nx@faY!<0?*4Y&wu#xYsTefMy+{c6M74JG|H|A!U5P^#hv+5WZk|iwV9!*; z5c?22m`So^8oB4VT!!DcNc#=XzRppu4^dmvX1}kg{sOgU^5~-Esv^X>2`f@a<25dc z{6>{46g0-$cE4Hs;gp!|x_O>CH?Vi({n3dG7x|7-bfgNrYj%G>s@IP=%)Y|j)~E(~FgVJ&y!CWfQqEHv0lvW;$nX1k z=r`Sc57N`|RZ)lus-Iezk>g{jSSRG_yy@kY5N&5(X_;G{l~FDxi80<&%l@U8)BkN1 zJ<#A&(-deACp!1tLdzdgSK7Ih7^7-ExmMvr?4dbFF$;Z!4(5L-z~|SwDSCACt?t(> zUI{h|_0{0#7}EBSH(|deXghD3MZ7NcJg-_-)mvyyTNdNvUKTQG~hQ-7xpxX z1)Zyt?v&GcJ!@MSSYo{K!E$5T_>)XU1tP0rkyRmyb(Tx<_8PUZyjpfnt16*~d7b9W z$I*ThTMJle={#fM?VVZ6Cq=g^W*f|SH$<7qS1XZ{6Oa8Ex+a0~U2k(wb!%L)d@!W{ z4YShZcw0@x(&bD&q^s$Lt{nnw67w98yn+C(41MIu)t;q?Miqm}4idUzXc7=H%l22! zYkO_a(&wd$?X0wusX8*Z{T-?kxX^SpK7f!i2_g-EFG3Sy2!bEqx^-Ty-~c1gL78o_g$#8j%;uSDUQX9jXU4do1T( zU9}-4e4N=2GgZSiG2gBgzAm`+9_##9O`R|dYHE&=)Tl{?e|=})a~<|I6SXbg_~z`f zLHrc^);Rs+Ubaes!q(TT;JQ-g7tFGD8bD68`u3&|<-HqnAnT#*olgr$3nWc}exT%Q zzr0;`VXRQo(h-IgCm%+f*y!qbqpVSAr*AEw>MsZYv8!Z%Q>dk%`F1$^Jib(8C=2%l zS({#YkhdfF)W)yAa6wkR3)aPcw7yqO-_Mm~G@)#tzqZSjV6dp>kB0It{c^hX8Z`YI zqX5DsHsHIrHq#~Lh2}}efE1)Iap9Co4D+1_1ta&!x!O(2RYErgVoW^6RIil4gaqXE zl>w_86Ru=b0%Q;657_NzMmu zxjgylovT6GrEE9Eti#ak92LNF_o>i1fg4!4ZaR2&|7zutpEa-oV{A?-F}PF&p2D&= z3=T-K1DmNBIlaQS+`A_AetBgEK#ZLt;?EL}p&)s!i$I9-8>bXd{b|1j0kubtY5=#f z>XB?g#WFpIU;|AH9iV>5sH|r_v8Z@TK<)ci9$cJtFq(Z3F1XSatyjs>tToPE7BtMu zTYNDX%JvP3do17S8WSArG2S7ez5OlJ;@Z zfR)9=sa&YVvI}O}gblj>p(VT_rTzRPW=8;zwfBU5&I^%fGMS^PJWUc9f}I(Xu~Cqf z0m7ol(Er*ZOyD~fS1#81LLf%CY8zlBK<#YAZoH9zb-2d!`p6 zn9LlSGSQoGI;;ce;F%jl;QHmn+G@YzY046GnvClNu@Hqmt lS7FW!b-~UH> PUT _component_template/http_template` + +Copy the http.mapping content [here](http.mapping) + +`>> PUT _component_template/communication_template` + +Copy the communication.mapping content [here](communication.mapping) + +`>> PUT _index_template/logs` + +Copy the logs.mapping content [here](logs.mapping) + +Now you can create an data-stream index (following the logs index pattern) that has the supported schema: + +`>> PUT _data_stream/sso_logs-dataset-test1` + +You can also directly start ingesting data without creating a data stream. +Because we have a matching index template with a data_stream object, OpenSearch automatically creates the data stream: + +`POST sso_logs-dataset-test1/_doc` +```json +{ + "body": "login attempt failed", + "@timestamp": "2013-03-01T00:00:00", + ... +} + +``` + +To see information about a that data stream: +`GET _data_stream/sso_logs-dataset-test1` + +Would respond the following: +```json +{ + "data_streams" : [ + { + "name" : "sso_logs-dataset-test1", + "timestamp_field" : { + "name" : "@timestamp" + }, + "indices" : [ + { + "index_name" : ".ds-sso_logs-dataset-test1-000001", + "index_uuid" : "-VhmuhrQQ6ipYCmBhn6vLw" + } + ], + "generation" : 1, + "status" : "GREEN", + "template" : "sso_logs-*-*" + } + ] +} +``` + +To see more insights about the data stream, use the `_stats` endpoint: +`GET _data_stream/sso_logs-dataset-test1/_stats` +Would respond the following: +```json +{ + "_shards" : { + "total" : 1, + "successful" : 1, + "failed" : 0 + }, + "data_stream_count" : 1, + "backing_indices" : 1, + "total_store_size_bytes" : 208, + "data_streams" : [ + { + "data_stream" : "sso_logs-dataset-test1", + "backing_indices" : 1, + "store_size_bytes" : 208, + "maximum_timestamp" : 0 + } + ] +} +``` +### Ingestion +To ingest data into a data stream, you can use the regular indexing APIs. Make sure every document that you index has a timestamp field. + +`POST sso_logs-dataset-test1/_doc` +```json +{ + "body": "login attempt failed", + "@timestamp": "2013-03-01T00:00:00", + ... +} + +``` +You can search a data stream just like you search a regular index or an index alias. The search operation applies to all of the backing indices (all data present in the stream). + +`GET sso_logs-dataset-test1/_search` +```json +{ + "query": { + "match": { + ... + } + } +} +``` + +### Manage data-streams in OpenSearh + +To manage data streams from OpenSearch Dashboards, open OpenSearch Dashboards, choose Index Management, select Indices or Policy managed indices. + +see additional information: + - https://opensearch.org/docs/latest/opensearch/data-streams/#step-6-manage-data-streams-in-opensearch-dashboards + - https://opensearch.org/docs/latest/opensearch/data-streams/#step-5-rollover-a-data-stream \ No newline at end of file diff --git a/src/main/resources/schema/observability/logs/aws_alb.mapping b/src/main/resources/schema/observability/logs/aws_alb.mapping new file mode 100644 index 000000000..a89ae1225 --- /dev/null +++ b/src/main/resources/schema/observability/logs/aws_alb.mapping @@ -0,0 +1,147 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "aws_alb" + }, + "aws": { + "type": "object", + "elb": { + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "target_group": { + "properties": { + "arn": { + "type": "keyword" + } + } + }, + "listener": { + "type": "keyword" + }, + "protocol": { + "type": "keyword" + }, + "request_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "backend_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "response_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "connection_time": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "tls_handshake_time": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "backend": { + "properties": { + "ip": { + "type": "keyword" + }, + "port": { + "type": "keyword" + }, + "http": { + "properties": { + "response": { + "properties": { + "status_code": { + "type": "long" + } + } + } + } + } + } + }, + "ssl_cipher": { + "type": "keyword" + }, + "ssl_protocol": { + "type": "keyword" + }, + "chosen_cert": { + "properties": { + "arn": { + "type": "keyword" + }, + "serial": { + "type": "keyword" + } + } + }, + "incoming_tls_alert": { + "type": "keyword" + }, + "tls_named_group": { + "type": "keyword" + }, + "trace_id": { + "type": "keyword" + }, + "matched_rule_priority": { + "type": "keyword" + }, + "action_executed": { + "type": "keyword" + }, + "redirect_url": { + "type": "keyword" + }, + "error": { + "properties": { + "reason": { + "type": "keyword" + } + } + }, + "target_port": { + "type": "keyword" + }, + "target_status_code": { + "type": "keyword" + }, + "classification": { + "type": "keyword" + }, + "classification_reason": { + "type": "keyword" + } + } + } + } + } + } +} diff --git a/src/main/resources/schema/observability/logs/aws_alb.schema b/src/main/resources/schema/observability/logs/aws_alb.schema new file mode 100644 index 000000000..025be426d --- /dev/null +++ b/src/main/resources/schema/observability/logs/aws_alb.schema @@ -0,0 +1,259 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://opensearch.org/schemas/observability/AWS_ALB", + "type": "object", + "properties": { + "@timestamp": { + "type": "string" + }, + "aws": { + "type": "object", + "properties": { + "elb": { + "type": "object", + "properties": { + "backend": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "response": { + "type": "object", + "properties": { + "status_code": { + "type": "integer" + } + }, + "required": [ + "status_code" + ] + } + }, + "required": [ + "response" + ] + }, + "ip": { + "type": "string" + }, + "port": { + "type": "string" + } + }, + "required": [ + "http", + "ip", + "port" + ] + }, + "backend_processing_time": { + "type": "object", + "properties": { + "sec": { + "type": "number" + } + }, + "required": [ + "sec" + ] + }, + "matched_rule_priority": { + "type": "string" + }, + "name": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "request_processing_time": { + "type": "object", + "properties": { + "sec": { + "type": "integer" + } + }, + "required": [ + "sec" + ] + }, + "response_processing_time": { + "type": "object", + "properties": { + "sec": { + "type": "integer" + } + }, + "required": [ + "sec" + ] + }, + "target_group": { + "type": "object", + "properties": { + "arn": { + "type": "string" + } + }, + "required": [ + "arn" + ] + }, + "target_port": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "target_status_code": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "traceId": { + "$ref": "https://opensearch.org/schemas/observability/Span#/properties/traceId" + }, + "type": { + "type": "string" + } + }, + "required": [ + "backend", + "backend_processing_time", + "matched_rule_priority", + "name", + "protocol", + "request_processing_time", + "response_processing_time", + "target_group", + "target_port", + "target_status_code", + "traceId", + "type" + ] + } + }, + "required": [ + "elb" + ] + }, + "cloud": { + "type": "object", + "properties": { + "provider": { + "type": "string" + } + }, + "required": [ + "provider" + ] + }, + "http": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "body": { + "type": "object", + "properties": { + "bytes": { + "type": "integer" + } + }, + "required": [ + "bytes" + ] + }, + "method": { + "type": "string" + } + }, + "required": [ + "body", + "method" + ] + }, + "response": { + "type": "object", + "properties": { + "body": { + "type": "object", + "properties": { + "bytes": { + "type": "integer" + } + }, + "required": [ + "bytes" + ] + }, + "status_code": { + "type": "integer" + } + }, + "required": [ + "body", + "status_code" + ] + }, + "url": { + "type": "string" + }, + "schema": { + "type": "string" + } + }, + "required": [ + "request", + "response", + "url", + "schema" + ] + }, + "communication": { + "type": "object", + "properties": { + "source": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + } + }, + "required": [ + "address", + "ip", + "port" + ] + } + }, + "required": [ + "source" + ] + }, + "traceId": { + "type": "string" + } + }, + "required": [ + "@timestamp", + "aws", + "cloud", + "http", + "communication", + "traceId" + ] +} \ No newline at end of file diff --git a/src/main/resources/schema/observability/logs/cloud.mapping b/src/main/resources/schema/observability/logs/cloud.mapping new file mode 100644 index 000000000..ece2a2c1f --- /dev/null +++ b/src/main/resources/schema/observability/logs/cloud.mapping @@ -0,0 +1,76 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "cloud" + }, + "properties": { + "cloud": { + "properties": { + "provider": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "machine": { + "type": "object", + "properties": { + "type": { + "type": "keyword" + } + } + }, + "account": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "service": { + "type": "object", + "properties": { + "name": { + "type": "keyword" + } + } + }, + "project": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "instance": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/schema/observability/logs/communication.mapping b/src/main/resources/schema/observability/logs/communication.mapping index f4325b4b2..73cb945b9 100644 --- a/src/main/resources/schema/observability/logs/communication.mapping +++ b/src/main/resources/schema/observability/logs/communication.mapping @@ -1,6 +1,12 @@ { "template": { "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "communication" + }, "properties": { "communication": { "properties": { diff --git a/src/main/resources/schema/observability/logs/communication.schema b/src/main/resources/schema/observability/logs/communication.schema index 44caf523b..76f0535eb 100644 --- a/src/main/resources/schema/observability/logs/communication.schema +++ b/src/main/resources/schema/observability/logs/communication.schema @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://opensearch.org/schemas/Communication", + "$id": "https://opensearch.org/schemas/observability/Communication", "title": "Communication", "type": "object", "properties": { @@ -11,10 +11,10 @@ "type": "string" }, "source": { - "$ref": "/schemas/Source" + "$ref": "#/definitions/Source" }, "destination": { - "$ref": "/schemas/Destination" + "$ref": "#/definitions/Destination" } } }, @@ -22,66 +22,67 @@ "type": "object", "properties": { } - }, - "$defs": { - "Source": { - "$id": "/schemas/Source", - "type": "object", - "additionalProperties": true, - "properties": { - "address": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "bytes": { - "type": "integer" - }, - "ip": { - "type": "string" - }, - "port": { - "type": "integer" - }, - "mac": { - "type": "string" - }, - "packets": { - "type": "integer" - } + } + }, + "definitions": { + "Source": { + "$id": "#/definitions/Source", + "type": "object", + "additionalProperties": true, + "properties": { + "address": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "bytes": { + "type": "integer" }, - "title": "Source" + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "mac": { + "type": "string" + }, + "packets": { + "type": "integer" + } }, - "Destination": { - "$id": "/schemas/Destination", - "type": "object", - "additionalProperties": true, - "properties": { - "address": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "bytes": { - "type": "integer" - }, - "ip": { - "type": "string" - }, - "port": { - "type": "integer" - }, - "mac": { - "type": "string" - }, - "packets": { - "type": "integer" - } + "title": "Source" + }, + "Destination": { + "$id": "#/definitions/Destination", + "type": "object", + "additionalProperties": true, + "properties": { + "address": { + "type": "string" }, - "title": "Destination" - } + "domain": { + "type": "string" + }, + "bytes": { + "type": "integer" + }, + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "mac": { + "type": "string" + }, + "packets": { + "type": "integer" + } + }, + "title": "Destination" } } + } diff --git a/src/main/resources/schema/observability/logs/container.mapping b/src/main/resources/schema/observability/logs/container.mapping new file mode 100644 index 000000000..edcff9897 --- /dev/null +++ b/src/main/resources/schema/observability/logs/container.mapping @@ -0,0 +1,67 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component" : "container" + }, + "properties": { + "container": { + "properties": { + "image": { + "type": "object", + "properties": { + "name": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + }, + "hash": { + "type": "keyword" + } + } + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "labels": { + "type": "keyword" + }, + "runtime": { + "type": "keyword" + }, + "memory.usage": { + "type": "float" + }, + "network": { + "type": "object", + "properties": { + "ingress.bytes": { + "type": "long" + }, + "egress.bytes": { + "type": "long" + } + } + }, + "cpu.usage": { + "type": "float" + }, + "disk.read.bytes": { + "type": "long" + }, + "disk.write.bytes": { + "type": "long" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/schema/observability/logs/http.mapping b/src/main/resources/schema/observability/logs/http.mapping index f55b3a5f0..cf70dac93 100644 --- a/src/main/resources/schema/observability/logs/http.mapping +++ b/src/main/resources/schema/observability/logs/http.mapping @@ -1,6 +1,12 @@ { "template": { "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "http" + }, "dynamic_templates": [ { "request_header_map": { diff --git a/src/main/resources/schema/observability/logs/http.schema b/src/main/resources/schema/observability/logs/http.schema index 85ace4283..b25fa81fc 100644 --- a/src/main/resources/schema/observability/logs/http.schema +++ b/src/main/resources/schema/observability/logs/http.schema @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://opensearch.org/schemas/Http", + "$id": "https://opensearch.org/schemas/observability/Http", "title": "Http", "type": "object", "properties": { "request": { - "$ref": "/schemas/Request" + "$ref": "#/definitions/Request" }, "response": { - "$ref": "/schemas/Response" + "$ref": "#/definitions/Response" }, "flavor": { "type": "string" @@ -35,9 +35,9 @@ "type": "integer" } }, - "$defs": { + "definitions": { "Request": { - "$id": "/schemas/Request", + "$id": "#/definitions/Request", "type": "object", "additionalProperties": true, "properties": { @@ -66,7 +66,7 @@ "title": "Request" }, "Response": { - "$id": "/schemas/Response", + "$id": "#/definitions/Response", "type": "object", "additionalProperties": true, "properties": { diff --git a/src/main/resources/schema/observability/logs/logs.mapping b/src/main/resources/schema/observability/logs/logs.mapping index 2fdac21f5..ad87bd4ad 100644 --- a/src/main/resources/schema/observability/logs/logs.mapping +++ b/src/main/resources/schema/observability/logs/logs.mapping @@ -6,7 +6,22 @@ "template": { "mappings": { "_meta": { - "version": "1.0.0" + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "log", + "correlations": [ + { + "field": "spanId", + "foreign-schema": "traces", + "foreign-field": "spanId" + }, + { + "field": "traceId", + "foreign-schema": "traces", + "foreign-field": "traceId" + } + ] }, "_source": { "enabled": true @@ -199,6 +214,18 @@ "_meta": { "description": "Simple Schema For Observability", "catalog": "observability", - "type": "logs" + "type": "logs", + "correlations": [ + { + "field": "spanId", + "foreign-schema": "traces", + "foreign-field": "spanId" + }, + { + "field": "traceId", + "foreign-schema": "traces", + "foreign-field": "traceId" + } + ] } } \ No newline at end of file diff --git a/src/main/resources/schema/observability/logs/logs.schema b/src/main/resources/schema/observability/logs/logs.schema index c014c12ac..10b10e647 100644 --- a/src/main/resources/schema/observability/logs/logs.schema +++ b/src/main/resources/schema/observability/logs/logs.schema @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://opensearch.org#/definitions/Logs", + "$id": "https://opensearch.org/schema/observability/Logs", "title": "OpenTelemetry Logs", "type": "object", "properties": { @@ -25,10 +25,10 @@ "format": "date-time" }, "traceId": { - "type": "string" + "$ref": "https://opensearch.org/schemas/observability/Span#/properties/traceId" }, "spanId": { - "type": "string" + "$ref": "https://opensearch.org/schemas/observability/Span#/properties/spanId" }, "schemaUrl": { "type": "string" @@ -165,7 +165,7 @@ "event", "metric", "state", - "pipeline_error", + "error", "signal" ] }, diff --git a/src/main/resources/schema/observability/metrics/README.md b/src/main/resources/schema/observability/metrics/README.md new file mode 100644 index 000000000..78e940f51 --- /dev/null +++ b/src/main/resources/schema/observability/metrics/README.md @@ -0,0 +1,105 @@ +# Metrics Schema Support + +Observability refers to the ability to monitor and diagnose systems and applications in real-time, in order to understand how they are behaving and identify potential issues. +Metrics present a critical component of observability, providing quantifiable data about the performance and behavior of systems and applications. +The importance of supporting metrics structured schema lies in the fact that it enables better analysis and understanding of system behavior. + +A structured schema provides a clear, consistent format, making it easier for observability tools to process and aggregate the data. +This in turn makes it easier for engineers to understand the performance and behavior of their systems, and quickly identify potential issues. + +When metrics are unstructured, it can be difficult for observability tools to extract meaningful information from them. +For example, if the data for a particular metric is not consistently recorded in the same format, it can be difficult to compare and analyze performance data over time. +Similarly, if metrics are not consistently named or categorized, it can be difficult to understand their context and significance. + +With a structured schema in place, observability tools can automatically extract and aggregate data, making it easier to understand system behavior at a high level. +This can help teams quickly identify performance bottlenecks, track changes in system behavior over time, and make informed decisions about system performance optimization. + +## Details +The next section provides the Simple Schema for Observability support which conforms with the OTEL specification. + +- metrics.mapping presents the template mapping for creating the Simple Schema for Observability index +- metrics.schema presents the json schema validation for verification of a metrics document conforms to the mapping structure + +## Metrics +see [OTEL metrics convention](https://opentelemetry.io/docs/reference/specification/metrics/) +see [OTEL metrics protobuf](https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto/metrics/v1) + +Simple Schema for Observability conforms with OTEL metrics protocol which defines the next data model: + +#### Timestamp field +As part of the data-stream definition the `@timestamp` is mandatory, if the field is not present in the original signal populate this field using `ObservedTimestamp` as value. + +### Instrumentation scope +This is a logical unit of the application with which the emitted telemetry can be associated. It is typically the developer’s choice to decide what denotes a reasonable instrumentation scope. +The most common approach is to use the instrumentation library as the scope, however other scopes are also common, e.g. a module, a package, or a class can be chosen as the instrumentation scope. + +The instrumentation scope may have zero or more additional attributes that provide additional information about the scope. As an example the field +`instrumentationScope.attributes.identification` is presented will be used to determine the resource origin of the signal and can be used to filter accordingly + +### Overview +Metrics are a specific kind of telemetry data. They represent a snapshot of the current state for a set of data. +Metrics are distinct from logs or events, which focus on records or information about individual events. + +Metrics expresses all system states as numerical values; counts, current values and such. +Metrics tend to aggregate data temporally, while this can lose information, the reduction in overhead is an engineering trade-off commonly chosen in many modern monitoring systems. + +Time series are a record of changing information over time. While time series can support arbitrary strings or binary data, only numeric data is in our scope. +Common examples of metric time series would be network interface counters, device temperatures, BGP connection states, and alert states. + +### Metric streams +In a similar way to the data_stream attribute field representing the category of a trace, the metric streams are grouped into individual Metric objects, identified by: + + - The originating Resource attributes + - The instrumentation Scope (e.g., instrumentation library name, version) + - The metric stream’s name + +### Metrics +Metric object is defined by the following properties: + + - The data point type (e.g. Sum, Gauge, Histogram ExponentialHistogram, Summary) + - The metric stream’s unit + - The data point properties, where applicable: AggregationTemporality, Monotonic + +The description is also present in the metrics object but is not part of the identification fields +_- The metric stream’s description_ + + +### Data Types + +**Values:** Metric values in MUST be either floating points or integers. + +**Attributes:** Labels are key-value pairs consisting of string as keys and Any type as values (strings, object, array) + +**MetricPoint:** Each MetricPoint consists of a set of values, depending on the MetricFamily type. + +**Metric** Metrics are defined by a unique attributes (dimensions) within a MetricFamily. + +--- + +Metrics MUST contain a list of one or more MetricPoints. Metrics with the same name for a given MetricFamily SHOULD have the same set of label names in their LabelSet. + +* Metrics.name: String value representation of the matrix purpose +* Metrics.type: Valid values are "gauge", "counter","histogram", and "summary". +* Metrics.Unit: specifies MetricFamily units. + +## Metric Types + +### Gauge +Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. +**_A MetricPoint in a Metric with the type gauge MUST have a single value._** +Gauges MAY increase, decrease, or stay constant over time. Even if they only ever go in one direction, they might still be gauges and not counters. + +### Counter +Counters measure discrete events. Common examples are the number of HTTP requests received, CPU seconds spent, or bytes sent. For counters how quickly they are increasing over time is what is of interest to a user. +**_A MetricPoint in a Metric with the type Counter MUST have one value called Total._** + +### Histogram / Exponential-Histogram +Histograms measure distributions of discrete events. Common examples are the latency of HTTP requests, function runtimes, or I/O request sizes. +**_A Histogram MetricPoint MUST contain at least one bucket_**, and SHOULD contain Sum, and Created values. Every bucket MUST have a threshold and a value. + +### Summary +Summaries also measure distributions of discrete events and MAY be used when Histograms are too expensive and/or an average event size is sufficient. +**_A Summary MetricPoint MAY consist of a Count, Sum, Created, and a set of quantiles._** +Semantically, Count and Sum values are counters & MUST be an integer. + + diff --git a/src/main/resources/schema/observability/metrics/metrics.mapping b/src/main/resources/schema/observability/metrics/metrics.mapping index 0e66087a0..f45a9d8f9 100644 --- a/src/main/resources/schema/observability/metrics/metrics.mapping +++ b/src/main/resources/schema/observability/metrics/metrics.mapping @@ -6,7 +6,22 @@ "template": { "mappings": { "_meta": { - "version": "1.0.0" + "version": "1.0.0", + "catalog": "observability", + "type": "metrics", + "component": "metrics", + "correlations" : [ + { + "field": "spanId", + "foreign-schema" : "traces", + "foreign-field" : "spanId" + }, + { + "field": "traceId", + "foreign-schema" : "traces", + "foreign-field" : "traceId" + } + ] }, "_source": { "enabled": true @@ -285,6 +300,18 @@ "_meta": { "description": "Observability Metrics Mapping Template", "catalog": "observability", - "type": "metrics" + "type": "metrics", + "correlations" : [ + { + "field": "spanId", + "foreign-schema" : "traces", + "foreign-field" : "spanId" + }, + { + "field": "traceId", + "foreign-schema" : "traces", + "foreign-field" : "traceId" + } + ] } } \ No newline at end of file diff --git a/src/main/resources/schema/observability/metrics/metrics.schema b/src/main/resources/schema/observability/metrics/metrics.schema index 27a8cd044..83d504a43 100644 --- a/src/main/resources/schema/observability/metrics/metrics.schema +++ b/src/main/resources/schema/observability/metrics/metrics.schema @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://opensearch.org#/definitions/Metrics", + "$id": "https://opensearch.org/schema/observability/Metrics", "title": "OpenTelemetry Metrics", "type": "object", "properties": { @@ -227,10 +227,10 @@ "format": "date-time" }, "spanId": { - "type": "string" + "$ref": "https://opensearch.org/schemas/observability/Span#/properties/spanId" }, "traceId": { - "type": "string" + "$ref": "https://opensearch.org/schemas/observability/Span#/properties/traceId" } }, "required": [ @@ -243,6 +243,9 @@ "type": "object", "additionalProperties": true, "properties": { + "serviceName": { + "type": "string" + }, "data_stream": { "$ref": "#/definitions/Dataflow" } diff --git a/src/main/resources/schema/observability/traces/README.md b/src/main/resources/schema/observability/traces/README.md new file mode 100644 index 000000000..dcf16adcb --- /dev/null +++ b/src/main/resources/schema/observability/traces/README.md @@ -0,0 +1,153 @@ +# Traces Schema Support +Observability in the software industry is the ability to monitor and diagnose systems and applications in real-time, in order to understand how they are behaving and identify potential issues. +Traces are a critical component of observability, providing detailed information about the flow of requests through a system, including timing information and any relevant contextual data. + +The importance of supporting traces schema lies in the fact that it enables better analysis and understanding of system behavior. +A structured schema provides a clear, consistent format for traces, making it easier for observability tools to process and aggregate the data. +This in turn makes it easier for engineers to understand the performance and behavior of their systems, and quickly identify potential issues. + +When traces are unstructured, it can be difficult for observability tools to extract meaningful information from them - For example, if the timing information for a particular request is not consistently represented in the same format, +it can be difficult to compare and analyze performance data over time. Similarly, if contextual data is not consistently recorded, it can be difficult to understand the context in which a particular request was executed. + +With a structured schema in place, observability tools can automatically extract and aggregate data, making it easier to understand system behavior at a high level. +This can help teams quickly identify performance bottlenecks, track the root cause of errors, and resolve issues more efficiently. + +## Details +The next section provides the Simple Schema for Observability support which conforms with the OTEL specification. + +- traces.mapping presents the template mapping for creating the Simple Schema for Observability index +- traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure + +### data-stream +[data-stream](https://opensearch.org/docs/latest/opensearch/data-streams/) Data streams simplify this process and enforce a setup that best suits time-series data, such as being designed primarily for append-only data and ensuring that each document has a timestamp field. +A data stream is internally composed of multiple backing indices. Search requests are routed to all the backing indices, while indexing requests are routed to the latest write index. + +As part of the Observability naming scheme, the value of the data stream fields combine to the name of the actual data stream : + +`{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. +This means the fields can only contain characters that are valid as part of names of data streams. + +- **type** conforms to one of the supported Observability signals (Traces, Logs, Metrics, Alerts) +- **dataset** user defined field that can mainly be utilized for describing the origin of the signal +- **namespace** user custom field that can be used to describe any customer domain specific classification + +#### Timestamp field +As part of the data-stream definition the `@timestamp` is mandatory, if the field is not present to begin with use `ObservedTimestamp` as value for this field +**Note** - `@timestamp` value is the actual signal happening time and `observedTimestamp` is the time the exporter reads the actual event record. + +### Instrumentation scope +This is a logical unit of the application with which the emitted telemetry can be associated. It is typically the developer’s choice to decide what denotes a reasonable instrumentation scope. +The most common approach is to use the instrumentation library as the scope, however other scopes are also common, e.g. a module, a package, or a class can be chosen as the instrumentation scope. + +The instrumentation scope may have zero or more additional attributes that provide additional information about the scope. As an example the field +`instrumentationScope.attributes.identification` is presented will be used to determine the resource origin of the signal and can be used to filter accordingly + +## Traces +see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) + +Traces are defined implicitly by their Spans - In particular, a Trace can be thought of as a directed acyclic graph (DAG) of Spans, where the edges between Spans are defined as parent/child relationship. + +## Spans +A span represents an operation within a transaction. Each Span encapsulates the following state: +Observability in the software industry is the ability to monitor and diagnose systems and applications in real-time, in order to understand how they are behaving and identify potential issues. +Traces are a critical component of observability, providing detailed information about the flow of requests through a system, including timing information and any relevant contextual data. + +The importance of supporting traces schema lies in the fact that it enables better analysis and understanding of system behavior. +A structured schema provides a clear, consistent format for traces, making it easier for observability tools to process and aggregate the data. +This in turn makes it easier for engineers to understand the performance and behavior of their systems, and quickly identify potential issues. + +When traces are unstructured, it can be difficult for observability tools to extract meaningful information from them - For example, if the timing information for a particular request is not consistently represented in the same format, +it can be difficult to compare and analyze performance data over time. Similarly, if contextual data is not consistently recorded, it can be difficult to understand the context in which a particular request was executed. + +With a structured schema in place, observability tools can automatically extract and aggregate data, making it easier to understand system behavior at a high level. +This can help teams quickly identify performance bottlenecks, track the root cause of errors, and resolve issues more efficiently. + +## Details +The next section provides the Simple Schema for Observability support which conforms with the OTEL specification. + +- traces.mapping presents the template mapping for creating the Simple Schema for Observability index +- traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure + +### data-stream +[data-stream](https://opensearch.org/docs/latest/opensearch/data-streams/) Data streams simplify this process and enforce a setup that best suits time-series data, such as being designed primarily for append-only data and ensuring that each document has a timestamp field. +A data stream is internally composed of multiple backing indices. Search requests are routed to all the backing indices, while indexing requests are routed to the latest write index. + +As part of the Observability naming scheme, the value of the data stream fields combine to the name of the actual data stream : + +`{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. +This means the fields can only contain characters that are valid as part of names of data streams. + +- **type** conforms to one of the supported Observability signals (Traces, Logs, Metrics, Alerts) +- **dataset** user defined field that can mainly be utilized for describing the origin of the signal +- **namespace** user custom field that can be used to describe any customer domain specific classification + +#### Timestamp field +As part of the data-stream definition the `@timestamp` is mandatory, if the field is not present to begin with use `ObservedTimestamp` as value for this field +**Note** - `@timestamp` value is the actual signal happening time and `observedTimestamp` is the time the exporter reads the actual event record. + +### Instrumentation scope +This is a logical unit of the application with which the emitted telemetry can be associated. It is typically the developer’s choice to decide what denotes a reasonable instrumentation scope. +The most common approach is to use the instrumentation library as the scope, however other scopes are also common, e.g. a module, a package, or a class can be chosen as the instrumentation scope. + +The instrumentation scope may have zero or more additional attributes that provide additional information about the scope. As an example the field +`instrumentationScope.attributes.identification` is presented will be used to determine the resource origin of the signal and can be used to filter accordingly + +## Traces +see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) + +Traces are defined implicitly by their Spans - In particular, a Trace can be thought of as a directed acyclic graph (DAG) of Spans, where the edges between Spans are defined as parent/child relationship. + +## Spans +A span represents an operation within a transaction. Each Span encapsulates the following state: + +* An operation name +* start and finish timestamp +* Attributes list of key-value pairs. +* Set of Events, each of which is itself a tuple (timestamp, name, Attributes) +* Parent's Span identifier. +* Links to causally-related Spans (via the SpanContext of those related Spans). +* SpanContext information required to reference a Span. + +### SpanContext +Represents all the information that identifies Span in the Trace and is propagated to child Spans and across process boundaries. +A **SpanContext** contains the tracing identifiers and the options that are propagated from parent to child Spans. + +* `TraceId` - It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +* `SpanId` - It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +* `Tracestate` - carries tracing-system specific context in a list of key value pairs . Trace-state allows different vendors propagate additional information and inter-operate with their legacy Id formats. For more details see this. + +Additional fields can be supported via the Attributes key/value store see [traces](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/README.md) + +### Structure +The default fields that are supported by the traces are +- **TraceId** : It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +- **SpanId** : It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +- **ParentId** : It is the identifier for a span's parent span. +- **TraceState** : carries tracing-system specific context in a list of key value pairs. Tracestate allows different vendors propagate additional information and inter-operate with their legacy Id formats. + +- **Name** : String representing the span's name +- **Kind** + - SpanKind.CLIENT + - SpanKind.SERVER + - SpanKind.CONSUMER + - SpanKind.PRODUCER + - SpanKind.INTERNAL + +- **StartTime** : Start time of the event +- **EndTime** : End time of the event +- **Attributes** + - An Attribute is a key-value pair, which has the following structure [Attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/b00980832b4b823155001df56dbf9203d4e53f98/specification/common/README.md#attribute) + +- **DroppedAttributesCount** : Integer counting the dropped attributes +- **Events** : A set of the next tuples (timestamp, name, Attributes) +- **DroppedEventsCount** : Integer counting the dropped events +- **Links** : links to causally-related Spans +- **DroppedLinksCount** : Integer counting the dropped links +- **Status** - + + _status code is the int value + status message is the text representation_ + + - `UNSET = 0` : The default status. + - `OK = 1` : The operation has been validated by an Application developer or Operator to have completed successfully. + - `ERROR = 2` : The operation contains an error. \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/Usage.md b/src/main/resources/schema/observability/traces/Usage.md new file mode 100644 index 000000000..31fe0b9b3 --- /dev/null +++ b/src/main/resources/schema/observability/traces/Usage.md @@ -0,0 +1,113 @@ +# Actual Index setup and usage +The next document describes the practicality of manually defining and initiating of the observability schema indices, it assumes the OpenSearch cluster +is up and running. + +### Setting Up the Mapping +Start the OpenSearch cluster and follow the next steps for manually setup of the Log mapping template: + +`>> PUT _component_template/tracegroups_template` + +Copy the [traceGroups.mapping](traceGroups.mapping) + +`>> PUT _index_template/traces` + +Copy the [traces.mapping](traces.mapping) + +Now you can create an data-stream index (following the logs index pattern) that has the supported schema: + +`>> PUT _data_stream/sso_traces-dataset-test` + +You can also directly start ingesting data without creating a data stream. +Because we have a matching index template with a data_stream object, OpenSearch automatically creates the data stream: + +`POST sso_traces-dataset-test/_doc` +```json +{ + "body": "login attempt failed", + "@timestamp": "2013-03-01T00:00:00", + ... +} + +``` + +To see information about a that data stream: +`GET _data_stream/sso_traces-dataset-test` + +Would respond the following: +```json +{ + "data_streams" : [ + { + "name" : "sso_traces-dataset-test", + "timestamp_field" : { + "name" : "@timestamp" + }, + "indices" : [ + { + "index_name" : ".ds-sso_traces-dataset-test-000001", + "index_uuid" : "-VhmuhrQQ6ipYCmBhn6vLw" + } + ], + "generation" : 1, + "status" : "GREEN", + "template" : "sso_traces-*-*" + } + ] +} +``` + +To see more insights about the data stream, use the `_stats` endpoint: +`GET _data_stream/sso_traces-dataset-test/_stats` +Would respond the following: +```json +{ + "_shards" : { + "total" : 1, + "successful" : 1, + "failed" : 0 + }, + "data_stream_count" : 1, + "backing_indices" : 1, + "total_store_size_bytes" : 208, + "data_streams" : [ + { + "data_stream" : "sso_traces-dataset-test", + "backing_indices" : 1, + "store_size_bytes" : 208, + "maximum_timestamp" : 0 + } + ] +} +``` +### Ingestion +To ingest data into a data stream, you can use the regular indexing APIs. Make sure every document that you index has a timestamp field. + +`POST sso_traces-dataset-test/_doc` +```json +{ + "body": "login attempt failed", + "@timestamp": "2013-03-01T00:00:00", + ... +} + +``` +You can search a data stream just like you search a regular index or an index alias. The search operation applies to all of the backing indices (all data present in the stream). + +`GET sso_traces-dataset-test/_search` +```json +{ + "query": { + "match": { + ... + } + } +} +``` + +### Manage data-streams in OpenSearh + +To manage data streams from OpenSearch Dashboards, open OpenSearch Dashboards, choose Index Management, select Indices or Policy managed indices. + +see additional information: + - https://opensearch.org/docs/latest/opensearch/data-streams/#step-6-manage-data-streams-in-opensearch-dashboards + - https://opensearch.org/docs/latest/opensearch/data-streams/#step-5-rollover-a-data-stream \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/services.mapping b/src/main/resources/schema/observability/traces/services.mapping index 06b9ab681..377596d7b 100644 --- a/src/main/resources/schema/observability/traces/services.mapping +++ b/src/main/resources/schema/observability/traces/services.mapping @@ -1,65 +1,102 @@ { + "index_patterns": [ + "sso_services-*-*" + ], "template": { "mappings": { - "mappings": { - "dynamic_templates": [ + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "traces", + "component": "services", + "correlations": [ { - "strings_as_keyword": { - "match_mapping_type": "string", - "mapping": { - "ignore_above": 1024, - "type": "keyword" - } - } + "field": "traceGroupName", + "foreign-schema": "traceGroups", + "foreign-field": "traceGroup" } - ], - "date_detection": false, - "properties": { - "services": { - "destination": { - "properties": { - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "resource": { - "type": "keyword", - "ignore_above": 1024 - } - } + ] + }, + "_source": { + "enabled": true + }, + "dynamic_templates": [ + { + "attributes_map": { + "mapping": { + "type": "keyword" }, - "hashId": { + "path_match": "attributes.*" + } + } + ], + "properties": { + "destination": { + "properties": { + "domain": { "type": "keyword", "ignore_above": 1024 }, - "kind": { + "resource": { "type": "keyword", "ignore_above": 1024 - }, - "serviceName": { + } + } + }, + "hashId": { + "type": "keyword", + "ignore_above": 1024 + }, + "serviceName": { + "type": "keyword", + "ignore_above": 1024 + }, + "kind": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "properties": { + "domain": { "type": "keyword", "ignore_above": 1024 }, - "target": { - "properties": { - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "resource": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "traceGroupName": { + "resource": { "type": "keyword", "ignore_above": 1024 } } + }, + "traceGroupName": { + "type": "keyword", + "ignore_above": 1024 } } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } } + }, + "composed_of": [ + ], + "version": 1, + "_meta": { + "description": "Simple Schema For Observability Service", + "catalog": "observability", + "type": "services", + "correlations": [ + { + "field": "traceGroupName", + "foreign-schema": "traceGroups", + "foreign-field": "traceGroup" + } + ] } -} } \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/services.schema b/src/main/resources/schema/observability/traces/services.schema new file mode 100644 index 000000000..607181c75 --- /dev/null +++ b/src/main/resources/schema/observability/traces/services.schema @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://opensearch.org/schemas/observability/Service", + "type": "object", + "properties": { + "serviceName": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "destination": { + "type": "object", + "properties": { + "resource": { + "type": "string" + }, + "domain": { + "type": "string" + } + }, + "required": [ + "resource", + "domain" + ] + }, + "target": { + "type": "null" + }, + "traceGroupName": { + "$ref": "https://opensearch.org/schemas/observability/TraceGroups#/properties/traceGroup" + }, + "hashId": { + "type": "string" + } + }, + "required": [ + "serviceName", + "kind", + "destination", + "target", + "traceGroupName", + "hashId" + ] +} diff --git a/src/main/resources/schema/observability/traces/traceGroups.mapping b/src/main/resources/schema/observability/traces/traceGroups.mapping new file mode 100644 index 000000000..6a564dff5 --- /dev/null +++ b/src/main/resources/schema/observability/traces/traceGroups.mapping @@ -0,0 +1,31 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "traces", + "component": "traceGroups" + }, + "properties": { + "traceGroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroupFields": { + "properties": { + "endTime": { + "type": "date_nanos" + }, + "durationInNanos": { + "type": "long" + }, + "statusCode": { + "type": "integer" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/traceGroups.schema b/src/main/resources/schema/observability/traces/traceGroups.schema new file mode 100644 index 000000000..15b5416a0 --- /dev/null +++ b/src/main/resources/schema/observability/traces/traceGroups.schema @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://opensearch.org/schemas/observability/TraceGroups", + "type": "object", + "properties": { + "traceGroupFields": { + "type": "object", + "properties": { + "endTime": { + "type": "string" + }, + "durationInNanos": { + "type": "integer" + }, + "statusCode": { + "type": "integer" + } + }, + "required": [ + "endTime", + "durationInNanos", + "statusCode" + ] + }, + "traceGroup": { + "type": "string" + } + }, + "required": [ + "traceGroupFields", + "traceGroup" + ] +} \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/traces.mapping b/src/main/resources/schema/observability/traces/traces.mapping index a6be2931f..eaeb0e71e 100644 --- a/src/main/resources/schema/observability/traces/traces.mapping +++ b/src/main/resources/schema/observability/traces/traces.mapping @@ -6,7 +6,17 @@ "template": { "mappings": { "_meta": { - "version": "1.0.0" + "version": "1.0.0", + "catalog": "observability", + "type": "traces", + "component": "trace", + "correlations": [ + { + "field": "serviceName", + "foreign-schema": "services", + "foreign-field": "spanId" + } + ] }, "dynamic_templates": [ { @@ -112,6 +122,9 @@ "attributes": { "type": "object", "properties": { + "serviceName": { + "type": "keyword" + }, "data_stream": { "properties": { "dataset": { @@ -190,11 +203,19 @@ } }, "composed_of": [ + "tracegroups_template" ], "version": 1, "_meta": { "description": "Observability Traces Mapping Template", "catalog": "observability", - "type": "traces" + "type": "traces", + "correlations": [ + { + "field": "serviceName", + "foreign-schema": "services", + "foreign-field": "spanId" + } + ] } } \ No newline at end of file diff --git a/src/main/resources/schema/observability/traces/traces.schema b/src/main/resources/schema/observability/traces/traces.schema index 6225c0ed4..518a81fbb 100644 --- a/src/main/resources/schema/observability/traces/traces.schema +++ b/src/main/resources/schema/observability/traces/traces.schema @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-06/schema#", - "$id": "https://opensearch.org/schemas/Span", + "$id": "https://opensearch.org/schemas/observability/Span", "type": "object", "additionalProperties": false, "properties": { @@ -21,7 +21,7 @@ "$ref": "#/definitions/Status" }, "parentSpanId": { - "type": "string" + "$ref": "#/properties/spanId" }, "name": { "type": "string" @@ -40,6 +40,10 @@ ] } }, + "@timestamp": { + "type": "string", + "format": "date-time" + }, "startTime": { "type": "string", "format": "date-time" @@ -67,10 +71,10 @@ "additionalProperties": false, "properties": { "traceId": { - "type": "string" + "$ref": "#/properties/traceId" }, "spanId": { - "type": "string" + "$ref": "#/properties/spanId" }, "traceState": { "type": "array", @@ -112,6 +116,7 @@ "required": [ "traceId", "spanId", + "@timestamp", "startTime", "endTime", "kind", @@ -188,6 +193,9 @@ "type": "object", "additionalProperties": true, "properties": { + "serviceName": { + "$ref": "https://opensearch.org/schemas/observability/Service#/properties/serviceName" + }, "data_stream": { "$ref": "#/definitions/Dataflow" } diff --git a/src/main/resources/schema/security/README.md b/src/main/resources/schema/security/README.md new file mode 100644 index 000000000..f5f87bf52 --- /dev/null +++ b/src/main/resources/schema/security/README.md @@ -0,0 +1,9 @@ +# Security Domain Schema + +OpenSearch Security is a [plugin](https://github.com/opensearch-project/security) for OpenSearch that offers encryption, authentication and authorization. When combined with OpenSearch Security-Advanced Modules, it supports authentication via Active Directory, LDAP, Kerberos, JSON web tokens, SAML, OpenID and more. It includes fine grained role-based access control to indices, documents and fields. It also provides multi-tenancy support in OpenSearch Dashboards. + +Another leading security specification if the [Open Cybersecurity Schema Framework](https://github.com/ocsf/ocsf-schema) +OCSF is a framework for creating schemas and it also delivers a cybersecurity event schema built with the framework. + +OCSF framework is made up of a set of categories, event classes, data types, and an attribute dictionary. The framework is not restricted to cybersecurity nor to events, however the initial focus of the framework has been a schema for cybersecurity events. A schema browser for the cybersecurity schema can be found at schema.ocsf.io. This is the recommended way to explore the schema. + diff --git a/src/main/resources/schema/system/README.md b/src/main/resources/schema/system/README.md new file mode 100644 index 000000000..20bf9b48a --- /dev/null +++ b/src/main/resources/schema/system/README.md @@ -0,0 +1,57 @@ +# Internal System Schema + +This folder contains internal representation of assets that are stored in the system indices of dashboard and integration. + - Application + - Datasource + - Index-Pattern + - Integration + - Notebook + - Operational-Panel + - SavedQuery + - Visualization + +### Application +[Application](https://opensearch.org/docs/2.5/observing-your-data/app-analytics/) enables creation of custom observability display to view the availability status of your systems, where you can combine log events with trace and metric data into a single view of overall system health. +This lets you quickly pivot between logs, traces, and metrics to dig into the source of any issues. + + - [Schema](application.schema) + +### Datasource +[Data-source](https://opensearch.org/docs/2.4/dashboards/discover/multi-data-sources/) Enables adding multiple data sources to a single dashboard. +OpenSearch Dashboards allows you to dynamically manage data sources, create index patterns based on those data sources, and execute queries against a specific data source and then combine visualizations in one dashboard. + + - [Schema](datasource.schema) + +### Index-Pattern +An Index Pattern allows to access data that you want to explore. An index pattern selects the data to use. An index pattern may point to multiple indices, data stream, or index aliases. + + - [Schema](index-pattern.schema) + +### Integration +Integration is a schematized and categorized bundle of assets grouped together to allow simple and coherent way to view, analyze and investigate different aspects of your data. +Integrations allow pre-defining dashboards, visualizations, index-templates, saved-queries and additional assets so that they provide a complete meaningful user experience. + + - [Schema](integration.schema) + +### Notebook +[Notebook](https://opensearch.org/docs/2.5/observing-your-data/notebooks/) A notebook is a document composed of two elements: code blocks (Markdown/SQL/PPL) and visualizations. +Choose multiple timelines to compare and contrast visualizations. +You can also generate reports directly from your notebooks. Common use cases include creating postmortem reports, designing runbooks, building live infrastructure reports, and writing documentation. + + - [Schema](notebook.schema) + +### Operational-Panel +[Operational Panels](https://opensearch.org/docs/2.5/observing-your-data/operational-panels/) in OpenSearch Dashboards are collections of visualizations generated using Piped Processing Language (PPL) queries. + + - [Schema](operational-panel.schema) + +### Saved-Query +A saved query (saved search) allows to reuse a search created in a dashboard for other dashboards. + + - [Schema](saved-query.schema) + +### Visualization +[Visualization](https://opensearch.org/docs/2.5/dashboards/visualize/viz-index/) allows translation of complex, high-volume, or numerical data into a visual representation that is easier to process. +OpenSearch Dashboards gives you data visualization tools to improve and automate the visual communication process. By using visual elements like charts, graphs, or maps to represent data, you can advance business intelligence and support data-driven decision-making and strategic planning. + + - [Schema](visualization.schema) diff --git a/src/main/resources/schema/system/integration-fields-list.schema b/src/main/resources/schema/system/integration-fields-list.schema new file mode 100644 index 000000000..7ff0925da --- /dev/null +++ b/src/main/resources/schema/system/integration-fields-list.schema @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "template-name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "description": { + "type": "string" + }, + "catalog": { + "type": "string" + }, + "collections": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "category": { + "type": "string" + }, + "components": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "container": { + "type": "boolean" + }, + "fields": { + "type": "object" + } + }, + "required": [ + "source", + "container", + "fields" + ] + } + ] + } + }, + "required": [ + "category", + "components" + ] + } + ] + } + }, + "required": [ + "template-name", + "version", + "description", + "catalog", + "collections" + ] +} \ No newline at end of file