From 69927489d2c1e8adfe2a69e93787dd8e61b5353b Mon Sep 17 00:00:00 2001 From: Nageshbansal <76246968+Nageshbansal@users.noreply.github.com> Date: Thu, 11 Jan 2024 17:48:26 +0530 Subject: [PATCH] Fixes Probe logging for all iterations (#676) * Fixes Probe logging for all iterations Signed-off-by: nagesh bansal --- go.mod | 24 +++--- go.sum | 40 ++++----- pkg/cerrors/custom_errors.go | 1 + pkg/probe/cmdprobe.go | 125 +++++++++++++++++++++-------- pkg/probe/comparator/comparator.go | 17 ++-- pkg/probe/comparator/float.go | 2 +- pkg/probe/comparator/integer.go | 2 +- pkg/probe/comparator/string.go | 2 +- pkg/probe/httpprobe.go | 64 +++++++++++---- pkg/probe/k8sprobe.go | 57 +++++++++---- pkg/probe/probe.go | 33 +++++++- pkg/probe/promProbe.go | 57 +++++++++---- pkg/types/types.go | 18 +++-- 13 files changed, 317 insertions(+), 125 deletions(-) diff --git a/go.mod b/go.mod index c68041e91..ae41927da 100644 --- a/go.mod +++ b/go.mod @@ -9,15 +9,15 @@ require ( github.com/aws/aws-sdk-go v1.38.59 github.com/containerd/cgroups v1.0.1 github.com/kyokomi/emoji v2.2.4+incompatible - github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f + github.com/litmuschaos/chaos-operator v0.0.0-20240104104915-2d8472873222 github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.1 google.golang.org/api v0.48.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.22.1 - k8s.io/apimachinery v0.22.1 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 ) @@ -38,7 +38,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -58,12 +58,12 @@ require ( github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 // indirect @@ -71,11 +71,11 @@ require ( google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect + k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect sigs.k8s.io/controller-runtime v0.10.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 93ce51aa9..fb2cb5834 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,10 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -354,8 +356,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kyokomi/emoji v2.2.4+incompatible h1:np0woGKwx9LiHAQmwZx79Oc0rHpNw3o+3evou4BEPv4= github.com/kyokomi/emoji v2.2.4+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= -github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f h1:RJMARNqpMgUHDrI9Oy4pMr8RTAh/CkYha4Wn7xXHt9Q= -github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f/go.mod h1:uIIUKHTPSEwSC52esrSSxgEe3pLoLlfOGXRmhLcgKy4= +github.com/litmuschaos/chaos-operator v0.0.0-20240104104915-2d8472873222 h1:e7QsO2cL0/aMlCNM2BRRg2gfv05C9ZeVF2U0BE7IPcE= +github.com/litmuschaos/chaos-operator v0.0.0-20240104104915-2d8472873222/go.mod h1:yDZVtAgRVgoQtf8tSN58tpus0kGFFJXTj/bppJCRrdo= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -528,8 +530,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b h1:Qwe1rC8PSniVfAFPFJeyUkB+zcysC3RgJBAGk7eqBEU= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -612,8 +614,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -700,13 +702,13 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -715,8 +717,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -958,15 +960,15 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -975,8 +977,8 @@ sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2M sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/cerrors/custom_errors.go b/pkg/cerrors/custom_errors.go index 143a099d0..32dca59fc 100644 --- a/pkg/cerrors/custom_errors.go +++ b/pkg/cerrors/custom_errors.go @@ -29,6 +29,7 @@ const ( ErrorTypePromProbe ErrorType = "PROM_PROBE_ERROR" FailureTypePromProbe ErrorType = "PROM_PROBE_FAILURE" ErrorTypeTimeout ErrorType = "TIMEOUT" + FailureTypeProbeTimeout ErrorType = "PROBE_TIMEOUT" ) type userFriendly interface { diff --git a/pkg/probe/cmdprobe.go b/pkg/probe/cmdprobe.go index f2fa9ac20..6d26b6a30 100644 --- a/pkg/probe/cmdprobe.go +++ b/pkg/probe/cmdprobe.go @@ -81,7 +81,7 @@ func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types. } rc := getAndIncrementRunCount(resultDetails, probe.Name) - description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(out.String()), rc) + description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, probe.RunProperties.Verbosity, strings.TrimSpace(out.String()), rc) if err != nil { if strings.TrimSpace(stdErr.String()) != "" { return cerrors.Error{ @@ -133,7 +133,7 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li } rc := getAndIncrementRunCount(resultDetails, probe.Name) - if description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(output), rc); err != nil { + if description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, probe.RunProperties.Verbosity, strings.TrimSpace(output), rc); err != nil { if strings.TrimSpace(stdErr) != "" { return cerrors.Error{ ErrorCode: cerrors.FailureTypeCmdProbe, @@ -334,22 +334,33 @@ func triggerInlineContinuousCmdProbe(probe v1alpha1.ProbeAttributes, clients cli // it marked the error for the probes, if any loop: for { - err := triggerInlineCmdProbe(probe, chaosresult) - // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails - if err != nil { - err = addProbePhase(err, string(chaosDetails.Phase)) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) for index := range chaosresult.ProbeDetails { if chaosresult.ProbeDetails[index].Name == probe.Name { - chaosresult.ProbeDetails[index].IsProbeFailedWithError = err - chaosresult.ProbeDetails[index].Status.Description = getDescription(err) - log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err) - isExperimentFailed = true - break loop + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + err := triggerInlineCmdProbe(probe, chaosresult) + // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails + if err != nil { + err = addProbePhase(err, string(chaosDetails.Phase)) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].IsProbeFailedWithError = err + chaosresult.ProbeDetails[index].Status.Description = getDescription(err) + log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err) + isExperimentFailed = true + break loop + } } } + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort // if experiment fails but stopOnfailure is provided as false then it will continue the execution @@ -376,16 +387,20 @@ func triggerInlineOnChaosCmdProbe(probe v1alpha1.ProbeAttributes, clients client var endTime <-chan time.Time timeDelay := time.Duration(duration) * time.Second - + endTime = time.After(timeDelay) // it trigger the inline cmd probe for the entire duration of chaos and it fails, if any err encounter // it marked the error for the probes, if any loop: for { - endTime = time.After(timeDelay) select { case <-endTime: log.Infof("[Chaos]: Time is up for the %v probe", probe.Name) endTime = nil + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } break loop default: // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails @@ -401,8 +416,19 @@ loop: } } } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) + } } } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort @@ -438,6 +464,11 @@ loop: case <-endTime: log.Infof("[Chaos]: Time is up for the %v probe", probe.Name) endTime = nil + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } break loop default: // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails @@ -453,8 +484,20 @@ loop: } } } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) + + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) + } } } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort @@ -483,22 +526,35 @@ func triggerSourceContinuousCmdProbe(probe v1alpha1.ProbeAttributes, execCommand // it marked the error for the probes, if any loop: for { - err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult) - // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails - if err != nil { - err = addProbePhase(err, string(chaosDetails.Phase)) + + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) for index := range chaosresult.ProbeDetails { if chaosresult.ProbeDetails[index].Name == probe.Name { - chaosresult.ProbeDetails[index].IsProbeFailedWithError = err - chaosresult.ProbeDetails[index].Status.Description = getDescription(err) - log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err) - isExperimentFailed = true - break loop + chaosresult.ProbeDetails[index].HasProbeCompleted = true } } + break loop + + default: + err = triggerSourceCmdProbe(probe, execCommandDetails, clients, chaosresult) + // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails + if err != nil { + err = addProbePhase(err, string(chaosDetails.Phase)) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].IsProbeFailedWithError = err + chaosresult.ProbeDetails[index].Status.Description = getDescription(err) + log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err) + isExperimentFailed = true + break loop + } + } + } + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort // if experiment fails but stopOnfailure is provided as false then it will continue the execution @@ -512,13 +568,14 @@ loop: // validateResult validate the probe result to specified comparison operation // it supports int, float, string operands -func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput string, rc int) (string, error) { +func validateResult(comparator v1alpha1.ComparatorInfo, probeName, probeVerbosity string, cmdOutput string, rc int) (string, error) { compare := cmp.RunCount(rc). FirstValue(cmdOutput). SecondValue(comparator.Value). Criteria(comparator.Criteria). - ProbeName(probeName) + ProbeName(probeName). + ProbeVerbosity(probeVerbosity) switch strings.ToLower(comparator.Type) { case "int": @@ -695,7 +752,7 @@ func postChaosCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu case "Continuous", "OnChaos": if isInlineProbe(probe.CmdProbeInputs) { // it will check for the error, It will detect the error if any error encountered in probe during chaos - if err = checkForErrorInContinuousProbe(resultDetails, probe.Name); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeCmdProbe { + if err = checkForErrorInContinuousProbe(resultDetails, probe.Name, chaosDetails.Delay, chaosDetails.Timeout); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeCmdProbe && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { return err } // failing the probe, if the success condition doesn't met after the retry & timeout combinations @@ -704,7 +761,7 @@ func postChaosCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu } } else { // it will check for the error, It will detect the error if any error encountered in probe during chaos - if err = checkForErrorInContinuousProbe(resultDetails, probe.Name); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeCmdProbe { + if err = checkForErrorInContinuousProbe(resultDetails, probe.Name, chaosDetails.Delay, chaosDetails.Timeout); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeCmdProbe && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { return err } diff --git a/pkg/probe/comparator/comparator.go b/pkg/probe/comparator/comparator.go index f87563c01..2c594f630 100644 --- a/pkg/probe/comparator/comparator.go +++ b/pkg/probe/comparator/comparator.go @@ -3,11 +3,12 @@ package comparator // Model contains operands and operator for the comparison operations // a and b attribute belongs to operands and operator attribute belongs to operator type Model struct { - a interface{} - b interface{} - operator string - rc int - probeName string + a interface{} + b interface{} + operator string + rc int + probeName string + probeVerbosity string } // RunCount sets the run counts @@ -51,3 +52,9 @@ func (model *Model) ProbeName(probeName string) *Model { model.probeName = probeName return model } + +// ProbeVerbosity sets the name of the probe under evaluation +func (model *Model) ProbeVerbosity(verbosity string) *Model { + model.probeVerbosity = verbosity + return model +} diff --git a/pkg/probe/comparator/float.go b/pkg/probe/comparator/float.go index e6b023240..1a0aa39da 100644 --- a/pkg/probe/comparator/float.go +++ b/pkg/probe/comparator/float.go @@ -17,7 +17,7 @@ func (model Model) CompareFloat(errorCode cerrors.ErrorType) error { obj := Float{} obj.setValues(reflect.ValueOf(model.a).String(), reflect.ValueOf(model.b).String()) - if model.rc == 1 { + if model.probeVerbosity != "info" || (model.probeVerbosity == "info" && model.rc == 1) { log.Infof("[Probe]: {Actual value: %v}, {Expected value: %v}, {Operator: %v}", obj.a, obj.b, model.operator) } diff --git a/pkg/probe/comparator/integer.go b/pkg/probe/comparator/integer.go index 81cc7fdc1..c947409b5 100644 --- a/pkg/probe/comparator/integer.go +++ b/pkg/probe/comparator/integer.go @@ -17,7 +17,7 @@ func (model Model) CompareInt(errorCode cerrors.ErrorType) error { obj := Integer{} obj.setValues(reflect.ValueOf(model.a).String(), reflect.ValueOf(model.b).String()) - if model.rc == 1 { + if model.probeVerbosity != "info" || (model.probeVerbosity == "info" && model.rc == 1) { log.Infof("[Probe]: {Actual value: %v}, {Expected value: %v}, {Operator: %v}", obj.a, obj.b, model.operator) } diff --git a/pkg/probe/comparator/string.go b/pkg/probe/comparator/string.go index b88f34b1f..a839c59f6 100644 --- a/pkg/probe/comparator/string.go +++ b/pkg/probe/comparator/string.go @@ -17,7 +17,7 @@ func (model Model) CompareString(errorCode cerrors.ErrorType) error { obj := String{} obj.setValues(reflect.ValueOf(model.a).String(), reflect.ValueOf(model.b).String()) - if model.rc == 1 { + if model.probeVerbosity != "info" || (model.probeVerbosity == "info" && model.rc == 1) { log.Infof("[Probe]: {Actual value: %v}, {Expected value: %v}, {Operator: %v}", obj.a, obj.b, model.operator) } diff --git a/pkg/probe/httpprobe.go b/pkg/probe/httpprobe.go index cbac3542b..8412adeae 100644 --- a/pkg/probe/httpprobe.go +++ b/pkg/probe/httpprobe.go @@ -33,7 +33,7 @@ func prepareHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets return err } case "postchaos": - if err := postChaosHTTPProbe(probe, resultDetails); err != nil { + if err := postChaosHTTPProbe(probe, resultDetails, chaosDetails.Delay, chaosDetails.Timeout); err != nil { return err } case "duringchaos": @@ -127,6 +127,7 @@ func httpGet(probe v1alpha1.ProbeAttributes, client *http.Client, resultDetails SecondValue(probe.HTTPProbeInputs.Method.Get.ResponseCode). Criteria(probe.HTTPProbeInputs.Method.Get.Criteria). ProbeName(probe.Name). + ProbeVerbosity(probe.RunProperties.Verbosity). CompareInt(cerrors.FailureTypeHttpProbe); err != nil { log.Errorf("The %v http probe get method has Failed, err: %v", probe.Name, err) return err @@ -169,6 +170,7 @@ func httpPost(probe v1alpha1.ProbeAttributes, client *http.Client, resultDetails SecondValue(probe.HTTPProbeInputs.Method.Post.ResponseCode). Criteria(probe.HTTPProbeInputs.Method.Post.Criteria). ProbeName(probe.Name). + ProbeVerbosity(probe.RunProperties.Verbosity). CompareInt(cerrors.FailureTypeHttpProbe); err != nil { log.Errorf("The %v http probe post method has Failed, err: %v", probe.Name, err) return err @@ -222,24 +224,36 @@ func triggerContinuousHTTPProbe(probe v1alpha1.ProbeAttributes, clients clients. // it triggers the http probe for the entire duration of chaos and it fails, if any error encounter // it marked the error for the probes, if any + loop: for { - err = triggerHTTPProbe(probe, chaosresult) - // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails - if err != nil { - err = addProbePhase(err, string(chaosDetails.Phase)) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) for index := range chaosresult.ProbeDetails { if chaosresult.ProbeDetails[index].Name == probe.Name { - chaosresult.ProbeDetails[index].IsProbeFailedWithError = err - chaosresult.ProbeDetails[index].Status.Description = getDescription(err) - log.Errorf("The %v http probe has been Failed, err: %v", probe.Name, err) - isExperimentFailed = true - break loop + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + err = triggerHTTPProbe(probe, chaosresult) + // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails + if err != nil { + err = addProbePhase(err, string(chaosDetails.Phase)) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].IsProbeFailedWithError = err + chaosresult.ProbeDetails[index].Status.Description = getDescription(err) + log.Errorf("The %v http probe has been Failed, err: %v", probe.Name, err) + isExperimentFailed = true + break loop + } } } + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort // if experiment fails but stopOnfailure is provided as false then it will continue the execution @@ -292,13 +306,15 @@ func preChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu "Mode": probe.Mode, "Phase": "PreChaos", }) + go triggerContinuousHTTPProbe(probe, clients, resultDetails, chaosDetails) + } return nil } // postChaosHTTPProbe trigger the http probe for postchaos phase -func postChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.ResultDetails) error { +func postChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.ResultDetails, delay int, timeout int) error { probeTimeout := getProbeTimeouts(probe.Name, resultDetails.ProbeDetails) switch probe.Mode { @@ -331,7 +347,7 @@ func postChaosHTTPProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Res } case "Continuous", "OnChaos": // it will check for the error, It will detect the error if any error encountered in probe during chaos - if err = checkForErrorInContinuousProbe(resultDetails, probe.Name); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeHttpProbe { + if err = checkForErrorInContinuousProbe(resultDetails, probe.Name, delay, timeout); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeHttpProbe && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { return err } // failing the probe, if the success condition doesn't met after the retry & timeout combinations @@ -365,6 +381,11 @@ loop: case <-endTime: log.Infof("[Chaos]: Time is up for the %v probe", probe.Name) endTime = nil + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } break loop default: err = triggerHTTPProbe(probe, chaosresult) @@ -381,8 +402,19 @@ loop: } } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) + } } } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort diff --git a/pkg/probe/k8sprobe.go b/pkg/probe/k8sprobe.go index 91b420586..aa33eaeb8 100644 --- a/pkg/probe/k8sprobe.go +++ b/pkg/probe/k8sprobe.go @@ -138,22 +138,34 @@ func triggerContinuousK8sProbe(probe v1alpha1.ProbeAttributes, clients clients.C // marked the error for the probes, if any loop: for { - err = triggerK8sProbe(probe, clients, chaosresult) - // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails - if err != nil { - err = addProbePhase(err, string(chaosDetails.Phase)) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) for index := range chaosresult.ProbeDetails { if chaosresult.ProbeDetails[index].Name == probe.Name { - chaosresult.ProbeDetails[index].IsProbeFailedWithError = err - chaosresult.ProbeDetails[index].Status.Description = getDescription(err) - log.Errorf("the %v k8s probe has been Failed, err: %v", probe.Name, err) - isExperimentFailed = true - break loop + chaosresult.ProbeDetails[index].HasProbeCompleted = true } } + break loop + + default: + err = triggerK8sProbe(probe, clients, chaosresult) + // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails + if err != nil { + err = addProbePhase(err, string(chaosDetails.Phase)) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].IsProbeFailedWithError = err + chaosresult.ProbeDetails[index].Status.Description = getDescription(err) + log.Errorf("the %v k8s probe has been Failed, err: %v", probe.Name, err) + isExperimentFailed = true + break loop + } + } + } + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort // if experiment fails but stopOnfailure is provided as false then it will continue the execution @@ -357,7 +369,7 @@ func postChaosK8sProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resu } case "continuous", "onchaos": // it will check for the error, It will detect the error if any error encountered in probe during chaos - if err = checkForErrorInContinuousProbe(resultDetails, probe.Name); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeK8sProbe { + if err = checkForErrorInContinuousProbe(resultDetails, probe.Name, chaosDetails.Delay, chaosDetails.Timeout); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeK8sProbe && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { return err } // failing the probe, if the success condition doesn't met after the retry & timeout combinations @@ -409,6 +421,11 @@ loop: select { case <-endTime: log.Infof("[Chaos]: Time is up for the %v probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } break loop default: err = triggerK8sProbe(probe, clients, chaosresult) @@ -425,8 +442,20 @@ loop: } } } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) + + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) + } } } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index d3e1ac41c..909bdaeca 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -6,6 +6,7 @@ import ( "fmt" "html/template" "strings" + "time" "github.com/kyokomi/emoji" "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" @@ -55,6 +56,8 @@ func RunProbes(chaosDetails *types.ChaosDetails, clients clients.ClientSets, res // it first evaluate the onchaos and continuous modes then it evaluates the other modes // as onchaos and continuous probes are already completed var probeError []string + // call cancel function from chaosDetails context + chaosDetails.ProbeContext.CancelFunc() for _, probe := range probes { // evaluate continuous and onchaos probes switch strings.ToLower(probe.Mode) { @@ -251,8 +254,29 @@ func getDescription(err error) string { } // CheckForErrorInContinuousProbe check for the error in the continuous probes -func checkForErrorInContinuousProbe(resultDetails *types.ResultDetails, probeName string) error { - +func checkForErrorInContinuousProbe(resultDetails *types.ResultDetails, probeName string, delay int, timeout int) error { + + probe := getProbeByName(probeName, resultDetails.ProbeDetails) + startTime := time.Now() + timeoutSignal := time.After(time.Duration(timeout) * time.Second) + +loop: + for { + select { + case <-timeoutSignal: + return cerrors.Error{ + ErrorCode: cerrors.FailureTypeProbeTimeout, + Target: fmt.Sprintf("{probe: %s, timeout: %ds}", probeName, timeout), + Reason: "Probe is failed due to timeout", + } + default: + if probe.HasProbeCompleted { + break loop + } + log.Infof("[Probe]: Waiting for %s probe to finish or timeout (Elapsed time: %v s)", probeName, time.Since(startTime).Seconds()) + time.Sleep(time.Duration(delay) * time.Second) + } + } for index, probe := range resultDetails.ProbeDetails { if probe.Name == probeName { return resultDetails.ProbeDetails[index].IsProbeFailedWithError @@ -281,7 +305,10 @@ func parseCommand(templatedCommand string, resultDetails *types.ResultDetails) ( // stopChaosEngine update the probe status and patch the chaosengine to stop state func stopChaosEngine(probe v1alpha1.ProbeAttributes, clients clients.ClientSets, chaosresult *types.ResultDetails, chaosDetails *types.ChaosDetails) error { // it will check for the error, It will detect the error if any error encountered in probe during chaos - err = checkForErrorInContinuousProbe(chaosresult, probe.Name) + if err = checkForErrorInContinuousProbe(chaosresult, probe.Name, chaosDetails.Timeout, chaosDetails.Delay); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { + return err + } + // failing the probe, if the success condition doesn't met after the retry & timeout combinations markedVerdictInEnd(err, chaosresult, probe, "PostChaos") //patch chaosengine's state to stop diff --git a/pkg/probe/promProbe.go b/pkg/probe/promProbe.go index 5e8d553f8..4e02aa7fc 100644 --- a/pkg/probe/promProbe.go +++ b/pkg/probe/promProbe.go @@ -134,7 +134,7 @@ func postChaosPromProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Res case "continuous", "onchaos": // it will check for the error, It will detect the error if any error encountered in probe during chaos - if err = checkForErrorInContinuousProbe(resultDetails, probe.Name); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypePromProbe { + if err = checkForErrorInContinuousProbe(resultDetails, probe.Name, chaosDetails.Delay, chaosDetails.Timeout); err != nil && cerrors.GetErrorType(err) != cerrors.FailureTypePromProbe && cerrors.GetErrorType(err) != cerrors.FailureTypeProbeTimeout { return err } @@ -216,6 +216,7 @@ func triggerPromProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.Resul SecondValue(probe.PromProbeInputs.Comparator.Value). Criteria(probe.PromProbeInputs.Comparator.Criteria). ProbeName(probe.Name). + ProbeVerbosity(probe.RunProperties.Verbosity). CompareFloat(cerrors.FailureTypePromProbe); err != nil { log.Errorf("The %v prom probe has been Failed, err: %v", probe.Name, err) return err @@ -244,22 +245,33 @@ func triggerContinuousPromProbe(probe v1alpha1.ProbeAttributes, clients clients. // it marked the error for the probes, if any loop: for { - err = triggerPromProbe(probe, chaosresult) - // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails - if err != nil { - err = addProbePhase(err, string(chaosDetails.Phase)) + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) for index := range chaosresult.ProbeDetails { if chaosresult.ProbeDetails[index].Name == probe.Name { - chaosresult.ProbeDetails[index].IsProbeFailedWithError = err - chaosresult.ProbeDetails[index].Status.Description = getDescription(err) - log.Errorf("The %v prom probe has been Failed, err: %v", probe.Name, err) - isExperimentFailed = true - break loop + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + err = triggerPromProbe(probe, chaosresult) + // record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails + if err != nil { + err = addProbePhase(err, string(chaosDetails.Phase)) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].IsProbeFailedWithError = err + chaosresult.ProbeDetails[index].Status.Description = getDescription(err) + log.Errorf("The %v prom probe has been Failed, err: %v", probe.Name, err) + isExperimentFailed = true + break loop + } } } + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort // if experiment fails but stopOnfailure is provided as false then it will continue the execution @@ -293,6 +305,11 @@ loop: select { case <-endTime: log.Infof("[Chaos]: Time is up for the %v probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } endTime = nil break loop default: @@ -309,8 +326,20 @@ loop: } } } - // waiting for the probe polling interval - time.Sleep(probeTimeout.ProbePollingInterval) + + select { + case <-chaosDetails.ProbeContext.Ctx.Done(): + log.Infof("Stopping %s continuous Probe", probe.Name) + for index := range chaosresult.ProbeDetails { + if chaosresult.ProbeDetails[index].Name == probe.Name { + chaosresult.ProbeDetails[index].HasProbeCompleted = true + } + } + break loop + default: + // waiting for the probe polling interval + time.Sleep(probeTimeout.ProbePollingInterval) + } } } // if experiment fails and stopOnfailure is provided as true then it will patch the chaosengine for abort diff --git a/pkg/types/types.go b/pkg/types/types.go index 79a0657b8..c83bd66ed 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -86,6 +86,7 @@ type ProbeDetails struct { Status v1alpha1.ProbeStatus IsProbeFailedWithError error Failed bool + HasProbeCompleted bool RunID string RunCount int Stopped bool @@ -132,6 +133,7 @@ type ChaosDetails struct { ImagePullSecrets []corev1.LocalObjectReference Labels map[string]string Phase ExperimentPhase + ProbeContext ProbeContext SideCar []SideCar } @@ -150,6 +152,11 @@ type ParentResource struct { Namespace string } +type ProbeContext struct { + Ctx context.Context + CancelFunc context.CancelFunc +} + // AppDetails contains all the application related envs type AppDetails struct { Namespace string @@ -190,7 +197,7 @@ func parse(val string) []string { return strings.Split(val, ",") } -//InitialiseChaosVariables initialise all the global variables +// InitialiseChaosVariables initialise all the global variables func InitialiseChaosVariables(chaosDetails *ChaosDetails) { targets := Getenv("TARGETS", "") chaosDetails.AppDetail = GetTargets(strings.TrimSpace(targets)) @@ -211,9 +218,10 @@ func InitialiseChaosVariables(chaosDetails *ChaosDetails) { chaosDetails.ParentsResources = []ParentResource{} chaosDetails.Targets = []v1alpha1.TargetDetails{} chaosDetails.Phase = PreChaosPhase + chaosDetails.ProbeContext.Ctx, chaosDetails.ProbeContext.CancelFunc = context.WithCancel(context.Background()) } -//SetResultAttributes initialise all the chaos result ENV +// SetResultAttributes initialise all the chaos result ENV func SetResultAttributes(resultDetails *ResultDetails, chaosDetails ChaosDetails) { resultDetails.Verdict = "Awaited" resultDetails.Phase = "Running" @@ -230,7 +238,7 @@ func SetResultAttributes(resultDetails *ResultDetails, chaosDetails ChaosDetails } -//SetResultAfterCompletion set all the chaos result ENV in the EOT +// SetResultAfterCompletion set all the chaos result ENV in the EOT func SetResultAfterCompletion(resultDetails *ResultDetails, verdict v1alpha1.ResultVerdict, phase v1alpha1.ResultPhase, failStep string, errorCode cerrors.ErrorType) { resultDetails.Verdict = verdict resultDetails.Phase = phase @@ -242,7 +250,7 @@ func SetResultAfterCompletion(resultDetails *ResultDetails, verdict v1alpha1.Res } } -//SetEngineEventAttributes initialise attributes for event generation in chaos engine +// SetEngineEventAttributes initialise attributes for event generation in chaos engine func SetEngineEventAttributes(eventsDetails *EventDetails, Reason, Message, Type string, chaosDetails *ChaosDetails) { eventsDetails.Reason = Reason @@ -253,7 +261,7 @@ func SetEngineEventAttributes(eventsDetails *EventDetails, Reason, Message, Type } -//SetResultEventAttributes initialise attributes for event generation in chaos result +// SetResultEventAttributes initialise attributes for event generation in chaos result func SetResultEventAttributes(eventsDetails *EventDetails, Reason, Message, Type string, resultDetails *ResultDetails) { eventsDetails.Reason = Reason