From 039adaa40c6c4c1df4c2abfd39adfca9a5783daf Mon Sep 17 00:00:00 2001 From: Vivek Singh Chauhan Date: Wed, 7 Aug 2024 17:44:50 +0100 Subject: [PATCH 1/2] APIGOV-28475 - TA config optimization - Use CENTRAL_METRICREPORTING_PUBLISH and CENTRAL_METRICREPORTING_SCHEDULE for configuring metric event processing - Remove TRACEABILITY_SAMPLING_REPORTALLERRORS and use TRACEABILITY_SAMPLING_ONLYERRORS - Replace CENTRAL_USAGEREPORTING_USAGESCHEDULE with CENTRAL_USAGEREPORTING_SCHEDULE --- build/traceability/kong_traceability_agent.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build/traceability/kong_traceability_agent.yml b/build/traceability/kong_traceability_agent.yml index 1aa2850..85eb68f 100644 --- a/build/traceability/kong_traceability_agent.yml +++ b/build/traceability/kong_traceability_agent.yml @@ -16,11 +16,12 @@ kong_traceability_agent: agentName: ${CENTRAL_AGENTNAME:""} platformURL: ${CENTRAL_PLATFORMURL:https://platform.axway.com} reportActivityFrequency: ${CENTRAL_REPORTACTIVITYFREQUENCY:5m} + metricReporting: + publish: ${CENTRAL_METRICREPORTING_PUBLISH} + schedule: ${CENTRAL_METRICREPORTING_SCHEDULE} usageReporting: publish: ${CENTRAL_USAGEREPORTING_PUBLISH} - publishMetric: ${CENTRAL_USAGEREPORTING_PUBLISHMETRIC} - interval: ${CENTRAL_USAGEREPORTING_INTERVAL} - usageSchedule: ${CENTRAL_USAGEREPORTING_USAGESCHEDULE} + schedule: ${CENTRAL_USAGEREPORTING_SCHEDULE} offline: ${CENTRAL_USAGEREPORTING_OFFLINE} offlineSchedule: ${CENTRAL_USAGEREPORTING_OFFLINESCHEDULE} auth: @@ -114,10 +115,10 @@ output.traceability: sanitize: ${TRACEABILITY_REDACTION_JMSPROPERTIES_SANITIZE:[]} maskingCharacters: ${TRACEABILITY_REDACTION_MASKING_CHARACTERS:"\u007B*\u007D"} # unicode for {*} sampling: - percentage: ${TRACEABILITY_SAMPLING_PERCENTAGE:10} + percentage: ${TRACEABILITY_SAMPLING_PERCENTAGE:0} per_api: ${TRACEABILITY_SAMPLING_PER_API:true} per_subscription: ${TRACEABILITY_SAMPLING_PER_SUBSCRIPTION:true} - reportAllErrors: ${TRACEABILITY_SAMPLING_REPORTALLERRORS:true} + onlyErrors: ${TRACEABILITY_SAMPLING_ONLYERRORS:false} apiExceptionsList: ${TRACEABILITY_EXCEPTION_LIST:[]} queue: From 43507d3c59741ee187581d4913c0d01bd3c31db5 Mon Sep 17 00:00:00 2001 From: Vivek Singh Chauhan Date: Thu, 8 Aug 2024 19:08:15 +0100 Subject: [PATCH 2/2] APIGOV-28475 - unit test fixes --- go.mod | 10 ++++---- go.sum | 25 ++++++++++---------- pkg/discovery/config/config.go | 5 +++- pkg/discovery/config/config_test.go | 3 ++- pkg/traceability/config/config.go | 3 ++- pkg/traceability/config/config_test.go | 3 ++- pkg/traceability/processor/processor_test.go | 9 ++++++- 7 files changed, 35 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 3d2d7d6..456b33e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 toolchain go1.21.3 require ( - github.com/Axway/agent-sdk v1.1.94-0.20240805174013-ce6ec6cca09d + github.com/Axway/agent-sdk v1.1.94-0.20240808174610-fde925d6eb18 github.com/elastic/beats/v7 v7.17.20 github.com/google/uuid v1.6.0 github.com/kong/go-kong v0.47.0 @@ -51,11 +51,11 @@ require ( github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect - github.com/getkin/kin-openapi v0.125.0 // indirect + github.com/getkin/kin-openapi v0.127.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/swag v0.22.8 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/flock v0.7.2-0.20190320160742-5135e617513b // indirect @@ -79,7 +79,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.2.0 // indirect + github.com/invopop/yaml v0.3.1 // indirect github.com/jcchavezs/porto v0.6.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index ab19c9a..eb5b8c7 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Axway/agent-sdk v1.1.94-0.20240805174013-ce6ec6cca09d h1:0Bij9P71f3s+/GaIG4CX1Q7ncNIzIbtHCGtMGgsR5rs= -github.com/Axway/agent-sdk v1.1.94-0.20240805174013-ce6ec6cca09d/go.mod h1:BC7A1o93nyku8PH/0xZ6XPq7XQSXeBnSZf2iWf5zCpM= +github.com/Axway/agent-sdk v1.1.94-0.20240808174610-fde925d6eb18 h1:4Mz01BD8Cl44/KsixH08nCMQqi/8NLmHoTL7VV+Yioo= +github.com/Axway/agent-sdk v1.1.94-0.20240808174610-fde925d6eb18/go.mod h1:7K35oTQ1TwHFA3fzYKzYSFxnmTPcg57Cz+EzKKsxsqE= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -167,8 +167,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= -github.com/getkin/kin-openapi v0.125.0 h1:jyQCyf2qXS1qvs2U00xQzkGCqYPhEhZDmSmVt65fXno= -github.com/getkin/kin-openapi v0.125.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= +github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -183,15 +183,15 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= -github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -293,8 +293,8 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -324,8 +324,8 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= -github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= +github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= github.com/jcchavezs/porto v0.6.0 h1:AgQLGwsXaxDkPj4Y+paFkVGLAR4n/1RRF0xV5UKinwg= github.com/jcchavezs/porto v0.6.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= @@ -978,7 +978,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/pkg/discovery/config/config.go b/pkg/discovery/config/config.go index 9646de8..dc47470 100644 --- a/pkg/discovery/config/config.go +++ b/pkg/discovery/config/config.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" + "github.com/Axway/agent-sdk/pkg/cmd/properties" corecfg "github.com/Axway/agent-sdk/pkg/config" "github.com/Axway/agent-sdk/pkg/util/log" ) @@ -12,7 +13,7 @@ import ( type props interface { AddStringProperty(name string, defaultVal string, description string) AddStringSliceProperty(name string, defaultVal []string, description string) - AddIntProperty(name string, defaultVal int, description string) + AddIntProperty(name string, defaultVal int, description string, options ...properties.IntOpt) AddBoolProperty(name string, defaultVal bool, description string) StringPropertyValue(name string) string StringSlicePropertyValue(name string) []string @@ -20,6 +21,8 @@ type props interface { BoolPropertyValue(name string) bool } +// Methods for adding yaml properties and command flag + const ( cfgKongACLDisable = "kong.acl.disable" cfgKongAdminUrl = "kong.admin.url" diff --git a/pkg/discovery/config/config_test.go b/pkg/discovery/config/config_test.go index ea6c98e..bc4e0fd 100644 --- a/pkg/discovery/config/config_test.go +++ b/pkg/discovery/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "testing" + "github.com/Axway/agent-sdk/pkg/cmd/properties" "github.com/stretchr/testify/assert" ) @@ -79,7 +80,7 @@ func (f *fakeProps) AddStringSliceProperty(name string, defaultVal []string, des f.props[name] = propData{"string", description, defaultVal} } -func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string) { +func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string, options ...properties.IntOpt) { f.props[name] = propData{"int", description, defaultVal} } diff --git a/pkg/traceability/config/config.go b/pkg/traceability/config/config.go index 1e240a3..ea129c1 100644 --- a/pkg/traceability/config/config.go +++ b/pkg/traceability/config/config.go @@ -4,12 +4,13 @@ import ( "fmt" "strings" + "github.com/Axway/agent-sdk/pkg/cmd/properties" corecfg "github.com/Axway/agent-sdk/pkg/config" ) type props interface { AddStringProperty(name string, defaultVal string, description string) - AddIntProperty(name string, defaultVal int, description string) + AddIntProperty(name string, defaultVal int, description string, options ...properties.IntOpt) StringPropertyValue(name string) string IntPropertyValue(name string) int } diff --git a/pkg/traceability/config/config_test.go b/pkg/traceability/config/config_test.go index e322af8..606cba6 100644 --- a/pkg/traceability/config/config_test.go +++ b/pkg/traceability/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "testing" + "github.com/Axway/agent-sdk/pkg/cmd/properties" "github.com/stretchr/testify/assert" ) @@ -35,7 +36,7 @@ func (f *fakeProps) AddStringProperty(name string, defaultVal string, descriptio f.props[name] = propData{"string", description, defaultVal} } -func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string) { +func (f *fakeProps) AddIntProperty(name string, defaultVal int, description string, options ...properties.IntOpt) { f.props[name] = propData{"int", description, defaultVal} } diff --git a/pkg/traceability/processor/processor_test.go b/pkg/traceability/processor/processor_test.go index 81c38ef..9644611 100644 --- a/pkg/traceability/processor/processor_test.go +++ b/pkg/traceability/processor/processor_test.go @@ -76,6 +76,7 @@ func TestNewHandler(t *testing.T) { constructorErr bool expectedEvents int expectedMetricDetails int + hasErrors bool }{ "expect error creating handler, when no data sent into handler": { data: []byte{}, @@ -91,6 +92,7 @@ func TestNewHandler(t *testing.T) { data: testErrorData, expectedEvents: 2, expectedMetricDetails: 1, + hasErrors: true, }, "handle data with sampling setup": { data: testData, @@ -105,7 +107,12 @@ func TestNewHandler(t *testing.T) { ctx := context.WithValue(context.Background(), "test", name) redaction.SetupGlobalRedaction(redaction.DefaultConfig()) - sampling.SetupSampling(sampling.DefaultConfig(), false) + sCfg := sampling.DefaultConfig() + sCfg.Percentage = 1 + if tc.hasErrors { + sCfg.OnlyErrors = true + } + sampling.SetupSampling(sCfg, false) // create the handler h, err := NewEventsHandler(ctx, tc.data)