From 2e207e58fd2b9026cd2506a35e741eed30d12dcf Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Thu, 20 Jul 2023 17:58:03 +0200 Subject: [PATCH 01/11] introduce custom formats module Signed-off-by: Peter Wilcsinszky --- .gitignore | 2 + Dockerfile | 6 +- Makefile | 9 +++ formats/custom/go.mod | 18 ++++++ formats/custom/go.sum | 138 +++++++++++++++++++++++++++++++++++++++++ formats/custom/stub.go | 20 ++++++ formats/log.go | 50 +++++++++++++++ loggen/loggen.go | 47 +++++--------- 8 files changed, 258 insertions(+), 32 deletions(-) create mode 100644 formats/custom/go.mod create mode 100644 formats/custom/go.sum create mode 100644 formats/custom/stub.go diff --git a/.gitignore b/.gitignore index cb09da2..8c7cf21 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,5 @@ conf/config.ini .licensei.cache .DS_Store + +go.work* diff --git a/Dockerfile b/Dockerfile index 1999407..c1abbc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM golang:1.20-alpine3.17 as builder +FROM golang:1.20-alpine3.18 as builder + +RUN apk -U add make WORKDIR /workspace @@ -14,7 +16,7 @@ RUN go mod download ADD . . # Build -RUN CGO_ENABLED=0 go build -a -o bin/loggen main.go +RUN make build # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index 48cba5f..08af3fc 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ BIN := ${PWD}/bin export PATH := ${BIN}:${PATH} +CUSTOM_FORMATS := formats/custom + LICENSEI := ${BIN}/licensei LICENSEI_VERSION = v0.8.0 @@ -20,6 +22,13 @@ license-cache: ${LICENSEI} ## Generate license cache .PHONY: check check: license-cache license-check +go.work: + go work init . ${CUSTOM_FORMATS} + +.PHONY: +build: go.work + CGO_ENABLED=0 go build -a -o ${BIN}/loggen main.go + ${LICENSEI}: ${LICENSEI}_${LICENSEI_VERSION} | ${BIN} ln -sf $(notdir $<) $@ diff --git a/formats/custom/go.mod b/formats/custom/go.mod new file mode 100644 index 0000000..338f420 --- /dev/null +++ b/formats/custom/go.mod @@ -0,0 +1,18 @@ +module github.com/kube-logging/log-generator/formats/custom + +go 1.20 + +require github.com/prometheus/client_golang v1.11.0 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) diff --git a/formats/custom/go.sum b/formats/custom/go.sum new file mode 100644 index 0000000..d8b3ec1 --- /dev/null +++ b/formats/custom/go.sum @@ -0,0 +1,138 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/formats/custom/stub.go b/formats/custom/stub.go new file mode 100644 index 0000000..637dfe0 --- /dev/null +++ b/formats/custom/stub.go @@ -0,0 +1,20 @@ +package custom + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type Log interface { + String() (string, float64) + Labels() prometheus.Labels +} + +// Formats returns supported template formats for a given type +func Formats() map[string][]string { + return map[string][]string{} +} + +// LogFactory creates log events for a log format and optionally randomises it +func LogFactory(logType string, format string, randomise bool) (Log, error) { + return nil, nil +} diff --git a/formats/log.go b/formats/log.go index 779fc34..e58e27f 100644 --- a/formats/log.go +++ b/formats/log.go @@ -15,6 +15,8 @@ package formats import ( + "errors" + "fmt" "io/fs" "log" "strings" @@ -22,6 +24,7 @@ import ( "github.com/prometheus/client_golang/prometheus" + "github.com/kube-logging/log-generator/formats/custom" "github.com/kube-logging/log-generator/formats/golang" "github.com/kube-logging/log-generator/formats/syslog" "github.com/kube-logging/log-generator/formats/web" @@ -60,6 +63,44 @@ func NewRandomSyslog(format string) (*LogTemplate, error) { return newLogTemplate(format, syslog.TemplateFS, syslog.RandomData()) } +func FormatsByType() map[string][]string { + response := map[string][]string{} + for t, f := range custom.Formats() { + response[t] = f + } + response["syslog"] = SyslogFormatNames() + response["web"] = WebFormatNames() + return response +} + +func LogFactory(logType string, format string, randomise bool) (Log, error) { + switch logType { + case "syslog": + if randomise { + return NewRandomSyslog(format) + } else { + return NewSyslog(format) + } + case "web": + if randomise { + return NewRandomWeb(format) + } else { + return NewWeb(format) + } + default: + for t, f := range custom.Formats() { + if t == logType { + if containsItem(f, format) { + return custom.LogFactory(logType, format, randomise) + } else { + return nil, errors.New(fmt.Sprintf("unsupported custom format %s", format)) + } + } + } + } + return nil, errors.New(fmt.Sprintf("unsupported type %s", logType)) +} + func SyslogFormatNames() []string { return formatNames(syslog.TemplateFS) } @@ -120,3 +161,12 @@ func (l *LogTemplate) Labels() prometheus.Labels { "severity": l.data.Severity(), } } + +func containsItem(list []string, item string) bool { + for _, i := range list { + if i == item { + return true + } + } + return false +} diff --git a/loggen/loggen.go b/loggen/loggen.go index d6219c2..5f4a1c0 100644 --- a/loggen/loggen.go +++ b/loggen/loggen.go @@ -86,11 +86,7 @@ func (l *List) MarshalJSON() ([]byte, error) { } func (l *LogGen) FormatsGetHandler(ctx *gin.Context) { - response := map[string][]string{} - response["syslog"] = formats.SyslogFormatNames() - response["web"] = formats.WebFormatNames() - - ctx.JSON(http.StatusOK, response) + ctx.JSON(http.StatusOK, formats.FormatsByType()) } func (l *LogGen) GetHandler(ctx *gin.Context) { @@ -122,38 +118,27 @@ func (l *LogGen) PostHandler(ctx *gin.Context) { } func (lr *LogGenRequest) Validate() error { - if _, exists := formats.Types[lr.Type]; !exists { - return fmt.Errorf("type %q does not exist", lr.Type) + for t, _ := range formats.FormatsByType() { + if t == lr.Type { + return nil + } } - - return nil + return fmt.Errorf("type %q does not exist", lr.Type) } -func (lr *LogGenRequest) process(l *LogGen) formats.Log { +func (lr *LogGenRequest) process(lg *LogGen) formats.Log { if lr.Count <= 0 { return nil } - // TODO: factory - var msg formats.Log - var err error - switch lr.Type { - case "syslog": - if l.Randomise { - msg, err = formats.NewRandomSyslog(lr.Format) - } else { - msg, err = formats.NewSyslog(lr.Format) - } - case "web": - if l.Randomise { - msg, err = formats.NewRandomWeb(lr.Format) - } else { - msg, err = formats.NewWeb(lr.Format) - } - case "golang": - msg = formats.NewGolangRandom(l.GolangLog) - default: - log.Panic("invalid LogGenRequest type") + // TODO configuration management for custom formats? + if lr.Type == "golang" { + return formats.NewGolangRandom(lg.GolangLog) + } + + msg, err := formats.LogFactory(lr.Type, lr.Format, lg.Randomise) + if err != nil { + log.Panic(err) } if err != nil { @@ -254,6 +239,8 @@ func (l *LogGen) Run() { go func() { + // TODO implement main loop for custom formats? + var counter = 0 var ticker *jitterbug.Ticker From bfd14506b3575348f63076fa9df1b55a0562e222 Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Wed, 26 Jul 2023 16:53:24 +0200 Subject: [PATCH 02/11] log error instead of panic Signed-off-by: Peter Wilcsinszky --- loggen/loggen.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loggen/loggen.go b/loggen/loggen.go index 5f4a1c0..22b8cfb 100644 --- a/loggen/loggen.go +++ b/loggen/loggen.go @@ -138,7 +138,8 @@ func (lr *LogGenRequest) process(lg *LogGen) formats.Log { msg, err := formats.LogFactory(lr.Type, lr.Format, lg.Randomise) if err != nil { - log.Panic(err) + log.Error(err) + return nil } if err != nil { From 922eb4fa662369f98748fcf94fb6d1580c55d29b Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Wed, 26 Jul 2023 16:53:39 +0200 Subject: [PATCH 03/11] make targets for custom formats build Signed-off-by: Peter Wilcsinszky --- .gitignore | 1 + Makefile | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 8c7cf21..912eff9 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,5 @@ conf/config.ini .DS_Store +formats/custom-impl go.work* diff --git a/Makefile b/Makefile index 08af3fc..ea7d7f9 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,19 @@ check: license-cache license-check go.work: go work init . ${CUSTOM_FORMATS} +.PHONY: +reinit: + rm -f go.work + @$(MAKE) go.work + .PHONY: build: go.work CGO_ENABLED=0 go build -a -o ${BIN}/loggen main.go +.PHONY: +docker-run: + docker build . -t log-generator:local && docker run -p 11000:11000 log-generator:local + ${LICENSEI}: ${LICENSEI}_${LICENSEI_VERSION} | ${BIN} ln -sf $(notdir $<) $@ From 9824e0fba1e04a32b5e189cde5b5ce985e260c0d Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Wed, 26 Jul 2023 16:57:01 +0200 Subject: [PATCH 04/11] make and ci fixes Signed-off-by: Peter Wilcsinszky --- .github/workflows/artifacts.yaml | 3 +++ .github/workflows/build-and-test.yaml | 4 ++-- Makefile | 10 +++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/artifacts.yaml b/.github/workflows/artifacts.yaml index 9d61582..c930fcc 100644 --- a/.github/workflows/artifacts.yaml +++ b/.github/workflows/artifacts.yaml @@ -22,6 +22,9 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Init + run: make go.work + - name: Gather metadata id: meta uses: docker/metadata-action@v4 diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index c64abf3..474cc3d 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v3 - name: Build - run: go build -v ./... + run: make build - name: Test - run: go test -v ./... + run: make test diff --git a/Makefile b/Makefile index ea7d7f9..1e6afe8 100644 --- a/Makefile +++ b/Makefile @@ -25,16 +25,20 @@ check: license-cache license-check go.work: go work init . ${CUSTOM_FORMATS} -.PHONY: +.PHONY: reinit reinit: rm -f go.work @$(MAKE) go.work -.PHONY: +.PHONY: build build: go.work CGO_ENABLED=0 go build -a -o ${BIN}/loggen main.go -.PHONY: +.PHONY: test +test: go.work + go test ./... + +.PHONY: docker-run docker-run: docker build . -t log-generator:local && docker run -p 11000:11000 log-generator:local From 9573ed28a673bbf3f61470a6f1b673e34a8a4233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 09:14:51 +0200 Subject: [PATCH 05/11] formats: add license header for custom stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- formats/custom/stub.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/formats/custom/stub.go b/formats/custom/stub.go index 637dfe0..5a97af4 100644 --- a/formats/custom/stub.go +++ b/formats/custom/stub.go @@ -1,3 +1,17 @@ +// Copyright © 2023 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.""" + package custom import ( From 8a9c8423fcf18b31c9d8af342e62275cb6e09345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:19:03 +0200 Subject: [PATCH 06/11] formats: move Log* to a separate package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- formats/log.go | 172 -------------------- formats/logfactory.go | 80 +++++++++ formats/{log_test.go => logfactory_test.go} | 7 +- log/log.go | 24 +++ {formats => log}/template.go | 60 ++++++- loggen/loggen.go | 25 +-- loggen/writers.go | 16 +- 7 files changed, 187 insertions(+), 197 deletions(-) delete mode 100644 formats/log.go create mode 100644 formats/logfactory.go rename formats/{log_test.go => logfactory_test.go} (88%) create mode 100644 log/log.go rename {formats => log}/template.go (54%) diff --git a/formats/log.go b/formats/log.go deleted file mode 100644 index e58e27f..0000000 --- a/formats/log.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright © 2023 Kube logging authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.""" - -package formats - -import ( - "errors" - "fmt" - "io/fs" - "log" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/kube-logging/log-generator/formats/custom" - "github.com/kube-logging/log-generator/formats/golang" - "github.com/kube-logging/log-generator/formats/syslog" - "github.com/kube-logging/log-generator/formats/web" -) - -// TODO: factory -type void = struct{} - -var Types = map[string]struct{}{ - "golang": void{}, - "syslog": void{}, - "web": void{}, -} - -type Log interface { - String() (string, float64) - Labels() prometheus.Labels -} - -type LogTemplateData interface { - Severity() string -} - -type LogTemplate struct { - Format string - - template *template.Template - data LogTemplateData -} - -func NewSyslog(format string) (*LogTemplate, error) { - return newLogTemplate(format, syslog.TemplateFS, syslog.SampleData()) -} - -func NewRandomSyslog(format string) (*LogTemplate, error) { - return newLogTemplate(format, syslog.TemplateFS, syslog.RandomData()) -} - -func FormatsByType() map[string][]string { - response := map[string][]string{} - for t, f := range custom.Formats() { - response[t] = f - } - response["syslog"] = SyslogFormatNames() - response["web"] = WebFormatNames() - return response -} - -func LogFactory(logType string, format string, randomise bool) (Log, error) { - switch logType { - case "syslog": - if randomise { - return NewRandomSyslog(format) - } else { - return NewSyslog(format) - } - case "web": - if randomise { - return NewRandomWeb(format) - } else { - return NewWeb(format) - } - default: - for t, f := range custom.Formats() { - if t == logType { - if containsItem(f, format) { - return custom.LogFactory(logType, format, randomise) - } else { - return nil, errors.New(fmt.Sprintf("unsupported custom format %s", format)) - } - } - } - } - return nil, errors.New(fmt.Sprintf("unsupported type %s", logType)) -} - -func SyslogFormatNames() []string { - return formatNames(syslog.TemplateFS) -} - -func NewWeb(format string) (*LogTemplate, error) { - return newLogTemplate(format, web.TemplateFS, web.SampleData()) -} - -func NewRandomWeb(format string) (*LogTemplate, error) { - return newLogTemplate(format, web.TemplateFS, web.RandomData()) -} - -func WebFormatNames() []string { - return formatNames(web.TemplateFS) -} - -func NewGolangRandom(i golang.GolangLogIntensity) Log { - return golang.NewGolangLogRandom(i) -} - -func newLogTemplate(format string, fs fs.FS, data LogTemplateData) (*LogTemplate, error) { - template, err := loadTemplate(format, fs) - if err != nil { - return nil, err - } - - return &LogTemplate{ - Format: format, - template: template, - data: data, - }, nil -} - -func formatNames(fs fs.FS) []string { - formats := []string{} - - for _, t := range loadAllTemplates(fs) { - formats = append(formats, strings.TrimSuffix(t.Name(), ".tmpl")) - } - - return formats -} - -func (l *LogTemplate) String() (string, float64) { - var b strings.Builder - if err := l.template.Execute(&b, l.data); err != nil { - log.Panic(err.Error()) - } - - str := strings.TrimSuffix(b.String(), "\n") - - return str, float64(len([]byte(str))) -} - -func (l *LogTemplate) Labels() prometheus.Labels { - return prometheus.Labels{ - "type": l.Format, - "severity": l.data.Severity(), - } -} - -func containsItem(list []string, item string) bool { - for _, i := range list { - if i == item { - return true - } - } - return false -} diff --git a/formats/logfactory.go b/formats/logfactory.go new file mode 100644 index 0000000..834368f --- /dev/null +++ b/formats/logfactory.go @@ -0,0 +1,80 @@ +// Copyright © 2023 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.""" + +package formats + +import ( + "github.com/kube-logging/log-generator/formats/custom" + "github.com/kube-logging/log-generator/formats/golang" + "github.com/kube-logging/log-generator/formats/syslog" + "github.com/kube-logging/log-generator/formats/web" + "github.com/kube-logging/log-generator/log" +) + +func FormatsByType() map[string][]string { + response := map[string][]string{} + for t, f := range custom.Formats() { + response[t] = f + } + response["syslog"] = SyslogFormatNames() + response["web"] = WebFormatNames() + return response +} + +func LogFactory(logType string, format string, randomise bool) (log.Log, error) { + switch logType { + case "syslog": + if randomise { + return NewRandomSyslog(format) + } else { + return NewSyslog(format) + } + case "web": + if randomise { + return NewRandomWeb(format) + } else { + return NewWeb(format) + } + default: + return custom.LogFactory(logType, format, randomise) + } +} + +func NewSyslog(format string) (*log.LogTemplate, error) { + return log.NewLogTemplate(format, syslog.TemplateFS, syslog.SampleData()) +} + +func NewRandomSyslog(format string) (*log.LogTemplate, error) { + return log.NewLogTemplate(format, syslog.TemplateFS, syslog.RandomData()) +} + +func SyslogFormatNames() []string { + return log.FormatNames(syslog.TemplateFS) +} + +func NewWeb(format string) (*log.LogTemplate, error) { + return log.NewLogTemplate(format, web.TemplateFS, web.SampleData()) +} + +func NewRandomWeb(format string) (*log.LogTemplate, error) { + return log.NewLogTemplate(format, web.TemplateFS, web.RandomData()) +} + +func WebFormatNames() []string { + return log.FormatNames(web.TemplateFS) +} + +func NewGolangRandom(i golang.GolangLogIntensity) log.Log { + return golang.NewGolangLogRandom(i) +} diff --git a/formats/log_test.go b/formats/logfactory_test.go similarity index 88% rename from formats/log_test.go rename to formats/logfactory_test.go index d6bbcc9..d1f3876 100644 --- a/formats/log_test.go +++ b/formats/logfactory_test.go @@ -21,15 +21,16 @@ import ( "github.com/kube-logging/log-generator/formats/syslog" "github.com/kube-logging/log-generator/formats/web" + "github.com/kube-logging/log-generator/log" ) -type LogConstructor func(string) (*LogTemplate, error) +type LogConstructor func(string) (*log.LogTemplate, error) func TestSyslogFormats(t *testing.T) { // Separate Template tree per directory because Go Template can not handle // multiple files with the same name in different directories. - syslogTemplates := loadAllTemplates(syslog.TemplateFS) + syslogTemplates := log.LoadAllTemplates(syslog.TemplateFS) assertFormatAll(t, syslogTemplates, NewSyslog) } @@ -37,7 +38,7 @@ func TestWebFormats(t *testing.T) { // Separate Template tree per directory because Go Template can not handle // multiple files with the same name in different directories. - webTemplates := loadAllTemplates(web.TemplateFS) + webTemplates := log.LoadAllTemplates(web.TemplateFS) assertFormatAll(t, webTemplates, NewWeb) } diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..a1454b5 --- /dev/null +++ b/log/log.go @@ -0,0 +1,24 @@ +// Copyright © 2023 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.""" + +package log + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type Log interface { + String() (string, float64) + Labels() prometheus.Labels +} diff --git a/formats/template.go b/log/template.go similarity index 54% rename from formats/template.go rename to log/template.go index d1c3a6f..6fa0313 100644 --- a/formats/template.go +++ b/log/template.go @@ -12,15 +12,71 @@ // See the License for the specific language governing permissions and // limitations under the License.""" -package formats +package log import ( "fmt" "io/fs" "strings" "text/template" + + log "github.com/sirupsen/logrus" + + "github.com/prometheus/client_golang/prometheus" ) +type LogTemplateData interface { + Severity() string +} + +type LogTemplate struct { + Format string + + template *template.Template + data LogTemplateData +} + +func NewLogTemplate(format string, fs fs.FS, data LogTemplateData) (*LogTemplate, error) { + template, err := loadTemplate(format, fs) + if err != nil { + return nil, err + } + + return &LogTemplate{ + Format: format, + template: template, + data: data, + }, nil +} + +func FormatNames(fs fs.FS) []string { + formats := []string{} + + for _, t := range LoadAllTemplates(fs) { + formats = append(formats, strings.TrimSuffix(t.Name(), ".tmpl")) + } + + return formats +} + +func (l *LogTemplate) String() (string, float64) { + var b strings.Builder + if err := l.template.Execute(&b, l.data); err != nil { + log.Panic(err.Error()) + } + + str := strings.TrimSuffix(b.String(), "\n") + + return str, float64(len([]byte(str))) +} + +func (l *LogTemplate) Labels() prometheus.Labels { + return prometheus.Labels{ + "type": l.Format, + "severity": l.data.Severity(), + } +} + func loadTemplate(name string, fs fs.FS) (*template.Template, error) { // syslog.rfc5424.sdata => syslog.tmpl templateFileName, _, structuredFormatName := strings.Cut(name, ".") @@ -44,6 +100,6 @@ func loadTemplate(name string, fs fs.FS) (*template.Template, error) { return nil, fmt.Errorf("could not find format %q", name) } -func loadAllTemplates(fs fs.FS) []*template.Template { +func LoadAllTemplates(fs fs.FS) []*template.Template { return template.Must(template.ParseFS(fs, "*.tmpl")).Templates() } diff --git a/loggen/loggen.go b/loggen/loggen.go index 22b8cfb..5234440 100644 --- a/loggen/loggen.go +++ b/loggen/loggen.go @@ -27,11 +27,12 @@ import ( "github.com/gin-gonic/gin" "github.com/lthibault/jitterbug" - log "github.com/sirupsen/logrus" + logger "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/kube-logging/log-generator/formats" "github.com/kube-logging/log-generator/formats/golang" + "github.com/kube-logging/log-generator/log" ) type List struct { @@ -99,7 +100,7 @@ func (l *LogGen) GetHandler(ctx *gin.Context) { func (l *LogGen) PostHandler(ctx *gin.Context) { var lr LogGenRequest if err := ctx.ShouldBindJSON(&lr); err != nil { - log.Error(err.Error()) + logger.Error(err.Error()) ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -126,7 +127,7 @@ func (lr *LogGenRequest) Validate() error { return fmt.Errorf("type %q does not exist", lr.Type) } -func (lr *LogGenRequest) process(lg *LogGen) formats.Log { +func (lr *LogGenRequest) process(lg *LogGen) log.Log { if lr.Count <= 0 { return nil } @@ -143,7 +144,7 @@ func (lr *LogGenRequest) process(lg *LogGen) formats.Log { } if err != nil { - log.Warnf("Error generating log from request %v, %v", lr, err) + logger.Warnf("Error generating log from request %v, %v", lr, err) return nil } @@ -171,7 +172,7 @@ func (l *LogGen) GolangGetHandler(c *gin.Context) { func (l *LogGen) GolangPatchHandler(c *gin.Context) { if err := c.ShouldBindJSON(&l.GolangLog); err != nil { - log.Error(err.Error()) + logger.Error(err.Error()) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -203,7 +204,7 @@ func (l *LogGen) golangSet() error { func (l *LogGen) processRequests() bool { l.m.Lock() - logs := make([]formats.Log, 0, l.ActiveRequests.Len()) + logs := make([]log.Log, 0, l.ActiveRequests.Len()) e := l.ActiveRequests.Front() for e != nil { @@ -266,7 +267,7 @@ func (l *LogGen) Run() { for range ticker.C { if viper.GetBool("nginx.enabled") { - l.sendIfCount(count, &counter, func() (formats.Log, error) { + l.sendIfCount(count, &counter, func() (log.Log, error) { if l.Randomise { return formats.NewRandomWeb("nginx") } else { @@ -275,7 +276,7 @@ func (l *LogGen) Run() { }) } if viper.GetBool("apache.enabled") { - l.sendIfCount(count, &counter, func() (formats.Log, error) { + l.sendIfCount(count, &counter, func() (log.Log, error) { if l.Randomise { return formats.NewRandomWeb("apache") } else { @@ -284,7 +285,7 @@ func (l *LogGen) Run() { }) } if viper.GetBool("golang.enabled") { - l.sendIfCount(count, &counter, func() (formats.Log, error) { + l.sendIfCount(count, &counter, func() (log.Log, error) { return formats.NewGolangRandom(l.GolangLog), nil }) } @@ -301,18 +302,18 @@ func (l *LogGen) Run() { select { case <-interrupt: - log.Infoln("Recieved interrupt") + logger.Infoln("Recieved interrupt") break case <-done: break } } -func (l *LogGen) sendIfCount(count int, counter *int, f func() (formats.Log, error)) { +func (l *LogGen) sendIfCount(count int, counter *int, f func() (log.Log, error)) { if count == -1 || *counter < count { n, err := f() if err != nil { - log.Panic(err) + logger.Panic(err) } l.writer.Send(n) *counter++ diff --git a/loggen/writers.go b/loggen/writers.go index 674b3d5..b1ddb77 100644 --- a/loggen/writers.go +++ b/loggen/writers.go @@ -19,16 +19,16 @@ import ( "net" "time" - log "github.com/sirupsen/logrus" + logger "github.com/sirupsen/logrus" "github.com/cenkalti/backoff/v4" - "github.com/kube-logging/log-generator/formats" + "github.com/kube-logging/log-generator/log" "github.com/kube-logging/log-generator/metrics" ) type LogWriter interface { - Send(formats.Log) + Send(log.Log) Close() } @@ -38,7 +38,7 @@ func newStdoutWriter() LogWriter { return &StdoutLogWriter{} } -func (*StdoutLogWriter) Send(l formats.Log) { +func (*StdoutLogWriter) Send(l log.Log) { msg, size := l.String() fmt.Println(msg) @@ -64,7 +64,7 @@ func newNetworkWriter(network string, address string) LogWriter { return nlw } -func (nlw *NetworkLogWriter) Send(l formats.Log) { +func (nlw *NetworkLogWriter) Send(l log.Log) { msg, size := l.String() msg += "\n" @@ -74,7 +74,7 @@ func (nlw *NetworkLogWriter) Send(l formats.Log) { n, err := nlw.conn.Write([]byte(data)) if err != nil { - log.Errorf("Error sending message (%q), reconnecting...", err.Error()) + logger.Errorf("Error sending message (%q), reconnecting...", err.Error()) nlw.reconnect() continue } @@ -103,11 +103,11 @@ func (nlw *NetworkLogWriter) reconnect() { bo.MaxElapsedTime = 0 backoff.RetryNotify(func() error { - log.Infof("Connecting to %s %s...", nlw.network, nlw.address) + logger.Infof("Connecting to %s %s...", nlw.network, nlw.address) var err error nlw.conn, err = net.DialTimeout(nlw.network, nlw.address, 5*time.Second) return err }, bo, func(err error, delay time.Duration) { - log.Errorf("Error connecting to server (%q), retrying in %s", err.Error(), delay.String()) + logger.Errorf("Error connecting to server (%q), retrying in %s", err.Error(), delay.String()) }) } From 1ed116dbdb171a4cf65a594988dab2a4da54d4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:21:11 +0200 Subject: [PATCH 07/11] loggen: lookup map instead of linear search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- loggen/loggen.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/loggen/loggen.go b/loggen/loggen.go index 5234440..a8917a0 100644 --- a/loggen/loggen.go +++ b/loggen/loggen.go @@ -119,12 +119,11 @@ func (l *LogGen) PostHandler(ctx *gin.Context) { } func (lr *LogGenRequest) Validate() error { - for t, _ := range formats.FormatsByType() { - if t == lr.Type { - return nil - } + if _, exists := formats.FormatsByType()[lr.Type]; !exists { + return fmt.Errorf("type %q does not exist", lr.Type) } - return fmt.Errorf("type %q does not exist", lr.Type) + + return nil } func (lr *LogGenRequest) process(lg *LogGen) log.Log { From 62a46b216ff5908df89eded8e409756d1a69111d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:21:34 +0200 Subject: [PATCH 08/11] loggen: remove redundant error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- loggen/loggen.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/loggen/loggen.go b/loggen/loggen.go index a8917a0..33d2e74 100644 --- a/loggen/loggen.go +++ b/loggen/loggen.go @@ -137,10 +137,6 @@ func (lr *LogGenRequest) process(lg *LogGen) log.Log { } msg, err := formats.LogFactory(lr.Type, lr.Format, lg.Randomise) - if err != nil { - log.Error(err) - return nil - } if err != nil { logger.Warnf("Error generating log from request %v, %v", lr, err) From 4db3e968fd58da8344a5e3b38e8e6ea979665f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:21:59 +0200 Subject: [PATCH 09/11] stub: cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- formats/custom/stub.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/formats/custom/stub.go b/formats/custom/stub.go index 5a97af4..febefb7 100644 --- a/formats/custom/stub.go +++ b/formats/custom/stub.go @@ -15,20 +15,15 @@ package custom import ( - "github.com/prometheus/client_golang/prometheus" + "github.com/kube-logging/log-generator/log" ) -type Log interface { - String() (string, float64) - Labels() prometheus.Labels -} - // Formats returns supported template formats for a given type func Formats() map[string][]string { return map[string][]string{} } // LogFactory creates log events for a log format and optionally randomises it -func LogFactory(logType string, format string, randomise bool) (Log, error) { +func LogFactory(logType string, format string, randomise bool) (log.Log, error) { return nil, nil } From 62a91f7f1fbe2140920bacd25a0090ae8ff11683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:32:17 +0200 Subject: [PATCH 10/11] ci: restore verbose output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .github/workflows/build-and-test.yaml | 4 ++-- Makefile | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 474cc3d..7237055 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v3 - name: Build - run: make build + run: make GOFLAGS="-v" build - name: Test - run: make test + run: make GOFLAGS="-v" test diff --git a/Makefile b/Makefile index 1e6afe8..68a933a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ CUSTOM_FORMATS := formats/custom LICENSEI := ${BIN}/licensei LICENSEI_VERSION = v0.8.0 +GOFLAGS = + ${BIN}: mkdir -p ${BIN} @@ -32,11 +34,11 @@ reinit: .PHONY: build build: go.work - CGO_ENABLED=0 go build -a -o ${BIN}/loggen main.go + CGO_ENABLED=0 go build ${GOFLAGS} -a -o ${BIN}/loggen main.go .PHONY: test test: go.work - go test ./... + go test ${GOFLAGS} ./... .PHONY: docker-run docker-run: From ecd0dc34f10b162c02e052d4678b0a6bedf94ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Fri, 4 Aug 2023 13:39:43 +0200 Subject: [PATCH 11/11] ci: fix license-cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .github/workflows/artifacts.yaml | 3 --- Makefile | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/artifacts.yaml b/.github/workflows/artifacts.yaml index c930fcc..9d61582 100644 --- a/.github/workflows/artifacts.yaml +++ b/.github/workflows/artifacts.yaml @@ -22,9 +22,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Init - run: make go.work - - name: Gather metadata id: meta uses: docker/metadata-action@v4 diff --git a/Makefile b/Makefile index 68a933a..52897cf 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ license-check: ${LICENSEI} ## Run license check ${LICENSEI} header .PHONY: license-cache -license-cache: ${LICENSEI} ## Generate license cache +license-cache: ${LICENSEI} go.work ## Generate license cache ${LICENSEI} cache .PHONY: check