Merge pull request #13 from logzio/feature/otlp-1.0-support

feature/otlp-1.0-support

.github/workflows/auto-test.yml

@@ -6,11 +6,10 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Go
-        uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: 1.15
+          go-version: '1.22.0'
       - name: Test
         run: |
           cd handler

.github/workflows/upload-zip.yaml

@@ -8,12 +8,10 @@ jobs:
   upload:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Set up Go
-        uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: '1.22.0'
       - name: Build function
         run: make function
       - name: Upload release asset
@@ -22,3 +20,4 @@ jobs:
           asset_path: ./function.zip
           asset_name: function.zip
           asset_content_type: application/zip
+          upload_url: ${{ github.event.release.upload_url }}

Makefile

@@ -2,5 +2,5 @@ build:
 	go build main.go
 
 function:
-	GOARCH=amd64 GOOS=linux go build main.go
-	zip -r function.zip main
+	GOOS=linux GOARCH=amd64 go build -tags lambda.norpc -o bootstrap main.go
+	zip function.zip bootstrap

README.md

@@ -1,28 +1,23 @@
 # metric-stream-lambda
-Lambda function that receives OTLP (0.7.0) data from AWS metric stream and exports the data to logz.io using prometheus remote write
+Lambda function that receives OTLP (1.0) data from AWS metric stream and exports the data to logz.io using prometheus remote write
 
 ### How to create function.zip
 ```
 make function
 ```
 
-### How it Works
-- The function begins by importing a number of packages, including those for handling AWS events, `encoding/decoding` `JSON`, working with OpenTelemetry data, and logging. 
-- The `firehoseResponse` struct is used to hold information about the response to the Lambda function's API Gateway trigger, including the request ID, a timestamp, and an error message. The `generateValidFirehoseResponse` function takes in a status code, request ID, error message, and error, and returns an `events.APIGatewayProxyResponse` with the given information and some default headers and other values.
-- The `initLogger` function initializes a logger using Uber's zap package, taking in a context and an `events.APIGatewayProxyRequest` as arguments. It extracts the AWS request ID and account ID from the context and request, respectively, and uses them to configure the logger.
-- The `HandleRequest` function is the main entry point for the Lambda function. It receives an `events.APIGatewayProxyRequest` and a context, and returns an `events.APIGatewayProxyResponse`. It first initializes the logger using the initLogger function. It then decodes the base64-encoded request body and unmarshals it into a `pb.ExportMetricsServiceRequest` protobuf message.
-- Next, the function creates a `pdata.Metrics` value from the protobuf message and a `componenttest.NewNopExporter` component to hold the metrics data. It then creates a `prometheusremotewriteexporter.Exporter` using the `getListenerUrl` function to get the logz.io listener url and the `LogzioToken` variable extracted from the `events.APIGatewayProxyRequest` headers.
-- The function then iterates through the metrics in the `pdata.Metrics` value, extracting their names and data points. It converts the data points into Prometheus Remote Write format and sends them to the exporter. If any errors are encountered during this process, they are collected in an `ErrorCollector`.
-- Finally, the function checks the length of the `ErrorCollector` and returns an appropriate response based on whether any errors were encountered. If there were no errors, a response with a status code of 200 and an empty error message is returned. If there were errors, a response with a status code of 400 and the concatenated error messages is returned.
-
 ### Limitations
 This function has the following limitations:
 
-- It can only process metrics data in OTLP 0.7 format.
+- It can only process metrics data in OTLP 1.0 format.
 - It can only forward the data to a Prometheus Remote Write endpoint.
 
 ### Changelog
-
+- v2.0.0
+  - Breaking changes:
+    - Move from otlp 0.7 -> otlp 1.0
+  - Add support for custom p8s_logzio_name label
+  - Performance improvements
 - v1.0.2
   - Stop trying to send bulks if encountered 401 status code
   - Add logzio identifier to each log (5 last chars of the shipping token)

go.mod

@@ -1,46 +1,80 @@
 module github.com/logzio/metric-stream-lambda
 
-go 1.17
+go 1.22.0
 
 require (
 	github.com/aws/aws-lambda-go v1.36.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.5.2
-	github.com/golang/snappy v0.0.3
-	github.com/prometheus/prometheus v1.8.2-0.20210621150501-ff58416a0b02
-	github.com/stretchr/testify v1.7.2
-	go.opentelemetry.io/collector v0.29.0
-	go.opentelemetry.io/otel/metric v0.28.0
-	go.opentelemetry.io/proto/otlp v0.15.0
-	go.uber.org/zap v1.17.0
+	github.com/golang/protobuf v1.5.4
+	github.com/golang/snappy v0.0.4
+	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.111.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.111.0
+	github.com/prometheus/prometheus v0.54.1
+	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/collector/component v0.111.0
+	go.opentelemetry.io/collector/config/confighttp v0.111.0
+	go.opentelemetry.io/collector/config/configopaque v1.17.0
+	go.opentelemetry.io/collector/config/configtelemetry v0.111.0
+	go.opentelemetry.io/collector/exporter v0.111.0
+	go.opentelemetry.io/collector/pdata v1.17.0
+	go.opentelemetry.io/otel/metric v1.30.0
+	go.opentelemetry.io/otel/sdk v1.30.0
+	go.opentelemetry.io/proto/otlp v1.3.1
+	go.uber.org/zap v1.27.0
 )
 
 require (
-	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
-	github.com/jaegertracing/jaeger v1.23.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.10 // indirect
 	github.com/knadh/koanf v1.1.0 // indirect
-	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/common v0.29.0 // indirect
-	github.com/rs/cors v1.7.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v1.6.0 // indirect
-	go.uber.org/atomic v1.8.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/net v0.0.0-20210610132358-84b48f89b13b // indirect
-	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.111.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.111.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.60.0 // indirect
+	github.com/rs/cors v1.11.1 // indirect
+	github.com/tidwall/gjson v1.10.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/tinylru v1.1.0 // indirect
+	github.com/tidwall/wal v1.1.7 // indirect
+	go.opentelemetry.io/collector/client v1.17.0 // indirect
+	go.opentelemetry.io/collector/config/configauth v0.111.0 // indirect
+	go.opentelemetry.io/collector/config/configcompression v1.17.0 // indirect
+	go.opentelemetry.io/collector/config/configretry v1.17.0 // indirect
+	go.opentelemetry.io/collector/config/configtls v1.17.0 // indirect
+	go.opentelemetry.io/collector/config/internal v0.111.0 // indirect
+	go.opentelemetry.io/collector/consumer v0.111.0 // indirect
+	go.opentelemetry.io/collector/extension v0.111.0 // indirect
+	go.opentelemetry.io/collector/extension/auth v0.111.0 // indirect
+	go.opentelemetry.io/collector/extension/experimental/storage v0.111.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.17.0 // indirect
+	go.opentelemetry.io/collector/internal/globalsignal v0.111.0 // indirect
+	go.opentelemetry.io/collector/pipeline v0.111.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.111.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect
+	go.opentelemetry.io/otel v1.30.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect
+	go.opentelemetry.io/otel/trace v1.30.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
-	google.golang.org/grpc v1.42.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	google.golang.org/grpc v1.67.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )