diff --git a/.github/tracing/local-smoke-docker-compose.yaml b/.github/tracing/local-smoke-docker-compose.yaml index e0e60a675e5..7f051f9551c 100644 --- a/.github/tracing/local-smoke-docker-compose.yaml +++ b/.github/tracing/local-smoke-docker-compose.yaml @@ -7,6 +7,7 @@ services: command: [ "--config=/etc/otel-collector.yaml" ] volumes: - ./otel-collector.yaml:/etc/otel-collector.yaml + - ./trace-data.json:/etc/trace-data.json ports: - "4317:4317" # otlp grpc depends_on: diff --git a/.github/tracing/otel-collector.yaml b/.github/tracing/otel-collector.yaml index fb8721cba20..1ae3e9c3c45 100644 --- a/.github/tracing/otel-collector.yaml +++ b/.github/tracing/otel-collector.yaml @@ -4,12 +4,13 @@ receivers: grpc: endpoint: "0.0.0.0:4317" exporters: - otlp: - endpoint: tempo:4317 - tls: - insecure: true + file: + path: /etc/trace-data.json service: + telemetry: + logs: + level: "debug" # Set log level to debug pipelines: traces: receivers: [otlp] - exporters: [otlp] \ No newline at end of file + exporters: [file] \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 98e59ce8d49..81501bd2930 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -340,67 +340,15 @@ jobs: # Create network docker network create --driver bridge tracing - # Start Grafana + # Switch directory cd ./.github/tracing - docker run -d --network=tracing --name=grafana -p 3000:3000 -v $PWD/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml -e GF_AUTH_ANONYMOUS_ENABLED=true -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin -e GF_AUTH_DISABLE_LOGIN_FORM=true -e GF_FEATURE_TOGGLES_ENABLE=traceqlEditor grafana/grafana:9.4.3 - # Start Tempo - docker run -d --network=tracing --name=tempo -v ./tempo.yaml:/etc/tempo.yaml -v $PWD/tempo-data:/tmp/tempo grafana/tempo:latest -config.file=/etc/tempo.yaml + # initialize trace data exporter file + mkdir /tmp/tracing + touch /tmp/tracing/trace-data.json # Start OpenTelemetry Collector - docker run -d --network=tracing --name=otel-collector -v $PWD/otel-collector.yaml:/etc/otel-collector.yaml -p 4317:4317 otel/opentelemetry-collector:0.61.0 --config=/etc/otel-collector.yaml - - - name: Generate port - id: generate-port - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - env: - GITHUB_PR_NUMBER: ${{ github.event.number }} - run: | - PORT_BASE=3001 - MAX_PORT=8000 - - # Use PR number as offset. Given GitHub PRs are incremental, this guarantees uniqueness for at least 5000 PRs. - OFFSET=$GITHUB_PR_NUMBER - echo "PR Number: $OFFSET" - - # Ensure that we don't exceed the max port - if (( OFFSET > (MAX_PORT - PORT_BASE) )); then - OFFSET=$((OFFSET % (MAX_PORT - PORT_BASE))) - fi - - # Map the offset to the port range - REMOTE_PORT=$((PORT_BASE + OFFSET)) - echo "REMOTE_PORT=$REMOTE_PORT" >> $GITHUB_OUTPUT - - name: Reverse SSH Tunneling - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - env: - TRACING_SSH_KEY: ${{ secrets.TRACING_SSH_KEY }} - TRACING_SSH_SERVER: ${{ secrets.TRACING_SSH_SERVER }} - REMOTE_PORT: ${{ steps.generate-port.outputs.REMOTE_PORT }} - run: | - eval $(ssh-agent) - echo "test" - echo "$TRACING_SSH_KEY" | wc -c - echo "$TRACING_SSH_KEY" | tr -d '\r' | wc -c - echo "$TRACING_SSH_KEY" | tr -d '\r' | base64 --decode | ssh-add - - # f: background process - # N: do not execute a remote command - # R: remote port forwarding - ssh -o StrictHostKeyChecking=no -f -N -R $REMOTE_PORT:127.0.0.1:3000 user-gha@$TRACING_SSH_SERVER - echo "To view Grafana locally:" - echo "ssh -N -L 8000:localhost:$REMOTE_PORT user-gha@$TRACING_SSH_SERVER" - echo "Then visit http://localhost:8000 in a browser." - echo "If you are unable to connect, check with the security team that you have access to the tracing server." - - name: Show Grafana Logs - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - docker logs grafana - docker logs tempo - docker logs otel-collector - - name: Set sleep time to use in future steps - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - echo "SLEEP_TIME=2400" >> "$GITHUB_ENV" + docker run -d --network=tracing --name=otel-collector -v $PWD/otel-collector.yaml:/etc/otel-collector.yaml -v /tmp/tracing/trace-data.json:./trace-data.json -p 4317:4317 otel/opentelemetry-collector:0.61.0 --config=/etc/otel-collector.yaml ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' @@ -447,11 +395,20 @@ jobs: this-job-name: ETH Smoke Tests ${{ matrix.product.name }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true - - name: Keep action running to view traces + - name: Show Grafana Logs if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' run: | - echo "Sleeping for $SLEEP_TIME seconds..." - sleep $SLEEP_TIME + docker logs otel-collector + - name: Wait for OTEL-Collector to flush buffered traces + if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' + run: | + sleep 60 + - name: Upload Trace Data + uses: actions/upload-artifact@v2 + with: + name: trace-data + path: /tmp/tracing/trace-data.json + ### Used to check the required checks box when the matrix completes eth-smoke-tests: