diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71723439631..3224b222842 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -255,6 +255,21 @@ jobs: - name: build docker image run: sudo docker build -f ./Ingestor/Dockerfile . + docker-build-open-telemetry-ingest: + runs-on: ubuntu-latest + env: + CI_PIPELINE_ID: ${{github.run_number}} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Preinstall + run: npm run prerun + + # build image probe api + - name: build docker image + run: sudo docker build -f ./OpenTelemetryIngest/Dockerfile . + docker-build-incoming-request-ingest: runs-on: ubuntu-latest env: diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index c639a85cb50..0b6152ebf56 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -216,6 +216,19 @@ jobs: - run: cd Common && npm install - run: cd Ingestor && npm install && npm run compile && npm run dep-check + compile-open-telemetry-ingest: + runs-on: ubuntu-latest + env: + CI_PIPELINE_ID: ${{github.run_number}} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 18.3.0 + - run: cd Common && npm install + - run: cd OpenTelemetryIngest && npm install && npm run compile && npm run dep-check + + compile-incoming-request-ingest: runs-on: ubuntu-latest env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c130694eb5..add7d6bca63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -613,6 +613,66 @@ jobs: GIT_SHA=${{ github.sha }} APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}} + open-telemetry-ingest-docker-image-deploy: + needs: [generate-build-number] + runs-on: ubuntu-latest + steps: + - name: Docker Meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + oneuptime/open-telemetry-ingest + ghcr.io/oneuptime/open-telemetry-ingest + tags: | + type=raw,value=release,enable=true + type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - uses: actions/setup-node@v2 + with: + node-version: 18.3.0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate Dockerfile from Dockerfile.tpl + run: npm run prerun + + # Build and deploy open-telemetry-ingest. + + - name: Login to Docker Hub + uses: docker/login-action@v2.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + file: ./OpenTelemetryIngest/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_SHA=${{ github.sha }} + APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}} + incoming-request-ingest-docker-image-deploy: needs: [generate-build-number] @@ -1530,7 +1590,7 @@ jobs: test-e2e-release-saas: runs-on: ubuntu-latest - needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy] + needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy] env: CI_PIPELINE_ID: ${{github.run_number}} steps: @@ -1583,7 +1643,7 @@ jobs: test-e2e-release-self-hosted: runs-on: ubuntu-latest # After all the jobs runs - needs: [copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy] + needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy] env: CI_PIPELINE_ID: ${{github.run_number}} steps: diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index 582df61abea..6d5f32ccd18 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -721,6 +721,128 @@ jobs: GIT_SHA=${{ github.sha }} APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}} + ingestor-docker-image-deploy: + needs: generate-build-number + runs-on: ubuntu-latest + steps: + - name: Docker Meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + oneuptime/ingestor + ghcr.io/oneuptime/ingestor + tags: | + type=raw,value=test,enable=true + type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true + + + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - uses: actions/setup-node@v2 + with: + node-version: 18.3.0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate Dockerfile from Dockerfile.tpl + run: npm run prerun + + # Build and deploy ingestor. + + - name: Login to Docker Hub + uses: docker/login-action@v2.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + file: ./Ingestor/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_SHA=${{ github.sha }} + APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}} + + open-telemetry-ingest-docker-image-deploy: + needs: generate-build-number + runs-on: ubuntu-latest + steps: + - name: Docker Meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + oneuptime/open-telemetry-ingest + ghcr.io/oneuptime/open-telemetry-ingest + tags: | + type=raw,value=test,enable=true + type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true + + + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - uses: actions/setup-node@v2 + with: + node-version: 18.3.0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate Dockerfile from Dockerfile.tpl + run: npm run prerun + + # Build and deploy incoming-request-ingest. + + - name: Login to Docker Hub + uses: docker/login-action@v2.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + file: ./OpenTelemetryIngest/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_SHA=${{ github.sha }} + APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}} + fluent-ingest-docker-image-deploy: needs: generate-build-number runs-on: ubuntu-latest diff --git a/.github/workflows/test.open-telemetry-ingest.yaml b/.github/workflows/test.open-telemetry-ingest.yaml new file mode 100644 index 00000000000..b047dea9941 --- /dev/null +++ b/.github/workflows/test.open-telemetry-ingest.yaml @@ -0,0 +1,21 @@ +name: OpenTelemetryIngest Test + +on: + pull_request: + push: + branches-ignore: + - 'hotfix-*' # excludes hotfix branches + - 'release' + +jobs: + test: + runs-on: ubuntu-latest + env: + CI_PIPELINE_ID: ${{github.run_number}} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 18.3.0 + - run: cd OpenTelemetryIngest && npm install && npm run test + diff --git a/.vscode/launch.json b/.vscode/launch.json index bd049076272..1c1a1f6ba13 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -189,11 +189,25 @@ "restart": true, "autoAttachChildProcesses": true }, + { + "address": "127.0.0.1", + "localRoot": "${workspaceFolder}/OpenTelemetryIngest", + "name": "OpenTelemetryIngest: Debug with Docker", + "port": 9937, + "remoteRoot": "/usr/src/app", + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node", + "restart": true, + "autoAttachChildProcesses": true + }, { "address": "127.0.0.1", "localRoot": "${workspaceFolder}/FluentIngest", "name": "Fluent Ingest: Debug with Docker", - "port": 9937, + "port": 9938, "remoteRoot": "/usr/src/app", "request": "attach", "skipFiles": [ diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts index b3260938089..a2aec97ef17 100644 --- a/Common/Server/EnvironmentConfig.ts +++ b/Common/Server/EnvironmentConfig.ts @@ -94,6 +94,12 @@ export const IngestorHostname: Hostname = Hostname.fromString( }`, ); +export const OpenTelemetryIngestHostname: Hostname = Hostname.fromString( + `${process.env["SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME"] || "localhost"}:${ + process.env["OPEN_TELEMETRY_INGEST_PORT"] || 80 + }`, +); + export const IncomingRequestIngestHostname: Hostname = Hostname.fromString( `${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${ process.env["INCOMING_REQUEST_INGEST_PORT"] || 80 diff --git a/OpenTelemetryIngest/Middleware/TelemetryIngest.ts b/Common/Server/Middleware/TelemetryIngest.ts similarity index 81% rename from OpenTelemetryIngest/Middleware/TelemetryIngest.ts rename to Common/Server/Middleware/TelemetryIngest.ts index 37413a25785..f4fe00bf07e 100644 --- a/OpenTelemetryIngest/Middleware/TelemetryIngest.ts +++ b/Common/Server/Middleware/TelemetryIngest.ts @@ -1,13 +1,13 @@ -import BadRequestException from "Common/Types/Exception/BadRequestException"; -import ProductType from "Common/Types/MeteredPlan/ProductType"; -import ObjectID from "Common/Types/ObjectID"; +import BadRequestException from "../../Types/Exception/BadRequestException"; +import ProductType from "../../Types/MeteredPlan/ProductType"; +import ObjectID from "../../Types/ObjectID"; import { ExpressRequest, ExpressResponse, NextFunction, -} from "Common/Server/Utils/Express"; -import TelemetryIngestionKeyService from "Common/Server/Services/TelemetryIngestionKeyService"; -import TelemetryIngestionKey from "Common/Models/DatabaseModels/TelemetryIngestionKey"; +} from "../../Server/Utils/Express"; +import TelemetryIngestionKeyService from "../../Server/Services/TelemetryIngestionKeyService"; +import TelemetryIngestionKey from "../../Models/DatabaseModels/TelemetryIngestionKey"; export interface TelemetryRequest extends ExpressRequest { projectId: ObjectID; // Project ID diff --git a/OpenTelemetryIngest/Service/OTelIngest.ts b/Common/Server/Services/OpenTelemetryIngestService.ts similarity index 100% rename from OpenTelemetryIngest/Service/OTelIngest.ts rename to Common/Server/Services/OpenTelemetryIngestService.ts diff --git a/Common/ServiceRoute.ts b/Common/ServiceRoute.ts index 8aedb26b2ca..24dac98d162 100644 --- a/Common/ServiceRoute.ts +++ b/Common/ServiceRoute.ts @@ -28,6 +28,10 @@ export const AdminDashboardRoute: Route = new Route("/admin"); export const IngestorRoute: Route = new Route("/ingestor"); +export const OpenTelemetryIngestRoute: Route = new Route( + "/open-telemetry-ingest", +); + export const IncomingRequestIngestRoute: Route = new Route( "/incoming-request-ingest", ); diff --git a/Common/UI/Config.ts b/Common/UI/Config.ts index 1c5cc62185b..be6e3c17f77 100644 --- a/Common/UI/Config.ts +++ b/Common/UI/Config.ts @@ -17,6 +17,7 @@ import { WorkflowRoute, FluentIngestRoute, IncomingRequestIngestRoute, + OpenTelemetryIngestRoute, } from "Common/ServiceRoute"; import Hostname from "Common/Types/API/Hostname"; import Protocol from "Common/Types/API/Protocol"; @@ -69,6 +70,9 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST); export const INGESTOR_HOSTNAME: Hostname = Hostname.fromString(HOST); +export const OPEN_TELEMETRY_INGEST_HOSTNAME: Hostname = + Hostname.fromString(HOST); + export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST); @@ -108,6 +112,12 @@ export const STATUS_PAGE_API_URL: URL = new URL( StatusPageApiRoute, ); +export const OPEN_TELEMETRY_INGEST_URL: URL = new URL( + HTTP_PROTOCOL, + OPEN_TELEMETRY_INGEST_HOSTNAME, + OpenTelemetryIngestRoute, +); + export const FLUENT_INGEST_URL: URL = new URL( HTTP_PROTOCOL, FLUENT_INGEST_HOSTNAME, diff --git a/E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts b/E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts new file mode 100644 index 00000000000..9c41792b496 --- /dev/null +++ b/E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts @@ -0,0 +1,47 @@ +import { BASE_URL } from "../../Config"; +import { Page, expect, test } from "@playwright/test"; +import URL from "Common/Types/API/URL"; + +test.describe("check live and health check of the open-telemetry-ingest", () => { + test("check if open-telemetry-ingest status is ok", async ({ + page, + }: { + page: Page; + }) => { + await page.goto( + `${URL.fromString(BASE_URL.toString()) + .addRoute("/open-telemetry-ingest/status") + .toString()}`, + ); + const content: string = await page.content(); + expect(content).toContain('{"status":"ok"}'); + }); + + test("check if open-telemetry-ingest is ready", async ({ + page, + }: { + page: Page; + }) => { + await page.goto( + `${URL.fromString(BASE_URL.toString()) + .addRoute("/open-telemetry-ingest/status/ready") + .toString()}`, + ); + const content: string = await page.content(); + expect(content).toContain('{"status":"ok"}'); + }); + + test("check if open-telemetry-ingest is live", async ({ + page, + }: { + page: Page; + }) => { + await page.goto( + `${URL.fromString(BASE_URL.toString()) + .addRoute("/open-telemetry-ingest/status/live") + .toString()}`, + ); + const content: string = await page.content(); + expect(content).toContain('{"status":"ok"}'); + }); +}); diff --git a/FluentIngest/API/FluentIngest.ts b/FluentIngest/API/FluentIngest.ts index 5e3302ae15c..cb5afc30255 100644 --- a/FluentIngest/API/FluentIngest.ts +++ b/FluentIngest/API/FluentIngest.ts @@ -1,6 +1,6 @@ import TelemetryIngest, { TelemetryRequest, -} from "../../Ingestor/Middleware/TelemetryIngest"; +} from "Common/Server/Middleware/TelemetryIngest"; import OneUptimeDate from "Common/Types/Date"; import { JSONObject } from "Common/Types/JSON"; import ProductType from "Common/Types/MeteredPlan/ProductType"; @@ -15,7 +15,7 @@ import logger from "Common/Server/Utils/Logger"; import Response from "Common/Server/Utils/Response"; import Log from "Common/Models/AnalyticsModels/Log"; import LogSeverity from "Common/Types/Log/LogSeverity"; -import OTelIngestService from "../../Ingestor/Service/OTelIngest"; +import OTelIngestService from "Common/Server/Services/OpenTelemetryIngestService"; import ObjectID from "Common/Types/ObjectID"; import JSONFunctions from "Common/Types/JSONFunctions"; diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl index 29c01aaa3a5..440efb0f9e5 100644 --- a/HelmChart/Public/oneuptime/templates/_helpers.tpl +++ b/HelmChart/Public/oneuptime/templates/_helpers.tpl @@ -60,6 +60,8 @@ Usage: value: {{ $.Release.Name }}-app.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_INGESTOR_HOSTNAME value: {{ $.Release.Name }}-ingestor.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} +- name: OPEN_TELEMETRY_INGEST_HOSTNAME + value: {{ $.Release.Name }}-open-telemetry-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_INCOMING_REQUEST_INGEST_HOSTNAME value: {{ $.Release.Name }}-incoming-request-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_FLUENT_INGEST_HOSTNAME @@ -81,6 +83,8 @@ Usage: value: {{ $.Values.port.app | squote }} - name: INGESTOR_PORT value: {{ $.Values.port.ingestor | squote }} +- name: OPEN_TELEMETRY_INGEST_PORT + value: {{ $.Values.port.openTelemetryIngest | squote }} - name: INCOMING_REQUEST_INGEST_PORT value: {{ $.Values.port.incomingRequestIngest | squote }} - name: FLUENT_INGEST_PORT diff --git a/HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml b/HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml new file mode 100644 index 00000000000..ce369f01aef --- /dev/null +++ b/HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml @@ -0,0 +1,111 @@ +# OneUptime open-telemetry-ingest Deployment + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + app.kubernetes.io/part-of: oneuptime + app.kubernetes.io/managed-by: Helm + appname: oneuptime + date: "{{ now | unixEpoch }}" +spec: + selector: + matchLabels: + app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + {{- if $.Values.deployment.open-telemetry-ingest.replicaCount }} + replicas: {{ $.Values.deployment.open-telemetry-ingest.replicaCount }} + {{- else }} + replicas: {{ $.Values.deployment.replicaCount }} + {{- end }} + template: + metadata: + labels: + app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + date: "{{ now | unixEpoch }}" + appname: oneuptime + spec: + volumes: + - name: greenlockrc + emptyDir: + sizeLimit: "1Gi" + {{- if $.Values.podSecurityContext }} + securityContext: {{- $.Values.podSecurityContext | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml $.Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if $.Values.affinity }} + affinity: {{- $.Values.affinity | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.tolerations }} + tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }} + {{- end }} + {{- if $.Values.nodeSelector }} + nodeSelector: {{- $.Values.nodeSelector | toYaml | nindent 8 }} + {{- end }} + containers: + - image: {{ printf "%s/%s/%s:%s" $.Values.image.registry $.Values.image.repository "open-telemetry-ingest" $.Values.image.tag }} + name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + {{- if $.Values.startupProbe.enabled }} + # Startup probe + startupProbe: + httpGet: + path: /status/live + port: {{ $.Values.port.openTelemetryIngest }} + periodSeconds: {{ $.Values.startupProbe.periodSeconds }} + failureThreshold: {{ $.Values.startupProbe.failureThreshold }} + {{- end }} + {{- if $.Values.livenessProbe.enabled }} + # Liveness probe + livenessProbe: + httpGet: + path: /status/live + port: {{ $.Values.port.openTelemetryIngest }} + periodSeconds: {{ $.Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }} + initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }} + {{- end }} + {{- if $.Values.readinessProbe.enabled }} + # Readyness Probe + readinessProbe: + httpGet: + path: /status/ready + port: {{ $.Values.port.openTelemetryIngest }} + periodSeconds: {{ $.Values.readinessProbe.periodSeconds }} + initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if $.Values.containerSecurityContext }} + securityContext: {{- $.Values.containerSecurityContext | toYaml | nindent 12 }} + {{- end }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + env: + {{- include "oneuptime.env.common" . | nindent 12 }} + {{- include "oneuptime.env.commonServer" . | nindent 12 }} + {{- include "oneuptime.env.oneuptimeSecret" . | nindent 12 }} + - name: OPENTELEMETRY_EXPORTER_OTLP_HEADERS + value: {{ $.Values.openTelemetryExporter.headers }} + - name: PORT + value: {{ $.Values.port.openTelemetryIngest | quote }} + ports: + - containerPort: {{ $.Values.port.openTelemetryIngest }} + protocol: TCP + name: http + restartPolicy: {{ $.Values.image.restartPolicy }} + +--- + +# OneUptime open-telemetry-ingest Service +{{- $open-telemetry-ingestPorts := dict "port" $.Values.port.openTelemetryIngest -}} +{{- $open-telemetry-ingestServiceArgs := dict "ServiceName" "open-telemetry-ingest" "Ports" $open-telemetry-ingestPorts "Release" $.Release "Values" $.Values -}} +{{- include "oneuptime.service" $open-telemetry-ingestServiceArgs }} +--- + +# OneUptime open-telemetry-ingest autoscaler +{{- $open-telemetry-ingestAutoScalerArgs := dict "ServiceName" "open-telemetry-ingest" "Release" $.Release "Values" $.Values -}} +{{- include "oneuptime.autoscaler" $open-telemetry-ingestAutoScalerArgs }} +--- \ No newline at end of file diff --git a/HelmChart/Public/oneuptime/values.yaml b/HelmChart/Public/oneuptime/values.yaml index 6dadec3a803..b7fa6f0613d 100644 --- a/HelmChart/Public/oneuptime/values.yaml +++ b/HelmChart/Public/oneuptime/values.yaml @@ -204,6 +204,7 @@ probes: port: app: 3002 ingestor: 3400 + openTelemetryIngest: 3403 fluentIngest: 3401 incomingRequestIngest: 3402 testServer: 3800 diff --git a/Nginx/default.conf.template b/Nginx/default.conf.template index 645a36fa2d2..864fb0125bd 100644 --- a/Nginx/default.conf.template +++ b/Nginx/default.conf.template @@ -10,6 +10,10 @@ upstream ingestor { server ${SERVER_INGESTOR_HOSTNAME}:${INGESTOR_PORT} weight=10 max_fails=3 fail_timeout=30s; } +upstream open-telemetry-ingest { + server ${SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME}:${OPEN_TELEMETRY_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s; +} + upstream incoming-reuqets-ingest { server ${SERVER_INCOMING_REQUEST_INGEST_HOSTNAME}:${INCOMING_REQUEST_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s; } diff --git a/OTelCollector/config.yaml b/OTelCollector/config.yaml index 164b46a2870..b00ae58718a 100644 --- a/OTelCollector/config.yaml +++ b/OTelCollector/config.yaml @@ -24,7 +24,7 @@ receivers: exporters: otlphttp: - endpoint: "http://${env:SERVER_INGESTOR_HOSTNAME}:${env:INGESTOR_PORT}/otlp" + endpoint: "http://${env:SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME}:${env:OPEN_TELEMETRY_INGEST_PORT}/otlp" headers: {"Content-Type": "application/json"} auth: authenticator: headers_setter diff --git a/OpenTelemetryIngest/API/OTelIngest.ts b/OpenTelemetryIngest/API/OTelIngest.ts index ada68b328e7..6c1f1b053b8 100644 --- a/OpenTelemetryIngest/API/OTelIngest.ts +++ b/OpenTelemetryIngest/API/OTelIngest.ts @@ -1,11 +1,11 @@ import ArrayUtil from "Common/Utils/Array"; import TelemetryIngest, { TelemetryRequest, -} from "../Middleware/TelemetryIngest"; +} from "Common/Server/Middleware/TelemetryIngest"; import OTelIngestService, { OtelAggregationTemporality, TelemetryServiceDataIngested, -} from "../Service/OTelIngest"; +} from "Common/Server/Services/OpenTelemetryIngestService"; import OneUptimeDate from "Common/Types/Date"; import BadRequestException from "Common/Types/Exception/BadRequestException"; import { JSONArray, JSONObject } from "Common/Types/JSON"; diff --git a/OpenTelemetryIngest/Dockerfile.tpl b/OpenTelemetryIngest/Dockerfile.tpl index 8992e1e6374..df8fcdce3f1 100644 --- a/OpenTelemetryIngest/Dockerfile.tpl +++ b/OpenTelemetryIngest/Dockerfile.tpl @@ -1,5 +1,5 @@ # -# OneUptime-Ingestor Dockerfile +# OneUptime-OpenTelemetryIngest Dockerfile # # Pull base image nodejs image. @@ -62,19 +62,19 @@ WORKDIR /usr/src/app RUN npx playwright install --with-deps # Install app dependencies -COPY ./Ingestor/package*.json /usr/src/app/ +COPY ./OpenTelemetryIngest/package*.json /usr/src/app/ RUN npm install # Expose ports. -# - 3400: OneUptime-ingestor -EXPOSE 3400 +# - 3403: OneUptime-OpenTelemetryIngest +EXPOSE 3403 {{ if eq .Env.ENVIRONMENT "development" }} #Run the app CMD [ "npm", "run", "dev" ] {{ else }} # Copy app source -COPY ./Ingestor /usr/src/app +COPY ./OpenTelemetryIngest /usr/src/app # Bundle app source RUN npm run compile #Run the app diff --git a/OpenTelemetryIngest/Index.ts b/OpenTelemetryIngest/Index.ts index d378e41ad10..e3a186c37af 100644 --- a/OpenTelemetryIngest/Index.ts +++ b/OpenTelemetryIngest/Index.ts @@ -1,9 +1,4 @@ -import IncomingRequestAPI from "../IncomingRequestIngest/API/IncomingRequest"; -import MonitorAPI from "./API/Monitor"; import OTelIngestAPI from "./API/OTelIngest"; -import Ingestor from "./API/Probe"; -import RegisterAPI from "./API/Register"; -import ServerMonitorAPI from "./API/ServerMonitor"; import { PromiseVoidFunction } from "Common/Types/FunctionTypes"; import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase"; import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase"; @@ -18,14 +13,9 @@ import "ejs"; const app: ExpressApplication = Express.getExpressApp(); -const APP_NAME: string = "ingestor"; +const APP_NAME: string = "open-telemetry-ingest"; -app.use([`/${APP_NAME}`, "/"], RegisterAPI); -app.use([`/${APP_NAME}`, "/"], MonitorAPI); -app.use([`/${APP_NAME}`, "/"], Ingestor); -app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI); app.use([`/${APP_NAME}`, "/"], OTelIngestAPI); -app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI); const init: PromiseVoidFunction = async (): Promise => { try { diff --git a/Tests/Scripts/status-check.sh b/Tests/Scripts/status-check.sh index 8fea1bbdcc4..09c164bd282 100644 --- a/Tests/Scripts/status-check.sh +++ b/Tests/Scripts/status-check.sh @@ -51,6 +51,8 @@ bash $scriptDir/endpoint-status.sh "Admin Dashboard (Ready Check)" $HOST_TO_CHEC bash $scriptDir/endpoint-status.sh "Ingestor (Ready Check)" $HOST_TO_CHECK/ingestor/status/ready +bash $scriptDir/endpoint-status.sh "OpenTelemetry Ingest (Ready Check)" $HOST_TO_CHECK/open-telemetry-ingest/status/ready + bash $scriptDir/endpoint-status.sh "Ingestor (Status Check)" $HOST_TO_CHECK/ingestor/status echo "🚀 OneUptime is up! 🚀" diff --git a/config.example.env b/config.example.env index 5c766304f5e..cd8f82f14f6 100644 --- a/config.example.env +++ b/config.example.env @@ -95,10 +95,12 @@ REDIS_TLS_SENTINEL_MODE=false INGESTOR_HOSTNAME=ingestor:3400 FLUENT_INGEST_HOSTNAME=fluent-ingest:3401 INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest:3402 +OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest:3403 SERVER_ACCOUNTS_HOSTNAME=accounts SERVER_APP_HOSTNAME=app SERVER_INGESTOR_HOSTNAME=ingestor +SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest SERVER_INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest SERVER_FLUENT_INGEST_HOSTNAME=fluent-ingest SERVER_TEST_SERVER_HOSTNAME=test-server @@ -114,6 +116,7 @@ SERVER_DOCS_HOSTNAME=docs APP_PORT=3002 INGESTOR_PORT=3400 +OPEN_TELEMETRY_INGEST_PORT=3403 FLUENT_INGEST_PORT=3401 INCOMING_REQUEST_INGEST_PORT=3402 PROBE_PORT=3500 @@ -269,6 +272,7 @@ LLM_SERVER_HUGGINGFACE_MODEL_NAME= DISABLE_TELEMETRY_FOR_ACCOUNTS=true DISABLE_TELEMETRY_FOR_APP=true DISABLE_TELEMETRY_FOR_INGESTOR=true +DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST=true DISABLE_TELEMETRY_FOR_FLUENT_INGEST=true DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST=true DISABLE_TELEMETRY_FOR_TEST_SERVER=true diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 310cdb74a87..5a3b719388b 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -28,6 +28,7 @@ x-common-variables: &common-variables SERVER_APP_HOSTNAME: app SERVER_ALERT_HOSTNAME: alert SERVER_INGESTOR_HOSTNAME: ingestor + SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME: open-telemetry-ingest SERVER_INCOMING_REQUEST_INGEST_HOSTNAME: incoming-request-ingest SERVER_FLUENT_INGEST_HOSTNAME: fluent-ingest SERVER_TEST_SERVER_HOSTNAME: test-server @@ -46,6 +47,7 @@ x-common-variables: &common-variables APP_PORT: ${APP_PORT} HOME_PORT: ${HOME_PORT} INGESTOR_PORT: ${INGESTOR_PORT} + OPEN_TELEMETRY_INGEST_PORT: ${OPEN_TELEMETRY_INGEST_PORT} INCOMING_REQUEST_INGEST_PORT: ${INCOMING_REQUEST_INGEST_PORT} FLUENT_INGEST_PORT: ${FLUENT_INGEST_PORT} PROBE_PORT: ${PROBE_PORT} @@ -450,6 +452,19 @@ services: options: max-size: "1000m" + open-telemetry-ingest: + networks: + - oneuptime + restart: always + environment: + <<: *common-server-variables + PORT: ${OPEN_TELEMETRY_INGEST_PORT} + DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST} + logging: + driver: "local" + options: + max-size: "1000m" + incoming-request-ingest: networks: - oneuptime diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1dc6abbddbc..f7f33e0bb40 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -351,6 +351,24 @@ services: context: . dockerfile: ./Ingestor/Dockerfile + open-telemetry-ingest: + volumes: + - ./OpenTelemetryIngest:/usr/src/app + # Use node modules of the container and not host system. + # https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder + - /usr/src/app/node_modules/ + - ./Common:/usr/src/Common + - /usr/src/Common/node_modules/ + ports: + - '9938:9229' # Debugging port. + extends: + file: ./docker-compose.base.yml + service: open-telemetry-ingest + build: + network: host + context: . + dockerfile: ./OpenTelemetryIngest/Dockerfile + incoming-request-ingest: volumes: - ./IncomingRequestIngest:/usr/src/app diff --git a/docker-compose.yml b/docker-compose.yml index 0a16feb8732..0b4f56ea707 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,12 @@ services: extends: file: ./docker-compose.base.yml service: ingestor + + open-telemetry-ingest: + image: oneuptime/open-telemetry-ingest:${APP_TAG} + extends: + file: ./docker-compose.base.yml + service: open-telemetry-ingest incoming-request-ingest: image: oneuptime/incoming-request-ingest:${APP_TAG}