From 92dbb53f9b2372e96c642922bebcbb65dbc95a69 Mon Sep 17 00:00:00 2001 From: chronark Date: Wed, 13 Nov 2024 00:36:24 +0100 Subject: [PATCH] chore: rip out tinybird from the API --- apps/api/package.json | 1 - apps/api/src/pkg/analytics.ts | 243 +----------------- apps/api/src/pkg/audit.ts | 11 +- apps/api/src/pkg/auth/root_key.ts | 21 +- apps/api/src/pkg/cache/index.ts | 2 +- apps/api/src/pkg/cache/namespaces.ts | 5 +- apps/api/src/pkg/env.ts | 2 +- apps/api/src/pkg/key_migration/handler.ts | 76 ------ apps/api/src/pkg/middleware/init.ts | 16 +- apps/api/src/routes/legacy_keys_createKey.ts | 24 +- apps/api/src/routes/v1_apis_createApi.ts | 19 +- apps/api/src/routes/v1_apis_deleteApi.ts | 42 +-- .../routes/v1_identities_createIdentity.ts | 48 +--- .../routes/v1_identities_deleteIdentity.ts | 49 +--- .../routes/v1_identities_updateIdentity.ts | 6 +- apps/api/src/routes/v1_keys_addPermissions.ts | 54 +--- apps/api/src/routes/v1_keys_addRoles.ts | 54 +--- apps/api/src/routes/v1_keys_createKey.ts | 12 +- apps/api/src/routes/v1_keys_deleteKey.ts | 19 +- .../src/routes/v1_keys_getVerifications.ts | 41 ++- .../src/routes/v1_keys_removePermissions.ts | 31 +-- apps/api/src/routes/v1_keys_removeRoles.ts | 31 +-- apps/api/src/routes/v1_keys_setPermissions.ts | 3 +- apps/api/src/routes/v1_keys_setRoles.ts | 3 +- apps/api/src/routes/v1_keys_updateKey.ts | 33 +-- .../api/src/routes/v1_keys_updateRemaining.ts | 28 +- apps/api/src/routes/v1_keys_verifyKey.ts | 26 -- apps/api/src/routes/v1_liveness.ts | 6 +- .../api/src/routes/v1_migrations_createKey.ts | 77 +----- .../routes/v1_permissions_createPermission.ts | 27 +- .../src/routes/v1_permissions_createRole.ts | 26 +- .../routes/v1_permissions_deletePermission.ts | 21 +- .../src/routes/v1_permissions_deleteRole.ts | 22 +- apps/api/src/routes/v1_ratelimit_limit.ts | 84 +----- apps/billing/package.json | 2 +- apps/billing/src/lib/clickhouse.ts | 4 + apps/billing/src/lib/env.ts | 2 +- apps/billing/src/lib/tinybird.ts | 64 ----- apps/billing/src/trigger/create-invoice.ts | 27 +- apps/dashboard/package.json | 1 - apps/semantic-cache/package.json | 1 - apps/semantic-cache/src/pkg/analytics.ts | 61 ----- apps/semantic-cache/src/pkg/hono/env.ts | 2 - .../semantic-cache/src/pkg/middleware/init.ts | 15 -- apps/semantic-cache/src/worker.ts | 40 +-- apps/workflows/lib/tinybird.ts | 64 ----- apps/workflows/package.json | 1 - apps/www/package.json | 1 - internal/clickhouse/src/index.ts | 12 + internal/clickhouse/src/ratelimits.ts | 22 +- internal/clickhouse/src/requests.ts | 30 +++ internal/clickhouse/src/telemetry.ts | 15 ++ internal/clickhouse/src/verifications.ts | 26 +- internal/tinybird/.tinyenv | 2 - .../active_keys_hourly_mv.datasource | 12 - .../datasources/aggregate_mv.datasource | 11 - .../datasources/audit_logs.datasource | 19 -- .../billable_verifications__mv.datasource | 8 - .../datasources/daily_usage.datasource | 14 - .../datasources/deployment_logs.datasource | 13 - .../datasources/key_verifications.datasource | 22 -- .../datasources/mv__daily_usage.datasource | 14 - .../mv__daily_verifications.datasource | 19 -- .../mv__latest_verifications.datasource | 17 -- .../mv__monthly_active_keys.datasource | 11 - .../datasources/mv__monthly_stats.datasource | 12 - .../mv__monthly_verifications.datasource | 18 -- .../mv__ratelimits_daily.datasource | 14 - .../mv__ratelimits_hourly.datasource | 15 -- .../mv__ratelimits_last_used.datasource | 10 - .../mv__ratelimits_minutely.datasource | 14 - .../mv__ratelimits_monthly.datasource | 15 -- .../mv__ratelimits_weekly.datasource | 15 -- .../mv__semantic_cache_daily.datasource | 22 -- .../mv__semantic_cache_hourly.datasource | 22 -- .../mv__total_verifications.datasource | 8 - ...rifications_for_analytics_daily.datasource | 16 -- ...ifications_for_analytics_hourly.datasource | 16 -- ...fications_for_analytics_monthly.datasource | 16 -- ...cations_for_analytics_weekly_v1.datasource | 15 -- ...ications_per_workspace_per_hour.datasource | 13 - .../mv_billing_hourly_active_keys.datasource | 11 - .../datasources/mv_key_total_usage.datasource | 9 - .../datasources/mv_keys_last_used.datasource | 8 - .../tinybird/datasources/pageviews.datasource | 16 -- .../q1_goal_distinct_workspaces.datasource | 8 - .../datasources/ratelimits.datasource | 26 -- .../datasources/sdk_telemetry.datasource | 9 - .../datasources/semantic_cache.datasource | 21 -- .../tinybird_proxy_errors.datasource | 8 - .../tinybird_proxy_metrics.datasource | 10 - .../verifications_daily_mv.datasource | 19 -- .../verifications_hourly_mv.datasource | 20 -- .../verifications_monthly_mv.datasource | 19 -- .../verifications_weekly_mv.datasource | 19 -- .../webhook_delivery_attempts.datasource | 17 -- .../tinybird/pipes/active_keys_hourly.pipe | 20 -- ...egate_active_keys_hourly_by_workspace.pipe | 18 -- ...gate_active_keys_monthly_by_workspace.pipe | 22 -- .../pipes/aggregate_active_keys_per_hour.pipe | 17 -- .../tinybird/pipes/aggregate_last_used.pipe | 15 -- .../pipes/aggregate_total_usage_per_key.pipe | 14 - .../aggregate_unique_visitors_per_hour.pipe | 11 - .../pipes/aggregate_verifications_daily.pipe | 28 -- ...aggregate_verifications_daily_per_api.pipe | 20 -- .../pipes/aggregate_verifications_hourly.pipe | 14 - ...ate_verifications_hourly_by_workspace.pipe | 18 -- .../aggregate_verifications_monthly.pipe | 21 -- ...te_verifications_monthly_by_workspace.pipe | 28 -- .../aggregate_verifications_per_day.pipe | 23 -- .../tinybird/pipes/all_verifications.pipe | 34 --- .../pipes/billable_verifications.pipe | 10 - .../pipes/copy_latest_verifications.pipe | 19 -- .../pipes/count_monthly_verifications.pipe | 12 - .../pipes/count_verifications_daily.pipe | 8 - .../tinybird/pipes/distinct_workspaces.pipe | 23 -- .../endpoint__active_by_workspace_daily.pipe | 34 --- .../endpoint__active_by_workspace_hourly.pipe | 34 --- ...endpoint__active_by_workspace_monthly.pipe | 34 --- .../endpoint__active_by_workspace_weekly.pipe | 34 --- .../endpoint__active_keys_by_workspace.pipe | 13 - .../pipes/endpoint__all_verifications.pipe | 31 --- .../endpoint__all_verifications__v1.pipe | 7 - ...nt__analytics_active_by_workspace_all.pipe | 26 -- .../tinybird/pipes/endpoint__audit_logs.pipe | 28 -- ...point__billing_verifications_per_hour.pipe | 9 - .../pipes/endpoint__get_active_keys.pipe | 15 -- .../endpoint__get_daily_verifications.pipe | 24 -- .../pipes/endpoint__get_last_used.pipe | 12 - .../endpoint__get_latest_verifications.pipe | 13 - .../endpoint__get_total_usage_for_key.pipe | 8 - .../pipes/endpoint__monthly_stats.pipe | 12 - ...ations_analytics_by_workspaceId_daily.pipe | 38 --- ...tions_analytics_by_workspaceId_hourly.pipe | 38 --- ...ations_analytics_by_workspace_monthly.pipe | 38 --- ...cations_analytics_by_workspace_weekly.pipe | 38 --- .../endpoint__verifications_by_workspace.pipe | 17 -- ...ndpoint_billing_get_active_keys_usage.pipe | 18 -- ..._verifications_per_workspace_per_hour.pipe | 11 - .../tinybird/pipes/endpoint_get_usage.pipe | 19 -- ...et__verifications_for_analytics_month.pipe | 25 -- ...t__verifications_for_analytics_weekly.pipe | 25 -- internal/tinybird/pipes/get_active_keys.pipe | 18 -- .../tinybird/pipes/get_active_keys_daily.pipe | 22 -- .../pipes/get_active_keys_hourly.pipe | 22 -- .../pipes/get_active_keys_monthly.pipe | 22 -- .../pipes/get_active_keys_weekly.pipe | 22 -- .../tinybird/pipes/get_billable_keys.pipe | 21 -- .../pipes/get_billable_verifications.pipe | 15 -- .../get_q1_goal_distinct_workspaces.pipe | 12 - .../tinybird/pipes/get_ratelimit_events.pipe | 33 --- .../get_ratelimit_identifiers_daily.pipe | 15 -- .../get_ratelimit_identifiers_hourly.pipe | 15 -- .../get_ratelimit_identifiers_minutely.pipe | 15 -- .../get_ratelimit_identifiers_monthly.pipe | 15 -- .../get_ratelimit_identifiers_weekly.pipe | 15 -- .../tinybird/pipes/get_ratelimits_daily.pipe | 35 --- .../tinybird/pipes/get_ratelimits_hourly.pipe | 35 --- .../pipes/get_ratelimits_last_used.pipe | 16 -- .../pipes/get_ratelimits_minutely.pipe | 31 --- .../pipes/get_ratelimits_monthly.pipe | 35 --- .../tinybird/pipes/get_ratelimits_weekly.pipe | 34 --- .../pipes/get_semantic_caches_daily.pipe | 39 --- .../pipes/get_semantic_caches_hourly.pipe | 39 --- .../pipes/get_verifications_daily.pipe | 37 --- ...get_verifications_for_analytics_daily.pipe | 25 -- ...et_verifications_for_analytics_hourly.pipe | 25 -- .../pipes/get_verifications_hourly.pipe | 38 --- .../pipes/get_verifications_monthly.pipe | 38 --- .../get_verifications_ownerid_hourly_v1.pipe | 40 --- .../pipes/get_verifications_weekly.pipe | 38 --- .../pipes/monthly_active_workspaces.pipe | 21 -- .../pipes/mv__daily_verifications.pipe | 5 - .../tinybird/pipes/ratelimit_last_used.pipe | 19 -- internal/tinybird/pipes/ratelimits_daily.pipe | 22 -- .../tinybird/pipes/ratelimits_hourly.pipe | 22 -- .../tinybird/pipes/ratelimits_minutely.pipe | 22 -- .../tinybird/pipes/ratelimits_monthly.pipe | 22 -- .../tinybird/pipes/ratelimits_weekly.pipe | 22 -- .../tinybird/pipes/semantic_cache_daily.pipe | 30 --- .../tinybird/pipes/semantic_cache_hourly.pipe | 30 --- .../tinybird/pipes/verifications_daily.pipe | 29 --- .../tinybird/pipes/verifications_hourly.pipe | 29 --- .../tinybird/pipes/verifications_monthly.pipe | 29 --- .../tinybird/pipes/verifications_weekly.pipe | 29 --- internal/tinybird/scripts/exec_test.sh | 21 -- pnpm-lock.yaml | 53 ++-- tools/migrate/package.json | 1 - 188 files changed, 221 insertions(+), 4104 deletions(-) create mode 100644 apps/billing/src/lib/clickhouse.ts delete mode 100644 apps/billing/src/lib/tinybird.ts delete mode 100644 apps/semantic-cache/src/pkg/analytics.ts delete mode 100644 apps/workflows/lib/tinybird.ts create mode 100644 internal/clickhouse/src/requests.ts create mode 100644 internal/clickhouse/src/telemetry.ts delete mode 100644 internal/tinybird/.tinyenv delete mode 100644 internal/tinybird/datasources/active_keys_hourly_mv.datasource delete mode 100644 internal/tinybird/datasources/aggregate_mv.datasource delete mode 100644 internal/tinybird/datasources/audit_logs.datasource delete mode 100644 internal/tinybird/datasources/billable_verifications__mv.datasource delete mode 100644 internal/tinybird/datasources/daily_usage.datasource delete mode 100644 internal/tinybird/datasources/deployment_logs.datasource delete mode 100644 internal/tinybird/datasources/key_verifications.datasource delete mode 100644 internal/tinybird/datasources/mv__daily_usage.datasource delete mode 100644 internal/tinybird/datasources/mv__daily_verifications.datasource delete mode 100644 internal/tinybird/datasources/mv__latest_verifications.datasource delete mode 100644 internal/tinybird/datasources/mv__monthly_active_keys.datasource delete mode 100644 internal/tinybird/datasources/mv__monthly_stats.datasource delete mode 100644 internal/tinybird/datasources/mv__monthly_verifications.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_daily.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_hourly.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_last_used.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_minutely.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_monthly.datasource delete mode 100644 internal/tinybird/datasources/mv__ratelimits_weekly.datasource delete mode 100644 internal/tinybird/datasources/mv__semantic_cache_daily.datasource delete mode 100644 internal/tinybird/datasources/mv__semantic_cache_hourly.datasource delete mode 100644 internal/tinybird/datasources/mv__total_verifications.datasource delete mode 100644 internal/tinybird/datasources/mv__verifications_for_analytics_daily.datasource delete mode 100644 internal/tinybird/datasources/mv__verifications_for_analytics_hourly.datasource delete mode 100644 internal/tinybird/datasources/mv__verifications_for_analytics_monthly.datasource delete mode 100644 internal/tinybird/datasources/mv__verifications_for_analytics_weekly_v1.datasource delete mode 100644 internal/tinybird/datasources/mv__verifications_per_workspace_per_hour.datasource delete mode 100644 internal/tinybird/datasources/mv_billing_hourly_active_keys.datasource delete mode 100644 internal/tinybird/datasources/mv_key_total_usage.datasource delete mode 100644 internal/tinybird/datasources/mv_keys_last_used.datasource delete mode 100644 internal/tinybird/datasources/pageviews.datasource delete mode 100644 internal/tinybird/datasources/q1_goal_distinct_workspaces.datasource delete mode 100644 internal/tinybird/datasources/ratelimits.datasource delete mode 100644 internal/tinybird/datasources/sdk_telemetry.datasource delete mode 100644 internal/tinybird/datasources/semantic_cache.datasource delete mode 100644 internal/tinybird/datasources/tinybird_proxy_errors.datasource delete mode 100644 internal/tinybird/datasources/tinybird_proxy_metrics.datasource delete mode 100644 internal/tinybird/datasources/verifications_daily_mv.datasource delete mode 100644 internal/tinybird/datasources/verifications_hourly_mv.datasource delete mode 100644 internal/tinybird/datasources/verifications_monthly_mv.datasource delete mode 100644 internal/tinybird/datasources/verifications_weekly_mv.datasource delete mode 100644 internal/tinybird/datasources/webhook_delivery_attempts.datasource delete mode 100644 internal/tinybird/pipes/active_keys_hourly.pipe delete mode 100644 internal/tinybird/pipes/aggregate_active_keys_hourly_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/aggregate_active_keys_monthly_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/aggregate_active_keys_per_hour.pipe delete mode 100644 internal/tinybird/pipes/aggregate_last_used.pipe delete mode 100644 internal/tinybird/pipes/aggregate_total_usage_per_key.pipe delete mode 100644 internal/tinybird/pipes/aggregate_unique_visitors_per_hour.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_daily.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_daily_per_api.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_hourly.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_hourly_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_monthly.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_monthly_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/aggregate_verifications_per_day.pipe delete mode 100644 internal/tinybird/pipes/all_verifications.pipe delete mode 100644 internal/tinybird/pipes/billable_verifications.pipe delete mode 100644 internal/tinybird/pipes/copy_latest_verifications.pipe delete mode 100644 internal/tinybird/pipes/count_monthly_verifications.pipe delete mode 100644 internal/tinybird/pipes/count_verifications_daily.pipe delete mode 100644 internal/tinybird/pipes/distinct_workspaces.pipe delete mode 100644 internal/tinybird/pipes/endpoint__active_by_workspace_daily.pipe delete mode 100644 internal/tinybird/pipes/endpoint__active_by_workspace_hourly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__active_by_workspace_monthly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__active_by_workspace_weekly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__active_keys_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/endpoint__all_verifications.pipe delete mode 100644 internal/tinybird/pipes/endpoint__all_verifications__v1.pipe delete mode 100644 internal/tinybird/pipes/endpoint__analytics_active_by_workspace_all.pipe delete mode 100644 internal/tinybird/pipes/endpoint__audit_logs.pipe delete mode 100644 internal/tinybird/pipes/endpoint__billing_verifications_per_hour.pipe delete mode 100644 internal/tinybird/pipes/endpoint__get_active_keys.pipe delete mode 100644 internal/tinybird/pipes/endpoint__get_daily_verifications.pipe delete mode 100644 internal/tinybird/pipes/endpoint__get_last_used.pipe delete mode 100644 internal/tinybird/pipes/endpoint__get_latest_verifications.pipe delete mode 100644 internal/tinybird/pipes/endpoint__get_total_usage_for_key.pipe delete mode 100644 internal/tinybird/pipes/endpoint__monthly_stats.pipe delete mode 100644 internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_daily.pipe delete mode 100644 internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_hourly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_monthly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_weekly.pipe delete mode 100644 internal/tinybird/pipes/endpoint__verifications_by_workspace.pipe delete mode 100644 internal/tinybird/pipes/endpoint_billing_get_active_keys_usage.pipe delete mode 100644 internal/tinybird/pipes/endpoint_billing_get_verifications_per_workspace_per_hour.pipe delete mode 100644 internal/tinybird/pipes/endpoint_get_usage.pipe delete mode 100644 internal/tinybird/pipes/get__verifications_for_analytics_month.pipe delete mode 100644 internal/tinybird/pipes/get__verifications_for_analytics_weekly.pipe delete mode 100644 internal/tinybird/pipes/get_active_keys.pipe delete mode 100644 internal/tinybird/pipes/get_active_keys_daily.pipe delete mode 100644 internal/tinybird/pipes/get_active_keys_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_active_keys_monthly.pipe delete mode 100644 internal/tinybird/pipes/get_active_keys_weekly.pipe delete mode 100644 internal/tinybird/pipes/get_billable_keys.pipe delete mode 100644 internal/tinybird/pipes/get_billable_verifications.pipe delete mode 100644 internal/tinybird/pipes/get_q1_goal_distinct_workspaces.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_events.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_identifiers_daily.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_identifiers_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_identifiers_minutely.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_identifiers_monthly.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimit_identifiers_weekly.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_daily.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_last_used.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_minutely.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_monthly.pipe delete mode 100644 internal/tinybird/pipes/get_ratelimits_weekly.pipe delete mode 100644 internal/tinybird/pipes/get_semantic_caches_daily.pipe delete mode 100644 internal/tinybird/pipes/get_semantic_caches_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_daily.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_for_analytics_daily.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_for_analytics_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_hourly.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_monthly.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_ownerid_hourly_v1.pipe delete mode 100644 internal/tinybird/pipes/get_verifications_weekly.pipe delete mode 100644 internal/tinybird/pipes/monthly_active_workspaces.pipe delete mode 100644 internal/tinybird/pipes/mv__daily_verifications.pipe delete mode 100644 internal/tinybird/pipes/ratelimit_last_used.pipe delete mode 100644 internal/tinybird/pipes/ratelimits_daily.pipe delete mode 100644 internal/tinybird/pipes/ratelimits_hourly.pipe delete mode 100644 internal/tinybird/pipes/ratelimits_minutely.pipe delete mode 100644 internal/tinybird/pipes/ratelimits_monthly.pipe delete mode 100644 internal/tinybird/pipes/ratelimits_weekly.pipe delete mode 100644 internal/tinybird/pipes/semantic_cache_daily.pipe delete mode 100644 internal/tinybird/pipes/semantic_cache_hourly.pipe delete mode 100644 internal/tinybird/pipes/verifications_daily.pipe delete mode 100644 internal/tinybird/pipes/verifications_hourly.pipe delete mode 100644 internal/tinybird/pipes/verifications_monthly.pipe delete mode 100644 internal/tinybird/pipes/verifications_weekly.pipe delete mode 100755 internal/tinybird/scripts/exec_test.sh diff --git a/apps/api/package.json b/apps/api/package.json index 2a0a98e142..8893cd42a9 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -22,7 +22,6 @@ }, "dependencies": { "@axiomhq/js": "1.0.0-rc.2", - "@chronark/zod-bird": "0.3.9", "@hono/zod-openapi": "^0.11.0", "@hono/zod-validator": "^0.2.1", "@planetscale/database": "^1.16.0", diff --git a/apps/api/src/pkg/analytics.ts b/apps/api/src/pkg/analytics.ts index 41782581dd..d87c7575d7 100644 --- a/apps/api/src/pkg/analytics.ts +++ b/apps/api/src/pkg/analytics.ts @@ -1,44 +1,13 @@ -import { NoopTinybird, Tinybird } from "@chronark/zod-bird"; import { ClickHouse } from "@unkey/clickhouse"; -import { newId } from "@unkey/id"; -import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog"; -import { ratelimitSchemaV1 } from "@unkey/schema/src/ratelimit-tinybird"; import { z } from "zod"; -import type { MaybeArray } from "./types/maybe"; -// const datetimeToUnixMilli = z.string().transform((t) => new Date(t).getTime()); - -/** - * `t` has the format `2021-01-01 00:00:00` - * - * If we transform it as is, we get `1609459200000` which is `2021-01-01 01:00:00` due to fun timezone stuff. - * So we split the string at the space and take the date part, and then parse that. - */ -const dateToUnixMilli = z.string().transform((t) => new Date(t.split(" ").at(0) ?? t).getTime()); export class Analytics { - public readonly readClient: Tinybird | NoopTinybird; - public readonly writeClient: Tinybird | NoopTinybird; - private clickhouse: ClickHouse; + private readonly clickhouse: ClickHouse; constructor(opts: { - tinybirdToken?: string; - tinybirdProxy?: { - url: string; - token: string; - }; - clickhouse?: { - url: string; - }; + clickhouseUrl: string; }) { - this.readClient = opts.tinybirdToken - ? new Tinybird({ token: opts.tinybirdToken }) - : new NoopTinybird(); - - this.writeClient = opts.tinybirdProxy - ? new Tinybird({ token: opts.tinybirdProxy.token, baseUrl: opts.tinybirdProxy.url }) - : this.readClient; - - this.clickhouse = new ClickHouse({ url: opts.clickhouse?.url }); + this.clickhouse = new ClickHouse({ url: opts.clickhouseUrl }); } public get insertSdkTelemetry() { @@ -53,218 +22,20 @@ export class Analytics { }), }); } - //tinybird, to be removed - public get ingestSdkTelemetry() { - return this.writeClient.buildIngestEndpoint({ - datasource: "sdk_telemetry__v1", - event: z.object({ - runtime: z.string(), - platform: z.string(), - versions: z.array(z.string()), - requestId: z.string(), - time: z.number(), - }), - }); - } - - //tinybird - public ingestUnkeyAuditLogsTinybird(logs: MaybeArray) { - return this.writeClient.buildIngestEndpoint({ - datasource: "audit_logs__v2", - event: auditLogSchemaV1 - .merge( - z.object({ - event: unkeyAuditLogEvents, - auditLogId: z.string().default(newId("auditLog")), - bucket: z.string().default("unkey_mutations"), - time: z.number().default(Date.now()), - }), - ) - .transform((l) => ({ - ...l, - meta: l.meta ? JSON.stringify(l.meta) : undefined, - actor: { - ...l.actor, - meta: l.actor.meta ? JSON.stringify(l.actor.meta) : undefined, - }, - resources: JSON.stringify(l.resources), - })), - })(logs); - } - //tinybird - public get ingestGenericAuditLogsTinybird() { - return this.writeClient.buildIngestEndpoint({ - datasource: "audit_logs__v2", - event: auditLogSchemaV1.transform((l) => ({ - ...l, - meta: l.meta ? JSON.stringify(l.meta) : undefined, - actor: { - ...l.actor, - meta: l.actor.meta ? JSON.stringify(l.actor.meta) : undefined, - }, - resources: JSON.stringify(l.resources), - })), - }); - } public get insertRatelimit() { - return this.clickhouse.client.insert({ - table: "ratelimits.raw_ratelimits_v1", - schema: z.object({ - request_id: z.string(), - time: z.number().int(), - workspace_id: z.string(), - namespace_id: z.string(), - identifier: z.string(), - passed: z.boolean(), - }), - }); - } - - //tinybird - public get ingestRatelimit() { - return this.writeClient.buildIngestEndpoint({ - datasource: "ratelimits__v2", - event: ratelimitSchemaV1, - }); + return this.clickhouse.ratelimits.insert; } public get insertKeyVerification() { - return this.clickhouse.client.insert({ - table: "verifications.raw_key_verifications_v1", - schema: z.object({ - request_id: z.string(), - time: z.number().int(), - workspace_id: z.string(), - key_space_id: z.string(), - key_id: z.string(), - region: z.string(), - outcome: z.enum([ - "VALID", - "RATE_LIMITED", - "EXPIRED", - "DISABLED", - "FORBIDDEN", - "USAGE_EXCEEDED", - "INSUFFICIENT_PERMISSIONS", - ]), - identity_id: z.string().optional().default(""), - }), - }); + return this.clickhouse.verifications.insert; } public get insertApiRequest() { - return this.clickhouse.client.insert({ - table: "metrics.raw_api_requests_v1", - schema: z.object({ - request_id: z.string(), - time: z.number().int(), - workspace_id: z.string(), - host: z.string(), - method: z.string(), - path: z.string(), - request_headers: z.array(z.string()), - request_body: z.string(), - response_status: z.number().int(), - response_headers: z.array(z.string()), - response_body: z.string(), - error: z.string().optional().default(""), - service_latency: z.number().int(), - user_agent: z.string(), - ip_address: z.string(), - continent: z.string().nullable().default(""), - city: z.string().nullable().default(""), - country: z.string().nullable().default(""), - colo: z.string().nullable().default(""), - }), - }); - } - // replaced by insertKeyVerification - public get ingestKeyVerification() { - return this.writeClient.buildIngestEndpoint({ - datasource: "key_verifications__v2", - event: z.object({ - workspaceId: z.string(), - apiId: z.string(), - keyId: z.string(), - deniedReason: z - .enum([ - "RATE_LIMITED", - "USAGE_EXCEEDED", - "FORBIDDEN", - "UNAUTHORIZED", - "DISABLED", - "INSUFFICIENT_PERMISSIONS", - "EXPIRED", - ]) - .optional(), - time: z.number(), - ipAddress: z.string().default(""), - userAgent: z.string().default(""), - requestedResource: z.string().default(""), - edgeRegion: z.string().default(""), - region: z.string(), - // deprecated, use deniedReason - ratelimited: z.boolean().default(false), - // deprecated, use deniedReason - usageExceeded: z.boolean().default(false), - ownerId: z.string().optional(), - keySpaceId: z.string(), - requestId: z.string().optional(), - requestBody: z.string().optional(), - responseBody: z.string().optional(), - }), - }); + return this.clickhouse.api.insert; } public get getVerificationsDaily() { - return this.readClient.buildPipe({ - pipe: "get_verifications_daily__v2", - parameters: z.object({ - workspaceId: z.string(), - apiId: z.string(), - keyId: z.string().optional(), - start: z.number().optional(), - end: z.number().optional(), - }), - data: z.object({ - time: dateToUnixMilli, - success: z.number(), - rateLimited: z.number(), - usageExceeded: z.number(), - }), - }); + return this.clickhouse.verifications.perDay; } } - -export type UnkeyAuditLog = { - workspaceId: string; - event: z.infer; - description: string; - actor: { - type: "user" | "key"; - name?: string; - id: string; - }; - resources: Array<{ - type: - | "key" - | "api" - | "workspace" - | "role" - | "permission" - | "keyAuth" - | "vercelBinding" - | "vercelIntegration" - | "ratelimitIdentifier" - | "ratelimitNamespace" - | "identity" - | "ratelimit"; - id: string; - meta?: Record; - }>; - context: { - userAgent?: string; - location: string; - }; -}; diff --git a/apps/api/src/pkg/audit.ts b/apps/api/src/pkg/audit.ts index 512dc541e5..dd79706d1d 100644 --- a/apps/api/src/pkg/audit.ts +++ b/apps/api/src/pkg/audit.ts @@ -4,7 +4,16 @@ import { type Transaction, schema } from "./db"; import { newId } from "@unkey/id"; import { z } from "zod"; -import type { UnkeyAuditLog } from "./analytics"; + +const unkeyAuditLogSchema = auditLogSchemaV1.merge( + z.object({ + event: unkeyAuditLogEvents, + auditLogId: z.string().default(newId("auditLog")), + bucket: z.string().default("unkey_mutations"), + time: z.number().default(Date.now()), + }), +); +export type UnkeyAuditLog = z.input; export async function insertUnkeyAuditLog( c: Context, diff --git a/apps/api/src/pkg/auth/root_key.ts b/apps/api/src/pkg/auth/root_key.ts index 8c33d893c1..a35ae54235 100644 --- a/apps/api/src/pkg/auth/root_key.ts +++ b/apps/api/src/pkg/auth/root_key.ts @@ -53,22 +53,15 @@ export async function rootKeyAuth(c: Context, permissionQuery?: Permiss // otherwise, they likely sent garbage to us and we can't associate it with anything c.executionCtx.waitUntil( - analytics.ingestKeyVerification({ - workspaceId: rootKey.key.workspaceId, - apiId: rootKey.api.id, - keyId: rootKey.key.id, + analytics.insertKeyVerification({ + workspace_id: rootKey.key.workspaceId, + key_id: rootKey.key.id, time: Date.now(), - deniedReason: rootKey.code, - ipAddress: c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP"), - userAgent: c.req.header("User-Agent"), - requestedResource: "", + outcome: rootKey.code ?? "VALID", + key_space_id: rootKey.key.keyAuthId, // @ts-expect-error - the cf object will be there on cloudflare - region: c.req.raw?.cf?.country ?? "", - ownerId: rootKey.key.ownerId ?? undefined, - // @ts-expect-error - the cf object will be there on cloudflare - edgeRegion: c.req.raw?.cf?.colo ?? "", - keySpaceId: rootKey.key.keyAuthId, - requestId: c.get("requestId"), + region: c.req.cf?.region, + request_id: c.get("requestId"), }), ); diff --git a/apps/api/src/pkg/cache/index.ts b/apps/api/src/pkg/cache/index.ts index 28cd758b57..9b546673bf 100644 --- a/apps/api/src/pkg/cache/index.ts +++ b/apps/api/src/pkg/cache/index.ts @@ -35,7 +35,7 @@ export function initCache(c: Context, metrics: Metrics): C; diff --git a/apps/api/src/pkg/env.ts b/apps/api/src/pkg/env.ts index 7c263cecf0..8ad19a463a 100644 --- a/apps/api/src/pkg/env.ts +++ b/apps/api/src/pkg/env.ts @@ -33,7 +33,7 @@ export const zEnv = z.object({ AGENT_URL: z.string().url(), AGENT_TOKEN: z.string(), - CLICKHOUSE_URL: z.string().optional(), + CLICKHOUSE_URL: z.string(), SYNC_RATELIMIT_ON_NO_DATA: z .string() diff --git a/apps/api/src/pkg/key_migration/handler.ts b/apps/api/src/pkg/key_migration/handler.ts index 1696a18463..452b30a4ad 100644 --- a/apps/api/src/pkg/key_migration/handler.ts +++ b/apps/api/src/pkg/key_migration/handler.ts @@ -2,7 +2,6 @@ import { BaseError, Err, Ok, type Result } from "@unkey/error"; import { newId } from "@unkey/id"; import { ConsoleLogger } from "@unkey/worker-logging"; -import { Analytics } from "../analytics"; import { createConnection, schema } from "../db"; import type { Env } from "../env"; import type { MessageBody } from "./message"; @@ -24,19 +23,6 @@ export async function migrateKey( logger: new ConsoleLogger({ requestId: "", application: "api", environment: env.ENVIRONMENT }), }); - const tinybirdProxy = - env.TINYBIRD_PROXY_URL && env.TINYBIRD_PROXY_TOKEN - ? { - url: env.TINYBIRD_PROXY_URL, - token: env.TINYBIRD_PROXY_TOKEN, - } - : undefined; - - const analytics = new Analytics({ - tinybirdProxy, - tinybirdToken: env.TINYBIRD_TOKEN, - }); - const keyId = newId("key"); // name -> id @@ -105,28 +91,6 @@ export async function migrateKey( environment: message.environment, }); - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: message.workspaceId, - event: "key.create", - actor: { - type: "key", - id: message.rootKeyId, - }, - description: `Created ${keyId} in ${message.keyAuthId}`, - resources: [ - { - type: "key", - id: keyId, - }, - { - type: "keyAuth", - id: message.keyAuthId, - }, - ], - - context: message.auditLogContext, - }); - if (message.encrypted) { await tx.insert(schema.encryptedKeys).values({ workspaceId: message.workspaceId, @@ -149,26 +113,6 @@ export async function migrateKey( })); await tx.insert(schema.keysRoles).values(roleConnections); - - await analytics.ingestUnkeyAuditLogsTinybird( - roleConnections.map((rc) => ({ - workspaceId: message.workspaceId, - actor: { type: "key", id: message.rootKeyId }, - event: "authorization.connect_role_and_key", - description: `Connected ${rc.roleId} and ${rc.keyId}`, - resources: [ - { - type: "key", - id: rc.keyId, - }, - { - type: "role", - id: rc.roleId, - }, - ], - context: message.auditLogContext, - })), - ); } /** @@ -184,26 +128,6 @@ export async function migrateKey( })); await tx.insert(schema.keysPermissions).values(permissionConnections); - - await analytics.ingestUnkeyAuditLogsTinybird( - permissionConnections.map((pc) => ({ - workspaceId: message.workspaceId, - actor: { type: "key", id: message.rootKeyId }, - event: "authorization.connect_permission_and_key", - description: `Connected ${pc.permissionId} and ${pc.keyId}`, - resources: [ - { - type: "key", - id: pc.keyId, - }, - { - type: "permission", - id: pc.permissionId, - }, - ], - context: message.auditLogContext, - })), - ); } }); } catch (e) { diff --git a/apps/api/src/pkg/middleware/init.ts b/apps/api/src/pkg/middleware/init.ts index 23289f7876..c53ddefc10 100644 --- a/apps/api/src/pkg/middleware/init.ts +++ b/apps/api/src/pkg/middleware/init.ts @@ -96,22 +96,8 @@ export function init(): MiddlewareHandler { }) : new NoopUsageLimiter(); - const tinybirdProxy = - c.env.TINYBIRD_PROXY_URL && c.env.TINYBIRD_PROXY_TOKEN - ? { - url: c.env.TINYBIRD_PROXY_URL, - token: c.env.TINYBIRD_PROXY_TOKEN, - } - : undefined; - const analytics = new Analytics({ - tinybirdProxy, - tinybirdToken: c.env.TINYBIRD_TOKEN, - clickhouse: c.env.CLICKHOUSE_URL - ? { - url: c.env.CLICKHOUSE_URL, - } - : undefined, + clickhouseUrl: c.env.CLICKHOUSE_URL, }); const rateLimiter = new AgentRatelimiter({ agent: { url: c.env.AGENT_URL, token: c.env.AGENT_TOKEN }, diff --git a/apps/api/src/routes/legacy_keys_createKey.ts b/apps/api/src/routes/legacy_keys_createKey.ts index 96ef64f307..dc431d5a3f 100644 --- a/apps/api/src/routes/legacy_keys_createKey.ts +++ b/apps/api/src/routes/legacy_keys_createKey.ts @@ -158,7 +158,7 @@ export type LegacyKeysCreateKeyResponse = z.infer< export const registerLegacyKeysCreate = (app: App) => app.openapi(route, async (c) => { - const { cache, db, analytics } = c.get("services"); + const { cache, db } = c.get("services"); const auth = await rootKeyAuth(c); const req = c.req.valid("json"); @@ -227,28 +227,6 @@ export const registerLegacyKeysCreate = (app: App) => remaining: req.remaining, deletedAt: null, }); - - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: authorizedWorkspaceId, - actor: { type: "key", id: rootKeyId }, - event: "key.create", - description: `Created ${keyId}`, - resources: [ - { - type: "key", - id: keyId, - }, - { - type: "keyAuth", - id: api.keyAuthId!, - }, - { type: "api", id: api.id }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }); await insertUnkeyAuditLog(c, tx, { workspaceId: authorizedWorkspaceId, actor: { type: "key", id: rootKeyId }, diff --git a/apps/api/src/routes/v1_apis_createApi.ts b/apps/api/src/routes/v1_apis_createApi.ts index 811328063a..fb92d8dfc1 100644 --- a/apps/api/src/routes/v1_apis_createApi.ts +++ b/apps/api/src/routes/v1_apis_createApi.ts @@ -57,7 +57,7 @@ export type V1ApisCreateApiResponse = z.infer< export const registerV1ApisCreateApi = (app: App) => app.openapi(route, async (c) => { - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const auth = await rootKeyAuth( c, @@ -108,23 +108,6 @@ export const registerV1ApisCreateApi = (app: App) => }, ], - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }); - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: authorizedWorkspaceId, - event: "api.create", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Created ${apiId}`, - resources: [ - { - type: "api", - id: apiId, - }, - ], - context: { location: c.get("location"), userAgent: c.get("userAgent") }, }); }); diff --git a/apps/api/src/routes/v1_apis_deleteApi.ts b/apps/api/src/routes/v1_apis_deleteApi.ts index c7eba212c7..32669ac461 100644 --- a/apps/api/src/routes/v1_apis_deleteApi.ts +++ b/apps/api/src/routes/v1_apis_deleteApi.ts @@ -62,7 +62,7 @@ export type V1ApisDeleteApiResponse = z.infer< export const registerV1ApisDeleteApi = (app: App) => app.openapi(route, async (c) => { const { apiId } = c.req.valid("json"); - const { cache, db, analytics } = c.get("services"); + const { cache, db } = c.get("services"); const auth = await rootKeyAuth( c, @@ -97,23 +97,6 @@ export const registerV1ApisDeleteApi = (app: App) => await db.primary.transaction(async (tx) => { await tx.update(schema.apis).set({ deletedAt: new Date() }).where(eq(schema.apis.id, apiId)); - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: authorizedWorkspaceId, - event: "api.delete", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Deleted ${apiId}`, - resources: [ - { - type: "api", - id: apiId, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }); await insertUnkeyAuditLog(c, tx, { workspaceId: authorizedWorkspaceId, event: "api.delete", @@ -169,29 +152,6 @@ export const registerV1ApisDeleteApi = (app: App) => context: { location: c.get("location"), userAgent: c.get("userAgent") }, })), ); - await analytics.ingestUnkeyAuditLogsTinybird( - keyIds.map((key) => ({ - workspaceId: authorizedWorkspaceId, - event: "key.delete", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Deleted ${key.id} as part of ${api.id} deletion`, - resources: [ - { - type: "keyAuth", - id: api.keyAuthId!, - }, - { - type: "key", - id: key.id, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - })), - ); }); await cache.apiById.remove(apiId); diff --git a/apps/api/src/routes/v1_identities_createIdentity.ts b/apps/api/src/routes/v1_identities_createIdentity.ts index 450470facf..0c405fc96a 100644 --- a/apps/api/src/routes/v1_identities_createIdentity.ts +++ b/apps/api/src/routes/v1_identities_createIdentity.ts @@ -102,7 +102,7 @@ export type V1IdentitiesCreateIdentityResponse = z.infer< export const registerV1IdentitiesCreateIdentity = (app: App) => app.openapi(route, async (c) => { - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const auth = await rootKeyAuth( c, @@ -202,52 +202,6 @@ export const registerV1IdentitiesCreateIdentity = (app: App) => context: { location: c.get("location"), userAgent: c.get("userAgent") }, })), ]); - - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird([ - { - workspaceId: authorizedWorkspaceId, - event: "identity.create", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Created ${identity.id}`, - resources: [ - { - type: "identity", - id: identity.id, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }, - ...ratelimits.map((r) => ({ - workspaceId: authorizedWorkspaceId, - event: "ratelimit.create" as const, - actor: { - type: "key" as const, - id: rootKeyId, - }, - description: `Created ${r.id}`, - resources: [ - { - type: "identity" as const, - id: identity.id, - }, - { - type: "ratelimit" as const, - id: r.id, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - })), - ]), - ); }) .catch((e) => { if (e instanceof UnkeyApiError) { diff --git a/apps/api/src/routes/v1_identities_deleteIdentity.ts b/apps/api/src/routes/v1_identities_deleteIdentity.ts index 1913572ab6..2833d34ba3 100644 --- a/apps/api/src/routes/v1_identities_deleteIdentity.ts +++ b/apps/api/src/routes/v1_identities_deleteIdentity.ts @@ -53,7 +53,7 @@ export type V1IdentitiesDeleteIdentityResponse = z.infer< export const registerV1IdentitiesDeleteIdentity = (app: App) => app.openapi(route, async (c) => { const { identityId } = c.req.valid("json"); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const auth = await rootKeyAuth( c, @@ -109,53 +109,6 @@ export const registerV1IdentitiesDeleteIdentity = (app: App) => }, }, - ...deleteRatelimitIds.map((ratelimitId) => ({ - workspaceId: authorizedWorkspaceId, - event: "ratelimit.delete" as const, - actor: { - type: "key" as const, - id: rootKeyId, - }, - description: `Deleted ${ratelimitId}`, - resources: [ - { - type: "identity" as const, - id: identity.id, - }, - { - type: "ratelimit" as const, - id: ratelimitId, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ]); - await analytics.ingestUnkeyAuditLogsTinybird([ - { - workspaceId: authorizedWorkspaceId, - event: "identity.delete", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Deleted ${identity.id}`, - resources: [ - { - type: "identity", - id: identity.id, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }, - ...deleteRatelimitIds.map((ratelimitId) => ({ workspaceId: authorizedWorkspaceId, event: "ratelimit.delete" as const, diff --git a/apps/api/src/routes/v1_identities_updateIdentity.ts b/apps/api/src/routes/v1_identities_updateIdentity.ts index ff20ff12c8..7101f1b896 100644 --- a/apps/api/src/routes/v1_identities_updateIdentity.ts +++ b/apps/api/src/routes/v1_identities_updateIdentity.ts @@ -1,8 +1,7 @@ import type { App } from "@/pkg/hono/app"; import { createRoute, z } from "@hono/zod-openapi"; -import type { UnkeyAuditLog } from "@/pkg/analytics"; -import { insertUnkeyAuditLog } from "@/pkg/audit"; +import { type UnkeyAuditLog, insertUnkeyAuditLog } from "@/pkg/audit"; import { rootKeyAuth } from "@/pkg/auth/root_key"; import { UnkeyApiError, openApiErrorResponses } from "@/pkg/errors"; import { type Ratelimit, eq, schema } from "@unkey/db"; @@ -139,7 +138,7 @@ export const registerV1IdentitiesUpdateIdentity = (app: App) => buildUnkeyQuery(({ or }) => or("identity.*.update_identity")), ); - const { db, analytics, cache } = c.get("services"); + const { db, cache } = c.get("services"); if (!req.identityId && !req.externalId) { throw new UnkeyApiError({ @@ -386,7 +385,6 @@ export const registerV1IdentitiesUpdateIdentity = (app: App) => }); await insertUnkeyAuditLog(c, undefined, auditLogs); - c.executionCtx.waitUntil(analytics.ingestUnkeyAuditLogsTinybird(auditLogs)); return c.json({ id: identity.id, diff --git a/apps/api/src/routes/v1_keys_addPermissions.ts b/apps/api/src/routes/v1_keys_addPermissions.ts index 5acbab8474..efe080f233 100644 --- a/apps/api/src/routes/v1_keys_addPermissions.ts +++ b/apps/api/src/routes/v1_keys_addPermissions.ts @@ -92,7 +92,7 @@ export const registerV1KeysAddPermissions = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.add_permission_to_key")), ); - const { db, analytics, rbac, cache } = c.get("services"); + const { db, rbac, cache } = c.get("services"); const requestedIds = req.permissions.filter((p) => "id" in p).map((p) => p.id!); const requestedNames = req.permissions.filter((p) => "name" in p).map((p) => p.name!); @@ -254,57 +254,5 @@ export const registerV1KeysAddPermissions = (app: App) => })), ]); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird([ - ...createPermissions.map((p) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "permission.create" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Created ${p.id}`, - resources: [ - { - type: "permission" as const, - id: p.id, - meta: { - name: p.name, - }, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ...addPermissions.map((p) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "authorization.connect_permission_and_key" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Connected ${p.id} and ${req.keyId}`, - resources: [ - { - type: "permission" as const, - id: p.id, - }, - { - type: "key" as const, - id: req.keyId, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ]), - ); - return c.json(allPermissions.map((p) => ({ id: p.id, name: p.name }))); }); diff --git a/apps/api/src/routes/v1_keys_addRoles.ts b/apps/api/src/routes/v1_keys_addRoles.ts index b19f2013fc..6d8cd5a8cf 100644 --- a/apps/api/src/routes/v1_keys_addRoles.ts +++ b/apps/api/src/routes/v1_keys_addRoles.ts @@ -105,7 +105,7 @@ export const registerV1KeysAddRoles = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.add_role_to_key")), ); - const { db, analytics, cache, rbac } = c.get("services"); + const { db, cache, rbac } = c.get("services"); const requestedIds = req.roles.filter((r) => "id" in r).map((r) => r.id!); const requestedNames = req.roles.filter((r) => "name" in r).map((r) => r.name!); @@ -264,57 +264,5 @@ export const registerV1KeysAddRoles = (app: App) => })), ]); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird([ - ...createRoles.map((r) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "role.create" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Created ${r.id}`, - resources: [ - { - type: "role" as const, - id: r.id, - meta: { - name: r.name, - }, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ...addRoles.map((r) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "authorization.connect_role_and_key" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Connected ${r.id} and ${req.keyId}`, - resources: [ - { - type: "role" as const, - id: r.id, - }, - { - type: "key" as const, - id: req.keyId, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ]), - ); - return c.json(allRoles.map((p) => ({ id: p.id, name: p.name }))); }); diff --git a/apps/api/src/routes/v1_keys_createKey.ts b/apps/api/src/routes/v1_keys_createKey.ts index 99d2e193d4..8fa343d099 100644 --- a/apps/api/src/routes/v1_keys_createKey.ts +++ b/apps/api/src/routes/v1_keys_createKey.ts @@ -1,12 +1,10 @@ -import type { App } from "@/pkg/hono/app"; -import { createRoute, z } from "@hono/zod-openapi"; - -import type { UnkeyAuditLog } from "@/pkg/analytics"; -import { insertUnkeyAuditLog } from "@/pkg/audit"; +import { type UnkeyAuditLog, insertUnkeyAuditLog } from "@/pkg/audit"; import { rootKeyAuth } from "@/pkg/auth/root_key"; import type { Database, Identity } from "@/pkg/db"; import { UnkeyApiError, openApiErrorResponses } from "@/pkg/errors"; +import type { App } from "@/pkg/hono/app"; import { retry } from "@/pkg/util/retry"; +import { createRoute, z } from "@hono/zod-openapi"; import { schema } from "@unkey/db"; import { sha256 } from "@unkey/hash"; import { newId } from "@unkey/id"; @@ -262,7 +260,7 @@ export type V1KeysCreateKeyResponse = z.infer< export const registerV1KeysCreateKey = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); - const { cache, db, analytics, logger, vault, rbac } = c.get("services"); + const { cache, db, logger, vault, rbac } = c.get("services"); const auth = await rootKeyAuth( c, @@ -525,8 +523,6 @@ export const registerV1KeysCreateKey = (app: App) => ]; await insertUnkeyAuditLog(c, undefined, auditLogs); - c.executionCtx.waitUntil(analytics.ingestUnkeyAuditLogsTinybird(auditLogs)); - // TODO: emit event to tinybird return c.json({ keyId: newKey.id, diff --git a/apps/api/src/routes/v1_keys_deleteKey.ts b/apps/api/src/routes/v1_keys_deleteKey.ts index e7ecb67ee5..2da564c135 100644 --- a/apps/api/src/routes/v1_keys_deleteKey.ts +++ b/apps/api/src/routes/v1_keys_deleteKey.ts @@ -54,7 +54,7 @@ export type V1KeysDeleteKeyResponse = z.infer< export const registerV1KeysDeleteKey = (app: App) => app.openapi(route, async (c) => { const { keyId } = c.req.valid("json"); - const { cache, db, analytics } = c.get("services"); + const { cache, db } = c.get("services"); const data = await cache.keyById.swr(keyId, async () => { const dbRes = await db.readonly.query.keys.findFirst({ @@ -144,23 +144,6 @@ export const registerV1KeysDeleteKey = (app: App) => context: { location: c.get("location"), userAgent: c.get("userAgent") }, }); }); - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: authorizedWorkspaceId, - event: "key.delete", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Deleted ${key.id}`, - resources: [ - { - type: "key", - id: key.id, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }); await Promise.all([cache.keyByHash.remove(key.hash), cache.keyById.remove(key.id)]); diff --git a/apps/api/src/routes/v1_keys_getVerifications.ts b/apps/api/src/routes/v1_keys_getVerifications.ts index c1928c15fe..9d40f034fb 100644 --- a/apps/api/src/routes/v1_keys_getVerifications.ts +++ b/apps/api/src/routes/v1_keys_getVerifications.ts @@ -22,11 +22,13 @@ const route = createRoute({ example: "chronark", }), start: z.coerce.number().int().optional().openapi({ - description: "The start of the period to fetch usage for as unix milliseconds timestamp", + description: + "The start of the period to fetch usage for as unix milliseconds timestamp, defaults to 24h ago.", example: 1620000000000, }), end: z.coerce.number().int().optional().openapi({ - description: "The end of the period to fetch usage for as unix milliseconds timestamp", + description: + "The end of the period to fetch usage for as unix milliseconds timestamp, defaults to now.", example: 1620000000000, }), granularity: z.enum(["day"]).optional().default("day").openapi({ @@ -83,6 +85,7 @@ export const registerV1KeysGetVerifications = (app: App) => const ids: { keyId: string; apiId: string; + keySpaceId: string; workspaceId: string; }[] = []; @@ -134,7 +137,12 @@ export const registerV1KeysGetVerifications = (app: App) => }); } - ids.push({ keyId, apiId: data.val.api.id, workspaceId: data.val.key.workspaceId }); + ids.push({ + keyId, + apiId: data.val.api.id, + keySpaceId: data.val.api.keyAuthId!, + workspaceId: data.val.key.workspaceId, + }); } else { if (!ownerId) { throw new UnkeyApiError({ @@ -172,6 +180,7 @@ export const registerV1KeysGetVerifications = (app: App) => ...(keys.val ?? []).map(({ key, api }) => ({ keyId: key.id, apiId: api.id, + keySpaceId: api.keyAuthId!, workspaceId: key.workspaceId, })), ); @@ -207,18 +216,18 @@ export const registerV1KeysGetVerifications = (app: App) => message: "you are not allowed to access this workspace", }); } + const now = Date.now(); const verificationsFromAllKeys = await Promise.all( - ids.map(({ keyId, apiId }) => { + ids.map(({ keyId, keySpaceId }) => { return cache.verificationsByKeyId.swr(`${keyId}:${start}-${end}`, async () => { - const res = await analytics.getVerificationsDaily({ + return await analytics.getVerificationsDaily({ workspaceId: authorizedWorkspaceId, - apiId: apiId, + keySpaceId: keySpaceId, keyId: keyId, - start: start ? start : undefined, - end: end ? end : undefined, + start: start ? start : now - 24 * 60 * 60 * 1000, + end: end ? end : now, }); - return res.data; }); }), ); @@ -231,9 +240,17 @@ export const registerV1KeysGetVerifications = (app: App) => if (!verifications[d.time]) { verifications[d.time] = { success: 0, rateLimited: 0, usageExceeded: 0 }; } - verifications[d.time].success += d.success; - verifications[d.time].rateLimited += d.rateLimited; - verifications[d.time].usageExceeded += d.usageExceeded; + switch (d.outcome) { + case "VALID": + verifications[d.time].success += d.count; + break; + case "RATE_LIMITED": + verifications[d.time].rateLimited += d.count; + break; + case "USAGE_EXCEEDED": + verifications[d.time].usageExceeded += d.count; + break; + } } } diff --git a/apps/api/src/routes/v1_keys_removePermissions.ts b/apps/api/src/routes/v1_keys_removePermissions.ts index 1bc26f3c7d..4631a24139 100644 --- a/apps/api/src/routes/v1_keys_removePermissions.ts +++ b/apps/api/src/routes/v1_keys_removePermissions.ts @@ -81,7 +81,7 @@ export const registerV1KeysRemovePermissions = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.remove_permission_from_key")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const [key, connectedPermissions] = await Promise.all([ db.primary.query.keys.findFirst({ @@ -161,34 +161,5 @@ export const registerV1KeysRemovePermissions = (app: App) => ); }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird( - deletePermissions.map((r) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "authorization.disconnect_permission_and_key" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Disonnected ${r.permissionId} and ${req.keyId}`, - resources: [ - { - type: "permission" as const, - id: r.permissionId, - }, - { - type: "key" as const, - id: req.keyId, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ), - ); - return c.json({}); }); diff --git a/apps/api/src/routes/v1_keys_removeRoles.ts b/apps/api/src/routes/v1_keys_removeRoles.ts index 9bead7da48..c0477b790d 100644 --- a/apps/api/src/routes/v1_keys_removeRoles.ts +++ b/apps/api/src/routes/v1_keys_removeRoles.ts @@ -81,7 +81,7 @@ export const registerV1KeysRemoveRoles = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.remove_role_from_key")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const [key, connectedRoles] = await Promise.all([ db.primary.query.keys.findFirst({ @@ -160,34 +160,5 @@ export const registerV1KeysRemoveRoles = (app: App) => ); }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird( - deleteRoles.map((r) => ({ - workspaceId: auth.authorizedWorkspaceId, - event: "authorization.disconnect_role_and_key" as const, - actor: { - type: "key" as const, - id: auth.key.id, - }, - description: `Disonnected ${r.roleId} and ${req.keyId}`, - resources: [ - { - type: "role" as const, - id: r.roleId, - }, - { - type: "key" as const, - id: req.keyId, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ), - ); - return c.json({}); }); diff --git a/apps/api/src/routes/v1_keys_setPermissions.ts b/apps/api/src/routes/v1_keys_setPermissions.ts index e469010963..b5cbd93af4 100644 --- a/apps/api/src/routes/v1_keys_setPermissions.ts +++ b/apps/api/src/routes/v1_keys_setPermissions.ts @@ -121,7 +121,7 @@ export async function setPermissions( create?: boolean; }>, ): Promise> { - const { db, analytics, cache, rbac } = c.get("services"); + const { db, cache, rbac } = c.get("services"); const requestedIds = requested.filter(({ id }) => !!id).map(({ id }) => id!); const requestedNames = requested @@ -388,6 +388,5 @@ export async function setPermissions( })), ]; await insertUnkeyAuditLog(c, undefined, auditLogs); - c.executionCtx.waitUntil(analytics.ingestUnkeyAuditLogsTinybird(auditLogs)); return allPermissions; } diff --git a/apps/api/src/routes/v1_keys_setRoles.ts b/apps/api/src/routes/v1_keys_setRoles.ts index 09f72338ab..00064ac103 100644 --- a/apps/api/src/routes/v1_keys_setRoles.ts +++ b/apps/api/src/routes/v1_keys_setRoles.ts @@ -117,7 +117,7 @@ export async function setRoles( keyId: string, requested: Array<{ id?: string; name?: string; create?: boolean }>, ): Promise> { - const { db, analytics, cache, rbac } = c.get("services"); + const { db, cache, rbac } = c.get("services"); const requestedIds = requested.filter(({ id }) => !!id).map(({ id }) => id!); const requestedNames = requested @@ -384,6 +384,5 @@ export async function setRoles( ]; await insertUnkeyAuditLog(c, undefined, auditLogs); - c.executionCtx.waitUntil(analytics.ingestUnkeyAuditLogsTinybird(auditLogs)); return allRoles; } diff --git a/apps/api/src/routes/v1_keys_updateKey.ts b/apps/api/src/routes/v1_keys_updateKey.ts index c28c0d3e0e..a6bfe48bac 100644 --- a/apps/api/src/routes/v1_keys_updateKey.ts +++ b/apps/api/src/routes/v1_keys_updateKey.ts @@ -279,7 +279,7 @@ export type V1KeysUpdateKeyResponse = z.infer< export const registerV1KeysUpdate = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); - const { cache, db, usageLimiter, analytics, rbac } = c.get("services"); + const { cache, db, usageLimiter, rbac } = c.get("services"); const auth = await rootKeyAuth(c); const key = await db.primary.query.keys.findFirst({ where: (table, { eq }) => eq(table.id, req.keyId), @@ -416,37 +416,6 @@ export const registerV1KeysUpdate = (app: App) => c.executionCtx.waitUntil(usageLimiter.revalidate({ keyId: key.id })); c.executionCtx.waitUntil(cache.keyByHash.remove(key.hash)); c.executionCtx.waitUntil(cache.keyById.remove(key.id)); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: authorizedWorkspaceId, - event: "key.update", - actor: { - type: "key", - id: rootKeyId, - }, - description: `Updated key ${key.id}`, - resources: [ - { - type: "key", - id: key.id, - meta: Object.entries(req) - .filter(([_key, value]) => typeof value !== "undefined") - .reduce( - (obj, [key, value]) => { - obj[key] = JSON.stringify(value); - - return obj; - }, - {} as Record, - ), - }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }), - ); await Promise.all([ typeof req.roles !== "undefined" diff --git a/apps/api/src/routes/v1_keys_updateRemaining.ts b/apps/api/src/routes/v1_keys_updateRemaining.ts index 5da08ab14a..46ebc3d813 100644 --- a/apps/api/src/routes/v1_keys_updateRemaining.ts +++ b/apps/api/src/routes/v1_keys_updateRemaining.ts @@ -65,7 +65,7 @@ export type V1KeysUpdateRemainingResponse = z.infer< export const registerV1KeysUpdateRemaining = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); - const { cache, db, analytics, usageLimiter } = c.get("services"); + const { cache, db, usageLimiter } = c.get("services"); const key = await db.readonly.query.keys.findFirst({ where: (table, { eq }) => eq(table.id, req.keyId), @@ -195,32 +195,6 @@ export const registerV1KeysUpdateRemaining = (app: App) => }, }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - actor: { - type: "key", - id: rootKeyId, - }, - event: "key.update", - workspaceId: authorizedWorkspaceId, - description: `Changed remaining to ${keyAfterUpdate.remaining}`, - resources: [ - { - type: "keyAuth", - id: key.keyAuthId, - }, - { - type: "key", - id: key.id, - }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }), - ); - return c.json({ remaining: keyAfterUpdate.remaining, }); diff --git a/apps/api/src/routes/v1_keys_verifyKey.ts b/apps/api/src/routes/v1_keys_verifyKey.ts index e80ce1201c..83a4c87f6d 100644 --- a/apps/api/src/routes/v1_keys_verifyKey.ts +++ b/apps/api/src/routes/v1_keys_verifyKey.ts @@ -365,31 +365,5 @@ export const registerV1KeysVerifyKey = (app: App) => }), ); - c.executionCtx.waitUntil( - // old tinybird - analytics.ingestKeyVerification({ - workspaceId: val.key.workspaceId, - apiId: val.api.id, - keyId: val.key.id, - time: Date.now(), - deniedReason: val.code, - ipAddress: c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP"), - userAgent: c.req.header("User-Agent"), - requestedResource: "", - // @ts-expect-error - the cf object will be there on cloudflare - region: c.req.raw?.cf?.country ?? "", - ownerId: val.key.ownerId ?? undefined, - // @ts-expect-error - the cf object will be there on cloudflare - edgeRegion: c.req.raw?.cf?.colo ?? "", - keySpaceId: val.key.keyAuthId, - requestId: c.get("requestId"), - requestBody: JSON.stringify({ - ...req, - key: "", - }), - responseBody: JSON.stringify(responseBody), - }), - ); - return c.json(responseBody); }); diff --git a/apps/api/src/routes/v1_liveness.ts b/apps/api/src/routes/v1_liveness.ts index c7db20f8f7..a66241b1e8 100644 --- a/apps/api/src/routes/v1_liveness.ts +++ b/apps/api/src/routes/v1_liveness.ts @@ -31,9 +31,6 @@ const route = createRoute({ usagelimit: z.string().openapi({ description: "The name of the connected usagelimit service", }), - analytics: z.string().openapi({ - description: "The name of the connected analytics service", - }), }), }), }, @@ -49,7 +46,7 @@ export type V1LivenessResponse = z.infer< export const registerV1Liveness = (app: App) => app.openapi(route, async (c) => { - const { analytics, logger, metrics, rateLimiter, usageLimiter } = c.get("services"); + const { logger, metrics, rateLimiter, usageLimiter } = c.get("services"); return c.json({ status: "we're so back", @@ -58,7 +55,6 @@ export const registerV1Liveness = (app: App) => logger: logger.constructor.name, ratelimit: rateLimiter.constructor.name, usagelimit: usageLimiter.constructor.name, - analytics: analytics.readClient.constructor.name, }, }); }); diff --git a/apps/api/src/routes/v1_migrations_createKey.ts b/apps/api/src/routes/v1_migrations_createKey.ts index 311c54fb1c..a9fd772504 100644 --- a/apps/api/src/routes/v1_migrations_createKey.ts +++ b/apps/api/src/routes/v1_migrations_createKey.ts @@ -251,11 +251,10 @@ export const registerV1MigrationsCreateKeys = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); - const { cache, db, analytics, rbac, vault, logger } = c.get("services"); + const { cache, db, rbac, vault, logger } = c.get("services"); const auth = await rootKeyAuth(c); const authorizedWorkspaceId = auth.authorizedWorkspaceId; - const rootKeyId = auth.key.id; const roleNames = req.filter((k) => k.roles).flatMap((k) => k.roles) as string[]; // name -> id @@ -536,80 +535,6 @@ export const registerV1MigrationsCreateKeys = (app: App) => }, })), ); - - await analytics.ingestUnkeyAuditLogsTinybird( - keys.map((k) => ({ - workspaceId: authorizedWorkspaceId, - event: "key.create", - actor: { - type: "key", - id: auth.key.id, - }, - description: `Created ${k.id} in ${k.keyAuthId}`, - resources: [ - { - type: "key", - id: k.id, - }, - { - type: "keyAuth", - id: k.keyAuthId!, - }, - ], - - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ); - - await insertUnkeyAuditLog( - c, - tx, - roleConnections.map((rc) => ({ - workspaceId: authorizedWorkspaceId, - actor: { type: "key", id: rootKeyId }, - event: "authorization.connect_role_and_key", - description: `Connected ${rc.roleId} and ${rc.keyId}`, - resources: [ - { - type: "key", - id: rc.keyId, - }, - { - type: "role", - id: rc.roleId, - }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ); - await analytics.ingestUnkeyAuditLogsTinybird( - roleConnections.map((rc) => ({ - workspaceId: authorizedWorkspaceId, - actor: { type: "key", id: rootKeyId }, - event: "authorization.connect_role_and_key", - description: `Connected ${rc.roleId} and ${rc.keyId}`, - resources: [ - { - type: "key", - id: rc.keyId, - }, - { - type: "role", - id: rc.roleId, - }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - })), - ); }); return c.json({ diff --git a/apps/api/src/routes/v1_permissions_createPermission.ts b/apps/api/src/routes/v1_permissions_createPermission.ts index a89c536059..64dd4ef3e4 100644 --- a/apps/api/src/routes/v1_permissions_createPermission.ts +++ b/apps/api/src/routes/v1_permissions_createPermission.ts @@ -75,7 +75,7 @@ export const registerV1PermissionsCreatePermission = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.create_permission")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const permission = { id: newId("permission"), @@ -116,31 +116,6 @@ export const registerV1PermissionsCreatePermission = (app: App) => context: { location: c.get("location"), userAgent: c.get("userAgent") }, }); }); - - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: auth.authorizedWorkspaceId, - event: "permission.create", - actor: { - type: "key", - id: auth.key.id, - }, - description: `Created ${permission.id}`, - resources: [ - { - type: "permission", - id: permission.id, - meta: { - name: permission.name, - description: permission.description, - }, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }), - ); - return c.json({ permissionId: permission.id, }); diff --git a/apps/api/src/routes/v1_permissions_createRole.ts b/apps/api/src/routes/v1_permissions_createRole.ts index e16834c72a..812eb0edbb 100644 --- a/apps/api/src/routes/v1_permissions_createRole.ts +++ b/apps/api/src/routes/v1_permissions_createRole.ts @@ -75,7 +75,7 @@ export const registerV1PermissionsCreateRole = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.create_role")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const role = { id: newId("test"), @@ -117,30 +117,6 @@ export const registerV1PermissionsCreateRole = (app: App) => }); }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: auth.authorizedWorkspaceId, - event: "role.create", - actor: { - type: "key", - id: auth.key.id, - }, - description: `Created ${role.id}`, - resources: [ - { - type: "role", - id: role.id, - meta: { - name: role.name, - description: role.description, - }, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }), - ); - return c.json({ roleId: role.id, }); diff --git a/apps/api/src/routes/v1_permissions_deletePermission.ts b/apps/api/src/routes/v1_permissions_deletePermission.ts index dda2fa5f05..48062237c5 100644 --- a/apps/api/src/routes/v1_permissions_deletePermission.ts +++ b/apps/api/src/routes/v1_permissions_deletePermission.ts @@ -57,7 +57,7 @@ export const registerV1PermissionsDeletePermission = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.delete_permission")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); await db.primary.transaction(async (tx) => { const permission = await tx.query.permissions.findFirst({ @@ -96,25 +96,6 @@ export const registerV1PermissionsDeletePermission = (app: App) => context: { location: c.get("location"), userAgent: c.get("userAgent") }, }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: auth.authorizedWorkspaceId, - event: "permission.delete", - actor: { - type: "key", - id: auth.key.id, - }, - description: `Deleted ${permission.id}`, - resources: [ - { - type: "permission", - id: permission.id, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }), - ); }); return c.json({}); diff --git a/apps/api/src/routes/v1_permissions_deleteRole.ts b/apps/api/src/routes/v1_permissions_deleteRole.ts index ce0222ab49..c9eba9e797 100644 --- a/apps/api/src/routes/v1_permissions_deleteRole.ts +++ b/apps/api/src/routes/v1_permissions_deleteRole.ts @@ -57,7 +57,7 @@ export const registerV1PermissionsDeleteRole = (app: App) => buildUnkeyQuery(({ or }) => or("*", "rbac.*.delete_role")), ); - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const role = await db.primary.query.roles.findFirst({ where: (table, { eq, and }) => @@ -99,25 +99,5 @@ export const registerV1PermissionsDeleteRole = (app: App) => }); }); - c.executionCtx.waitUntil( - analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: auth.authorizedWorkspaceId, - event: "role.delete", - actor: { - type: "key", - id: auth.key.id, - }, - description: `Deleted ${role.id}`, - resources: [ - { - type: "role", - id: role.id, - }, - ], - - context: { location: c.get("location"), userAgent: c.get("userAgent") }, - }), - ); - return c.json({}); }); diff --git a/apps/api/src/routes/v1_ratelimit_limit.ts b/apps/api/src/routes/v1_ratelimit_limit.ts index 81e3c46bb9..031d432058 100644 --- a/apps/api/src/routes/v1_ratelimit_limit.ts +++ b/apps/api/src/routes/v1_ratelimit_limit.ts @@ -148,7 +148,7 @@ export type V1RatelimitLimitResponse = z.infer< export const registerV1RatelimitLimit = (app: App) => app.openapi(route, async (c) => { const req = c.req.valid("json"); - const { cache, logger, db, rateLimiter, analytics, rbac } = c.get("services"); + const { cache, db, rateLimiter, analytics, rbac } = c.get("services"); const rootKey = await rootKeyAuth(c); @@ -195,25 +195,6 @@ export const registerV1RatelimitLimit = (app: App) => }; try { await db.primary.insert(schema.ratelimitNamespaces).values(namespace); - await analytics.ingestUnkeyAuditLogsTinybird({ - workspaceId: rootKey.authorizedWorkspaceId, - actor: { - type: "key", - id: rootKey.key.id, - }, - event: "ratelimitNamespace.create", - description: `Created ${namespace.id}`, - resources: [ - { - type: "ratelimitNamespace", - id: namespace.id, - }, - ], - context: { - location: c.get("location"), - userAgent: c.get("userAgent"), - }, - }); await insertUnkeyAuditLog(c, undefined, { workspaceId: rootKey.authorizedWorkspaceId, actor: { @@ -353,44 +334,6 @@ export const registerV1RatelimitLimit = (app: App) => passed: ratelimitResponse.passed, }), ); - c.executionCtx.waitUntil( - analytics - .ingestRatelimit({ - workspaceId: rootKey.authorizedWorkspaceId, - - namespaceId: namespace.id, - requestId: c.get("requestId"), - identifier: req.identifier, - - time: Date.now(), - serviceLatency: -1, - success: ratelimitResponse.passed, - remaining, - config: { - limit, - duration, - async: async ?? false, - sharding: shard, - }, - context: { - ipAddress: c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP") ?? "", - userAgent: c.req.header("User-Agent") ?? "", - // @ts-expect-error - the cf object will be there on cloudflare - country: c.req.raw?.cf?.country ?? "", - // @ts-expect-error - the cf object will be there on cloudflare - continent: c.req.raw?.cf?.continent ?? "", - // @ts-expect-error - the cf object will be there on cloudflare - city: c.req.raw?.cf?.city ?? "", - // @ts-expect-error - the cf object will be there on cloudflare - colo: c.req.raw?.cf?.colo ?? "", - }, - }) - .catch((e) => { - logger.error("unable to ingest ratelimit event", { - error: (e as Error).message, - }); - }), - ); if (req.resources && req.resources.length > 0) { c.executionCtx.waitUntil( @@ -418,31 +361,6 @@ export const registerV1RatelimitLimit = (app: App) => }, }), ); - c.executionCtx.waitUntil( - analytics.ingestGenericAuditLogsTinybird({ - auditLogId: newId("auditLog"), - workspaceId: rootKey.authorizedWorkspaceId, - bucket: namespace.id, - actor: { - type: "key", - id: rootKey.key.id, - }, - description: "ratelimit", - event: ratelimitResponse.passed ? "ratelimit.success" : "ratelimit.denied", - meta: { - requestId: c.get("requestId"), - namespacId: namespace.id, - identifier: req.identifier, - success: ratelimitResponse.passed, - }, - time: Date.now(), - resources: req.resources, - context: { - location: c.req.header("True-Client-IP") ?? "", - userAgent: c.req.header("User-Agent") ?? "", - }, - }), - ); } return c.json({ diff --git a/apps/billing/package.json b/apps/billing/package.json index 26c44e9d69..7756fbe181 100644 --- a/apps/billing/package.json +++ b/apps/billing/package.json @@ -18,7 +18,6 @@ "dependencies": { "@agentic/serper": "^7.0.0", "@ai-sdk/openai": "^0.0.63", - "@chronark/zod-bird": "0.3.9", "@clerk/nextjs": "^4.29.10", "@mendable/firecrawl-js": "^1.5.2", "@octokit/rest": "^21.0.2", @@ -27,6 +26,7 @@ "@trigger.dev/sdk": "3.1.2", "@trigger.dev/slack": "3.1.2", "@unkey/billing": "workspace:^", + "@unkey/clickhouse": "workspace:^", "@unkey/db": "workspace:^", "@unkey/error": "workspace:^", "@unkey/id": "workspace:^", diff --git a/apps/billing/src/lib/clickhouse.ts b/apps/billing/src/lib/clickhouse.ts new file mode 100644 index 0000000000..aae86e7b04 --- /dev/null +++ b/apps/billing/src/lib/clickhouse.ts @@ -0,0 +1,4 @@ +import { ClickHouse } from "@unkey/clickhouse"; +import { env } from "./env"; + +export const clickhouse = new ClickHouse({ url: env().CLICKHOUSE_URL }); diff --git a/apps/billing/src/lib/env.ts b/apps/billing/src/lib/env.ts index e06f2b8bd2..fd4b835073 100644 --- a/apps/billing/src/lib/env.ts +++ b/apps/billing/src/lib/env.ts @@ -6,7 +6,7 @@ export function env() { DATABASE_HOST: z.string(), DATABASE_USERNAME: z.string(), DATABASE_PASSWORD: z.string(), - TINYBIRD_TOKEN: z.string(), + CLICKHOUSE_URL: z.string(), STRIPE_SECRET_KEY: z.string(), STRIPE_PRODUCT_ID_KEY_VERIFICATIONS: z.string(), diff --git a/apps/billing/src/lib/tinybird.ts b/apps/billing/src/lib/tinybird.ts deleted file mode 100644 index 4eaf645b81..0000000000 --- a/apps/billing/src/lib/tinybird.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Tinybird as Client } from "@chronark/zod-bird"; -import { z } from "zod"; - -export class Tinybird { - private readonly tb: Client; - - constructor(token: string) { - this.tb = new Client({ token }); - } - - public get activeKeys() { - return this.tb.buildPipe({ - pipe: "endpoint__active_keys_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - data: z.object({ - keys: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } - public get verifications() { - return this.tb.buildPipe({ - pipe: "endpoint__verifications_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - - data: z.object({ - success: z.number().int().nullable().default(0), - ratelimited: z.number().int().nullable().default(0), - usageExceeded: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } - public get ratelimits() { - return this.tb.buildPipe({ - pipe: "endpoint__ratelimits_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - - data: z.object({ - success: z.number().int().nullable().default(0), - total: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } -} diff --git a/apps/billing/src/trigger/create-invoice.ts b/apps/billing/src/trigger/create-invoice.ts index a81482fb73..2bc0314ea6 100644 --- a/apps/billing/src/trigger/create-invoice.ts +++ b/apps/billing/src/trigger/create-invoice.ts @@ -1,4 +1,4 @@ -import { Tinybird } from "@/lib/tinybird"; +import { clickhouse } from "@/lib/clickhouse"; import { type FixedSubscription, type TieredSubscription, @@ -28,7 +28,6 @@ export const createInvoiceTask = task({ apiVersion: "2023-10-16", typescript: true, }); - const tinybird = new Tinybird(env().TINYBIRD_TOKEN); const workspace = await db.query.workspaces.findFirst({ where: (table, { and, eq, isNull }) => @@ -109,13 +108,11 @@ export const createInvoiceTask = task({ * Verifications */ if (workspace.subscriptions?.verifications) { - const verifications = await tinybird - .verifications({ - workspaceId: workspace.id, - year, - month, - }) - .then((res) => res.data.at(0)?.success ?? 0); + const verifications = await clickhouse.billing.billableVerifications({ + workspaceId: workspace.id, + year, + month, + }); await createTieredInvoiceItem({ stripe, @@ -131,13 +128,11 @@ export const createInvoiceTask = task({ * Ratelimits */ if (workspace.subscriptions?.ratelimits) { - const ratelimits = await tinybird - .ratelimits({ - workspaceId: workspace.id, - year, - month, - }) - .then((res) => res.data.at(0)?.success ?? 0); + const ratelimits = await clickhouse.billing.billableRatelimits({ + workspaceId: workspace.id, + year, + month, + }); await createTieredInvoiceItem({ stripe, diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index 6b1861334a..0682e0ba07 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -11,7 +11,6 @@ "dependencies": { "@ant-design/graphs": "^1.4.1", "@ant-design/plots": "^1.2.5", - "@chronark/zod-bird": "0.3.9", "@clerk/nextjs": "^4.29.10", "@faker-js/faker": "^8.4.1", "@hookform/resolvers": "^3.3.4", diff --git a/apps/semantic-cache/package.json b/apps/semantic-cache/package.json index 5c9b027d3e..048eb49815 100644 --- a/apps/semantic-cache/package.json +++ b/apps/semantic-cache/package.json @@ -26,7 +26,6 @@ "wrangler": "^3.80.5" }, "dependencies": { - "@chronark/zod-bird": "^0.3.9", "@planetscale/database": "^1.16.0", "@unkey/cache": "workspace:^", "@unkey/db": "workspace:^", diff --git a/apps/semantic-cache/src/pkg/analytics.ts b/apps/semantic-cache/src/pkg/analytics.ts deleted file mode 100644 index d7bb6c606c..0000000000 --- a/apps/semantic-cache/src/pkg/analytics.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { NoopTinybird, Tinybird } from "@chronark/zod-bird"; -import { z } from "zod"; - -export class Analytics { - public readonly client: Tinybird | NoopTinybird; - - constructor(opts: { - tinybirdToken?: string; - tinybirdProxy?: { - url: string; - token: string; - }; - }) { - this.client = opts.tinybirdProxy - ? new Tinybird({ - token: opts.tinybirdProxy.token, - baseUrl: opts.tinybirdProxy.url, - }) - : opts.tinybirdToken - ? new Tinybird({ token: opts.tinybirdToken }) - : new NoopTinybird(); - } - - public get ingestLogs() { - return this.client.buildIngestEndpoint({ - datasource: "semantic_cache__v1", - event: eventSchema, - }); - } -} - -export const eventSchema = z.object({ - time: z.number(), - model: z.string(), - stream: z.boolean(), - query: z.string(), - vector: z.array(z.number()).default([]), - response: z.string(), - cache: z.boolean(), - latency: z.object({ - /** - * End to end latency of our running code - */ - service: z.number().int(), - embeddings: z.number().int(), - vectorize: z.number().int(), - inference: z.number().int().optional(), - cache: z.number().int(), - }), - tokens: z.number(), - requestId: z.string(), - workspaceId: z.string(), - gatewayId: z.string(), -}); - -export type AnalyticsEvent = z.infer; - -export type InitialAnalyticsEvent = Pick< - AnalyticsEvent, - "time" | "model" | "stream" | "query" | "vector" ->; diff --git a/apps/semantic-cache/src/pkg/hono/env.ts b/apps/semantic-cache/src/pkg/hono/env.ts index c145efae15..0c3719f72f 100644 --- a/apps/semantic-cache/src/pkg/hono/env.ts +++ b/apps/semantic-cache/src/pkg/hono/env.ts @@ -1,6 +1,5 @@ import type { Env } from "@/pkg/env"; import type { Logger } from "@unkey/worker-logging"; -import type { Analytics } from "../analytics"; import type { Database } from "../db"; import type { Metrics } from "../metrics"; @@ -8,7 +7,6 @@ export type ServiceContext = { db: Database; metrics: Metrics; logger: Logger; - analytics: Analytics; }; export type HonoEnv = { diff --git a/apps/semantic-cache/src/pkg/middleware/init.ts b/apps/semantic-cache/src/pkg/middleware/init.ts index 2a25e08aee..d69b31613f 100644 --- a/apps/semantic-cache/src/pkg/middleware/init.ts +++ b/apps/semantic-cache/src/pkg/middleware/init.ts @@ -3,7 +3,6 @@ import { ConsoleLogger } from "@unkey/worker-logging"; import { newId } from "@unkey/id"; import type { MiddlewareHandler } from "hono"; -import { Analytics } from "../analytics"; import type { HonoEnv } from "../hono/env"; import { type Metrics, NoopMetrics } from "../metrics"; import { LogdrainMetrics } from "../metrics/logdrain"; @@ -31,24 +30,10 @@ export function init(): MiddlewareHandler { environment: c.env.ENVIRONMENT, }); - const tinybirdProxy = - c.env.TINYBIRD_PROXY_URL && c.env.TINYBIRD_PROXY_TOKEN - ? { - url: c.env.TINYBIRD_PROXY_URL, - token: c.env.TINYBIRD_PROXY_TOKEN, - } - : undefined; - - const analytics = new Analytics({ - tinybirdProxy, - tinybirdToken: c.env.TINYBIRD_TOKEN, - }); - c.set("services", { db, metrics, logger, - analytics, }); await next(); diff --git a/apps/semantic-cache/src/worker.ts b/apps/semantic-cache/src/worker.ts index a634b6c599..1a84482919 100644 --- a/apps/semantic-cache/src/worker.ts +++ b/apps/semantic-cache/src/worker.ts @@ -12,7 +12,6 @@ app.use("*", ratelimit()); app.use("*", cors()); app.all("*", async (c) => { - const time = Date.now(); const url = new URL(c.req.url); let subdomain = url.hostname.replace(`.${c.env.APEX_DOMAIN}`, ""); if (subdomain === url.hostname || (subdomain === "" && c.env.FALLBACK_SUBDOMAIN)) { @@ -32,7 +31,7 @@ app.all("*", async (c) => { baseURL: c.req.header("X-Base-Url"), }); const request = (await c.req.json()) as OpenAI.Chat.Completions.ChatCompletionCreateParams; - const { db, analytics } = c.get("services"); + const { db } = c.get("services"); const gw = await db.query.llmGateways.findFirst({ where: (table, { eq }) => eq(table.subdomain, subdomain), @@ -41,41 +40,10 @@ app.all("*", async (c) => { return c.text("No gateway found", { status: 404 }); } - try { - if (request.stream) { - return await handleStreamingRequest(c, request, openai); - } - return await handleNonStreamingRequest(c, request, openai); - } finally { - c.executionCtx.waitUntil( - (async () => { - const p = c.get("response"); - const t = c.get("tokens"); - const tokens = t ? await t : -1; - const response = p ? await p : ""; - await analytics.ingestLogs({ - requestId: c.get("requestId"), - time, - latency: { - cache: c.get("cacheLatency") ?? -1, - inference: c.get("inferenceLatency") ?? -1, - service: Date.now() - time, - vectorize: c.get("vectorizeLatency") ?? -1, - embeddings: c.get("embeddingsLatency") ?? -1, - }, - gatewayId: gw.id, - workspaceId: gw.workspaceId, - stream: request.stream ?? false, - tokens: tokens, - cache: c.get("cacheHit") ?? false, - model: request.model, - query: c.get("query") ?? "", - vector: c.get("vector") ?? [], - response, - }); - })(), - ); + if (request.stream) { + return await handleStreamingRequest(c, request, openai); } + return await handleNonStreamingRequest(c, request, openai); }); const handler = { diff --git a/apps/workflows/lib/tinybird.ts b/apps/workflows/lib/tinybird.ts deleted file mode 100644 index 4eaf645b81..0000000000 --- a/apps/workflows/lib/tinybird.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Tinybird as Client } from "@chronark/zod-bird"; -import { z } from "zod"; - -export class Tinybird { - private readonly tb: Client; - - constructor(token: string) { - this.tb = new Client({ token }); - } - - public get activeKeys() { - return this.tb.buildPipe({ - pipe: "endpoint__active_keys_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - data: z.object({ - keys: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } - public get verifications() { - return this.tb.buildPipe({ - pipe: "endpoint__verifications_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - - data: z.object({ - success: z.number().int().nullable().default(0), - ratelimited: z.number().int().nullable().default(0), - usageExceeded: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } - public get ratelimits() { - return this.tb.buildPipe({ - pipe: "endpoint__ratelimits_by_workspace__v1", - parameters: z.object({ - workspaceId: z.string(), - year: z.number().int(), - month: z.number().int().min(1).max(12), - }), - - data: z.object({ - success: z.number().int().nullable().default(0), - total: z.number().int().nullable().default(0), - }), - opts: { - cache: "no-store", - }, - }); - } -} diff --git a/apps/workflows/package.json b/apps/workflows/package.json index 3e621df7ab..47901f3ec7 100644 --- a/apps/workflows/package.json +++ b/apps/workflows/package.json @@ -9,7 +9,6 @@ "lint": "next lint" }, "dependencies": { - "@chronark/zod-bird": "0.3.9", "@clerk/nextjs": "^4.29.10", "@planetscale/database": "^1.16.0", "@trigger.dev/nextjs": "^3.1.0", diff --git a/apps/www/package.json b/apps/www/package.json index b3ca68e60f..a454565ea0 100644 --- a/apps/www/package.json +++ b/apps/www/package.json @@ -10,7 +10,6 @@ "lint": "next lint" }, "dependencies": { - "@chronark/zod-bird": "^0.3.9", "@faker-js/faker": "^8.4.1", "@hookform/resolvers": "^3.3.4", "@next/bundle-analyzer": "^14.2.3", diff --git a/internal/clickhouse/src/index.ts b/internal/clickhouse/src/index.ts index a323f5b651..cf17b6482f 100644 --- a/internal/clickhouse/src/index.ts +++ b/internal/clickhouse/src/index.ts @@ -10,12 +10,16 @@ import { getRatelimitsPerHour, getRatelimitsPerMinute, getRatelimitsPerMonth, + insertRatelimit, } from "./ratelimits"; +import { insertApiRequest } from "./requests"; import { getActiveWorkspacesPerMonth } from "./success"; +import { insertSDKTelemetry } from "./telemetry"; import { getVerificationsPerDay, getVerificationsPerHour, getVerificationsPerMonth, + insertVerification, } from "./verifications"; export type ClickHouseConfig = { @@ -38,6 +42,7 @@ export class ClickHouse { } public get verifications() { return { + insert: insertVerification(this.client), logs: getLatestVerifications(this.client), perHour: getVerificationsPerHour(this.client), perDay: getVerificationsPerDay(this.client), @@ -54,6 +59,7 @@ export class ClickHouse { } public get ratelimits() { return { + insert: insertRatelimit(this.client), logs: getRatelimitLogs(this.client), latest: getRatelimitLastUsed(this.client), perMinute: getRatelimitsPerMinute(this.client), @@ -70,6 +76,7 @@ export class ClickHouse { } public get api() { return { + insert: insertApiRequest(this.client), logs: getLogs(this.client), }; } @@ -78,4 +85,9 @@ export class ClickHouse { activeWorkspaces: getActiveWorkspacesPerMonth(this.client), }; } + public get telemetry() { + return { + insert: insertSDKTelemetry(this.client), + }; + } } diff --git a/internal/clickhouse/src/ratelimits.ts b/internal/clickhouse/src/ratelimits.ts index c36547d2f7..e540c5e1ca 100644 --- a/internal/clickhouse/src/ratelimits.ts +++ b/internal/clickhouse/src/ratelimits.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import type { Querier } from "./client"; +import type { Inserter, Querier } from "./client"; import { dateTimeToUnix } from "./util"; const params = z.object({ @@ -10,8 +10,22 @@ const params = z.object({ end: z.number().default(() => Date.now()), }); +export function insertRatelimit(ch: Inserter) { + return ch.insert({ + table: "ratelimits.raw_ratelimits_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + namespace_id: z.string(), + identifier: z.string(), + passed: z.boolean(), + }), + }); +} + export function getRatelimitsPerMinute(ch: Querier) { - return async (args: z.infer) => { + return async (args: z.input) => { const query = ch.query({ query: ` SELECT @@ -94,7 +108,7 @@ export function getRatelimitsPerDay(ch: Querier) { WITH FILL FROM toStartOfDay(fromUnixTimestamp64Milli({start: Int64})) TO toStartOfDay(fromUnixTimestamp64Milli({end: Int64})) - STEP INTERVAL 1 DAY + STEP INTERVAL 1 DAY ;`, params, schema: z.object({ @@ -194,7 +208,7 @@ export function getRatelimitLastUsed(ch: Querier) { identifier, max(time) as time FROM ratelimits.ratelimits_last_used_v1 - WHERE + WHERE workspace_id = {workspaceId: String} AND namespace_id = {namespaceId: String} ${args.identifier ? "AND multiSearchAny(identifier, {identifier: Array(String)}) > 0" : ""} diff --git a/internal/clickhouse/src/requests.ts b/internal/clickhouse/src/requests.ts new file mode 100644 index 0000000000..f9dc60a475 --- /dev/null +++ b/internal/clickhouse/src/requests.ts @@ -0,0 +1,30 @@ +import type { Inserter } from "./client/interface"; + +import { z } from "zod"; + +export function insertApiRequest(ch: Inserter) { + return ch.insert({ + table: "metrics.raw_api_requests_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + host: z.string(), + method: z.string(), + path: z.string(), + request_headers: z.array(z.string()), + request_body: z.string(), + response_status: z.number().int(), + response_headers: z.array(z.string()), + response_body: z.string(), + error: z.string().optional().default(""), + service_latency: z.number().int(), + user_agent: z.string(), + ip_address: z.string(), + continent: z.string().nullable().default(""), + city: z.string().nullable().default(""), + country: z.string().nullable().default(""), + colo: z.string().nullable().default(""), + }), + }); +} diff --git a/internal/clickhouse/src/telemetry.ts b/internal/clickhouse/src/telemetry.ts new file mode 100644 index 0000000000..69304185c5 --- /dev/null +++ b/internal/clickhouse/src/telemetry.ts @@ -0,0 +1,15 @@ +import { z } from "zod"; +import type { Inserter } from "./client/interface"; + +export function insertSDKTelemetry(ch: Inserter) { + return ch.insert({ + table: "telemetry.raw_sdks_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + runtime: z.string(), + platform: z.string(), + versions: z.array(z.string()), + }), + }); +} diff --git a/internal/clickhouse/src/verifications.ts b/internal/clickhouse/src/verifications.ts index bb06f49c68..d97a6d76fc 100644 --- a/internal/clickhouse/src/verifications.ts +++ b/internal/clickhouse/src/verifications.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import type { Querier } from "./client"; +import type { Inserter, Querier } from "./client"; import { dateTimeToUnix } from "./util"; const outcome = z.enum([ @@ -26,6 +26,30 @@ const schema = z.object({ count: z.number().int(), }); +export function insertVerification(ch: Inserter) { + return ch.insert({ + table: "verifications.raw_key_verifications_v1", + schema: z.object({ + request_id: z.string(), + time: z.number().int(), + workspace_id: z.string(), + key_space_id: z.string(), + key_id: z.string(), + region: z.string(), + outcome: z.enum([ + "VALID", + "RATE_LIMITED", + "EXPIRED", + "DISABLED", + "FORBIDDEN", + "USAGE_EXCEEDED", + "INSUFFICIENT_PERMISSIONS", + ]), + identity_id: z.string().optional().default(""), + }), + }); +} + export function getVerificationsPerHour(ch: Querier) { return async (args: z.input) => { const query = ` diff --git a/internal/tinybird/.tinyenv b/internal/tinybird/.tinyenv deleted file mode 100644 index 5e312c69e6..0000000000 --- a/internal/tinybird/.tinyenv +++ /dev/null @@ -1,2 +0,0 @@ -VERSION=0.0.0 -TB_SKIP_REGRESSION=1 diff --git a/internal/tinybird/datasources/active_keys_hourly_mv.datasource b/internal/tinybird/datasources/active_keys_hourly_mv.datasource deleted file mode 100644 index a3fc173aa1..0000000000 --- a/internal/tinybird/datasources/active_keys_hourly_mv.datasource +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'active_keys_hourly__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time, apiId, keyId" diff --git a/internal/tinybird/datasources/aggregate_mv.datasource b/internal/tinybird/datasources/aggregate_mv.datasource deleted file mode 100644 index ec07ae6b38..0000000000 --- a/internal/tinybird/datasources/aggregate_mv.datasource +++ /dev/null @@ -1,11 +0,0 @@ -# Data Source created from Pipe 't_d1b4586e4a134937bc6588f8a0e7c8b9' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time, apiId, keyId" diff --git a/internal/tinybird/datasources/audit_logs.datasource b/internal/tinybird/datasources/audit_logs.datasource deleted file mode 100644 index 832ccbb412..0000000000 --- a/internal/tinybird/datasources/audit_logs.datasource +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 2 -SCHEMA > - `workspaceId` String `json:$.workspaceId`, - `bucket` String `json:$.bucket`, - `auditLogId` String `json:$.auditLogId`, - `event` String `json:$.event`, - `time` Int64 `json:$.time`, - `actorType` String `json:$.actor.type`, - `actorId` String `json:$.actor.id`, - `actorName` Nullable(String) `json:$.actor.name`, - `actorMeta` Nullable(String) `json:$.actor.meta`, - `description` String `json:$.description`, - `resources` String `json:$.resources`, - `userAgent` Nullable(String) `json:$.context.userAgent`, - `location` Nullable(String) `json:$.context.location` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId,bucket,time" - diff --git a/internal/tinybird/datasources/billable_verifications__mv.datasource b/internal/tinybird/datasources/billable_verifications__mv.datasource deleted file mode 100644 index 04495a2a32..0000000000 --- a/internal/tinybird/datasources/billable_verifications__mv.datasource +++ /dev/null @@ -1,8 +0,0 @@ -# Data Source created from Pipe 'billable_verifications__v1' - -SCHEMA > - `workspaceId` String, - `time` Int64 - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId, time" diff --git a/internal/tinybird/datasources/daily_usage.datasource b/internal/tinybird/datasources/daily_usage.datasource deleted file mode 100644 index 9c483fd231..0000000000 --- a/internal/tinybird/datasources/daily_usage.datasource +++ /dev/null @@ -1,14 +0,0 @@ -# Data Source created from Pipe 'aggregate_verifications_per_day' -DESCRIPTION > - Daily usage aggregated by workspaceId, apiId and keyId - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `count` AggregateFunction(count), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/datasources/deployment_logs.datasource b/internal/tinybird/datasources/deployment_logs.datasource deleted file mode 100644 index de30789691..0000000000 --- a/internal/tinybird/datasources/deployment_logs.datasource +++ /dev/null @@ -1,13 +0,0 @@ - -SCHEMA > - `workspaceId` String `json:$.workspaceId`, - `gatewayId` String `json:$.gatewayId`, - `branchId` String `json:$.branchId`, - `deploymentId` String `json:$.deploymentId`, - `message` String `json:$.message`, - `time` Int64 `json:$.time`, - `level` String `json:$.level` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId, gatewayId, branchId, deploymentId" -ENGINE_TTL "toDateTime(fromUnixTimestamp64Milli(time) + toIntervalDay(90))" diff --git a/internal/tinybird/datasources/key_verifications.datasource b/internal/tinybird/datasources/key_verifications.datasource deleted file mode 100644 index da4d2159bf..0000000000 --- a/internal/tinybird/datasources/key_verifications.datasource +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 2 - -SCHEMA > - `apiId` String `json:$.apiId`, - `edgeRegion` String `json:$.edgeRegion`, - `ipAddress` String `json:$.ipAddress`, - `keyId` String `json:$.keyId`, - `ratelimited` UInt8 `json:$.ratelimited`, - `region` String `json:$.region`, - `requestedResource` String `json:$.requestedResource`, - `time` Int64 `json:$.time`, - `usageExceeded` UInt8 `json:$.usageExceeded`, - `userAgent` String `json:$.userAgent`, - `workspaceId` String `json:$.workspaceId`, - `deniedReason` Nullable(String) `json:$.deniedReason`, - `ownerId` Nullable(String) `json:$.ownerId`, - `keySpaceId` Nullable(String) `json:$.keySpaceId`, - `requestBody` Nullable(String) `json:$.requestBody`, - `responseBody` Nullable(String) `json:$.responseBody`, - `requestId` Nullable(String) `json:$.requestId` -ENGINE "MergeTree" -ENGINE_SORTING_KEY "apiId, usageExceeded, userAgent, workspaceId" diff --git a/internal/tinybird/datasources/mv__daily_usage.datasource b/internal/tinybird/datasources/mv__daily_usage.datasource deleted file mode 100644 index 1e102d23ba..0000000000 --- a/internal/tinybird/datasources/mv__daily_usage.datasource +++ /dev/null @@ -1,14 +0,0 @@ -VERSION 2 -# Data Source created from Pipe 'aggregate_verifications_per_day__v2' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `ratelimited` SimpleAggregateFunction(sum, Int64), - `count` AggregateFunction(count), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/datasources/mv__daily_verifications.datasource b/internal/tinybird/datasources/mv__daily_verifications.datasource deleted file mode 100644 index 3360e4c9ae..0000000000 --- a/internal/tinybird/datasources/mv__daily_verifications.datasource +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 3 -# Data Source created from Pipe 'aggregate_verifications_daily__v2' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/datasources/mv__latest_verifications.datasource b/internal/tinybird/datasources/mv__latest_verifications.datasource deleted file mode 100644 index c56ad64cf3..0000000000 --- a/internal/tinybird/datasources/mv__latest_verifications.datasource +++ /dev/null @@ -1,17 +0,0 @@ -VERSION 3 - -SCHEMA > - `time` Int64, - `requestedResource` String, - `outcome` String, - `region` String, - `ipAddress` String, - `userAgent` String, - `workspaceId` String, - `apiId` String, - `keyId` String, - `requestBody` Nullable(String), - `responseBody` Nullable(String) - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/datasources/mv__monthly_active_keys.datasource b/internal/tinybird/datasources/mv__monthly_active_keys.datasource deleted file mode 100644 index 7b8fc194f8..0000000000 --- a/internal/tinybird/datasources/mv__monthly_active_keys.datasource +++ /dev/null @@ -1,11 +0,0 @@ -VERSION 1 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/datasources/mv__monthly_stats.datasource b/internal/tinybird/datasources/mv__monthly_stats.datasource deleted file mode 100644 index 8766059aec..0000000000 --- a/internal/tinybird/datasources/mv__monthly_stats.datasource +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'aggregate_verifications_monthly__v1' - -SCHEMA > - `verifications` AggregateFunction(count), - `activeKeys` AggregateFunction(uniq, String), - `activeWorkspaces` AggregateFunction(uniq, String), - `month` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(month)" -ENGINE_SORTING_KEY "month" diff --git a/internal/tinybird/datasources/mv__monthly_verifications.datasource b/internal/tinybird/datasources/mv__monthly_verifications.datasource deleted file mode 100644 index a0c5ae4428..0000000000 --- a/internal/tinybird/datasources/mv__monthly_verifications.datasource +++ /dev/null @@ -1,18 +0,0 @@ -VERSION 3 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time, apiId, keyId" diff --git a/internal/tinybird/datasources/mv__ratelimits_daily.datasource b/internal/tinybird/datasources/mv__ratelimits_daily.datasource deleted file mode 100644 index 58519860a5..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_daily.datasource +++ /dev/null @@ -1,14 +0,0 @@ -# Data Source created from Pipe 'ratelimits_daily__v1' -VERSION 1 -SCHEMA > - `workspaceId` String, - `namespaceId` String, - `identifier` String, - `total` AggregateFunction(count), - `success` AggregateFunction(countIf, UInt8), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier, time" -ENGINE_TTL "time + toIntervalDay(400)" diff --git a/internal/tinybird/datasources/mv__ratelimits_hourly.datasource b/internal/tinybird/datasources/mv__ratelimits_hourly.datasource deleted file mode 100644 index f0e393d06a..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_hourly.datasource +++ /dev/null @@ -1,15 +0,0 @@ -# Data Source created from Pipe 'ratelimits_hourly__v1' -VERSION 1 - -SCHEMA > - `workspaceId` String, - `namespaceId` String, - `identifier` String, - `total` AggregateFunction(count), - `success` AggregateFunction(countIf, UInt8), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier, time" -ENGINE_TTL "time + toIntervalDay(90)" diff --git a/internal/tinybird/datasources/mv__ratelimits_last_used.datasource b/internal/tinybird/datasources/mv__ratelimits_last_used.datasource deleted file mode 100644 index 2253768ab4..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_last_used.datasource +++ /dev/null @@ -1,10 +0,0 @@ -# Data Source created from Pipe 'ratelimit_last_used__v1' -VERSION 1 -SCHEMA > - `namespaceId` String, - `workspaceId` String, - `identifier` String, - `lastUsed` SimpleAggregateFunction(max, Int64) - -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier" diff --git a/internal/tinybird/datasources/mv__ratelimits_minutely.datasource b/internal/tinybird/datasources/mv__ratelimits_minutely.datasource deleted file mode 100644 index c2a5be33bd..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_minutely.datasource +++ /dev/null @@ -1,14 +0,0 @@ -# Data Source created from Pipe 'ratelimits_daily__v1' -VERSION 1 -SCHEMA > - `workspaceId` String, - `namespaceId` String, - `identifier` String, - `total` AggregateFunction(count), - `success` AggregateFunction(countIf, UInt8), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier, time" -ENGINE_TTL "time + toIntervalDay(1)" diff --git a/internal/tinybird/datasources/mv__ratelimits_monthly.datasource b/internal/tinybird/datasources/mv__ratelimits_monthly.datasource deleted file mode 100644 index 1cdd0584d8..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_monthly.datasource +++ /dev/null @@ -1,15 +0,0 @@ -# Data Source created from Pipe 'ratelimits_monthly__v1' -VERSION 1 - -SCHEMA > - `workspaceId` String, - `namespaceId` String, - `identifier` String, - `total` AggregateFunction(count), - `success` AggregateFunction(countIf, UInt8), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier, time" -ENGINE_TTL "time + toIntervalDay(800)" diff --git a/internal/tinybird/datasources/mv__ratelimits_weekly.datasource b/internal/tinybird/datasources/mv__ratelimits_weekly.datasource deleted file mode 100644 index 1532119ed8..0000000000 --- a/internal/tinybird/datasources/mv__ratelimits_weekly.datasource +++ /dev/null @@ -1,15 +0,0 @@ -# Data Source created from Pipe 'ratelimits_weekly__v1' -VERSION 1 - -SCHEMA > - `workspaceId` String, - `namespaceId` String, - `identifier` String, - `total` AggregateFunction(count), - `success` AggregateFunction(countIf, UInt8), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, namespaceId, identifier, time" -ENGINE_TTL "time + toIntervalDay(400)" diff --git a/internal/tinybird/datasources/mv__semantic_cache_daily.datasource b/internal/tinybird/datasources/mv__semantic_cache_daily.datasource deleted file mode 100644 index 4b01c37a4d..0000000000 --- a/internal/tinybird/datasources/mv__semantic_cache_daily.datasource +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 4 -# Data Source created from Pipe 'semantic_cache_hourly__v1' - -SCHEMA > - `hit` AggregateFunction(countIf, UInt8), - `model` String, - `total` AggregateFunction(count), - `avgServiceLatency` AggregateFunction(avg, Int64), - `avgEmbeddingsLatency` AggregateFunction(avg, Int64), - `avgVectorizeLatency` AggregateFunction(avg, Int64), - `avgInferenceLatency` AggregateFunction(avg, Int64), - `avgCacheLatency` AggregateFunction(avg, Int64), - `avgTokens` AggregateFunction(avg, Int64), - `sumTokens` AggregateFunction(sum, Int64), - `cachedTokens` AggregateFunction(sum, Int64), - `workspaceId` String, - `gatewayId` String, - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, gatewayId, time" diff --git a/internal/tinybird/datasources/mv__semantic_cache_hourly.datasource b/internal/tinybird/datasources/mv__semantic_cache_hourly.datasource deleted file mode 100644 index 4b01c37a4d..0000000000 --- a/internal/tinybird/datasources/mv__semantic_cache_hourly.datasource +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 4 -# Data Source created from Pipe 'semantic_cache_hourly__v1' - -SCHEMA > - `hit` AggregateFunction(countIf, UInt8), - `model` String, - `total` AggregateFunction(count), - `avgServiceLatency` AggregateFunction(avg, Int64), - `avgEmbeddingsLatency` AggregateFunction(avg, Int64), - `avgVectorizeLatency` AggregateFunction(avg, Int64), - `avgInferenceLatency` AggregateFunction(avg, Int64), - `avgCacheLatency` AggregateFunction(avg, Int64), - `avgTokens` AggregateFunction(avg, Int64), - `sumTokens` AggregateFunction(sum, Int64), - `cachedTokens` AggregateFunction(sum, Int64), - `workspaceId` String, - `gatewayId` String, - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, gatewayId, time" diff --git a/internal/tinybird/datasources/mv__total_verifications.datasource b/internal/tinybird/datasources/mv__total_verifications.datasource deleted file mode 100644 index 1502f00490..0000000000 --- a/internal/tinybird/datasources/mv__total_verifications.datasource +++ /dev/null @@ -1,8 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'count_monthly_verifications__v1' - -SCHEMA > - `count` UInt64 - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "count" diff --git a/internal/tinybird/datasources/mv__verifications_for_analytics_daily.datasource b/internal/tinybird/datasources/mv__verifications_for_analytics_daily.datasource deleted file mode 100644 index 86d039f001..0000000000 --- a/internal/tinybird/datasources/mv__verifications_for_analytics_daily.datasource +++ /dev/null @@ -1,16 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'get_verifications_for_analytics_daily__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `ownerId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, ownerId, keyId, time" diff --git a/internal/tinybird/datasources/mv__verifications_for_analytics_hourly.datasource b/internal/tinybird/datasources/mv__verifications_for_analytics_hourly.datasource deleted file mode 100644 index e85bac9525..0000000000 --- a/internal/tinybird/datasources/mv__verifications_for_analytics_hourly.datasource +++ /dev/null @@ -1,16 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'get_verifications_for_analytics_hourly__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `ownerId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, ownerId, keyId, time" diff --git a/internal/tinybird/datasources/mv__verifications_for_analytics_monthly.datasource b/internal/tinybird/datasources/mv__verifications_for_analytics_monthly.datasource deleted file mode 100644 index 2a8b5bab6b..0000000000 --- a/internal/tinybird/datasources/mv__verifications_for_analytics_monthly.datasource +++ /dev/null @@ -1,16 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'get__verifications_for_analytics_month__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `ownerId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, ownerId, keyId, time" diff --git a/internal/tinybird/datasources/mv__verifications_for_analytics_weekly_v1.datasource b/internal/tinybird/datasources/mv__verifications_for_analytics_weekly_v1.datasource deleted file mode 100644 index 7ae9397a2e..0000000000 --- a/internal/tinybird/datasources/mv__verifications_for_analytics_weekly_v1.datasource +++ /dev/null @@ -1,15 +0,0 @@ -# Data Source created from Pipe 'get__verifications_for_analytics_weekly__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `ownerId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, ownerId, keyId, time" diff --git a/internal/tinybird/datasources/mv__verifications_per_workspace_per_hour.datasource b/internal/tinybird/datasources/mv__verifications_per_workspace_per_hour.datasource deleted file mode 100644 index 14b4ef553e..0000000000 --- a/internal/tinybird/datasources/mv__verifications_per_workspace_per_hour.datasource +++ /dev/null @@ -1,13 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'aggregate_verifications_hourly_by_workspace__v1' -DESCRIPTION > - Hourly usage per workspace - -SCHEMA > - `workspaceId` String, - `verifications` AggregateFunction(count), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time" diff --git a/internal/tinybird/datasources/mv_billing_hourly_active_keys.datasource b/internal/tinybird/datasources/mv_billing_hourly_active_keys.datasource deleted file mode 100644 index 5993cb4e40..0000000000 --- a/internal/tinybird/datasources/mv_billing_hourly_active_keys.datasource +++ /dev/null @@ -1,11 +0,0 @@ -VERSION 2 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, time, keyId" diff --git a/internal/tinybird/datasources/mv_key_total_usage.datasource b/internal/tinybird/datasources/mv_key_total_usage.datasource deleted file mode 100644 index 26579e7f83..0000000000 --- a/internal/tinybird/datasources/mv_key_total_usage.datasource +++ /dev/null @@ -1,9 +0,0 @@ -VERSION 1 -# Data Source created from Pipe 'aggregate_total_usage_per_key__v1' - -SCHEMA > - `totalUsage` AggregateFunction(count), - `keyId` String - -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "keyId" diff --git a/internal/tinybird/datasources/mv_keys_last_used.datasource b/internal/tinybird/datasources/mv_keys_last_used.datasource deleted file mode 100644 index 7dea38982c..0000000000 --- a/internal/tinybird/datasources/mv_keys_last_used.datasource +++ /dev/null @@ -1,8 +0,0 @@ -# Data Source created from Pipe 'aggregate_last_used__v1' - -SCHEMA > - `keyId` String, - `lastUsed` SimpleAggregateFunction(max, Int64) - -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "keyId" diff --git a/internal/tinybird/datasources/pageviews.datasource b/internal/tinybird/datasources/pageviews.datasource deleted file mode 100644 index d0c26d1a92..0000000000 --- a/internal/tinybird/datasources/pageviews.datasource +++ /dev/null @@ -1,16 +0,0 @@ -VERSION 1 - -SCHEMA > - `sessionId` String `json:$.sessionId`, - `userId` Nullable(String) `json:$.userId`, - `tenantId` Nullable(String) `json:$.tenantId`, - `time` Int64 `json:$.time`, - `path` String `json:$.path`, - `region` Nullable(String) `json:$.region`, - `country` Nullable(String) `json:$.country`, - `city` Nullable(String) `json:$.city`, - `userAgent` Nullable(String) `json:$.userAgent`, - `referrer` Nullable(String) `json:$.referrer` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "sessionId, time, path" diff --git a/internal/tinybird/datasources/q1_goal_distinct_workspaces.datasource b/internal/tinybird/datasources/q1_goal_distinct_workspaces.datasource deleted file mode 100644 index cd5755aaff..0000000000 --- a/internal/tinybird/datasources/q1_goal_distinct_workspaces.datasource +++ /dev/null @@ -1,8 +0,0 @@ - -SCHEMA > - `workspaces` UInt64, - `time` DateTime - -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYear(time)" -ENGINE_SORTING_KEY "time, workspaces" diff --git a/internal/tinybird/datasources/ratelimits.datasource b/internal/tinybird/datasources/ratelimits.datasource deleted file mode 100644 index 5f74fbdc53..0000000000 --- a/internal/tinybird/datasources/ratelimits.datasource +++ /dev/null @@ -1,26 +0,0 @@ -VERSION 2 -SCHEMA > - `workspaceId` String `json:$.workspaceId`, - `namespaceId` String `json:$.namespaceId`, - `requestId` String `json:$.requestId`, - `identifier` String `json:$.identifier`, - `time` Int64 `json:$.time`, - `serviceLatency` Int64 `json:$.serviceLatency`, - `success` Int8 `json:$.success`, - `remaining` Int64 `json:$.remaining`, - `limit` Int64 `json:$.config.limit`, - `duration` Int64 `json:$.config.duration`, - `async` Int8 `json:$.config.async`, - `sharding` String `json:$.config.sharding`, - `ipAddress` String `json:$.context.ipAddress`, - `userAgent` String `json:$.context.userAgent`, - `country` String `json:$.context.country`, - `continent` String `json:$.context.continent`, - `city` String `json:$.context.city`, - `colo` String `json:$.context.colo` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId,namespaceId,identifier,time" - - - diff --git a/internal/tinybird/datasources/sdk_telemetry.datasource b/internal/tinybird/datasources/sdk_telemetry.datasource deleted file mode 100644 index b446ac4ee4..0000000000 --- a/internal/tinybird/datasources/sdk_telemetry.datasource +++ /dev/null @@ -1,9 +0,0 @@ -VERSION 1 -SCHEMA > - `runtime` String `json:$.runtime`, - `platform` String `json:$.platform`, - `versions` Array(String) `json:$.versions[:]`, - `requestId` String `json:$.requestId`, - `time` Int64 `json:$.time` -ENGINE “MergeTree” -ENGINE_SORTING_KEY “time” \ No newline at end of file diff --git a/internal/tinybird/datasources/semantic_cache.datasource b/internal/tinybird/datasources/semantic_cache.datasource deleted file mode 100644 index 29acdb4fdd..0000000000 --- a/internal/tinybird/datasources/semantic_cache.datasource +++ /dev/null @@ -1,21 +0,0 @@ -VERSION 1 -SCHEMA > - `cache` UInt8 `json:$.cache`, - `model` String `json:$.model`, - `query` String `json:$.query`, - `requestId` String `json:$.requestId`, - `response` String `json:$.response`, - `stream` UInt8 `json:$.stream`, - `time` Int64 `json:$.time`, - `serviceLatency` Int64 `json:$.latency.service`, - `embeddingsLatency` Int64 `json:$.latency.embeddings`, - `vectorizeLatency` Int64 `json:$.latency.vectorize`, - `inferenceLatency` Nullable(Int64) `json:$.latency.inference`, - `cacheLatency` Int64 `json:$.latency.cache`, - `tokens` Int64 `json:$.tokens`, - `vector` Array(Float64) `json:$.vector[:]`, - `workspaceId` String `json:$.workspaceId`, - `gatewayId` String `json:$.gatewayId` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId, gatewayId, time" diff --git a/internal/tinybird/datasources/tinybird_proxy_errors.datasource b/internal/tinybird/datasources/tinybird_proxy_errors.datasource deleted file mode 100644 index 081908e1d6..0000000000 --- a/internal/tinybird/datasources/tinybird_proxy_errors.datasource +++ /dev/null @@ -1,8 +0,0 @@ -VERSION 1 -SCHEMA > - `nodeId` String `json:$.nodeId`, - `time` Int64 `json:$.time`, - `message` String `json:$.message` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "nodeId, time" diff --git a/internal/tinybird/datasources/tinybird_proxy_metrics.datasource b/internal/tinybird/datasources/tinybird_proxy_metrics.datasource deleted file mode 100644 index 5143c746d5..0000000000 --- a/internal/tinybird/datasources/tinybird_proxy_metrics.datasource +++ /dev/null @@ -1,10 +0,0 @@ -VERSION 1 -SCHEMA > - `nodeId` String `json:$.nodeId`, - `requests` Int64 `json:$.requests`, - `rows` Int64 `json:$.rows`, - `flushes` Int64 `json:$.flushes`, - `time` Int64 `json:$.time` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "nodeId, time" diff --git a/internal/tinybird/datasources/verifications_daily_mv.datasource b/internal/tinybird/datasources/verifications_daily_mv.datasource deleted file mode 100644 index 98969a0930..0000000000 --- a/internal/tinybird/datasources/verifications_daily_mv.datasource +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 2 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, time, keyId" -ENGINE_TTL "time + toIntervalDay(90)" diff --git a/internal/tinybird/datasources/verifications_hourly_mv.datasource b/internal/tinybird/datasources/verifications_hourly_mv.datasource deleted file mode 100644 index 46fcfbbd2e..0000000000 --- a/internal/tinybird/datasources/verifications_hourly_mv.datasource +++ /dev/null @@ -1,20 +0,0 @@ -VERSION 2 -# Data Source created from Pipe 'verifications_hourly__v1' - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` DateTime - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, time, keyId" -ENGINE_TTL "time + toIntervalDay(30)" diff --git a/internal/tinybird/datasources/verifications_monthly_mv.datasource b/internal/tinybird/datasources/verifications_monthly_mv.datasource deleted file mode 100644 index a36ea7121e..0000000000 --- a/internal/tinybird/datasources/verifications_monthly_mv.datasource +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 2 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" -ENGINE_TTL "time + toIntervalDay(365)" diff --git a/internal/tinybird/datasources/verifications_weekly_mv.datasource b/internal/tinybird/datasources/verifications_weekly_mv.datasource deleted file mode 100644 index 65f5515c00..0000000000 --- a/internal/tinybird/datasources/verifications_weekly_mv.datasource +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 2 - -SCHEMA > - `workspaceId` String, - `apiId` String, - `keyId` String, - `success` AggregateFunction(countIf, Nullable(UInt8)), - `rateLimited` AggregateFunction(countIf, Nullable(UInt8)), - `usageExceeded` AggregateFunction(countIf, Nullable(UInt8)), - `disabled` AggregateFunction(countIf, Nullable(UInt8)), - `insufficientPermissions` AggregateFunction(countIf, Nullable(UInt8)), - `forbidden` AggregateFunction(countIf, Nullable(UInt8)), - `expired` AggregateFunction(countIf, Nullable(UInt8)), - `time` Date - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" -ENGINE_TTL "time + toIntervalDay(180)" diff --git a/internal/tinybird/datasources/webhook_delivery_attempts.datasource b/internal/tinybird/datasources/webhook_delivery_attempts.datasource deleted file mode 100644 index fd9235b6a9..0000000000 --- a/internal/tinybird/datasources/webhook_delivery_attempts.datasource +++ /dev/null @@ -1,17 +0,0 @@ -VERSION 1 -SCHEMA > - `workspaceId` String `json:$.workspaceId`, - `webhookId` String `json:$.webhookId`, - `eventId` String `json:$.eventId`, - `event` String `json:$.event`, - `deliveryId` String `json:$.deliveryId`, - `time` Int64 `json:$.time`, - `attempt` Int8 `json:$.attempt`, - `nextAttemptAt` Int64 `json:$.nextAttemptAt`, - `success` UInt8 `json:$.success`, - `internalError` String `json:$.internalError`, - `responseStatus` Int16 `json:$.response.status`, - `responseBody` String `json:$.response.body` - -ENGINE "MergeTree" -ENGINE_SORTING_KEY "workspaceId, webhookId, event, time" diff --git a/internal/tinybird/pipes/active_keys_hourly.pipe b/internal/tinybird/pipes/active_keys_hourly.pipe deleted file mode 100644 index fca1ba49b2..0000000000 --- a/internal/tinybird/pipes/active_keys_hourly.pipe +++ /dev/null @@ -1,20 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE active_keys_hourly_mv - - diff --git a/internal/tinybird/pipes/aggregate_active_keys_hourly_by_workspace.pipe b/internal/tinybird/pipes/aggregate_active_keys_hourly_by_workspace.pipe deleted file mode 100644 index da74ceeb7c..0000000000 --- a/internal/tinybird/pipes/aggregate_active_keys_hourly_by_workspace.pipe +++ /dev/null @@ -1,18 +0,0 @@ -VERSION 3 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE mv_billing_hourly_active_keys__v2 diff --git a/internal/tinybird/pipes/aggregate_active_keys_monthly_by_workspace.pipe b/internal/tinybird/pipes/aggregate_active_keys_monthly_by_workspace.pipe deleted file mode 100644 index ee24690b3c..0000000000 --- a/internal/tinybird/pipes/aggregate_active_keys_monthly_by_workspace.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 - -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE mv__monthly_active_keys__v1 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/pipes/aggregate_active_keys_per_hour.pipe b/internal/tinybird/pipes/aggregate_active_keys_per_hour.pipe deleted file mode 100644 index a5ff3d9853..0000000000 --- a/internal/tinybird/pipes/aggregate_active_keys_per_hour.pipe +++ /dev/null @@ -1,17 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - keyId, - time - - diff --git a/internal/tinybird/pipes/aggregate_last_used.pipe b/internal/tinybird/pipes/aggregate_last_used.pipe deleted file mode 100644 index 4db8445b7a..0000000000 --- a/internal/tinybird/pipes/aggregate_last_used.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_1411_0 -SQL > - - SELECT - keyId, - maxSimpleState(time) AS lastUsed - FROM key_verifications - GROUP BY keyId - -TYPE materialized -DATASOURCE mv_keys_last_used -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "keyId" - diff --git a/internal/tinybird/pipes/aggregate_total_usage_per_key.pipe b/internal/tinybird/pipes/aggregate_total_usage_per_key.pipe deleted file mode 100644 index 85a3f5bed0..0000000000 --- a/internal/tinybird/pipes/aggregate_total_usage_per_key.pipe +++ /dev/null @@ -1,14 +0,0 @@ -VERSION 1 -NODE group_by_keyId -SQL > - - SELECT - countState() AS totalUsage, - keyId - FROM key_verifications - GROUP BY keyId - -TYPE materialized -DATASOURCE mv_key_total_usage -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "keyId" diff --git a/internal/tinybird/pipes/aggregate_unique_visitors_per_hour.pipe b/internal/tinybird/pipes/aggregate_unique_visitors_per_hour.pipe deleted file mode 100644 index faa870edcb..0000000000 --- a/internal/tinybird/pipes/aggregate_unique_visitors_per_hour.pipe +++ /dev/null @@ -1,11 +0,0 @@ -VERSION 1 -NODE pageviews_pipe_3663_0 -SQL > - - SELECT - count() as views, - path, - toStartOfHour(fromUnixTimestamp64Milli(time)) as time - FROM pageviews - group by sessionId, time, path - order by time desc diff --git a/internal/tinybird/pipes/aggregate_verifications_daily.pipe b/internal/tinybird/pipes/aggregate_verifications_daily.pipe deleted file mode 100644 index a694a94091..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_daily.pipe +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE mv__daily_verifications__3 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/pipes/aggregate_verifications_daily_per_api.pipe b/internal/tinybird/pipes/aggregate_verifications_daily_per_api.pipe deleted file mode 100644 index 7fa1cf133e..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_daily_per_api.pipe +++ /dev/null @@ -1,20 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - countState(deniedReason IS NULL) AS success, - countState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - time diff --git a/internal/tinybird/pipes/aggregate_verifications_hourly.pipe b/internal/tinybird/pipes/aggregate_verifications_hourly.pipe deleted file mode 100644 index 6ca081fd1b..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_hourly.pipe +++ /dev/null @@ -1,14 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - countState() AS count, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - time diff --git a/internal/tinybird/pipes/aggregate_verifications_hourly_by_workspace.pipe b/internal/tinybird/pipes/aggregate_verifications_hourly_by_workspace.pipe deleted file mode 100644 index b89936d535..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_hourly_by_workspace.pipe +++ /dev/null @@ -1,18 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - countState() AS verifications, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - time - -TYPE materialized -DATASOURCE mv__verifications_per_workspace_per_hour__v1 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time" diff --git a/internal/tinybird/pipes/aggregate_verifications_monthly.pipe b/internal/tinybird/pipes/aggregate_verifications_monthly.pipe deleted file mode 100644 index 278f280b6c..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_monthly.pipe +++ /dev/null @@ -1,21 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_7424_0 -DESCRIPTION > - Internal Stats - How many legit verifications we do per month - -SQL > - - SELECT - countState() AS verifications, - uniqState(keyId) AS activeKeys, - uniqState(workspaceId) AS activeWorkspaces, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS month - FROM key_verifications - WHERE (workspaceId != 'ws_QXaikfWEdwe1uvYw1QXpSa') AND (workspaceId != 'ws_4GKWTRr6vyEyh4wEAZJsPg') AND (workspaceId != 'ws_GcQBeZ51Z1VPDxX3TSSi9t') - GROUP BY month - -TYPE materialized -DATASOURCE mv__monthly_stats -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(month)" -ENGINE_SORTING_KEY "month" diff --git a/internal/tinybird/pipes/aggregate_verifications_monthly_by_workspace.pipe b/internal/tinybird/pipes/aggregate_verifications_monthly_by_workspace.pipe deleted file mode 100644 index 27d6540ec2..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_monthly_by_workspace.pipe +++ /dev/null @@ -1,28 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE mv__monthly_verifications__v3 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, time, apiId, keyId" diff --git a/internal/tinybird/pipes/aggregate_verifications_per_day.pipe b/internal/tinybird/pipes/aggregate_verifications_per_day.pipe deleted file mode 100644 index 7bee4d575c..0000000000 --- a/internal/tinybird/pipes/aggregate_verifications_per_day.pipe +++ /dev/null @@ -1,23 +0,0 @@ -VERSION 2 -NODE aggregate_daily -SQL > - - SELECT - workspaceId, - apiId, - keyId, - sumSimpleState(ratelimited) AS ratelimited, - countState() AS count, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE mv__daily_usage -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" diff --git a/internal/tinybird/pipes/all_verifications.pipe b/internal/tinybird/pipes/all_verifications.pipe deleted file mode 100644 index 77af106689..0000000000 --- a/internal/tinybird/pipes/all_verifications.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 2 -NODE all_verifications_in_v2 -DESCRIPTION > - undefined - -SQL > - - SELECT countIfMerge(success) + countIfMerge(rateLimited) + countIfMerge(usageExceeded) as totalVerifications - FROM mv__daily_verifications - where - time > toUnixTimestamp64Milli(toDateTime64('2023-09-07 07:00:00', 3)) - and workspaceId != 'ws_QXaikfWEdwe1uvYw1QXpSa' - and workspaceId != 'ws_4GKWTRr6vyEyh4wEAZJsPg' - and workspaceId != 'ws_GcQBeZ51Z1VPDxX3TSSi9t' - - - -NODE all_verifications_in_v1 -SQL > - - SELECT count(*) - FROM key_verifications - where - time < toUnixTimestamp64Milli(toDateTime64('2023-09-07 07:00:00', 3)) - and workspaceId != 'ws_QXaikfWEdwe1uvYw1QXpSa' - and workspaceId != 'ws_4GKWTRr6vyEyh4wEAZJsPg' - and workspaceId != 'ws_GcQBeZ51Z1VPDxX3TSSi9t' - - - -NODE endpoint -SQL > - - (SELECT * FROM all_verifications_in_v1 ) UNION DISTINCT ( SELECT * FROM all_verifications_in_v2 ) diff --git a/internal/tinybird/pipes/billable_verifications.pipe b/internal/tinybird/pipes/billable_verifications.pipe deleted file mode 100644 index 5245ecc3f0..0000000000 --- a/internal/tinybird/pipes/billable_verifications.pipe +++ /dev/null @@ -1,10 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT workspaceId,time from key_verifications where deniedReason is null - -TYPE materialized -DATASOURCE billable_verifications__mv - - diff --git a/internal/tinybird/pipes/copy_latest_verifications.pipe b/internal/tinybird/pipes/copy_latest_verifications.pipe deleted file mode 100644 index 6ab2360737..0000000000 --- a/internal/tinybird/pipes/copy_latest_verifications.pipe +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 2 -NODE copy -SQL > -SELECT - time, - requestedResource, - region, - ipAddress, - userAgent, - workspaceId, - apiId, - keyId, - ifNull(deniedReason, 'VALID') as outcome, - requestBody, - responseBody - FROM key_verifications__v2 - -TYPE materialized -DATASOURCE mv__latest_verifications__v3 diff --git a/internal/tinybird/pipes/count_monthly_verifications.pipe b/internal/tinybird/pipes/count_monthly_verifications.pipe deleted file mode 100644 index 3149773543..0000000000 --- a/internal/tinybird/pipes/count_monthly_verifications.pipe +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -NODE count -SQL > - - SELECT count(*) as count - FROM key_verifications - where (deniedReason is null or deniedReason = '') and time > toUnixTimestamp64Milli(toDateTime64('2023-09-07 07:00:00', 3)) - -TYPE materialized -DATASOURCE mv__total_verifications -ENGINE "MergeTree" -ENGINE_SORTING_KEY "count" diff --git a/internal/tinybird/pipes/count_verifications_daily.pipe b/internal/tinybird/pipes/count_verifications_daily.pipe deleted file mode 100644 index 2691ee23a4..0000000000 --- a/internal/tinybird/pipes/count_verifications_daily.pipe +++ /dev/null @@ -1,8 +0,0 @@ -VERSION 1 -NODE count_daily -DESCRIPTION > - undefined - -SQL > - - SELECT countState() as verifications, toStartOfDay(fromUnixTimestamp64Milli(time)) as time FROM key_verifications group by time diff --git a/internal/tinybird/pipes/distinct_workspaces.pipe b/internal/tinybird/pipes/distinct_workspaces.pipe deleted file mode 100644 index 1f01b20843..0000000000 --- a/internal/tinybird/pipes/distinct_workspaces.pipe +++ /dev/null @@ -1,23 +0,0 @@ -VERSION 1 -TOKEN "scheduled_copy_t_e4f46a3c87904ac8bab2049dd6797ede" READ - -NODE get_total -SQL > - - SELECT count(DISTINCT workspaceId) as workspaces, toStartOfDay(now()) as time - FROM - ( - SELECT workspaceId - from verifications_monthly_mv__v1 - where not startsWith(workspaceId, 'test_') - UNION ALL - SELECT workspaceId - from mv__ratelimits_monthly__v1 - where not startsWith(workspaceId, 'test_') - ) - -TYPE copy -TARGET_DATASOURCE q1_goal_distinct_workspaces -COPY_SCHEDULE 0 0 * * * - - diff --git a/internal/tinybird/pipes/endpoint__active_by_workspace_daily.pipe b/internal/tinybird/pipes/endpoint__active_by_workspace_daily.pipe deleted file mode 100644 index b9ccc56604..0000000000 --- a/internal/tinybird/pipes/endpoint__active_by_workspace_daily.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 1 -TOKEN "endpoint__active_by_workspace_daily__v1_endpoint_read_4337" READ - -NODE mv__verifications_for_analytics_daily__v1_pipe_5183_0 -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM mv__verifications_for_analytics_daily__v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId) }} - {%end if %} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY - - diff --git a/internal/tinybird/pipes/endpoint__active_by_workspace_hourly.pipe b/internal/tinybird/pipes/endpoint__active_by_workspace_hourly.pipe deleted file mode 100644 index 06c746ea9a..0000000000 --- a/internal/tinybird/pipes/endpoint__active_by_workspace_hourly.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 1 -TOKEN "endpoint__active_by_workspace_hourly__v1_endpoint_read_9943" READ - -NODE mv__verifications_for_analytics_hourly__v1_pipe_1084_0 -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM mv__verifications_for_analytics_hourly__v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId) }} - {%end if %} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfHour(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfHour(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 HOUR - - diff --git a/internal/tinybird/pipes/endpoint__active_by_workspace_monthly.pipe b/internal/tinybird/pipes/endpoint__active_by_workspace_monthly.pipe deleted file mode 100644 index 9f08beea69..0000000000 --- a/internal/tinybird/pipes/endpoint__active_by_workspace_monthly.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 1 -TOKEN "endpoint__active_by_workspace_monthly__v1_endpoint_read_3143" READ - -NODE mv__verifications_for_analytics_monthly__v1_pipe_5603_0 -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM mv__verifications_for_analytics_monthly__v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId) }} - {%end if %} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 MONTH - - diff --git a/internal/tinybird/pipes/endpoint__active_by_workspace_weekly.pipe b/internal/tinybird/pipes/endpoint__active_by_workspace_weekly.pipe deleted file mode 100644 index 6c59e25096..0000000000 --- a/internal/tinybird/pipes/endpoint__active_by_workspace_weekly.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 1 -TOKEN "endpoint__active_by_workspace_weekly__v1_endpoint_read_4172" READ - -NODE mv__verifications_for_analytics_weekly_v1_pipe_9812_0 -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM mv__verifications_for_analytics_weekly_v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId) }} - {%end if %} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 WEEK - - diff --git a/internal/tinybird/pipes/endpoint__active_keys_by_workspace.pipe b/internal/tinybird/pipes/endpoint__active_keys_by_workspace.pipe deleted file mode 100644 index 716b96eb6a..0000000000 --- a/internal/tinybird/pipes/endpoint__active_keys_by_workspace.pipe +++ /dev/null @@ -1,13 +0,0 @@ -VERSION 1 -TOKEN "endpoint__active_keys_by_workspace__v1_endpoint_read_5754" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys - FROM mv__monthly_active_keys - where - workspaceId = {{ String(workspaceId, required=True) }} - and time = makeDate({{ Int64(year) }}, {{ Int64(month) }}, 1) - group by time diff --git a/internal/tinybird/pipes/endpoint__all_verifications.pipe b/internal/tinybird/pipes/endpoint__all_verifications.pipe deleted file mode 100644 index e16fbb2da0..0000000000 --- a/internal/tinybird/pipes/endpoint__all_verifications.pipe +++ /dev/null @@ -1,31 +0,0 @@ -VERSION 1 -NODE all_verifications_in_v2 -SQL > - - SELECT count(*) as totalUsage - FROM key_verifications__v2 - where - time > toUnixTimestamp64Milli(toDateTime64('2023-09-07 07:00:00', 3)) - and workspaceId != 'ws_QXaikfWEdwe1uvYw1QXpSa' - and workspaceId != 'ws_4GKWTRr6vyEyh4wEAZJsPg' - and workspaceId != 'ws_GcQBeZ51Z1VPDxX3TSSi9t' - - - -NODE all_verifications_in_v1 -SQL > - - SELECT count(*) as totalUsage - FROM key_verifications__v1 - where - time < toUnixTimestamp64Milli(toDateTime64('2023-09-07 07:00:00', 3)) - and workspaceId != 'ws_QXaikfWEdwe1uvYw1QXpSa' - and workspaceId != 'ws_4GKWTRr6vyEyh4wEAZJsPg' - and workspaceId != 'ws_GcQBeZ51Z1VPDxX3TSSi9t' - - - -NODE endpoint -SQL > - - (SELECT * FROM all_verifications_in_v1 ) UNION DISTINCT ( SELECT * FROM all_verifications_in_v2 ) diff --git a/internal/tinybird/pipes/endpoint__all_verifications__v1.pipe b/internal/tinybird/pipes/endpoint__all_verifications__v1.pipe deleted file mode 100644 index 67465a464e..0000000000 --- a/internal/tinybird/pipes/endpoint__all_verifications__v1.pipe +++ /dev/null @@ -1,7 +0,0 @@ -VERSION 1 -NODE all_verifications -DESCRIPTION > - Sum up all verifications since the beginning of time and until the heat death of the universe - -SQL > - SELECT countMerge(verifications) as verifications FROM mv__monthly_stats__v1 diff --git a/internal/tinybird/pipes/endpoint__analytics_active_by_workspace_all.pipe b/internal/tinybird/pipes/endpoint__analytics_active_by_workspace_all.pipe deleted file mode 100644 index 1958b6440b..0000000000 --- a/internal/tinybird/pipes/endpoint__analytics_active_by_workspace_all.pipe +++ /dev/null @@ -1,26 +0,0 @@ -VERSION 1 -TOKEN "endpoint__analytics_active_by_workspace_all__v1_endpoint_read_3215" READ - -NODE mv__verifications_for_analytics_daily__v1_pipe_7718_0 -SQL > - - % - - SELECT count(DISTINCT keyId) as keys - FROM mv__verifications_for_analytics_daily__v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId ) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - - diff --git a/internal/tinybird/pipes/endpoint__audit_logs.pipe b/internal/tinybird/pipes/endpoint__audit_logs.pipe deleted file mode 100644 index 698d70fff8..0000000000 --- a/internal/tinybird/pipes/endpoint__audit_logs.pipe +++ /dev/null @@ -1,28 +0,0 @@ -TOKEN "endpoint__audit_logs__v1_endpoint_read_1148" READ -VERSION 1 -NODE audit_logs__v1_pipe_7439_0 -SQL > - - % - SELECT * FROM audit_logs__v2 - where workspaceId={{String(workspaceId, required=True)}} - {% if defined(before) %} - and time < {{Int64(before)}} - {%end if %} - {% if defined(after) %} - and time >= {{Int64(after)}} - {%end if %} - {% if defined(events) %} - and event in {{Array(String(events))}} - {%end if %} - {% if defined(actorIds) %} - and actorId in {{Array(String(actorIds))}} - {%end if %} - {% if defined(bucket) %} - and bucket = {{String(bucket)}} - {%end if %} - - ORDER BY time DESC - LIMIT 100 - - diff --git a/internal/tinybird/pipes/endpoint__billing_verifications_per_hour.pipe b/internal/tinybird/pipes/endpoint__billing_verifications_per_hour.pipe deleted file mode 100644 index 7f8db4b532..0000000000 --- a/internal/tinybird/pipes/endpoint__billing_verifications_per_hour.pipe +++ /dev/null @@ -1,9 +0,0 @@ -VERSION 1 - -NODE endpoint -SQL > - % - SELECT workspaceId, countMerge(verifications) as verifications, time - FROM mv__verifications_per_workspace_per_hour__v1 - where time > {{ Int64(since, 0) }} - group by workspaceId, time diff --git a/internal/tinybird/pipes/endpoint__get_active_keys.pipe b/internal/tinybird/pipes/endpoint__get_active_keys.pipe deleted file mode 100644 index d2e45b30a3..0000000000 --- a/internal/tinybird/pipes/endpoint__get_active_keys.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -TOKEN "endpoint__get_active_keys_endpoint_read_5876" READ - -NODE endpoint__get_active_keys_0 -SQL > - - % - SELECT count(DISTINCT keyId) as active, toStartOfDay(time) as time from mv_billing_hourly_active_keys where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - - and time > now() - INTERVAL 30 day - GROUP BY time,keyId diff --git a/internal/tinybird/pipes/endpoint__get_daily_verifications.pipe b/internal/tinybird/pipes/endpoint__get_daily_verifications.pipe deleted file mode 100644 index a28b0780e5..0000000000 --- a/internal/tinybird/pipes/endpoint__get_daily_verifications.pipe +++ /dev/null @@ -1,24 +0,0 @@ -VERSION 1 -TOKEN "endpoint__get_daily_verifications__v1_endpoint_read_0343" READ - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded - FROM mv__daily_verifications - - - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - and time > now() - INTERVAL 30 day - GROUP BY time diff --git a/internal/tinybird/pipes/endpoint__get_last_used.pipe b/internal/tinybird/pipes/endpoint__get_last_used.pipe deleted file mode 100644 index 92253e09a0..0000000000 --- a/internal/tinybird/pipes/endpoint__get_last_used.pipe +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -TOKEN "endpoint__get_last_used__v1_endpoint_read_2224" READ - -NODE mv_keys_last_used_pipe_0365_0 -SQL > - - % - SELECT lastUsed - FROM mv_keys_last_used - where keyId = {{ String(keyId, required=True) }} - ORDER BY lastUsed desc - LIMIT 1 \ No newline at end of file diff --git a/internal/tinybird/pipes/endpoint__get_latest_verifications.pipe b/internal/tinybird/pipes/endpoint__get_latest_verifications.pipe deleted file mode 100644 index 46088fc487..0000000000 --- a/internal/tinybird/pipes/endpoint__get_latest_verifications.pipe +++ /dev/null @@ -1,13 +0,0 @@ -VERSION 3 -TOKEN "endpoint__get_latest_verifications_endpoint_read_5933" READ - -NODE endpoint -SQL > -% - SELECT time,requestedResource, region, ipAddress, outcome, userAgent - FROM mv__latest_verifications__v3 - where - workspaceId = {{String(workspaceId, required=True)}} - and apiId = {{String(apiId, required=True)}} - and keyId = {{String(keyId, required=True)}} - order by time desc limit 20 diff --git a/internal/tinybird/pipes/endpoint__get_total_usage_for_key.pipe b/internal/tinybird/pipes/endpoint__get_total_usage_for_key.pipe deleted file mode 100644 index 8b7e597bea..0000000000 --- a/internal/tinybird/pipes/endpoint__get_total_usage_for_key.pipe +++ /dev/null @@ -1,8 +0,0 @@ -VERSION 1 -TOKEN "endpoint__get_total_usage_for_key__v1_endpoint_read_6359" READ - -NODE mv_key_total_usage_pipe_5275_0 -SQL > - - % - SELECT countMerge(totalUsage) as totalUsage FROM mv_key_total_usage where keyId = {{String(keyId, required=True)}} diff --git a/internal/tinybird/pipes/endpoint__monthly_stats.pipe b/internal/tinybird/pipes/endpoint__monthly_stats.pipe deleted file mode 100644 index 7413f2585d..0000000000 --- a/internal/tinybird/pipes/endpoint__monthly_stats.pipe +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -TOKEN "mv__monthly_stats_pipe_5169_endpoint_read_4286" READ - -NODE endpoint -SQL > - - SELECT - countMerge(verifications) as verifications, - uniqMerge(activeKeys) as activeKeys, - uniqMerge(activeWorkspaces) as activeWorkspaces, - month - FROM mv__monthly_stats group by month diff --git a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_daily.pipe b/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_daily.pipe deleted file mode 100644 index df95c6ce99..0000000000 --- a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_daily.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 1 -TOKEN "endpoint__verifications_analytics_by_workspaceId_daily__v1_endpoint_read_2232" READ - -NODE mv__verifications_for_analytics_daily__v1_pipe_5513_0 -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded - FROM mv__verifications_for_analytics_daily__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId ) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY - - diff --git a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_hourly.pipe b/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_hourly.pipe deleted file mode 100644 index f29cd0a9f0..0000000000 --- a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspaceId_hourly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 1 -TOKEN "endpoint__verifications_analytics_by_workspaceId_hourly__v1_endpoint_read_7199" READ - -NODE mv__verifications_for_analytics_hourly__v1_pipe_8073_0 -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded - FROM mv__verifications_for_analytics_hourly__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId ) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfHour(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfHour(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 HOUR - - diff --git a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_monthly.pipe b/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_monthly.pipe deleted file mode 100644 index 564a4c7494..0000000000 --- a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_monthly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 1 -TOKEN "endpoint__verifications_analytics_by_workspace_monthly__v1_endpoint_read_4403" READ - -NODE mv__verifications_for_analytics_monthly__v1_pipe_6776_0 -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded - FROM mv__verifications_for_analytics_monthly__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId ) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 Month - - diff --git a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_weekly.pipe b/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_weekly.pipe deleted file mode 100644 index fc483937c0..0000000000 --- a/internal/tinybird/pipes/endpoint__verifications_analytics_by_workspace_weekly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 1 -TOKEN "endpoint__verifications_analytics_by_workspace_weekly__v1_endpoint_read_8616" READ - -NODE mv__verifications_for_analytics_weekly_v1_pipe_8517_0 -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded - FROM mv__verifications_for_analytics_weekly_v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(ownerId) %} - and ownerId ={{ String(ownerId ) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 WEEK - - diff --git a/internal/tinybird/pipes/endpoint__verifications_by_workspace.pipe b/internal/tinybird/pipes/endpoint__verifications_by_workspace.pipe deleted file mode 100644 index f619e5e73f..0000000000 --- a/internal/tinybird/pipes/endpoint__verifications_by_workspace.pipe +++ /dev/null @@ -1,17 +0,0 @@ -VERSION 1 -TOKEN "endpoint__verifications_by_workspace__v1_endpoint_read_7956" READ - -NODE endpoint -SQL > - - % - SELECT - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - time - FROM mv__monthly_verifications - where - workspaceId = {{ String(workspaceId, required=True) }} - and time = makeDate({{ Int64(year) }}, {{ Int64(month) }}, 1) - group by time diff --git a/internal/tinybird/pipes/endpoint_billing_get_active_keys_usage.pipe b/internal/tinybird/pipes/endpoint_billing_get_active_keys_usage.pipe deleted file mode 100644 index 5351753c20..0000000000 --- a/internal/tinybird/pipes/endpoint_billing_get_active_keys_usage.pipe +++ /dev/null @@ -1,18 +0,0 @@ -VERSION 2 -TOKEN "endpoint__billing_get_active_keys_usage__v1_endpoint_read_7945" READ - -TOKEN "endpoint_billing_get_active_keys_usage__v2_endpoint_read_9814" READ - -NODE get_usage -SQL > - - % - SELECT count(DISTINCT keyId) as usage - FROM mv_billing_hourly_active_keys - where - workspaceId = {{ String(workspaceId, required=True) }} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - and time >= toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start, required=True) }})) - and time <= toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end, required=True) }}))+INTERVAL 1 DAY diff --git a/internal/tinybird/pipes/endpoint_billing_get_verifications_per_workspace_per_hour.pipe b/internal/tinybird/pipes/endpoint_billing_get_verifications_per_workspace_per_hour.pipe deleted file mode 100644 index be73a47d73..0000000000 --- a/internal/tinybird/pipes/endpoint_billing_get_verifications_per_workspace_per_hour.pipe +++ /dev/null @@ -1,11 +0,0 @@ -VERSION 1 -TOKEN "endpoint_billing_get_verifications_per_workspace_per_day__v1_endpoint_read_5992" READ - -NODE get_usage -SQL > - - % - SELECT countMerge(count) as usage, workspaceId, time - FROM mv_billing_hourly_verifications - where time > now() - INTERVAL 32 DAY and time <= now() - group by workspaceId, time diff --git a/internal/tinybird/pipes/endpoint_get_usage.pipe b/internal/tinybird/pipes/endpoint_get_usage.pipe deleted file mode 100644 index 3e25ce015e..0000000000 --- a/internal/tinybird/pipes/endpoint_get_usage.pipe +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 3 -TOKEN "endpoint_get_usage__v3_endpoint_read_2506" READ - -NODE get_usage -SQL > - - % - SELECT time, countMerge(count) as usage, sum(ratelimited) as ratelimited - FROM mv__daily_usage - where - workspaceId = {{ String(workspaceId, required=True )}} - {% if defined(apiId) %} - and apiId ={{ String(apiId) }} - {%end if %} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - and time > now() - INTERVAL 30 day - GROUP BY time diff --git a/internal/tinybird/pipes/get__verifications_for_analytics_month.pipe b/internal/tinybird/pipes/get__verifications_for_analytics_month.pipe deleted file mode 100644 index 18abcfde18..0000000000 --- a/internal/tinybird/pipes/get__verifications_for_analytics_month.pipe +++ /dev/null @@ -1,25 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_8204_0 -SQL > - - SELECT - workspaceId, - apiId, - ifNull(ownerId, '') as ownerId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - ownerId, - keyId, - time - -TYPE materialized -DATASOURCE mv__verifications_for_analytics_monthly__v1 - - diff --git a/internal/tinybird/pipes/get__verifications_for_analytics_weekly.pipe b/internal/tinybird/pipes/get__verifications_for_analytics_weekly.pipe deleted file mode 100644 index 7bfc0ffe03..0000000000 --- a/internal/tinybird/pipes/get__verifications_for_analytics_weekly.pipe +++ /dev/null @@ -1,25 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_0701_0 -SQL > - - SELECT - workspaceId, - apiId, - ifNull(ownerId, '') as ownerId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - toStartOfWeek(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - ownerId, - keyId, - time - -TYPE materialized -DATASOURCE mv__verifications_for_analytics_weekly_v1 - - diff --git a/internal/tinybird/pipes/get_active_keys.pipe b/internal/tinybird/pipes/get_active_keys.pipe deleted file mode 100644 index 0564a14507..0000000000 --- a/internal/tinybird/pipes/get_active_keys.pipe +++ /dev/null @@ -1,18 +0,0 @@ -VERSION 1 -TOKEN "get_active_keys__v1_endpoint_read_3584" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys - FROM verifications_hourly_mv - where - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} diff --git a/internal/tinybird/pipes/get_active_keys_daily.pipe b/internal/tinybird/pipes/get_active_keys_daily.pipe deleted file mode 100644 index 84755fea7e..0000000000 --- a/internal/tinybird/pipes/get_active_keys_daily.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -TOKEN "get_activeKeys_daily__v1_endpoint_read_7257" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM verifications_daily_mv - where - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time diff --git a/internal/tinybird/pipes/get_active_keys_hourly.pipe b/internal/tinybird/pipes/get_active_keys_hourly.pipe deleted file mode 100644 index 1095365524..0000000000 --- a/internal/tinybird/pipes/get_active_keys_hourly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -TOKEN "get_active_keys_hourly__v1_endpoint_read_3914" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM verifications_hourly_mv - where - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time diff --git a/internal/tinybird/pipes/get_active_keys_monthly.pipe b/internal/tinybird/pipes/get_active_keys_monthly.pipe deleted file mode 100644 index 727d85a14a..0000000000 --- a/internal/tinybird/pipes/get_active_keys_monthly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -TOKEN "get_active_keys_monthly__v1_endpoint_read_5863" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM verifications_monthly_mv - where - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time diff --git a/internal/tinybird/pipes/get_active_keys_weekly.pipe b/internal/tinybird/pipes/get_active_keys_weekly.pipe deleted file mode 100644 index da7b1c6783..0000000000 --- a/internal/tinybird/pipes/get_active_keys_weekly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -TOKEN "get_active_keys_weekly__v1_endpoint_read_0608" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as keys, time - FROM verifications_weekly_mv - where - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - group by time diff --git a/internal/tinybird/pipes/get_billable_keys.pipe b/internal/tinybird/pipes/get_billable_keys.pipe deleted file mode 100644 index 91ae24ba65..0000000000 --- a/internal/tinybird/pipes/get_billable_keys.pipe +++ /dev/null @@ -1,21 +0,0 @@ -VERSION 1 -TOKEN "get_billable_keys__v1_endpoint_read_4636" READ - -NODE endpoint -SQL > - - % - SELECT count(DISTINCT keyId) as activeKeys - from active_keys_hourly_mv - where - workspaceId - = {{ - String( - workspaceId, - required=True, - ) - }} - and time >= fromUnixTimestamp64Milli({{ Int64(start, required=True) }}) - and time < fromUnixTimestamp64Milli({{ Int64(end, required=True) }}) - - diff --git a/internal/tinybird/pipes/get_billable_verifications.pipe b/internal/tinybird/pipes/get_billable_verifications.pipe deleted file mode 100644 index 2a1efd71fc..0000000000 --- a/internal/tinybird/pipes/get_billable_verifications.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -TOKEN "get_billable_verifications__v1_endpoint_read_9391" READ - -NODE endpoint -SQL > - - % - SELECT countIfMerge(success) as verifications - FROM verifications_hourly - where - workspaceId = {{ String(workspaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{Int64(start, required=True)}}) - and time < fromUnixTimestamp64Milli({{Int64(end, required=True)}}) - - diff --git a/internal/tinybird/pipes/get_q1_goal_distinct_workspaces.pipe b/internal/tinybird/pipes/get_q1_goal_distinct_workspaces.pipe deleted file mode 100644 index a627ae11e6..0000000000 --- a/internal/tinybird/pipes/get_q1_goal_distinct_workspaces.pipe +++ /dev/null @@ -1,12 +0,0 @@ -VERSION 1 -TOKEN "get_q1_goal_distinct_workspaces__v1_endpoint_read_1745" READ - -NODE endpoint -SQL > - - SELECT * - FROM q1_goal_distinct_workspaces - ORDER BY time ASC - WITH FILL FROM toStartOfDay(toDateTime('2023-12-19')) STEP INTERVAL 1 DAY - - diff --git a/internal/tinybird/pipes/get_ratelimit_events.pipe b/internal/tinybird/pipes/get_ratelimit_events.pipe deleted file mode 100644 index 492af6d125..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_events.pipe +++ /dev/null @@ -1,33 +0,0 @@ -TOKEN "get_ratelimit_events__v1_endpoint_read_4484" READ -VERSION 1 -NODE endpoint -SQL > - - % - SELECT * FROM ratelimits__v2 - where workspaceId={{String(workspaceId, required=True)}} - and namespaceId={{String(namespaceId, required=True)}} - {% if defined(before) %} - and time < {{Int64(before)}} - {%end if %} - {% if defined(after) %} - and time >= {{Int64(after)}} - {%end if %} - {% if defined(success) %} - and success = {{Boolean(success)}} - {%end if %} - - {% if defined(identifier) %} - and multiSearchAny(ratelimits__v2.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(country) %} - and country in {{Array(String(country))}} - {%end if %} - {% if defined(ipAddress) %} - and ipAddress in {{Array(String(ipAddress))}} - {%end if %} - - ORDER BY time DESC - LIMIT {{Int64(limit, 100, description="How many rows to return", required=False)}} - - diff --git a/internal/tinybird/pipes/get_ratelimit_identifiers_daily.pipe b/internal/tinybird/pipes/get_ratelimit_identifiers_daily.pipe deleted file mode 100644 index 5ecd1cd05f..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_identifiers_daily.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE endpoint -SQL > - % - SELECT identifier, countMerge(success) as success, countMerge(total) as total - FROM mv__ratelimits_daily__v1 - where - workspaceId = {{ String(workspaceId, required=True) }} - and namespaceId = {{ String(namespaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - and time <= fromUnixTimestamp64Milli({{ Int64(end) }}) - GROUP BY identifier - ORDER BY {{ column(orderBy, 'total') }} DESC - - diff --git a/internal/tinybird/pipes/get_ratelimit_identifiers_hourly.pipe b/internal/tinybird/pipes/get_ratelimit_identifiers_hourly.pipe deleted file mode 100644 index ce25e208d5..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_identifiers_hourly.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE endpoint -SQL > - % - SELECT identifier, countMerge(success) as success, countMerge(total) as total - FROM mv__ratelimits_hourly__v1 - where - workspaceId = {{ String(workspaceId, required=True) }} - and namespaceId = {{ String(namespaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - and time <= fromUnixTimestamp64Milli({{ Int64(end) }}) - GROUP BY identifier - ORDER BY {{ column(orderBy, 'total') }} DESC - - diff --git a/internal/tinybird/pipes/get_ratelimit_identifiers_minutely.pipe b/internal/tinybird/pipes/get_ratelimit_identifiers_minutely.pipe deleted file mode 100644 index 67abe94744..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_identifiers_minutely.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE endpoint -SQL > - % - SELECT identifier, countMerge(success) as success, countMerge(total) as total - FROM mv__ratelimits_minutely__v1 - where - workspaceId = {{ String(workspaceId, required=True) }} - and namespaceId = {{ String(namespaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - and time <= fromUnixTimestamp64Milli({{ Int64(end) }}) - GROUP BY identifier - ORDER BY {{ column(orderBy, 'total') }} DESC - - diff --git a/internal/tinybird/pipes/get_ratelimit_identifiers_monthly.pipe b/internal/tinybird/pipes/get_ratelimit_identifiers_monthly.pipe deleted file mode 100644 index 8457c02459..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_identifiers_monthly.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE endpoint -SQL > - % - SELECT identifier, countMerge(success) as success, countMerge(total) as total - FROM mv__ratelimits_monthly__v1 - where - workspaceId = {{ String(workspaceId, required=True) }} - and namespaceId = {{ String(namespaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - and time <= fromUnixTimestamp64Milli({{ Int64(end) }}) - GROUP BY identifier - ORDER BY {{ column(orderBy, 'total') }} DESC - - diff --git a/internal/tinybird/pipes/get_ratelimit_identifiers_weekly.pipe b/internal/tinybird/pipes/get_ratelimit_identifiers_weekly.pipe deleted file mode 100644 index e6bbf763f1..0000000000 --- a/internal/tinybird/pipes/get_ratelimit_identifiers_weekly.pipe +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 1 -NODE endpoint -SQL > - % - SELECT identifier, countMerge(success) as success, countMerge(total) as total - FROM mv__ratelimits_weekly__v1 - where - workspaceId = {{ String(workspaceId, required=True) }} - and namespaceId = {{ String(namespaceId, required=True) }} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - and time <= fromUnixTimestamp64Milli({{ Int64(end) }}) - GROUP BY identifier - ORDER BY {{ column(orderBy, 'total') }} DESC - - diff --git a/internal/tinybird/pipes/get_ratelimits_daily.pipe b/internal/tinybird/pipes/get_ratelimits_daily.pipe deleted file mode 100644 index 4b97ca9e1b..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_daily.pipe +++ /dev/null @@ -1,35 +0,0 @@ -TOKEN "get_ratelimits_daily__v1_endpoint_read_5562" READ -VERSION 1 - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countMerge(total) as total - FROM mv__ratelimits_daily__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - and namespaceId ={{ String(namespaceId) }} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_daily__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY - - diff --git a/internal/tinybird/pipes/get_ratelimits_hourly.pipe b/internal/tinybird/pipes/get_ratelimits_hourly.pipe deleted file mode 100644 index fa73c2ab1f..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_hourly.pipe +++ /dev/null @@ -1,35 +0,0 @@ -TOKEN "mv__ratelimits_hourly__v1_pipe_9552_endpoint_read_2686" READ -VERSION 1 - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countMerge(total) as total - FROM mv__ratelimits_hourly__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - and namespaceId ={{ String(namespaceId) }} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_hourly__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfHour(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfHour(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 HOUR - - diff --git a/internal/tinybird/pipes/get_ratelimits_last_used.pipe b/internal/tinybird/pipes/get_ratelimits_last_used.pipe deleted file mode 100644 index 96ee2d002a..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_last_used.pipe +++ /dev/null @@ -1,16 +0,0 @@ -TOKEN "get_ratelimits_last_used__v1_endpoint_read_4376" READ -VERSION 1 -NODE endpoint -SQL > - - % - SELECT lastUsed - FROM mv__ratelimits_last_used__v1 - where workspaceId={{String(workspaceId, required=True)}} - and namespaceId={{String(namespaceId, required=True)}} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_last_used__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - order by lastUsed desc limit 1 - - diff --git a/internal/tinybird/pipes/get_ratelimits_minutely.pipe b/internal/tinybird/pipes/get_ratelimits_minutely.pipe deleted file mode 100644 index 1a341850cb..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_minutely.pipe +++ /dev/null @@ -1,31 +0,0 @@ -VERSION 1 - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countMerge(total) as total - FROM mv__ratelimits_minutely__v1 - where - workspaceId = {{ String(workspaceId, required=True )}} - and namespaceId ={{ String(namespaceId) }} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_minutely__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfMinute(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfMinute(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 MINUTE \ No newline at end of file diff --git a/internal/tinybird/pipes/get_ratelimits_monthly.pipe b/internal/tinybird/pipes/get_ratelimits_monthly.pipe deleted file mode 100644 index 491d18349e..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_monthly.pipe +++ /dev/null @@ -1,35 +0,0 @@ -TOKEN "get_ratelimits_monthly__v1_endpoint_read_7071" READ -VERSION 1 - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countMerge(total) as total - FROM mv__ratelimits_monthly__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - and namespaceId ={{ String(namespaceId) }} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_monthly__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 MONTH - - diff --git a/internal/tinybird/pipes/get_ratelimits_weekly.pipe b/internal/tinybird/pipes/get_ratelimits_weekly.pipe deleted file mode 100644 index 23ca385d32..0000000000 --- a/internal/tinybird/pipes/get_ratelimits_weekly.pipe +++ /dev/null @@ -1,34 +0,0 @@ -VERSION 1 - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countMerge(total) as total - FROM mv__ratelimits_weekly__v1 - - where - workspaceId = {{ String(workspaceId, required=True )}} - and namespaceId ={{ String(namespaceId) }} - {% if defined(identifier) %} - and multiSearchAny(mv__ratelimits_weekly__v1.identifier, {{Array(String(identifier))}}) > 0 - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 WEEK - - diff --git a/internal/tinybird/pipes/get_semantic_caches_daily.pipe b/internal/tinybird/pipes/get_semantic_caches_daily.pipe deleted file mode 100644 index 41b0a0c9ea..0000000000 --- a/internal/tinybird/pipes/get_semantic_caches_daily.pipe +++ /dev/null @@ -1,39 +0,0 @@ -VERSION 4 - -NODE endpoint -SQL > - - % - SELECT time, - model, - countIfMerge(hit) as hit, - countMerge(total) as total, - avgMerge(avgServiceLatency) as avgServiceLatency, - avgMerge(avgEmbeddingsLatency) as avgEmbeddingsLatency, - avgMerge(avgVectorizeLatency) as avgVectorizeLatency, - avgMerge(avgInferenceLatency) as avgInferenceLatency, - avgMerge(avgCacheLatency) as avgCacheLatency, - avgMerge(avgTokens) as avgTokens, - sumMerge(sumTokens) as sumTokens, - sumIfMerge(cachedTokens) as cachedTokens - FROM mv__semantic_cache_daily__v4 - where - workspaceId = {{ String(workspaceId, required=True )}} - and gatewayId ={{ String(gatewayId, required=True ) }} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY model, time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY - - diff --git a/internal/tinybird/pipes/get_semantic_caches_hourly.pipe b/internal/tinybird/pipes/get_semantic_caches_hourly.pipe deleted file mode 100644 index 5749fb828f..0000000000 --- a/internal/tinybird/pipes/get_semantic_caches_hourly.pipe +++ /dev/null @@ -1,39 +0,0 @@ -VERSION 4 - -NODE endpoint -SQL > - - % - SELECT time, - model, - countIfMerge(hit) as hit, - countMerge(total) as total, - avgMerge(avgServiceLatency) as avgServiceLatency, - avgMerge(avgEmbeddingsLatency) as avgEmbeddingsLatency, - avgMerge(avgVectorizeLatency) as avgVectorizeLatency, - avgMerge(avgInferenceLatency) as avgInferenceLatency, - avgMerge(avgCacheLatency) as avgCacheLatency, - avgMerge(avgTokens) as avgTokens, - sumMerge(sumTokens) as sumTokens, - sumIfMerge(cachedTokens) as cachedTokens - FROM mv__semantic_cache_hourly__v4 - where - workspaceId = {{ String(workspaceId, required=True )}} - and gatewayId ={{ String(gatewayId, required=True ) }} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY model, time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 HOUR - - diff --git a/internal/tinybird/pipes/get_verifications_daily.pipe b/internal/tinybird/pipes/get_verifications_daily.pipe deleted file mode 100644 index 21e37b2a73..0000000000 --- a/internal/tinybird/pipes/get_verifications_daily.pipe +++ /dev/null @@ -1,37 +0,0 @@ -VERSION 2 -TOKEN "get_verifications_daily__v1_endpoint_read_9561" READ - -NODE endpoint -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - countIfMerge(disabled) as disabled, - countIfMerge(insufficientPermissions) as insufficientPermissions, - countIfMerge(forbidden) as forbidden, - countIfMerge(expired) as expired - FROM verifications_daily_mv - WHERE - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY diff --git a/internal/tinybird/pipes/get_verifications_for_analytics_daily.pipe b/internal/tinybird/pipes/get_verifications_for_analytics_daily.pipe deleted file mode 100644 index 598e7af2d5..0000000000 --- a/internal/tinybird/pipes/get_verifications_for_analytics_daily.pipe +++ /dev/null @@ -1,25 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_9516_0 -SQL > - - SELECT - workspaceId, - apiId, - ifNull(ownerId, '') as ownerId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - ownerId, - keyId, - time - -TYPE materialized -DATASOURCE mv__verifications_for_analytics_daily__v1 - - diff --git a/internal/tinybird/pipes/get_verifications_for_analytics_hourly.pipe b/internal/tinybird/pipes/get_verifications_for_analytics_hourly.pipe deleted file mode 100644 index fd91d7c763..0000000000 --- a/internal/tinybird/pipes/get_verifications_for_analytics_hourly.pipe +++ /dev/null @@ -1,25 +0,0 @@ -VERSION 1 -NODE key_verifications_pipe_4229_0 -SQL > - - SELECT - workspaceId, - apiId, - ifNull(ownerId, '') as ownerId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications - GROUP BY - workspaceId, - apiId, - ownerId, - keyId, - time - -TYPE materialized -DATASOURCE mv__verifications_for_analytics_hourly__v1 - - diff --git a/internal/tinybird/pipes/get_verifications_hourly.pipe b/internal/tinybird/pipes/get_verifications_hourly.pipe deleted file mode 100644 index 303ebdf1e4..0000000000 --- a/internal/tinybird/pipes/get_verifications_hourly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 2 -TOKEN "get_verifications_hourly__v1_endpoint_read_0976" READ - -NODE endpoint -SQL > - - % - SELECT - time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - countIfMerge(disabled) as disabled, - countIfMerge(insufficientPermissions) as insufficientPermissions, - countIfMerge(forbidden) as forbidden, - countIfMerge(expired) as expired - FROM verifications_hourly_mv - WHERE - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfHour(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfHour(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 HOUR diff --git a/internal/tinybird/pipes/get_verifications_monthly.pipe b/internal/tinybird/pipes/get_verifications_monthly.pipe deleted file mode 100644 index 205f4ef8b5..0000000000 --- a/internal/tinybird/pipes/get_verifications_monthly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 2 -TOKEN "get_verifications_monthly__v1_endpoint_read_2525" READ - -NODE endpoint -SQL > - - % - SELECT - time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - countIfMerge(disabled) as disabled, - countIfMerge(insufficientPermissions) as insufficientPermissions, - countIfMerge(forbidden) as forbidden, - countIfMerge(expired) as expired - FROM verifications_monthly_mv - WHERE - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfMonth(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 MONTH diff --git a/internal/tinybird/pipes/get_verifications_ownerid_hourly_v1.pipe b/internal/tinybird/pipes/get_verifications_ownerid_hourly_v1.pipe deleted file mode 100644 index 30590d772c..0000000000 --- a/internal/tinybird/pipes/get_verifications_ownerid_hourly_v1.pipe +++ /dev/null @@ -1,40 +0,0 @@ -VERSION 2 -DESCRIPTION > - Gets verifications based on OwnerId for analytics page - - -NODE get_verifications_ownerid_hourly_v1_0 -SQL > - - % - SELECT time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - countIfMerge(disabled) as disabled, - countIfMerge(insufficientPermissions) as insufficientPermissions, - countIfMerge(forbidden) as forbidden, - countIfMerge(expired) as expired - FROM verifications_daily_mv - - where - workspaceId = {{ String(workspaceId, required=True )}} - and ownerId ={{ String(ownerId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfDay(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfDay(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 DAY diff --git a/internal/tinybird/pipes/get_verifications_weekly.pipe b/internal/tinybird/pipes/get_verifications_weekly.pipe deleted file mode 100644 index 5b4cbe2a52..0000000000 --- a/internal/tinybird/pipes/get_verifications_weekly.pipe +++ /dev/null @@ -1,38 +0,0 @@ -VERSION 2 -TOKEN "get_verifications_weekly__v1_endpoint_read_1956" READ - -NODE endpoint -SQL > - - % - SELECT - time, - countIfMerge(success) as success, - countIfMerge(rateLimited) as rateLimited, - countIfMerge(usageExceeded) as usageExceeded, - countIfMerge(disabled) as disabled, - countIfMerge(insufficientPermissions) as insufficientPermissions, - countIfMerge(forbidden) as forbidden, - countIfMerge(expired) as expired - FROM verifications_weekly_mv - WHERE - workspaceId = {{ String(workspaceId, required=True )}} - and apiId ={{ String(apiId) }} - {% if defined(keyId) %} - and keyId ={{ String(keyId) }} - {%end if %} - {% if defined(start)%} - and time >= fromUnixTimestamp64Milli({{ Int64(start) }}) - {%end if %} - {% if defined(end)%} - and time <= fromUnixTimestamp64Milli({{ Int64(end)}}) - {%end if %} - GROUP BY time - ORDER BY time ASC - WITH FILL {% if defined(start)%} - FROM toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(start) }})) - {%end if %} - {% if defined(end)%} - TO toStartOfWeek(fromUnixTimestamp64Milli({{ Int64(end) }})) - {%end if %} - STEP INTERVAL 1 WEEK diff --git a/internal/tinybird/pipes/monthly_active_workspaces.pipe b/internal/tinybird/pipes/monthly_active_workspaces.pipe deleted file mode 100644 index a0772ebe62..0000000000 --- a/internal/tinybird/pipes/monthly_active_workspaces.pipe +++ /dev/null @@ -1,21 +0,0 @@ -VERSION 1 -NODE all_workspaces -SQL > - - SELECT workspaceId, time - FROM verifications_monthly_mv__v1 UNION ALL - SELECT workspaceId, time - FROM mv__ratelimits_monthly__v1 - - - -NODE endpoint -SQL > - - SELECT count(DISTINCT workspaceId) as workspaces, time - FROM all_workspaces - where not startsWith(workspaceId, 'test_') - group by time - order by time asc - - diff --git a/internal/tinybird/pipes/mv__daily_verifications.pipe b/internal/tinybird/pipes/mv__daily_verifications.pipe deleted file mode 100644 index 0055a81726..0000000000 --- a/internal/tinybird/pipes/mv__daily_verifications.pipe +++ /dev/null @@ -1,5 +0,0 @@ -VERSION 2 -NODE mv__daily_verifications_pipe_5598_0 -SQL > - - SELECT apiId,countMerge(success), countMerge(rateLimited), countMerge(usageExceeded) FROM mv__daily_verifications where apiId='api_XsjtsbGw8fG6XyRrX8SnQ2' group by apiId diff --git a/internal/tinybird/pipes/ratelimit_last_used.pipe b/internal/tinybird/pipes/ratelimit_last_used.pipe deleted file mode 100644 index 25911d136a..0000000000 --- a/internal/tinybird/pipes/ratelimit_last_used.pipe +++ /dev/null @@ -1,19 +0,0 @@ -VERSION 1 -NODE key_verifications__v2_pipe_1411_0 -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - maxSimpleState(time) AS lastUsed - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier - -TYPE materialized -DATASOURCE mv__ratelimits_last_used__v1 - - diff --git a/internal/tinybird/pipes/ratelimits_daily.pipe b/internal/tinybird/pipes/ratelimits_daily.pipe deleted file mode 100644 index 78575454af..0000000000 --- a/internal/tinybird/pipes/ratelimits_daily.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - countState(*) AS total, - countIfState(success = 1) AS success, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier, - time - -TYPE materialized -DATASOURCE mv__ratelimits_daily__v1 - - diff --git a/internal/tinybird/pipes/ratelimits_hourly.pipe b/internal/tinybird/pipes/ratelimits_hourly.pipe deleted file mode 100644 index 509e5caf27..0000000000 --- a/internal/tinybird/pipes/ratelimits_hourly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - countState(*) AS total, - countIfState(success = 1) AS success, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier, - time - -TYPE materialized -DATASOURCE mv__ratelimits_hourly__v1 - - diff --git a/internal/tinybird/pipes/ratelimits_minutely.pipe b/internal/tinybird/pipes/ratelimits_minutely.pipe deleted file mode 100644 index f1e7c7a610..0000000000 --- a/internal/tinybird/pipes/ratelimits_minutely.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - countState(*) AS total, - countIfState(success = 1) AS success, - toStartOfMinute(fromUnixTimestamp64Milli(time)) AS time - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier, - time - -TYPE materialized -DATASOURCE mv__ratelimits_minutely__v1 - - diff --git a/internal/tinybird/pipes/ratelimits_monthly.pipe b/internal/tinybird/pipes/ratelimits_monthly.pipe deleted file mode 100644 index 4f14ed1aa5..0000000000 --- a/internal/tinybird/pipes/ratelimits_monthly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - countState(*) AS total, - countIfState(success = 1) AS success, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier, - time - -TYPE materialized -DATASOURCE mv__ratelimits_monthly__v1 - - diff --git a/internal/tinybird/pipes/ratelimits_weekly.pipe b/internal/tinybird/pipes/ratelimits_weekly.pipe deleted file mode 100644 index 58ac5d176b..0000000000 --- a/internal/tinybird/pipes/ratelimits_weekly.pipe +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 1 -NODE aggregate -SQL > - - SELECT - workspaceId, - namespaceId, - identifier, - countState(*) AS total, - countIfState(success = 1) AS success, - toStartOfWeek(fromUnixTimestamp64Milli(time)) AS time - FROM ratelimits__v2 - GROUP BY - workspaceId, - namespaceId, - identifier, - time - -TYPE materialized -DATASOURCE mv__ratelimits_weekly__v1 - - diff --git a/internal/tinybird/pipes/semantic_cache_daily.pipe b/internal/tinybird/pipes/semantic_cache_daily.pipe deleted file mode 100644 index 2314e536ca..0000000000 --- a/internal/tinybird/pipes/semantic_cache_daily.pipe +++ /dev/null @@ -1,30 +0,0 @@ -VERSION 4 -NODE aggregate -SQL > - - SELECT - model, - countIfState(cache) AS hit, - countState(*) AS total, - avgState(serviceLatency) AS avgServiceLatency, - avgState(embeddingsLatency) AS avgEmbeddingsLatency, - avgState(vectorizeLatency) AS avgVectorizeLatency, - avgState(inferenceLatency) AS avgInferenceLatency, - avgState(cacheLatency) AS avgCacheLatency, - avgState(tokens) AS avgTokens, - sumState(tokens) AS sumTokens, - sumIfState(tokens, cache) AS cachedTokens, - workspaceId, - gatewayId, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM semantic_cache__v1 - GROUP BY - workspaceId, - gatewayId, - model, - time - -TYPE materialized -DATASOURCE mv__semantic_cache_daily__v4 - - diff --git a/internal/tinybird/pipes/semantic_cache_hourly.pipe b/internal/tinybird/pipes/semantic_cache_hourly.pipe deleted file mode 100644 index 743fdff693..0000000000 --- a/internal/tinybird/pipes/semantic_cache_hourly.pipe +++ /dev/null @@ -1,30 +0,0 @@ -VERSION 4 -NODE aggregate -SQL > - - SELECT - model, - countIfState(cache) AS hit, - countState(*) AS total, - avgState(serviceLatency) AS avgServiceLatency, - avgState(embeddingsLatency) AS avgEmbeddingsLatency, - avgState(vectorizeLatency) AS avgVectorizeLatency, - avgState(inferenceLatency) AS avgInferenceLatency, - avgState(cacheLatency) AS avgCacheLatency, - avgState(tokens) AS avgTokens, - sumState(tokens) AS sumTokens, - sumIfState(tokens, cache) AS cachedTokens, - workspaceId, - gatewayId, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM semantic_cache__v1 - GROUP BY - workspaceId, - gatewayId, - model, - time - -TYPE materialized -DATASOURCE mv__semantic_cache_hourly__v4 - - diff --git a/internal/tinybird/pipes/verifications_daily.pipe b/internal/tinybird/pipes/verifications_daily.pipe deleted file mode 100644 index bde1f9068d..0000000000 --- a/internal/tinybird/pipes/verifications_daily.pipe +++ /dev/null @@ -1,29 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfDay(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE verifications_daily_mv__v2 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, time, keyId" -ENGINE_TTL "time + toIntervalDay(90)" diff --git a/internal/tinybird/pipes/verifications_hourly.pipe b/internal/tinybird/pipes/verifications_hourly.pipe deleted file mode 100644 index bcd9a718e5..0000000000 --- a/internal/tinybird/pipes/verifications_hourly.pipe +++ /dev/null @@ -1,29 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfHour(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE verifications_hourly_mv__v2 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, time, keyId" -ENGINE_TTL "time + toIntervalDay(30)" diff --git a/internal/tinybird/pipes/verifications_monthly.pipe b/internal/tinybird/pipes/verifications_monthly.pipe deleted file mode 100644 index d4d583a5af..0000000000 --- a/internal/tinybird/pipes/verifications_monthly.pipe +++ /dev/null @@ -1,29 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfMonth(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE verifications_monthly_mv__v2 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" -ENGINE_TTL "time + toIntervalDay(365)" diff --git a/internal/tinybird/pipes/verifications_weekly.pipe b/internal/tinybird/pipes/verifications_weekly.pipe deleted file mode 100644 index 5e233506c1..0000000000 --- a/internal/tinybird/pipes/verifications_weekly.pipe +++ /dev/null @@ -1,29 +0,0 @@ -VERSION 2 -NODE aggregate -SQL > - - SELECT - workspaceId, - apiId, - keyId, - countIfState((deniedReason = '') OR (deniedReason IS NULL)) AS success, - countIfState(deniedReason = 'RATE_LIMITED') AS rateLimited, - countIfState(deniedReason = 'USAGE_EXCEEDED') AS usageExceeded, - countIfState(deniedReason = 'DISABLED') AS disabled, - countIfState(deniedReason = 'INSUFFICIENT_PERMISSIONS') AS insufficientPermissions, - countIfState(deniedReason = 'FORBIDDEN') AS forbidden, - countIfState(deniedReason = 'EXPIRED') AS expired, - toStartOfWeek(fromUnixTimestamp64Milli(time)) AS time - FROM key_verifications__v2 - GROUP BY - workspaceId, - apiId, - keyId, - time - -TYPE materialized -DATASOURCE verifications_weekly_mv__v2 -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(time)" -ENGINE_SORTING_KEY "workspaceId, apiId, keyId, time" -ENGINE_TTL "time + toIntervalDay(180)" diff --git a/internal/tinybird/scripts/exec_test.sh b/internal/tinybird/scripts/exec_test.sh deleted file mode 100755 index 09d6d2b5f8..0000000000 --- a/internal/tinybird/scripts/exec_test.sh +++ /dev/null @@ -1,21 +0,0 @@ - -#!/usr/bin/env bash - -fail=0; - -for t in `find ./tests -name "*.test"`; do - echo "** Running $t **" - echo "** $(cat $t)" - if res=$(bash $t $1 | diff -B ${t}.result -); then - echo 'OK'; - else - echo "failed, diff:"; - echo "$res"; - fail=1 - fi - echo "" -done; - -if [ $fail == 1 ]; then - exit -1; -fi diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bac1f20de6..b4e578f115 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,9 +64,6 @@ importers: '@axiomhq/js': specifier: 1.0.0-rc.2 version: 1.0.0-rc.2 - '@chronark/zod-bird': - specifier: 0.3.9 - version: 0.3.9 '@hono/zod-openapi': specifier: ^0.11.0 version: 0.11.0(hono@4.6.3)(zod@3.23.8) @@ -158,9 +155,6 @@ importers: '@ai-sdk/openai': specifier: ^0.0.63 version: 0.0.63(zod@3.23.8) - '@chronark/zod-bird': - specifier: 0.3.9 - version: 0.3.9 '@clerk/nextjs': specifier: ^4.29.10 version: 4.29.10(next@14.2.10)(react-dom@18.3.1)(react@18.3.1) @@ -185,6 +179,9 @@ importers: '@unkey/billing': specifier: workspace:^ version: link:../../internal/billing + '@unkey/clickhouse': + specifier: workspace:^ + version: link:../../internal/clickhouse '@unkey/db': specifier: workspace:^ version: link:../../internal/db @@ -243,9 +240,6 @@ importers: '@ant-design/plots': specifier: ^1.2.5 version: 1.2.5(react-dom@18.3.1)(react@18.3.1) - '@chronark/zod-bird': - specifier: 0.3.9 - version: 0.3.9 '@clerk/nextjs': specifier: ^4.29.10 version: 4.29.10(next@14.2.10)(react-dom@18.3.1)(react@18.3.1) @@ -834,9 +828,6 @@ importers: apps/semantic-cache: dependencies: - '@chronark/zod-bird': - specifier: ^0.3.9 - version: 0.3.9 '@planetscale/database': specifier: ^1.16.0 version: 1.18.0 @@ -907,9 +898,6 @@ importers: apps/workflows: dependencies: - '@chronark/zod-bird': - specifier: 0.3.9 - version: 0.3.9 '@clerk/nextjs': specifier: ^4.29.10 version: 4.29.10(next@14.2.10)(react-dom@18.3.1)(react@18.3.1) @@ -974,9 +962,6 @@ importers: apps/www: dependencies: - '@chronark/zod-bird': - specifier: ^0.3.9 - version: 0.3.9 '@faker-js/faker': specifier: ^8.4.1 version: 8.4.1 @@ -1697,9 +1682,6 @@ importers: '@axiomhq/js': specifier: 1.0.0-rc.2 version: 1.0.0-rc.2 - '@chronark/zod-bird': - specifier: ^0.3.9 - version: 0.3.9 '@unkey/clickhouse': specifier: workspace:^ version: link:../../internal/clickhouse @@ -2851,12 +2833,6 @@ packages: prettier: 2.8.8 dev: true - /@chronark/zod-bird@0.3.9: - resolution: {integrity: sha512-5+W0KUBDZKWl86OFgKZJ9/pLRg1jbhAEPKDIKBJUEAAJblwRU3FBrnkcEAqW4/JfPBFiiEC5wxG16A8UzgphAQ==} - dependencies: - zod: 3.23.8 - dev: false - /@clack/core@0.3.4: resolution: {integrity: sha512-H4hxZDXgHtWTwV3RAVenqcC4VbJZNegbBjlPvzOzCouXtS2y3sDvlO3IsbrPNWuLWPPlYVYPghQdSF64683Ldw==} dependencies: @@ -6074,8 +6050,8 @@ packages: - typescript dev: true - /@oclif/core@4.0.31: - resolution: {integrity: sha512-7oyIZv/C1TP+fPc2tSzVPYqG1zU+nel1QvJxjAWyVhud0J8B5SpKZnryedxs3nlSVPJ6K1MT31C9esupCBYgZw==} + /@oclif/core@4.0.32: + resolution: {integrity: sha512-O3jfIAhqaJxXI2dzF81PLTMhKpFFA0Nyz8kfBnc9WYDJnvdmXK0fVAOSpwpi2mHTow/9FXxY6Kww8+Kbe7/sag==} engines: {node: '>=18.0.0'} dependencies: ansi-escapes: 4.3.2 @@ -6128,7 +6104,7 @@ packages: resolution: {integrity: sha512-p30fo3JPtbOqTJOX9A/8qKV/14XWt8xFgG/goVfIkuKBAO+cdY78ag8pYatlpzsYzJhO27X1MFn0WkkPWo36Ww==} engines: {node: '>=18.0.0'} dependencies: - '@oclif/core': 4.0.31 + '@oclif/core': 4.0.32 ansis: 3.3.2 debug: 4.3.7(supports-color@8.1.1) npm: 10.9.0 @@ -7060,7 +7036,7 @@ packages: resolution: {integrity: sha512-coUfuoMeIB7B8/NMekxaDzLhaYmp0HZNPEjYRm9goRou8UZIC3z21s0sL9AWoCw4EG876QyO3kYrc61WNF9B/w==} engines: {node: '>=8.0.0'} dependencies: - tslib: 2.4.1 + tslib: 2.8.1 dev: false /@peculiar/webcrypto@1.4.1: @@ -7070,7 +7046,7 @@ packages: '@peculiar/asn1-schema': 2.3.13 '@peculiar/json-schema': 1.1.12 pvtsutils: 1.3.5 - tslib: 2.4.1 + tslib: 2.8.1 webcrypto-core: 1.8.1 dev: false @@ -13631,7 +13607,7 @@ packages: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} dependencies: no-case: 3.0.4 - tslib: 2.4.1 + tslib: 2.8.1 dev: false /dot-prop@6.0.1: @@ -15017,7 +14993,7 @@ packages: engines: {node: '>= 10.17.0'} hasBin: true dependencies: - debug: 4.3.4 + debug: 4.3.7(supports-color@8.1.1) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -17586,7 +17562,7 @@ packages: /lower-case@2.0.2: resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} dependencies: - tslib: 2.4.1 + tslib: 2.8.1 dev: false /lowercase-keys@3.0.0: @@ -18999,6 +18975,7 @@ packages: /minipass@6.0.2: resolution: {integrity: sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==} engines: {node: '>=16 || 14 >=14.17'} + dev: true /minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} @@ -19395,7 +19372,7 @@ packages: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} dependencies: lower-case: 2.0.2 - tslib: 2.4.1 + tslib: 2.8.1 dev: false /node-addon-api@7.1.1: @@ -20108,7 +20085,7 @@ packages: engines: {node: '>=16 || 14 >=14.18'} dependencies: lru-cache: 10.4.3 - minipass: 6.0.2 + minipass: 7.1.2 /path-to-regexp@0.1.10: resolution: {integrity: sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==} @@ -22143,7 +22120,7 @@ packages: resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==} dependencies: dot-case: 3.0.4 - tslib: 2.4.1 + tslib: 2.8.1 dev: false /snakecase-keys@3.2.1: diff --git a/tools/migrate/package.json b/tools/migrate/package.json index c2c802dd1d..79bd1933e4 100644 --- a/tools/migrate/package.json +++ b/tools/migrate/package.json @@ -9,7 +9,6 @@ "license": "AGPL-3.0", "dependencies": { "@axiomhq/js": "1.0.0-rc.2", - "@chronark/zod-bird": "^0.3.9", "@unkey/clickhouse": "workspace:^", "@unkey/db": "workspace:^", "@unkey/id": "workspace:^",