feat: clickhouse-buffer (#2672)

* feat: clickhouse-buffer

* docs: engineering docs v1

* fix: accept lower case queries too

apps/api/src/pkg/analytics.ts

@@ -6,12 +6,20 @@ export class Analytics {
 
   constructor(opts: {
     clickhouseUrl: string;
+    clickhouseInsertUrl?: string;
   }) {
-    this.clickhouse = new ClickHouse({ url: opts.clickhouseUrl });
+    if (opts.clickhouseInsertUrl) {
+      this.clickhouse = new ClickHouse({
+        insertUrl: opts.clickhouseInsertUrl,
+        queryUrl: opts.clickhouseUrl,
+      });
+    } else {
+      this.clickhouse = new ClickHouse({ url: opts.clickhouseUrl });
+    }
   }
 
   public get insertSdkTelemetry() {
-    return this.clickhouse.client.insert({
+    return this.clickhouse.inserter.insert({
       table: "telemetry.raw_sdks_v1",
       schema: z.object({
         request_id: z.string(),

apps/api/src/pkg/env.ts

@@ -31,6 +31,7 @@ export const zEnv = z.object({
   AGENT_TOKEN: z.string(),
 
   CLICKHOUSE_URL: z.string(),
+  CLICKHOUSE_INSERT_URL: z.string().optional(),
 
   SYNC_RATELIMIT_ON_NO_DATA: z
     .string()

apps/api/src/pkg/middleware/init.ts

@@ -98,6 +98,7 @@ export function init(): MiddlewareHandler<HonoEnv> {
 
     const analytics = new Analytics({
       clickhouseUrl: c.env.CLICKHOUSE_URL,
+      clickhouseInsertUrl: c.env.CLICKHOUSE_INSERT_URL,
     });
     const rateLimiter = new AgentRatelimiter({
       agent: { url: c.env.AGENT_URL, token: c.env.AGENT_TOKEN },

apps/api/src/routes/v1_ratelimits_limit.ts

@@ -148,7 +148,7 @@ export type V1RatelimitLimitResponse = z.infer<
 export const registerV1RatelimitLimit = (app: App) =>
   app.openapi(route, async (c) => {
     const req = c.req.valid("json");
-    const { cache, db, rateLimiter, analytics, rbac } = c.get("services");
+    const { cache, db, rateLimiter, analytics, rbac, logger } = c.get("services");
 
     const rootKey = await rootKeyAuth(c);
 
@@ -325,14 +325,22 @@ export const registerV1RatelimitLimit = (app: App) =>
     const remaining = Math.max(0, limit - ratelimitResponse.current);
 
     c.executionCtx.waitUntil(
-      analytics.insertRatelimit({
-        workspace_id: rootKey.authorizedWorkspaceId,
-        namespace_id: namespace.id,
-        request_id: c.get("requestId"),
-        identifier: req.identifier,
-        time: Date.now(),
-        passed: ratelimitResponse.passed,
-      }),
+      analytics
+        .insertRatelimit({
+          workspace_id: rootKey.authorizedWorkspaceId,
+          namespace_id: namespace.id,
+          request_id: c.get("requestId"),
+          identifier: req.identifier,
+          time: Date.now(),
+          passed: ratelimitResponse.passed,
+        })
+        .then(({ err }) => {
+          if (err) {
+            logger.error("inserting ratelimit event failed", {
+              error: err.message,
+            });
+          }
+        }),
     );
 
     if (req.resources && req.resources.length > 0) {

apps/chproxy/Dockerfile

@@ -0,0 +1,37 @@
+FROM node:lts AS base
+
+RUN npm i -g pnpm turbo bun
+RUN corepack enable
+
+
+FROM base AS builder
+
+# Set working directory
+WORKDIR /unkey
+
+COPY . .
+RUN turbo prune chproxy --docker
+
+# Add lockfile and package.json's of isolated subworkspace
+FROM base AS installer
+WORKDIR /unkey
+
+# First install dependencies (as they change less often)
+COPY .gitignore .gitignore
+COPY --from=builder /unkey/out/json/ .
+COPY --from=builder /unkey/out/pnpm-lock.yaml ./pnpm-lock.yaml
+RUN pnpm install
+
+# Build the project and its dependencies
+COPY --from=builder /unkey/out/full/ .
+COPY turbo.json turbo.json
+
+RUN pnpm turbo build --filter=chproxy...
+
+FROM base AS runner
+WORKDIR /unkey
+
+COPY --from=installer /unkey .
+
+WORKDIR /unkey/apps/chproxy
+CMD bun run ./src/main.ts

apps/chproxy/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "chproxy",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/main.ts",
+  "keywords": [],
+  "author": "Andreas Thomas",
+  "license": "ISC",
+  "dependencies": {
+    "@types/bun": "^1.1.13",
+    "@unkey/clickhouse": "workspace:^",
+    "zod": "^3.23.8"
+  }
+}

apps/chproxy/src/main.ts

@@ -0,0 +1,127 @@
+import { z } from "zod";
+
+const MAX_BATCH_SIZE = 10000;
+const MAX_BATCH_TIME = 3000; // milliseconds
+
+type Batch = {
+  createdAt: number;
+  rows: string[];
+  params: URLSearchParams;
+};
+
+const env = z
+  .object({
+    // url including basic auth
+    // ie: https://user:[email protected]
+    CLICKHOUSE_URL: z.string(),
+    PORT: z
+      .string()
+      .optional()
+      .default("7123")
+      .transform((s) => Number.parseInt(s)),
+    // user:pass
+    // We receive a base64 of this in the authorization header ie: `Basic ZGVmYXVsdDo=`
+    BASIC_AUTH: z.string(),
+  })
+  .parse(process.env);
+
+const requiredAuthorization = `Basic ${btoa(env.BASIC_AUTH)}`;
+
+const buffer = new Map<string, Batch>();
+
+async function flush(force?: boolean): Promise<void> {
+  const now = Date.now();
+
+  for (const [key, batch] of buffer.entries()) {
+    if (force || now >= batch.createdAt + MAX_BATCH_TIME) {
+      await persist(key);
+    }
+  }
+}
+
+// persist inserts the data into clickhouse and removes it from the buffer
+async function persist(key: string): Promise<void> {
+  const batch = buffer.get(key);
+  if (!batch) {
+    return;
+  }
+
+  const url = new URL(env.CLICKHOUSE_URL);
+  batch.params.forEach((v, k) => {
+    url.searchParams.set(k, v);
+  });
+
+  const res = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "text/plain",
+      Authorization: `Basic ${btoa([url.username, url.password].join(":"))}`,
+    },
+    body: batch.rows.join("\n"),
+  });
+  if (res.ok) {
+    buffer.delete(key);
+    console.info(`persisted ${batch.rows.length} rows`);
+  } else {
+    console.error("unable to persist", await res.text(), JSON.stringify(batch));
+  }
+}
+
+setInterval(flush, 1000);
+
+const server = Bun.serve({
+  port: env.PORT,
+  fetch: async (req: Request): Promise<Response> => {
+    const url = new URL(req.url);
+
+    if (url.pathname === "/v1/liveness") {
+      return new Response("I'm alive");
+    }
+
+    if (req.headers.get("Authorization") !== requiredAuthorization) {
+      return new Response("unauthorized", { status: 401 });
+    }
+
+    const query = url.searchParams.get("query");
+    if (!query || !query.toLowerCase().startsWith("insert into")) {
+      return new Response("wrong query", { status: 400 });
+    }
+
+    const params = url.searchParams;
+    params.delete("query_id");
+    params.sort();
+
+    const key = params.toString();
+
+    const rows = (await req.text()).split("\n");
+
+    const existing = buffer.get(key);
+    if (existing) {
+      const size = existing.rows.push(...rows);
+      if (size >= MAX_BATCH_SIZE) {
+        await persist(key);
+      }
+    } else {
+      buffer.set(key, {
+        createdAt: Date.now(),
+        params,
+        rows,
+      });
+    }
+
+    return new Response("ok");
+  },
+  error: (err) => {
+    console.error(err);
+    return new Response("internal server error", { status: 500 });
+  },
+});
+
+console.info("listening on", server.hostname, server.port);
+process.on("SIGTERM", async (s) => {
+  console.warn("Received signal", s);
+
+  server.stop();
+  await flush(true);
+  process.exit(0);
+});

apps/chproxy/tsconfig.json

@@ -0,0 +1,108 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs" /* Specify what module code is generated. */,
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "isolatedDeclarations": true,                     /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+
+    /* Type Checking */
+    "strict": true /* Enable all strict type-checking options. */,
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */
+  }
+}

apps/dashboard/lib/hooks/useDelayLoader.tsx

@@ -1,11 +1,11 @@
 import { useEffect, useState } from "react";
-
 export const useDelayLoader = (isPending: boolean, delay = 50) => {
   const [showLoader, setShowLoader] = useState(false);
 
   useEffect(() => {
     let timeout: NodeJS.Timeout;
     if (isPending) {
+      // @ts-expect-error bun types are colliding with node
       timeout = setTimeout(() => {
         setShowLoader(true);
       }, delay);

apps/engineering/app/(home)/layout.tsx

@@ -1,4 +1,4 @@
-import { HomeLayout } from "fumadocs-ui/home-layout";
+import { HomeLayout } from "fumadocs-ui/layouts/home";
 import type { ReactNode } from "react";
 import { baseOptions } from "../layout.config";