From 83fed429971fcc758ada9af12d52a2936b537456 Mon Sep 17 00:00:00 2001 From: Yidadaa Date: Tue, 19 Sep 2023 03:18:34 +0800 Subject: [PATCH] feat: add upstash redis cloud sync --- app/components/settings.tsx | 39 ++++++++++++++++++- app/locales/cn.ts | 6 +++ app/locales/en.ts | 6 +++ app/store/sync.ts | 4 +- app/utils/cloud/upstash.ts | 74 ++++++++++++++++++++++++++++++++++--- app/utils/cloud/webdav.ts | 2 - app/utils/format.ts | 15 ++++++++ app/utils/sync.ts | 3 ++ 8 files changed, 137 insertions(+), 12 deletions(-) diff --git a/app/components/settings.tsx b/app/components/settings.tsx index fb1d688f01c..8ed6b77383c 100644 --- a/app/components/settings.tsx +++ b/app/components/settings.tsx @@ -50,7 +50,7 @@ import Locale, { } from "../locales"; import { copyToClipboard } from "../utils"; import Link from "next/link"; -import { Path, RELEASE_URL, UPDATE_URL } from "../constant"; +import { Path, RELEASE_URL, STORAGE_KEY, UPDATE_URL } from "../constant"; import { Prompt, SearchService, usePromptStore } from "../store/prompt"; import { ErrorBoundary } from "./error"; import { InputRange } from "./input-range"; @@ -413,7 +413,42 @@ function SyncConfigModal(props: { onClose?: () => void }) { {syncStore.provider === ProviderType.UpStash && ( - + + { + syncStore.update( + (config) => + (config.upstash.endpoint = e.currentTarget.value), + ); + }} + > + + + + { + syncStore.update( + (config) => + (config.upstash.username = e.currentTarget.value), + ); + }} + > + + + { + syncStore.update( + (config) => (config.upstash.apiKey = e.currentTarget.value), + ); + }} + > + )} diff --git a/app/locales/cn.ts b/app/locales/cn.ts index ac4a1777f5d..b2afc753457 100644 --- a/app/locales/cn.ts +++ b/app/locales/cn.ts @@ -207,6 +207,12 @@ const cn = { UserName: "用户名", Password: "密码", }, + + UpStash: { + Endpoint: "UpStash Redis REST Url", + UserName: "备份名称", + Password: "UpStash Redis REST Token", + }, }, LocalState: "本地数据", diff --git a/app/locales/en.ts b/app/locales/en.ts index 3f3fa7ce920..697d09d1f4e 100644 --- a/app/locales/en.ts +++ b/app/locales/en.ts @@ -210,6 +210,12 @@ const en: LocaleType = { UserName: "User Name", Password: "Password", }, + + UpStash: { + Endpoint: "UpStash Redis REST Url", + UserName: "Backup Name", + Password: "UpStash Redis REST Token", + }, }, LocalState: "Local Data", diff --git a/app/store/sync.ts b/app/store/sync.ts index 29b6a82c235..ff9f650c083 100644 --- a/app/store/sync.ts +++ b/app/store/sync.ts @@ -1,5 +1,5 @@ import { Updater } from "../typing"; -import { ApiPath, StoreKey } from "../constant"; +import { ApiPath, STORAGE_KEY, StoreKey } from "../constant"; import { createPersistStore } from "../utils/store"; import { AppState, @@ -36,7 +36,7 @@ export const useSyncStore = createPersistStore( upstash: { endpoint: "", - username: "", + username: STORAGE_KEY, apiKey: "", }, diff --git a/app/utils/cloud/upstash.ts b/app/utils/cloud/upstash.ts index 6f9b30f6b5e..5f5b9fc7925 100644 --- a/app/utils/cloud/upstash.ts +++ b/app/utils/cloud/upstash.ts @@ -1,25 +1,87 @@ +import { STORAGE_KEY } from "@/app/constant"; import { SyncStore } from "@/app/store/sync"; +import { corsFetch } from "../cors"; +import { chunks } from "../format"; export type UpstashConfig = SyncStore["upstash"]; export type UpStashClient = ReturnType; -export function createUpstashClient(config: UpstashConfig) { +export function createUpstashClient(store: SyncStore) { + const config = store.upstash; + const storeKey = config.username.length === 0 ? STORAGE_KEY : config.username; + const chunkCountKey = `${storeKey}-chunk-count`; + const chunkIndexKey = (i: number) => `${storeKey}-chunk-${i}`; + + const proxyUrl = + store.useProxy && store.proxyUrl.length > 0 ? store.proxyUrl : undefined; + return { async check() { - return true; + try { + const res = await corsFetch(this.path(`get/${storeKey}`), { + method: "GET", + headers: this.headers(), + proxyUrl, + }); + console.log("[Upstash] check", res.status, res.statusText); + return [200].includes(res.status); + } catch (e) { + console.error("[Upstash] failed to check", e); + } + return false; + }, + + async redisGet(key: string) { + const res = await corsFetch(this.path(`get/${key}`), { + method: "GET", + headers: this.headers(), + proxyUrl, + }); + + console.log("[Upstash] get key = ", key, res.status, res.statusText); + const resJson = (await res.json()) as { result: string }; + + return resJson.result; + }, + + async redisSet(key: string, value: string) { + const res = await corsFetch(this.path(`set/${key}`), { + method: "POST", + headers: this.headers(), + body: value, + proxyUrl, + }); + + console.log("[Upstash] set key = ", key, res.status, res.statusText); }, async get() { - throw Error("[Sync] not implemented"); + const chunkCount = Number(await this.redisGet(chunkCountKey)); + if (!Number.isInteger(chunkCount)) return; + + const chunks = await Promise.all( + new Array(chunkCount) + .fill(0) + .map((_, i) => this.redisGet(chunkIndexKey(i))), + ); + console.log("[Upstash] get full chunks", chunks); + return chunks.join(""); }, - async set() { - throw Error("[Sync] not implemented"); + async set(_: string, value: string) { + // upstash limit the max request size which is 1Mb for “Free” and “Pay as you go” + // so we need to split the data to chunks + let index = 0; + for await (const chunk of chunks(value)) { + await this.redisSet(chunkIndexKey(index), chunk); + index += 1; + } + await this.redisSet(chunkCountKey, index.toString()); }, headers() { return { - Authorization: `Basic ${config.apiKey}`, + Authorization: `Bearer ${config.apiKey}`, }; }, path(path: string) { diff --git a/app/utils/cloud/webdav.ts b/app/utils/cloud/webdav.ts index 6c96c9062a4..c87fdd71e1e 100644 --- a/app/utils/cloud/webdav.ts +++ b/app/utils/cloud/webdav.ts @@ -20,9 +20,7 @@ export function createWebDavClient(store: SyncStore) { headers: this.headers(), proxyUrl, }); - console.log("[WebDav] check", res.status, res.statusText); - return [201, 200, 404, 401].includes(res.status); } catch (e) { console.error("[WebDav] failed to check", e); diff --git a/app/utils/format.ts b/app/utils/format.ts index 450d66696d9..2e8a382b95a 100644 --- a/app/utils/format.ts +++ b/app/utils/format.ts @@ -11,3 +11,18 @@ export function prettyObject(msg: any) { } return ["```json", msg, "```"].join("\n"); } + +export function* chunks(s: string, maxBytes = 1000 * 1000) { + const decoder = new TextDecoder("utf-8"); + let buf = new TextEncoder().encode(s); + while (buf.length) { + let i = buf.lastIndexOf(32, maxBytes + 1); + // If no space found, try forward search + if (i < 0) i = buf.indexOf(32, maxBytes); + // If there's no space at all, take all + if (i < 0) i = buf.length; + // This is a safe cut-off point; never half-way a multi-byte + yield decoder.decode(buf.slice(0, i)); + buf = buf.slice(i + 1); // Skip space (if any) + } +} diff --git a/app/utils/sync.ts b/app/utils/sync.ts index ab1f1f44918..1acfc1289de 100644 --- a/app/utils/sync.ts +++ b/app/utils/sync.ts @@ -69,6 +69,9 @@ const MergeStates: StateMerger = { localState.sessions.forEach((s) => (localSessions[s.id] = s)); remoteState.sessions.forEach((remoteSession) => { + // skip empty chats + if (remoteSession.messages.length === 0) return; + const localSession = localSessions[remoteSession.id]; if (!localSession) { // if remote session is new, just merge it