From f412a875554dee5d1e4a6f01038211e160bde283 Mon Sep 17 00:00:00 2001 From: Juha Paananen Date: Sat, 24 Feb 2024 16:17:40 +0200 Subject: [PATCH] Manage CRDT local persistence and connection in crdt-store --- YJS_CRDT_WIP.md | 2 - frontend/package.json | 1 + frontend/src/board/BoardView.tsx | 1 + frontend/src/board/CollaborativeTextView.tsx | 31 ++++-------- frontend/src/board/ItemView.tsx | 5 +- frontend/src/store/board-store.ts | 5 ++ frontend/src/store/crdt-store.ts | 53 ++++++++++++++++++++ frontend/src/store/server-connection.ts | 7 +-- yarn.lock | 14 ++++++ 9 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 frontend/src/store/crdt-store.ts diff --git a/YJS_CRDT_WIP.md b/YJS_CRDT_WIP.md index cef56a633..51b1dd7de 100644 --- a/YJS_CRDT_WIP.md +++ b/YJS_CRDT_WIP.md @@ -14,7 +14,6 @@ The Y.js based collaborative editing support is under construction. - Build: Jest fails with the lib0 imports - Persistence: consider storing CRDT snapshot - Persistence: make sure the compactor works -- Persistence: implement local/offline persistence - Persistence: storing bundles with zero events (crdt only) - Domain: Migrate existing boards to CRDT or only apply CRDTs for new boards? - Domain: Consider if CRDT field values should also be included in the JSON presentation, maybe on save @@ -23,7 +22,6 @@ The Y.js based collaborative editing support is under construction. - UI: Add a toolbar. Needs some styling - if you now enable toolbar in Quill, it looks broken - UI: Clean up CollaborativeTextView. Apply appropriate color etc. - Undo buffer integration -- Manage session on client side: connect only when we have a sessionId. When it changes, reconnect. - Manage session on the server side: terminate YJS sockets when websocket session is terminated - Performance testing - Storage requirement measurements diff --git a/frontend/package.json b/frontend/package.json index 3278b691a..2d787d8bc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -39,6 +39,7 @@ "sanitize-html": "^2.3.2", "sass": "^1.32.8", "uuid": "^8.3.0", + "y-indexeddb": "^9.0.12", "y-quill": "^0.1.5", "y-websocket": "^1.5.3", "yjs": "^13.6.12" diff --git a/frontend/src/board/BoardView.tsx b/frontend/src/board/BoardView.tsx index 64d7711c4..d4f8e932f 100644 --- a/frontend/src/board/BoardView.tsx +++ b/frontend/src/board/BoardView.tsx @@ -383,6 +383,7 @@ export const BoardView = ({ dispatch, toolController, accessLevel, + boardStore, }} /> ) diff --git a/frontend/src/board/CollaborativeTextView.tsx b/frontend/src/board/CollaborativeTextView.tsx index 6cab43515..d239d9d72 100644 --- a/frontend/src/board/CollaborativeTextView.tsx +++ b/frontend/src/board/CollaborativeTextView.tsx @@ -1,16 +1,16 @@ import { h } from "harmaja" import * as L from "lonna" +import Quill from "quill" +import QuillCursors from "quill-cursors" +import { QuillBinding } from "y-quill" import { Board, getItemBackground, TextItem } from "../../../common/src/domain" import { emptySet } from "../../../common/src/sets" import { Dispatch } from "../store/board-store" import { BoardFocus, getSelectedItemIds } from "./board-focus" import { contrastingColor } from "./contrasting-color" import { ToolController } from "./tool-selection" -import * as Y from "yjs" -import { QuillBinding } from "y-quill" -import Quill from "quill" -import QuillCursors from "quill-cursors" -import { WebsocketProvider } from "y-websocket" +import { CRDTStore } from "../store/crdt-store" + Quill.register("modules/cursors", QuillCursors) interface CollaborativeTextViewProps { @@ -21,6 +21,7 @@ interface CollaborativeTextViewProps { toolController: ToolController focus: L.Atom itemFocus: L.Property<"none" | "selected" | "dragging" | "editing"> + crdtStore: CRDTStore } export function CollaborativeTextView({ id, @@ -30,6 +31,7 @@ export function CollaborativeTextView({ toolController, focus, itemFocus, + crdtStore, }: CollaborativeTextViewProps) { const textAtom = L.atom(L.view(item, "text"), (text) => dispatch({ action: "item.update", boardId: board.get().id, items: [{ id, text }] }), @@ -74,23 +76,10 @@ export function CollaborativeTextView({ }, theme: "snow", // 'bubble' is also great }) - // A Yjs document holds the shared data - const ydoc = new Y.Doc() - // Define a shared text type on the document - const ytext = ydoc.getText(`${id}.text`) - - // connect to the public demo server (not in production!) - const provider = new WebsocketProvider( // TODO: get socket address from server-connection.ts - `ws://localhost:1337/socket/yjs`, `board/${board.get().id}`, ydoc, { connect: true }) - - provider.on("status", (event: any) => { - console.log("YJS Provider status", event.status) - }) - - // Create an editor-binding which - // "binds" the quill editor to a Y.Text type. - const binding = new QuillBinding(ytext, quill, provider.awareness) + const crdt = crdtStore.getBoardCrdt(board.get().id) + const ytext = crdt.getField(id, "text") + const binding = new QuillBinding(ytext, quill, crdt.awareness) quillEditor.set(quill) } diff --git a/frontend/src/board/ItemView.tsx b/frontend/src/board/ItemView.tsx index 0bd4eb42f..b5d36e8f7 100644 --- a/frontend/src/board/ItemView.tsx +++ b/frontend/src/board/ItemView.tsx @@ -17,7 +17,7 @@ import { } from "../../../common/src/domain" import { emptySet } from "../../../common/src/sets" import { HTMLEditableSpan } from "../components/HTMLEditableSpan" -import { Dispatch } from "../store/board-store" +import { BoardStore, Dispatch } from "../store/board-store" import { autoFontSize } from "./autoFontSize" import { BoardCoordinateHelper } from "./board-coordinates" import { BoardFocus, getSelectedItemIds } from "./board-focus" @@ -41,6 +41,7 @@ export const ItemView = ({ latestConnection, dispatch, toolController, + boardStore, }: { board: L.Property accessLevel: L.Property @@ -53,6 +54,7 @@ export const ItemView = ({ latestConnection: L.Property dispatch: Dispatch toolController: ToolController + boardStore: BoardStore }) => { const element = L.atom(null) @@ -148,6 +150,7 @@ export const ItemView = ({ toolController={toolController} focus={focus} itemFocus={itemFocus} + crdtStore={boardStore.crdtStore} /> )} diff --git a/frontend/src/store/board-store.ts b/frontend/src/store/board-store.ts index a25cd153f..02620e62f 100644 --- a/frontend/src/store/board-store.ts +++ b/frontend/src/store/board-store.ts @@ -35,6 +35,8 @@ import { import { BoardLocalStore, LocalStorageBoard } from "./board-local-store" import { ServerConnection } from "./server-connection" import { UserSessionState, isLoginInProgress } from "./user-session-store" +import { CRDTStore } from "./crdt-store" +import { serve } from "esbuild" export type Dispatch = (e: UIEvent) => void export type BoardStore = ReturnType export type BoardAccessStatus = @@ -536,6 +538,8 @@ export function BoardStore( } } + const crdtStore = CRDTStore(L.view(state, (s) => s.status === "online")) + return { state, events, @@ -543,6 +547,7 @@ export function BoardStore( dispatch, canUndo: undoStack.canPop, canRedo: redoStack.canPop, + crdtStore, } } diff --git a/frontend/src/store/crdt-store.ts b/frontend/src/store/crdt-store.ts new file mode 100644 index 000000000..71bb04e56 --- /dev/null +++ b/frontend/src/store/crdt-store.ts @@ -0,0 +1,53 @@ +import { IndexeddbPersistence } from "y-indexeddb" +import { WebsocketProvider } from "y-websocket" +import * as Y from "yjs" +import { Id } from "../../../common/src/domain" +import { ServerConnection, WS_ROOT } from "./server-connection" +import * as L from "lonna" + +type BoardCRDT = ReturnType + +function BoardCRDT(boardId: Id, online: L.Property) { + const doc = new Y.Doc() + + const persistence = new IndexeddbPersistence(`b/${boardId}`, doc) + + persistence.on("synced", () => { + console.log("CRDT data from indexedDB is loaded") + }) + + const provider = new WebsocketProvider(`${WS_ROOT}/socket/yjs`, `board/${boardId}`, doc, { + connect: online.get(), + }) + + online.onChange((c) => (c ? provider.connect() : provider.disconnect())) + + provider.on("status", (event: any) => { + console.log("YJS Provider status", event.status) + }) + + function getField(itemId: Id, field: string) { + return doc.getText(`items.${itemId}.${field}`) + } + + return { + doc, + getField, + awareness: provider.awareness, + } +} + +export type CRDTStore = ReturnType +export function CRDTStore(online: L.Property) { + const boards = new Map() + function getBoardCrdt(boardId: Id): BoardCRDT { + let doc = boards.get(boardId) + if (!doc) { + doc = BoardCRDT(boardId, online) + } + return doc + } + return { + getBoardCrdt, + } +} diff --git a/frontend/src/store/server-connection.ts b/frontend/src/store/server-connection.ts index a2fcef4d0..701ecfd79 100644 --- a/frontend/src/store/server-connection.ts +++ b/frontend/src/store/server-connection.ts @@ -12,16 +12,17 @@ export type ServerConnection = ReturnType export type ConnectionStatus = "connecting" | "connected" | "sleeping" | "reconnecting" +export const WS_PROTOCOL = location.protocol === "http:" ? "ws:" : "wss:" +export const WS_ROOT = `${WS_PROTOCOL}//${location.host}` + export function BrowserSideServerConnection() { const documentHidden = L.fromEvent(document, "visibilitychange").pipe( L.toStatelessProperty(() => document.hidden || false), ) - const protocol = location.protocol === "http:" ? "ws:" : "wss:" - const root = `${protocol}//${location.host}` //const root = "wss://www.ourboard.io" //const root = "ws://localhost:1339" - return GenericServerConnection(`${root}/socket/lobby`, documentHidden, (s) => new WebSocket(s)) + return GenericServerConnection(`${WS_ROOT}/socket/lobby`, documentHidden, (s) => new WebSocket(s)) } export function GenericServerConnection( diff --git a/yarn.lock b/yarn.lock index 765a99759..1a76c8499 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4725,6 +4725,13 @@ lib0@^0.2.31, lib0@^0.2.42, lib0@^0.2.52, lib0@^0.2.85, lib0@^0.2.86: dependencies: isomorphic.js "^0.2.4" +lib0@^0.2.74: + version "0.2.89" + resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.89.tgz#f695ba69be34e28f73b3eeb5da92006f3897a470" + integrity sha512-5j19vcCjsQhvLG6mcDD+nprtJUCbmqLz5Hzt5xgi9SV6RIW/Dty7ZkVZHGBuPOADMKjQuKDvuQTH495wsmw8DQ== + dependencies: + isomorphic.js "^0.2.4" + lie@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" @@ -7822,6 +7829,13 @@ xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.0: resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== +y-indexeddb@^9.0.12: + version "9.0.12" + resolved "https://registry.yarnpkg.com/y-indexeddb/-/y-indexeddb-9.0.12.tgz#73657f31d52886d7532256610babf5cca4ad5e58" + integrity sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg== + dependencies: + lib0 "^0.2.74" + y-leveldb@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/y-leveldb/-/y-leveldb-0.1.2.tgz#43f6c5004b6891b57926d8a1e0eb0c883003e34b"