Manage CRDT local persistence and connection in crdt-store

YJS_CRDT_WIP.md

@@ -14,7 +14,6 @@ The Y.js based collaborative editing support is under construction.
 - Build: Jest fails with the lib0 imports
 - Persistence: consider storing CRDT snapshot
 - Persistence: make sure the compactor works
-- Persistence: implement local/offline persistence
 - Persistence: storing bundles with zero events (crdt only)
 - Domain: Migrate existing boards to CRDT or only apply CRDTs for new boards?
 - Domain: Consider if CRDT field values should also be included in the JSON presentation, maybe on save
@@ -23,7 +22,6 @@ The Y.js based collaborative editing support is under construction.
 - UI: Add a toolbar. Needs some styling - if you now enable toolbar in Quill, it looks broken
 - UI: Clean up CollaborativeTextView. Apply appropriate color etc.
 - Undo buffer integration
-- Manage session on client side: connect only when we have a sessionId. When it changes, reconnect.
 - Manage session on the server side: terminate YJS sockets when websocket session is terminated
 - Performance testing
 - Storage requirement measurements

frontend/package.json

@@ -39,6 +39,7 @@
  "sanitize-html": "^2.3.2",
  "sass": "^1.32.8",
  "uuid": "^8.3.0",
+ "y-indexeddb": "^9.0.12",
  "y-quill": "^0.1.5",
  "y-websocket": "^1.5.3",
  "yjs": "^13.6.12"

frontend/src/board/BoardView.tsx

@@ -383,6 +383,7 @@ export const BoardView = ({
  dispatch,
  toolController,
  accessLevel,
+ boardStore,
  }}
  />
  )

frontend/src/board/CollaborativeTextView.tsx

@@ -1,16 +1,16 @@
 import { h } from "harmaja"
 import * as L from "lonna"
+import Quill from "quill"
+import QuillCursors from "quill-cursors"
+import { QuillBinding } from "y-quill"
 import { Board, getItemBackground, TextItem } from "../../../common/src/domain"
 import { emptySet } from "../../../common/src/sets"
 import { Dispatch } from "../store/board-store"
 import { BoardFocus, getSelectedItemIds } from "./board-focus"
 import { contrastingColor } from "./contrasting-color"
 import { ToolController } from "./tool-selection"
-import * as Y from "yjs"
-import { QuillBinding } from "y-quill"
-import Quill from "quill"
-import QuillCursors from "quill-cursors"
-import { WebsocketProvider } from "y-websocket"
+import { CRDTStore } from "../store/crdt-store"
+
 Quill.register("modules/cursors", QuillCursors)
 
 interface CollaborativeTextViewProps {
@@ -21,6 +21,7 @@ interface CollaborativeTextViewProps {
  toolController: ToolController
  focus: L.Atom<BoardFocus>
  itemFocus: L.Property<"none" | "selected" | "dragging" | "editing">
+ crdtStore: CRDTStore
 }
 export function CollaborativeTextView({
  id,
@@ -30,6 +31,7 @@ export function CollaborativeTextView({
  toolController,
  focus,
  itemFocus,
+ crdtStore,
 }: CollaborativeTextViewProps) {
  const textAtom = L.atom(L.view(item, "text"), (text) =>
  dispatch({ action: "item.update", boardId: board.get().id, items: [{ id, text }] }),
@@ -74,23 +76,10 @@ export function CollaborativeTextView({
  },
  theme: "snow", // 'bubble' is also great
  })
- // A Yjs document holds the shared data
- const ydoc = new Y.Doc()
- // Define a shared text type on the document
- const ytext = ydoc.getText(`${id}.text`)
-
- // connect to the public demo server (not in production!)
- const provider = new WebsocketProvider( // TODO: get socket address from server-connection.ts
- `ws://localhost:1337/socket/yjs`, `board/${board.get().id}`, ydoc, { connect: true })
-
- provider.on("status", (event: any) => {
- console.log("YJS Provider status", event.status)
- })
-
- // Create an editor-binding which
- // "binds" the quill editor to a Y.Text type.
- const binding = new QuillBinding(ytext, quill, provider.awareness)
 
+ const crdt = crdtStore.getBoardCrdt(board.get().id)
+ const ytext = crdt.getField(id, "text")
+ const binding = new QuillBinding(ytext, quill, crdt.awareness)
  quillEditor.set(quill)
  }
 

frontend/src/board/ItemView.tsx

@@ -17,7 +17,7 @@ import {
 } from "../../../common/src/domain"
 import { emptySet } from "../../../common/src/sets"
 import { HTMLEditableSpan } from "../components/HTMLEditableSpan"
-import { Dispatch } from "../store/board-store"
+import { BoardStore, Dispatch } from "../store/board-store"
 import { autoFontSize } from "./autoFontSize"
 import { BoardCoordinateHelper } from "./board-coordinates"
 import { BoardFocus, getSelectedItemIds } from "./board-focus"
@@ -41,6 +41,7 @@ export const ItemView = ({
  latestConnection,
  dispatch,
  toolController,
+ boardStore,
 }: {
  board: L.Property<Board>
  accessLevel: L.Property<AccessLevel>
@@ -53,6 +54,7 @@ export const ItemView = ({
  latestConnection: L.Property<Connection | null>
  dispatch: Dispatch
  toolController: ToolController
+ boardStore: BoardStore
 }) => {
  const element = L.atom<HTMLElement | null>(null)
 
@@ -148,6 +150,7 @@ export const ItemView = ({
  toolController={toolController}
  focus={focus}
  itemFocus={itemFocus}
+ crdtStore={boardStore.crdtStore}
  />
  )}
 

frontend/src/store/board-store.ts

@@ -35,6 +35,8 @@ import {
 import { BoardLocalStore, LocalStorageBoard } from "./board-local-store"
 import { ServerConnection } from "./server-connection"
 import { UserSessionState, isLoginInProgress } from "./user-session-store"
+import { CRDTStore } from "./crdt-store"
+import { serve } from "esbuild"
 export type Dispatch = (e: UIEvent) => void
 export type BoardStore = ReturnType<typeof BoardStore>
 export type BoardAccessStatus =
@@ -536,13 +538,16 @@ export function BoardStore(
  }
  }
 
+ const crdtStore = CRDTStore(L.view(state, (s) => s.status === "online"))
+
  return {
  state,
  events,
  eventsFromServer: connection.bufferedServerEvents,
  dispatch,
  canUndo: undoStack.canPop,
  canRedo: redoStack.canPop,
+ crdtStore,
  }
 }
 

frontend/src/store/crdt-store.ts

@@ -0,0 +1,53 @@
+import { IndexeddbPersistence } from "y-indexeddb"
+import { WebsocketProvider } from "y-websocket"
+import * as Y from "yjs"
+import { Id } from "../../../common/src/domain"
+import { ServerConnection, WS_ROOT } from "./server-connection"
+import * as L from "lonna"
+
+type BoardCRDT = ReturnType<typeof BoardCRDT>
+
+function BoardCRDT(boardId: Id, online: L.Property<boolean>) {
+ const doc = new Y.Doc()
+
+ const persistence = new IndexeddbPersistence(`b/${boardId}`, doc)
+
+ persistence.on("synced", () => {
+ console.log("CRDT data from indexedDB is loaded")
+ })
+
+ const provider = new WebsocketProvider(`${WS_ROOT}/socket/yjs`, `board/${boardId}`, doc, {
+ connect: online.get(),
+ })
+
+ online.onChange((c) => (c ? provider.connect() : provider.disconnect()))
+
+ provider.on("status", (event: any) => {
+ console.log("YJS Provider status", event.status)
+ })
+
+ function getField(itemId: Id, field: string) {
+ return doc.getText(`items.${itemId}.${field}`)
+ }
+
+ return {
+ doc,
+ getField,
+ awareness: provider.awareness,
+ }
+}
+
+export type CRDTStore = ReturnType<typeof CRDTStore>
+export function CRDTStore(online: L.Property<boolean>) {
+ const boards = new Map<Id, BoardCRDT>()
+ function getBoardCrdt(boardId: Id): BoardCRDT {
+ let doc = boards.get(boardId)
+ if (!doc) {
+ doc = BoardCRDT(boardId, online)
+ }
+ return doc
+ }
+ return {
+ getBoardCrdt,
+ }
+}

frontend/src/store/server-connection.ts

@@ -12,16 +12,17 @@ export type ServerConnection = ReturnType<typeof GenericServerConnection>
 
 export type ConnectionStatus = "connecting" | "connected" | "sleeping" | "reconnecting"
 
+export const WS_PROTOCOL = location.protocol === "http:" ? "ws:" : "wss:"
+export const WS_ROOT = `${WS_PROTOCOL}//${location.host}`
+
 export function BrowserSideServerConnection() {
  const documentHidden = L.fromEvent(document, "visibilitychange").pipe(
  L.toStatelessProperty(() => document.hidden || false),
  )
 
- const protocol = location.protocol === "http:" ? "ws:" : "wss:"
- const root = `${protocol}//${location.host}`
  //const root = "wss://www.ourboard.io"
  //const root = "ws://localhost:1339"
- return GenericServerConnection(`${root}/socket/lobby`, documentHidden, (s) => new WebSocket(s))
+ return GenericServerConnection(`${WS_ROOT}/socket/lobby`, documentHidden, (s) => new WebSocket(s))
 }
 
 export function GenericServerConnection(

yarn.lock

@@ -4725,6 +4725,13 @@ lib0@^0.2.31, lib0@^0.2.42, lib0@^0.2.52, lib0@^0.2.85, lib0@^0.2.86:
  dependencies:
  isomorphic.js "^0.2.4"
 
+lib0@^0.2.74:
+ version "0.2.89"
+ resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.89.tgz#f695ba69be34e28f73b3eeb5da92006f3897a470"
+ integrity sha512-5j19vcCjsQhvLG6mcDD+nprtJUCbmqLz5Hzt5xgi9SV6RIW/Dty7ZkVZHGBuPOADMKjQuKDvuQTH495wsmw8DQ==
+ dependencies:
+ isomorphic.js "^0.2.4"
+
 [email protected]:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
@@ -7822,6 +7829,13 @@ xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.0:
  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
 
+y-indexeddb@^9.0.12:
+ version "9.0.12"
+ resolved "https://registry.yarnpkg.com/y-indexeddb/-/y-indexeddb-9.0.12.tgz#73657f31d52886d7532256610babf5cca4ad5e58"
+ integrity sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==
+ dependencies:
+ lib0 "^0.2.74"
+
 y-leveldb@^0.1.0:
  version "0.1.2"
  resolved "https://registry.yarnpkg.com/y-leveldb/-/y-leveldb-0.1.2.tgz#43f6c5004b6891b57926d8a1e0eb0c883003e34b"