From c5f562bae34da770544b8ccb5bd41c5a506920b0 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 9 Aug 2024 10:33:58 +0100 Subject: [PATCH 01/47] Yjs-codemirror base example --- examples/yjs-codemirror/.eslintrc.cjs | 41 ++ examples/yjs-codemirror/.gitignore | 2 + examples/yjs-codemirror/.prettierrc | 5 + .../db/migrations/01-create_yjs_tables.sql | 13 + examples/yjs-codemirror/index.html | 62 +++ examples/yjs-codemirror/package.json | 35 ++ examples/yjs-codemirror/src/codemirror.css | 344 +++++++++++++++ examples/yjs-codemirror/src/codemirror.html | 56 +++ examples/yjs-codemirror/src/codemirror.js | 42 ++ examples/yjs-codemirror/src/y-electric.js | 393 ++++++++++++++++++ 10 files changed, 993 insertions(+) create mode 100644 examples/yjs-codemirror/.eslintrc.cjs create mode 100644 examples/yjs-codemirror/.gitignore create mode 100644 examples/yjs-codemirror/.prettierrc create mode 100644 examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql create mode 100644 examples/yjs-codemirror/index.html create mode 100644 examples/yjs-codemirror/package.json create mode 100644 examples/yjs-codemirror/src/codemirror.css create mode 100644 examples/yjs-codemirror/src/codemirror.html create mode 100644 examples/yjs-codemirror/src/codemirror.js create mode 100644 examples/yjs-codemirror/src/y-electric.js diff --git a/examples/yjs-codemirror/.eslintrc.cjs b/examples/yjs-codemirror/.eslintrc.cjs new file mode 100644 index 0000000000..8aeb375841 --- /dev/null +++ b/examples/yjs-codemirror/.eslintrc.cjs @@ -0,0 +1,41 @@ +module.exports = { + env: { + browser: true, + es2021: true, + node: true, + }, + extends: [ + `eslint:recommended`, + `plugin:@typescript-eslint/recommended`, + `plugin:prettier/recommended`, + ], + parserOptions: { + ecmaVersion: 2022, + requireConfigFile: false, + sourceType: `module`, + ecmaFeatures: { + jsx: true, + }, + }, + parser: `@typescript-eslint/parser`, + plugins: [`prettier`], + rules: { + quotes: [`error`, `backtick`], + "no-unused-vars": `off`, + "@typescript-eslint/no-unused-vars": [ + `error`, + { + argsIgnorePattern: `^_`, + varsIgnorePattern: `^_`, + caughtErrorsIgnorePattern: `^_`, + }, + ], + }, + ignorePatterns: [ + `**/node_modules/**`, + `**/dist/**`, + `tsup.config.ts`, + `vitest.config.ts`, + `.eslintrc.js`, + ], +}; diff --git a/examples/yjs-codemirror/.gitignore b/examples/yjs-codemirror/.gitignore new file mode 100644 index 0000000000..9829edff9a --- /dev/null +++ b/examples/yjs-codemirror/.gitignore @@ -0,0 +1,2 @@ +dist +.env.local diff --git a/examples/yjs-codemirror/.prettierrc b/examples/yjs-codemirror/.prettierrc new file mode 100644 index 0000000000..eaff0359ca --- /dev/null +++ b/examples/yjs-codemirror/.prettierrc @@ -0,0 +1,5 @@ +{ + "trailingComma": "es5", + "semi": false, + "tabWidth": 2 +} diff --git a/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql new file mode 100644 index 0000000000..eed8ee6001 --- /dev/null +++ b/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql @@ -0,0 +1,13 @@ +CREATE TABLE ydoc_updates( + id uuid PRIMARY KEY, + name TEXT, + op TEXT NOT NULL +); + +CREATE TABLE ydoc_awareness( + id SERIAL, + client_id TEXT, + name TEXT, + op TEXT NOT NULL, + PRIMARY KEY (id, client_id, name) +); \ No newline at end of file diff --git a/examples/yjs-codemirror/index.html b/examples/yjs-codemirror/index.html new file mode 100644 index 0000000000..a1a50b0e07 --- /dev/null +++ b/examples/yjs-codemirror/index.html @@ -0,0 +1,62 @@ + + + + + + Yjs CodeMirror Example + + + + + + + +

+

+ This is a demo of the Yjs ⇔ + CodeMirror binding: + y-codemirror. +

+

+ The content of this editor is shared with every client that visits this + domain. +

+ + + \ No newline at end of file diff --git a/examples/yjs-codemirror/package.json b/examples/yjs-codemirror/package.json new file mode 100644 index 0000000000..43d8ea98ae --- /dev/null +++ b/examples/yjs-codemirror/package.json @@ -0,0 +1,35 @@ +{ + "name": "@electric-examples/yjs-codemirror", + "private": true, + "version": "0.0.1", + "author": "ElectricSQL", + "license": "Apache-2.0", + "type": "module", + "scripts": { + "backend:up": "PROJECT_NAME=yjs-codemirror pnpm -C ../../ run example-backend:up && pnpm db:migrate", + "backend:down": "PROJECT_NAME=yjs-codemirror pnpm -C ../../ run example-backend:down", + "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", + "dev": "vite", + "build": "vite build", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "preview": "vite preview", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@electric-sql/client": "workspace:*", + "abort-controller": "^3.0.0", + "js-base64": "^3.7.7", + "lib0": "^0.2.96", + "y-protocols": "^1.0.5", + "codemirror": "^5.64.0", + "y-codemirror": "^2.1.1", + "yjs": "^13.5.22" + + }, + "devDependencies": { + "@databases/pg-migrations": "^5.0.3", + "dotenv": "^16.4.5", + "eslint": "^8.57.0", + "vite": "^5.3.4" + } +} diff --git a/examples/yjs-codemirror/src/codemirror.css b/examples/yjs-codemirror/src/codemirror.css new file mode 100644 index 0000000000..f4d5718a78 --- /dev/null +++ b/examples/yjs-codemirror/src/codemirror.css @@ -0,0 +1,344 @@ +/* BASICS */ + +.CodeMirror { + /* Set height, width, borders, and global font properties here */ + font-family: monospace; + height: 300px; + color: black; + direction: ltr; +} + +/* PADDING */ + +.CodeMirror-lines { + padding: 4px 0; /* Vertical padding around content */ +} +.CodeMirror pre.CodeMirror-line, +.CodeMirror pre.CodeMirror-line-like { + padding: 0 4px; /* Horizontal padding of content */ +} + +.CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { + background-color: white; /* The little square between H and V scrollbars */ +} + +/* GUTTER */ + +.CodeMirror-gutters { + border-right: 1px solid #ddd; + background-color: #f7f7f7; + white-space: nowrap; +} +.CodeMirror-linenumbers {} +.CodeMirror-linenumber { + padding: 0 3px 0 5px; + min-width: 20px; + text-align: right; + color: #999; + white-space: nowrap; +} + +.CodeMirror-guttermarker { color: black; } +.CodeMirror-guttermarker-subtle { color: #999; } + +/* CURSOR */ + +.CodeMirror-cursor { + border-left: 1px solid black; + border-right: none; + width: 0; +} +/* Shown when moving in bi-directional text */ +.CodeMirror div.CodeMirror-secondarycursor { + border-left: 1px solid silver; +} +.cm-fat-cursor .CodeMirror-cursor { + width: auto; + border: 0 !important; + background: #7e7; +} +.cm-fat-cursor div.CodeMirror-cursors { + z-index: 1; +} +.cm-fat-cursor .CodeMirror-line::selection, +.cm-fat-cursor .CodeMirror-line > span::selection, +.cm-fat-cursor .CodeMirror-line > span > span::selection { background: transparent; } +.cm-fat-cursor .CodeMirror-line::-moz-selection, +.cm-fat-cursor .CodeMirror-line > span::-moz-selection, +.cm-fat-cursor .CodeMirror-line > span > span::-moz-selection { background: transparent; } +.cm-fat-cursor { caret-color: transparent; } +@-moz-keyframes blink { + 0% {} + 50% { background-color: transparent; } + 100% {} +} +@-webkit-keyframes blink { + 0% {} + 50% { background-color: transparent; } + 100% {} +} +@keyframes blink { + 0% {} + 50% { background-color: transparent; } + 100% {} +} + +/* Can style cursor different in overwrite (non-insert) mode */ +.CodeMirror-overwrite .CodeMirror-cursor {} + +.cm-tab { display: inline-block; text-decoration: inherit; } + +.CodeMirror-rulers { + position: absolute; + left: 0; right: 0; top: -50px; bottom: 0; + overflow: hidden; +} +.CodeMirror-ruler { + border-left: 1px solid #ccc; + top: 0; bottom: 0; + position: absolute; +} + +/* DEFAULT THEME */ + +.cm-s-default .cm-header {color: blue;} +.cm-s-default .cm-quote {color: #090;} +.cm-negative {color: #d44;} +.cm-positive {color: #292;} +.cm-header, .cm-strong {font-weight: bold;} +.cm-em {font-style: italic;} +.cm-link {text-decoration: underline;} +.cm-strikethrough {text-decoration: line-through;} + +.cm-s-default .cm-keyword {color: #708;} +.cm-s-default .cm-atom {color: #219;} +.cm-s-default .cm-number {color: #164;} +.cm-s-default .cm-def {color: #00f;} +.cm-s-default .cm-variable, +.cm-s-default .cm-punctuation, +.cm-s-default .cm-property, +.cm-s-default .cm-operator {} +.cm-s-default .cm-variable-2 {color: #05a;} +.cm-s-default .cm-variable-3, .cm-s-default .cm-type {color: #085;} +.cm-s-default .cm-comment {color: #a50;} +.cm-s-default .cm-string {color: #a11;} +.cm-s-default .cm-string-2 {color: #f50;} +.cm-s-default .cm-meta {color: #555;} +.cm-s-default .cm-qualifier {color: #555;} +.cm-s-default .cm-builtin {color: #30a;} +.cm-s-default .cm-bracket {color: #997;} +.cm-s-default .cm-tag {color: #170;} +.cm-s-default .cm-attribute {color: #00c;} +.cm-s-default .cm-hr {color: #999;} +.cm-s-default .cm-link {color: #00c;} + +.cm-s-default .cm-error {color: #f00;} +.cm-invalidchar {color: #f00;} + +.CodeMirror-composing { border-bottom: 2px solid; } + +/* Default styles for common addons */ + +div.CodeMirror span.CodeMirror-matchingbracket {color: #0b0;} +div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #a22;} +.CodeMirror-matchingtag { background: rgba(255, 150, 0, .3); } +.CodeMirror-activeline-background {background: #e8f2ff;} + +/* STOP */ + +/* The rest of this file contains styles related to the mechanics of + the editor. You probably shouldn't touch them. */ + +.CodeMirror { + position: relative; + overflow: hidden; + background: white; +} + +.CodeMirror-scroll { + overflow: scroll !important; /* Things will break if this is overridden */ + /* 50px is the magic margin used to hide the element's real scrollbars */ + /* See overflow: hidden in .CodeMirror */ + margin-bottom: -50px; margin-right: -50px; + padding-bottom: 50px; + height: 100%; + outline: none; /* Prevent dragging from highlighting the element */ + position: relative; + z-index: 0; +} +.CodeMirror-sizer { + position: relative; + border-right: 50px solid transparent; +} + +/* The fake, visible scrollbars. Used to force redraw during scrolling + before actual scrolling happens, thus preventing shaking and + flickering artifacts. */ +.CodeMirror-vscrollbar, .CodeMirror-hscrollbar, .CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { + position: absolute; + z-index: 6; + display: none; + outline: none; +} +.CodeMirror-vscrollbar { + right: 0; top: 0; + overflow-x: hidden; + overflow-y: scroll; +} +.CodeMirror-hscrollbar { + bottom: 0; left: 0; + overflow-y: hidden; + overflow-x: scroll; +} +.CodeMirror-scrollbar-filler { + right: 0; bottom: 0; +} +.CodeMirror-gutter-filler { + left: 0; bottom: 0; +} + +.CodeMirror-gutters { + position: absolute; left: 0; top: 0; + min-height: 100%; + z-index: 3; +} +.CodeMirror-gutter { + white-space: normal; + height: 100%; + display: inline-block; + vertical-align: top; + margin-bottom: -50px; +} +.CodeMirror-gutter-wrapper { + position: absolute; + z-index: 4; + background: none !important; + border: none !important; +} +.CodeMirror-gutter-background { + position: absolute; + top: 0; bottom: 0; + z-index: 4; +} +.CodeMirror-gutter-elt { + position: absolute; + cursor: default; + z-index: 4; +} +.CodeMirror-gutter-wrapper ::selection { background-color: transparent } +.CodeMirror-gutter-wrapper ::-moz-selection { background-color: transparent } + +.CodeMirror-lines { + cursor: text; + min-height: 1px; /* prevents collapsing before first draw */ +} +.CodeMirror pre.CodeMirror-line, +.CodeMirror pre.CodeMirror-line-like { + /* Reset some styles that the rest of the page might have set */ + -moz-border-radius: 0; -webkit-border-radius: 0; border-radius: 0; + border-width: 0; + background: transparent; + font-family: inherit; + font-size: inherit; + margin: 0; + white-space: pre; + word-wrap: normal; + line-height: inherit; + color: inherit; + z-index: 2; + position: relative; + overflow: visible; + -webkit-tap-highlight-color: transparent; + -webkit-font-variant-ligatures: contextual; + font-variant-ligatures: contextual; +} +.CodeMirror-wrap pre.CodeMirror-line, +.CodeMirror-wrap pre.CodeMirror-line-like { + word-wrap: break-word; + white-space: pre-wrap; + word-break: normal; +} + +.CodeMirror-linebackground { + position: absolute; + left: 0; right: 0; top: 0; bottom: 0; + z-index: 0; +} + +.CodeMirror-linewidget { + position: relative; + z-index: 2; + padding: 0.1px; /* Force widget margins to stay inside of the container */ +} + +.CodeMirror-widget {} + +.CodeMirror-rtl pre { direction: rtl; } + +.CodeMirror-code { + outline: none; +} + +/* Force content-box sizing for the elements where we expect it */ +.CodeMirror-scroll, +.CodeMirror-sizer, +.CodeMirror-gutter, +.CodeMirror-gutters, +.CodeMirror-linenumber { + -moz-box-sizing: content-box; + box-sizing: content-box; +} + +.CodeMirror-measure { + position: absolute; + width: 100%; + height: 0; + overflow: hidden; + visibility: hidden; +} + +.CodeMirror-cursor { + position: absolute; + pointer-events: none; +} +.CodeMirror-measure pre { position: static; } + +div.CodeMirror-cursors { + visibility: hidden; + position: relative; + z-index: 3; +} +div.CodeMirror-dragcursors { + visibility: visible; +} + +.CodeMirror-focused div.CodeMirror-cursors { + visibility: visible; +} + +.CodeMirror-selected { background: #d9d9d9; } +.CodeMirror-focused .CodeMirror-selected { background: #d7d4f0; } +.CodeMirror-crosshair { cursor: crosshair; } +.CodeMirror-line::selection, .CodeMirror-line > span::selection, .CodeMirror-line > span > span::selection { background: #d7d4f0; } +.CodeMirror-line::-moz-selection, .CodeMirror-line > span::-moz-selection, .CodeMirror-line > span > span::-moz-selection { background: #d7d4f0; } + +.cm-searching { + background-color: #ffa; + background-color: rgba(255, 255, 0, .4); +} + +/* Used to force a border model for a node */ +.cm-force-border { padding-right: .1px; } + +@media print { + /* Hide the cursor when printing */ + .CodeMirror div.CodeMirror-cursors { + visibility: hidden; + } +} + +/* See issue #2901 */ +.cm-tab-wrap-hack:after { content: ''; } + +/* Help users use markselection to safely style text background */ +span.CodeMirror-selectedtext { background: none; } diff --git a/examples/yjs-codemirror/src/codemirror.html b/examples/yjs-codemirror/src/codemirror.html new file mode 100644 index 0000000000..6f80c41b56 --- /dev/null +++ b/examples/yjs-codemirror/src/codemirror.html @@ -0,0 +1,56 @@ + + + + + Yjs CodeMirror Example + + + + + + +

+

+ This is a demo of the Yjs ⇔ + CodeMirror binding: + y-codemirror. +

+

+ The content of this editor is shared with every client that visits this + domain. +

+ + diff --git a/examples/yjs-codemirror/src/codemirror.js b/examples/yjs-codemirror/src/codemirror.js new file mode 100644 index 0000000000..ee15f70809 --- /dev/null +++ b/examples/yjs-codemirror/src/codemirror.js @@ -0,0 +1,42 @@ +/* eslint-env browser */ + +// @ts-ignore +import CodeMirror from 'codemirror' +import * as Y from 'yjs' +import { ElectricProvider } from './y-electric.js' +import { CodemirrorBinding } from 'y-codemirror' +import 'codemirror/mode/javascript/javascript.js' + +window.addEventListener('load', () => { + const ydoc = new Y.Doc() + const provider = new ElectricProvider( + `http://localhost:3000/`, + 'codemirror-demo-2024/06', + ydoc + ) + const ytext = ydoc.getText('codemirror') + const editorContainer = document.createElement('div') + editorContainer.setAttribute('id', 'editor') + document.body.insertBefore(editorContainer, null) + + const editor = CodeMirror(editorContainer, { + mode: 'javascript', + lineNumbers: true + }) + + const binding = new CodemirrorBinding(ytext, editor, provider.awareness) + + const connectBtn = /** @type {HTMLElement} */ (document.getElementById('y-connect-btn')) + connectBtn.addEventListener('click', () => { + if (provider.shouldConnect) { + provider.disconnect() + connectBtn.textContent = 'Connect' + } else { + provider.connect() + connectBtn.textContent = 'Disconnect' + } + }) + + // @ts-ignore + window.example = { provider, ydoc, ytext, binding, Y } +}) diff --git a/examples/yjs-codemirror/src/y-electric.js b/examples/yjs-codemirror/src/y-electric.js new file mode 100644 index 0000000000..f97dc003fa --- /dev/null +++ b/examples/yjs-codemirror/src/y-electric.js @@ -0,0 +1,393 @@ +/** + * @module provider/websocket + */ + +/* eslint-env browser */ + +import * as Y from 'yjs' // eslint-disable-line +import * as time from 'lib0/time' +import * as encoding from 'lib0/encoding' +import * as decoding from 'lib0/decoding' +import * as syncProtocol from 'y-protocols/sync' +import * as awarenessProtocol from 'y-protocols/awareness' +import { Observable } from 'lib0/observable' +import * as url from 'lib0/url' +import * as env from 'lib0/environment' + +export const messageSync = 0 +export const messageQueryAwareness = 3 +export const messageAwareness = 1 + +import { ShapeStream } from '@electric-sql/client' + +// Check if we can handle encoding another way +import {Base64} from 'js-base64'; + +/** + * encoder, decoder, provider, emitSynced, messageType + * @type {Array} + */ +const messageHandlers = [] + +messageHandlers[messageSync] = ( + encoder, + decoder, + provider, + emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + provider.doc, + provider + ) + if ( + emitSynced && syncMessageType === syncProtocol.messageYjsSyncStep2 && + !provider.synced + ) { + provider.synced = true + } +} + +messageHandlers[messageQueryAwareness] = ( + encoder, + _decoder, + provider, + _emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate( + provider.awareness, + Array.from(provider.awareness.getStates().keys()) + ) + ) +} + +messageHandlers[messageAwareness] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType +) => { + awarenessProtocol.applyAwarenessUpdate( + provider.awareness, + decoding.readVarUint8Array(decoder), + provider + ) +} + +/** + * @param {ElectricProvider} provider + */ +const setupShapeStream = (provider) => { + if (provider.shouldConnect && provider.stream === null) { + provider.connecting = true + provider.connected = false + provider.synced = false + + provider.stream = new ShapeStream({ + url: provider.url, + signal: new AbortController().signal + }) + + const readMessage = (provider, buf, emitSynced) => { + const decoder = decoding.createDecoder(buf) + const encoder = encoding.createEncoder() + const messageType = decoding.readVarUint(decoder) + const messageHandler = provider.messageHandlers[messageType] + if (/** @type {any} */ (messageHandler)) { + messageHandler(encoder, decoder, provider, emitSynced, messageType) + } else { + console.error('Unable to compute message') + } + return encoder + } + + const handleSyncMessage = (messages) => { + provider.lastMessageReceived = time.getUnixTime() + messages.forEach(message => { + if(message['key']){ + const buf = Base64.toUint8Array(message['value']['op']) + readMessage(provider, buf, true) + } + }) + } + + const handleError = (event) => { + console.warn('fetch shape error', event) + provider.emit('connection-error', [event, provider]) + } + + const unsubscribeSyncHandler = provider.stream.subscribe( + handleSyncMessage, + handleError + ) + + if (provider.awareness.getLocalState() !== null) { + const encoderAwarenessState = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessState, messageAwareness) + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ + provider.doc.clientID + ]) + ) + // websocket.send(encoding.toUint8Array(encoderAwarenessState)) + } + + + provider.closeHandler = (event) => { + provider.stream = null + provider.connecting = false + if (provider.connected) { + provider.connected = false + provider.synced = false + // update awareness (all users except local left) + awarenessProtocol.removeAwarenessStates( + provider.awareness, + Array.from(provider.awareness.getStates().keys()).filter((client) => + client !== provider.doc.clientID + ), + provider + ) + provider.emit('status', [{ + status: 'disconnected' + }]) + } + + unsubscribeSyncHandler() + provider.closeHandler = null + provider.emit('connection-close', [event, provider]) + } + + const handleOnceUpToDate = () => { + provider.lastMessageReceived = time.getUnixTime() + provider.connecting = false + provider.connected = true + provider.emit('status', [{ + status: 'connected' + }]) + + provider.pending.splice(0).forEach( + (buf) => broadcastMessage(provider, buf)) + + } + + provider.stream.subscribeOnceToUpToDate( + () => handleOnceUpToDate(), + () => handleError() + ) + + provider.emit('status', [{ + status: 'connecting' + }]) + } +} + +/** + * @param {ElectricProvider} provider + * @param {Uint8Array} buf + */ +const broadcastMessage = (provider, buf) => { + if (provider.connected && provider.stream !== null) { + const clientId = provider.doc.clientID + const name = provider.roomname + const op = Base64.fromUint8Array(buf) + + const mutation = { + action: `insert`, + schema: `public`, + tablename: `ydoc_updates`, + row: {name, op} + } + const req = buildRequest(clientId, new Date().getTime(), [mutation]) + fetch(req) + } +} + +function buildRequest( + clientId, + requestId, + mutations +) { + const url = `http://localhost:8080/` + return new Request(url, { + method: `POST`, + headers: { + 'Content-Type': `application/json`, + 'X-Electric-Request-Id': requestId, + 'X-Electric-User-Id': clientId, // TODO: drop this + }, + body: JSON.stringify(mutations), + }) +} + +/** + * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document. + * The document name is attached to the provided url. I.e. the following example + * creates a websocket connection to http://localhost:1234/my-document-name + * + * @example + * import * as Y from 'yjs' + * import { ElectricProvider } from 'y-websocket' + * const doc = new Y.Doc() + * const provider = new ElectricProvider('http://localhost:1234', 'my-document-name', doc) + * + * @extends {Observable} + */ +export class ElectricProvider extends Observable { + /** + * @param {string} serverUrl + * @param {string} roomname + * @param {Y.Doc} doc + * @param {object} opts + * @param {boolean} [opts.connect] + * @param {awarenessProtocol.Awareness} [opts.awareness] + * @param {Object} [opts.params] specify url parameters + * @param {Array} [opts.protocols] specify websocket protocols + * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff) + */ + constructor (serverUrl, roomname, doc, { + connect = true, + awareness = new awarenessProtocol.Awareness(doc) + } = {}) { + super() + // ensure that url is always ends with / + while (serverUrl[serverUrl.length - 1] === '/') { + serverUrl = serverUrl.slice(0, serverUrl.length - 1) + } + this.serverUrl = serverUrl + this.roomname = roomname + this.doc = doc + this.awareness = awareness + this.connected = false + this.connecting = false + + this.messageHandlers = messageHandlers.slice() + /** + * @type {boolean} + */ + this._synced = false + + this.lastMessageReceived = 0 + /** + * Whether to connect to other peers or not + * @type {boolean} + */ + this.shouldConnect = connect + + + /** + * @type {ShapeStream?} + */ + + this.stream = null + + this.pending = new Array() + + this.closeHandler = null + + /** + * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel) + * @param {Uint8Array} update + * @param {any} origin + */ + this._updateHandler = (update, origin) => { + // TODO would be nice to skip updates that are already included + if (origin !== this) { + if (!this.connected) { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeUpdate(encoder, update) + + this.pending.push( encoding.toUint8Array(encoder)) + } else { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeUpdate(encoder, update) + broadcastMessage(this, encoding.toUint8Array(encoder)) + } + } + } + this.doc.on('update', this._updateHandler) + + /** + * @param {any} changed + * @param {any} _origin + */ + this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { + const changedClients = added.concat(updated).concat(removed) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) + ) + // broadcastMessage(this, encoding.toUint8Array(encoder)) + } + this._exitHandler = () => { + awarenessProtocol.removeAwarenessStates( + this.awareness, + [doc.clientID], + 'app closed' + ) + } + if (env.isNode && typeof process !== 'undefined') { + process.on('exit', this._exitHandler) + } + awareness.on('update', this._awarenessUpdateHandler) + + if (connect) { + this.connect() + } + } + + get url () { + const params = {"where": `name = '${this.roomname}'`} + const encodedParams = url.encodeQueryParams(params) + return this.serverUrl + '/v1/shape/ydoc_updates?' + encodedParams + } + + /** + * @type {boolean} + */ + get synced () { + return this._synced + } + + set synced (state) { + if (this._synced !== state) { + this._synced = state + this.emit('synced', [state]) + this.emit('sync', [state]) + } + } + + destroy () { + this.disconnect() + if (env.isNode && typeof process !== 'undefined') { + process.off('exit', this._exitHandler) + } + this.awareness.off('update', this._awarenessUpdateHandler) + this.doc.off('update', this._updateHandler) + super.destroy() + } + + disconnect () { + this.shouldConnect = false + this.closeHandler() + } + + connect () { + this.shouldConnect = true + if (!this.connected && this.stream === null) { + setupShapeStream(this) + } + } +} \ No newline at end of file From 87fbdb999548cd6cd53761a661f86659702e11f6 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 9 Aug 2024 10:49:11 +0100 Subject: [PATCH 02/47] Changed schema --- examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql index eed8ee6001..a9e5aab189 100644 --- a/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql +++ b/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql @@ -1,5 +1,5 @@ CREATE TABLE ydoc_updates( - id uuid PRIMARY KEY, + id SERIAL PRIMARY KEY, name TEXT, op TEXT NOT NULL ); From 3a5003aaad0f8f6ae65dcc9d74d026e3cd847dea Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 9 Aug 2024 10:56:27 +0100 Subject: [PATCH 03/47] formatted --- examples/yjs-codemirror/package.json | 2 +- examples/yjs-codemirror/src/y-electric.js | 195 +++++++++++----------- 2 files changed, 100 insertions(+), 97 deletions(-) diff --git a/examples/yjs-codemirror/package.json b/examples/yjs-codemirror/package.json index 43d8ea98ae..73f72f2efc 100644 --- a/examples/yjs-codemirror/package.json +++ b/examples/yjs-codemirror/package.json @@ -11,7 +11,7 @@ "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", "dev": "vite", "build": "vite build", - "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "lint": "eslint . --ext js,ts,tsx --report-unused-disable-directives --max-warnings 0", "preview": "vite preview", "typecheck": "tsc --noEmit" }, diff --git a/examples/yjs-codemirror/src/y-electric.js b/examples/yjs-codemirror/src/y-electric.js index f97dc003fa..55ed67cc47 100644 --- a/examples/yjs-codemirror/src/y-electric.js +++ b/examples/yjs-codemirror/src/y-electric.js @@ -5,23 +5,23 @@ /* eslint-env browser */ import * as Y from 'yjs' // eslint-disable-line -import * as time from 'lib0/time' -import * as encoding from 'lib0/encoding' -import * as decoding from 'lib0/decoding' -import * as syncProtocol from 'y-protocols/sync' -import * as awarenessProtocol from 'y-protocols/awareness' -import { Observable } from 'lib0/observable' -import * as url from 'lib0/url' -import * as env from 'lib0/environment' +import * as time from "lib0/time" +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" +import * as syncProtocol from "y-protocols/sync" +import * as awarenessProtocol from "y-protocols/awareness" +import { Observable } from "lib0/observable" +import * as url from "lib0/url" +import * as env from "lib0/environment" export const messageSync = 0 export const messageQueryAwareness = 3 export const messageAwareness = 1 -import { ShapeStream } from '@electric-sql/client' +import { ShapeStream } from "@electric-sql/client" // Check if we can handle encoding another way -import {Base64} from 'js-base64'; +import { Base64 } from "js-base64" /** * encoder, decoder, provider, emitSynced, messageType @@ -44,7 +44,8 @@ messageHandlers[messageSync] = ( provider ) if ( - emitSynced && syncMessageType === syncProtocol.messageYjsSyncStep2 && + emitSynced && + syncMessageType === syncProtocol.messageYjsSyncStep2 && !provider.synced ) { provider.synced = true @@ -93,39 +94,39 @@ const setupShapeStream = (provider) => { provider.stream = new ShapeStream({ url: provider.url, - signal: new AbortController().signal + signal: new AbortController().signal, }) - const readMessage = (provider, buf, emitSynced) => { - const decoder = decoding.createDecoder(buf) - const encoder = encoding.createEncoder() - const messageType = decoding.readVarUint(decoder) - const messageHandler = provider.messageHandlers[messageType] - if (/** @type {any} */ (messageHandler)) { - messageHandler(encoder, decoder, provider, emitSynced, messageType) - } else { - console.error('Unable to compute message') + const readMessage = (provider, buf, emitSynced) => { + const decoder = decoding.createDecoder(buf) + const encoder = encoding.createEncoder() + const messageType = decoding.readVarUint(decoder) + const messageHandler = provider.messageHandlers[messageType] + if (/** @type {any} */ (messageHandler)) { + messageHandler(encoder, decoder, provider, emitSynced, messageType) + } else { + console.error(`Unable to compute message`) + } + return encoder } - return encoder - } const handleSyncMessage = (messages) => { - provider.lastMessageReceived = time.getUnixTime() - messages.forEach(message => { - if(message['key']){ - const buf = Base64.toUint8Array(message['value']['op']) + provider.lastMessageReceived = time.getUnixTime() + messages.forEach((message) => { + if (message[`key`]) { + const buf = Base64.toUint8Array(message[`value`][`op`]) readMessage(provider, buf, true) } }) } const handleError = (event) => { - console.warn('fetch shape error', event) - provider.emit('connection-error', [event, provider]) + console.warn(`fetch shape error`, event) + provider.emit(`connection-error`, [event, provider]) } const unsubscribeSyncHandler = provider.stream.subscribe( - handleSyncMessage, + handleSyncMessage, handleError ) @@ -135,12 +136,11 @@ const setupShapeStream = (provider) => { encoding.writeVarUint8Array( encoderAwarenessState, awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ - provider.doc.clientID + provider.doc.clientID, ]) ) // websocket.send(encoding.toUint8Array(encoderAwarenessState)) } - provider.closeHandler = (event) => { provider.stream = null @@ -151,42 +151,48 @@ const setupShapeStream = (provider) => { // update awareness (all users except local left) awarenessProtocol.removeAwarenessStates( provider.awareness, - Array.from(provider.awareness.getStates().keys()).filter((client) => - client !== provider.doc.clientID + Array.from(provider.awareness.getStates().keys()).filter( + (client) => client !== provider.doc.clientID ), provider ) - provider.emit('status', [{ - status: 'disconnected' - }]) + provider.emit(`status`, [ + { + status: `disconnected`, + }, + ]) } unsubscribeSyncHandler() provider.closeHandler = null - provider.emit('connection-close', [event, provider]) + provider.emit(`connection-close`, [event, provider]) } const handleOnceUpToDate = () => { provider.lastMessageReceived = time.getUnixTime() provider.connecting = false provider.connected = true - provider.emit('status', [{ - status: 'connected' - }]) - - provider.pending.splice(0).forEach( - (buf) => broadcastMessage(provider, buf)) - + provider.emit(`status`, [ + { + status: `connected`, + }, + ]) + + provider.pending + .splice(0) + .forEach((buf) => broadcastMessage(provider, buf)) } - provider.stream.subscribeOnceToUpToDate( - () => handleOnceUpToDate(), - () => handleError() + provider.stream.subscribeOnceToUpToDate( + () => handleOnceUpToDate(), + () => handleError() ) - provider.emit('status', [{ - status: 'connecting' - }]) + provider.emit(`status`, [ + { + status: `connecting`, + }, + ]) } } @@ -204,25 +210,21 @@ const broadcastMessage = (provider, buf) => { action: `insert`, schema: `public`, tablename: `ydoc_updates`, - row: {name, op} + row: { name, op }, } const req = buildRequest(clientId, new Date().getTime(), [mutation]) fetch(req) } } -function buildRequest( - clientId, - requestId, - mutations -) { +function buildRequest(clientId, requestId, mutations) { const url = `http://localhost:8080/` return new Request(url, { method: `POST`, headers: { - 'Content-Type': `application/json`, - 'X-Electric-Request-Id': requestId, - 'X-Electric-User-Id': clientId, // TODO: drop this + "Content-Type": `application/json`, + "X-Electric-Request-Id": requestId, + "X-Electric-User-Id": clientId, // TODO: drop this }, body: JSON.stringify(mutations), }) @@ -253,13 +255,15 @@ export class ElectricProvider extends Observable { * @param {Array} [opts.protocols] specify websocket protocols * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff) */ - constructor (serverUrl, roomname, doc, { - connect = true, - awareness = new awarenessProtocol.Awareness(doc) - } = {}) { + constructor( + serverUrl, + roomname, + doc, + { connect = true, awareness = new awarenessProtocol.Awareness(doc) } = {} + ) { super() // ensure that url is always ends with / - while (serverUrl[serverUrl.length - 1] === '/') { + while (serverUrl[serverUrl.length - 1] === `/`) { serverUrl = serverUrl.slice(0, serverUrl.length - 1) } this.serverUrl = serverUrl @@ -268,7 +272,7 @@ export class ElectricProvider extends Observable { this.awareness = awareness this.connected = false this.connecting = false - + this.messageHandlers = messageHandlers.slice() /** * @type {boolean} @@ -282,16 +286,15 @@ export class ElectricProvider extends Observable { */ this.shouldConnect = connect - /** * @type {ShapeStream?} - */ - - this.stream = null - - this.pending = new Array() + */ + + this.stream = null + + this.pending = [] - this.closeHandler = null + this.closeHandler = null /** * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel) @@ -305,8 +308,8 @@ export class ElectricProvider extends Observable { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) syncProtocol.writeUpdate(encoder, update) - - this.pending.push( encoding.toUint8Array(encoder)) + + this.pending.push(encoding.toUint8Array(encoder)) } else { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) @@ -315,7 +318,7 @@ export class ElectricProvider extends Observable { } } } - this.doc.on('update', this._updateHandler) + this.doc.on(`update`, this._updateHandler) /** * @param {any} changed @@ -335,59 +338,59 @@ export class ElectricProvider extends Observable { awarenessProtocol.removeAwarenessStates( this.awareness, [doc.clientID], - 'app closed' + `app closed` ) } - if (env.isNode && typeof process !== 'undefined') { - process.on('exit', this._exitHandler) + if (env.isNode && typeof process !== `undefined`) { + process.on(`exit`, this._exitHandler) } - awareness.on('update', this._awarenessUpdateHandler) - + awareness.on(`update`, this._awarenessUpdateHandler) + if (connect) { this.connect() } } - get url () { - const params = {"where": `name = '${this.roomname}'`} + get url() { + const params = { where: `name = '${this.roomname}'` } const encodedParams = url.encodeQueryParams(params) - return this.serverUrl + '/v1/shape/ydoc_updates?' + encodedParams + return this.serverUrl + `/v1/shape/ydoc_updates?` + encodedParams } /** * @type {boolean} */ - get synced () { + get synced() { return this._synced } - set synced (state) { + set synced(state) { if (this._synced !== state) { this._synced = state - this.emit('synced', [state]) - this.emit('sync', [state]) + this.emit(`synced`, [state]) + this.emit(`sync`, [state]) } } - destroy () { + destroy() { this.disconnect() - if (env.isNode && typeof process !== 'undefined') { - process.off('exit', this._exitHandler) + if (env.isNode && typeof process !== `undefined`) { + process.off(`exit`, this._exitHandler) } - this.awareness.off('update', this._awarenessUpdateHandler) - this.doc.off('update', this._updateHandler) + this.awareness.off(`update`, this._awarenessUpdateHandler) + this.doc.off(`update`, this._updateHandler) super.destroy() } - disconnect () { + disconnect() { this.shouldConnect = false this.closeHandler() } - connect () { + connect() { this.shouldConnect = true if (!this.connected && this.stream === null) { setupShapeStream(this) } } -} \ No newline at end of file +} From 8bb837e613c57811d606d73409829320db153f9c Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sat, 10 Aug 2024 01:47:17 +0100 Subject: [PATCH 04/47] Improvements --- examples/nextjs-yjs-codemirror/.eslintignore | 1 + .../.eslintrc.cjs | 1 + examples/nextjs-yjs-codemirror/.gitignore | 10 + .../.prettierrc | 0 examples/nextjs-yjs-codemirror/README.md | 22 ++ examples/nextjs-yjs-codemirror/app/App.css | 25 ++ .../nextjs-yjs-codemirror/app/Example.css | 25 ++ .../app/api/awareness/route.ts | 16 + .../app/api/operation/route.ts | 12 + examples/nextjs-yjs-codemirror/app/db.ts | 14 + examples/nextjs-yjs-codemirror/app/layout.tsx | 26 ++ examples/nextjs-yjs-codemirror/app/page.tsx | 104 ++++++ .../app/shape-proxy/[...table]/route.ts | 29 ++ examples/nextjs-yjs-codemirror/app/style.css | 52 +++ .../app}/y-electric.js | 157 ++++---- .../db/migrations/01-create_yjs_tables.sql | 27 ++ examples/nextjs-yjs-codemirror/index.html | 18 + examples/nextjs-yjs-codemirror/package.json | 48 +++ .../nextjs-yjs-codemirror/public/favicon.ico | Bin 0 -> 1659 bytes .../nextjs-yjs-codemirror/public/logo.svg | 11 + .../nextjs-yjs-codemirror/public/robots.txt | 3 + examples/nextjs-yjs-codemirror/tsconfig.json | 28 ++ examples/remix-basic/.eslintrc.cjs | 1 + examples/yjs-codemirror/.gitignore | 2 - .../db/migrations/01-create_yjs_tables.sql | 13 - examples/yjs-codemirror/index.html | 62 ---- examples/yjs-codemirror/package.json | 35 -- examples/yjs-codemirror/src/codemirror.css | 344 ------------------ examples/yjs-codemirror/src/codemirror.html | 56 --- examples/yjs-codemirror/src/codemirror.js | 42 --- 30 files changed, 558 insertions(+), 626 deletions(-) create mode 100644 examples/nextjs-yjs-codemirror/.eslintignore rename examples/{yjs-codemirror => nextjs-yjs-codemirror}/.eslintrc.cjs (98%) create mode 100644 examples/nextjs-yjs-codemirror/.gitignore rename examples/{yjs-codemirror => nextjs-yjs-codemirror}/.prettierrc (100%) create mode 100644 examples/nextjs-yjs-codemirror/README.md create mode 100644 examples/nextjs-yjs-codemirror/app/App.css create mode 100644 examples/nextjs-yjs-codemirror/app/Example.css create mode 100644 examples/nextjs-yjs-codemirror/app/api/awareness/route.ts create mode 100644 examples/nextjs-yjs-codemirror/app/api/operation/route.ts create mode 100644 examples/nextjs-yjs-codemirror/app/db.ts create mode 100644 examples/nextjs-yjs-codemirror/app/layout.tsx create mode 100644 examples/nextjs-yjs-codemirror/app/page.tsx create mode 100644 examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts create mode 100644 examples/nextjs-yjs-codemirror/app/style.css rename examples/{yjs-codemirror/src => nextjs-yjs-codemirror/app}/y-electric.js (71%) create mode 100644 examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql create mode 100644 examples/nextjs-yjs-codemirror/index.html create mode 100644 examples/nextjs-yjs-codemirror/package.json create mode 100644 examples/nextjs-yjs-codemirror/public/favicon.ico create mode 100644 examples/nextjs-yjs-codemirror/public/logo.svg create mode 100644 examples/nextjs-yjs-codemirror/public/robots.txt create mode 100644 examples/nextjs-yjs-codemirror/tsconfig.json delete mode 100644 examples/yjs-codemirror/.gitignore delete mode 100644 examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql delete mode 100644 examples/yjs-codemirror/index.html delete mode 100644 examples/yjs-codemirror/package.json delete mode 100644 examples/yjs-codemirror/src/codemirror.css delete mode 100644 examples/yjs-codemirror/src/codemirror.html delete mode 100644 examples/yjs-codemirror/src/codemirror.js diff --git a/examples/nextjs-yjs-codemirror/.eslintignore b/examples/nextjs-yjs-codemirror/.eslintignore new file mode 100644 index 0000000000..e32a3e1834 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/.eslintignore @@ -0,0 +1 @@ +/build/** diff --git a/examples/yjs-codemirror/.eslintrc.cjs b/examples/nextjs-yjs-codemirror/.eslintrc.cjs similarity index 98% rename from examples/yjs-codemirror/.eslintrc.cjs rename to examples/nextjs-yjs-codemirror/.eslintrc.cjs index 8aeb375841..c5c99d0cd9 100644 --- a/examples/yjs-codemirror/.eslintrc.cjs +++ b/examples/nextjs-yjs-codemirror/.eslintrc.cjs @@ -37,5 +37,6 @@ module.exports = { `tsup.config.ts`, `vitest.config.ts`, `.eslintrc.js`, + `**/*.css`, ], }; diff --git a/examples/nextjs-yjs-codemirror/.gitignore b/examples/nextjs-yjs-codemirror/.gitignore new file mode 100644 index 0000000000..7a06c3b2bd --- /dev/null +++ b/examples/nextjs-yjs-codemirror/.gitignore @@ -0,0 +1,10 @@ +dist +.env.local + +# Turborepo +.turbo + +# next.js +/.next/ +/out/ +next-env.d.ts diff --git a/examples/yjs-codemirror/.prettierrc b/examples/nextjs-yjs-codemirror/.prettierrc similarity index 100% rename from examples/yjs-codemirror/.prettierrc rename to examples/nextjs-yjs-codemirror/.prettierrc diff --git a/examples/nextjs-yjs-codemirror/README.md b/examples/nextjs-yjs-codemirror/README.md new file mode 100644 index 0000000000..49a66ab337 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/README.md @@ -0,0 +1,22 @@ +# Basic Remix example + +## Setup + +1. Make sure you've installed all dependencies for the monorepo and built packages + +From the root directory: + +- `pnpm i` +- `pnpm run -r build` + +2. Start the docker containers + +`pnpm run backend:up` + +3. Start the dev server + +`pnpm run dev` + +4. When done, tear down the backend containers so you can run other examples + +`pnpm run backend:down` diff --git a/examples/nextjs-yjs-codemirror/app/App.css b/examples/nextjs-yjs-codemirror/app/App.css new file mode 100644 index 0000000000..620047949a --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/App.css @@ -0,0 +1,25 @@ +.App { + text-align: center; +} + +.App-logo { + height: min(160px, 30vmin); + pointer-events: none; + margin-top: min(30px, 5vmin); + margin-bottom: min(30px, 5vmin); +} + +.App-header { + background-color: #1c1e20; + min-height: 100vh; + display: flex; + flex-direction: column; + align-items: top; + justify-content: top; + font-size: calc(10px + 2vmin); + color: white; +} + +.App-link { + color: #61dafb; +} \ No newline at end of file diff --git a/examples/nextjs-yjs-codemirror/app/Example.css b/examples/nextjs-yjs-codemirror/app/Example.css new file mode 100644 index 0000000000..b6fb457c58 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/Example.css @@ -0,0 +1,25 @@ +.controls { + margin-bottom: 1.5rem; +} + +.button { + display: inline-block; + line-height: 1.3; + text-align: center; + text-decoration: none; + vertical-align: middle; + cursor: pointer; + user-select: none; + width: calc(15vw + 100px); + margin-right: 0.5rem !important; + margin-left: 0.5rem !important; + border-radius: 32px; + text-shadow: 2px 6px 20px rgba(0, 0, 0, 0.4); + box-shadow: rgba(0, 0, 0, 0.5) 1px 2px 8px 0px; + background: #1e2123; + border: 2px solid #229089; + color: #f9fdff; + font-size: 16px; + font-weight: 500; + padding: 10px 18px; +} \ No newline at end of file diff --git a/examples/nextjs-yjs-codemirror/app/api/awareness/route.ts b/examples/nextjs-yjs-codemirror/app/api/awareness/route.ts new file mode 100644 index 0000000000..41a0b68a6e --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/api/awareness/route.ts @@ -0,0 +1,16 @@ +import { db } from "../../db" +import { NextResponse } from "next/server" + +export async function POST(request: Request) { + const body = await request.json() + + await db.query( + `INSERT INTO ydoc_awareness (client, name, op) + VALUES ($1, $2, $3) + ON CONFLICT (client, name) + DO UPDATE SET op = $3`, + [body.client, body.name, body.op] + ) + + return NextResponse.json({}) +} diff --git a/examples/nextjs-yjs-codemirror/app/api/operation/route.ts b/examples/nextjs-yjs-codemirror/app/api/operation/route.ts new file mode 100644 index 0000000000..68ad7ba3f3 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/api/operation/route.ts @@ -0,0 +1,12 @@ +import { db } from "../../db" +import { NextResponse } from "next/server" + +export async function POST(request: Request) { + const body = await request.json() + await db.query( + `INSERT INTO ydoc_updates (name, op) + VALUES ($1, $2)`, + [body.name, body.op] + ) + return NextResponse.json({}) +} diff --git a/examples/nextjs-yjs-codemirror/app/db.ts b/examples/nextjs-yjs-codemirror/app/db.ts new file mode 100644 index 0000000000..1571b1341c --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/db.ts @@ -0,0 +1,14 @@ +import pgPkg from "pg" +const { Client } = pgPkg + +const db = new Client({ + host: `localhost`, + port: 54321, + password: `password`, + user: `postgres`, + database: `electric`, +}) + +db.connect() + +export { db } diff --git a/examples/nextjs-yjs-codemirror/app/layout.tsx b/examples/nextjs-yjs-codemirror/app/layout.tsx new file mode 100644 index 0000000000..dded27eb6a --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/layout.tsx @@ -0,0 +1,26 @@ +import "./style.css" +import "./App.css" + +export const metadata = { + title: `Next.js Forms Example`, + description: `Example application with forms and Postgres.`, +} + +export default function RootLayout({ + children, +}: { + children: React.ReactNode +}) { + return ( + + +
+
+ logo + {children} +
+
+ + + ) +} diff --git a/examples/nextjs-yjs-codemirror/app/page.tsx b/examples/nextjs-yjs-codemirror/app/page.tsx new file mode 100644 index 0000000000..14688b2ca4 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/page.tsx @@ -0,0 +1,104 @@ +"use client" + +import "./Example.css" +import { useEffect, useRef, useState } from "react" + +import * as Y from "yjs" +import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" +import { ElectricProvider } from "./y-electric" + +import { EditorState, EditorView, basicSetup } from "@codemirror/basic-setup" +import { keymap } from "@codemirror/view" +import { javascript } from "@codemirror/lang-javascript" + +import * as random from "lib0/random" + +const usercolors = [ + { color: `#30bced`, light: `#30bced33` }, + { color: `#6eeb83`, light: `#6eeb8333` }, + { color: `#ffbc42`, light: `#ffbc4233` }, + { color: `#ecd444`, light: `#ecd44433` }, + { color: `#ee6352`, light: `#ee635233` }, + { color: `#9ac2c9`, light: `#9ac2c933` }, + { color: `#8acb88`, light: `#8acb8833` }, + { color: `#1be7ff`, light: `#1be7ff33` }, +] + +const userColor = usercolors[random.uint32() % usercolors.length] + +const theme = EditorView.theme( + { + ".cm-content": { + minWidth: `400px`, + textAlign: `left`, + backgroundColor: `#223239`, + }, + ".cm-content .cm-gutter": { + minHeight: `200px`, + }, + }, + { dark: true } +) + +const ydoc = new Y.Doc() +const provider = new ElectricProvider( + `http://localhost:3000/`, + `electric-demo`, + ydoc +) + +export default function Home() { + const editor = useRef(null) + + const [connect, setConnect] = useState(`connected`) + + const toggle = () => { + if (connect === `connected`) { + provider.disconnect() + setConnect(`disconnected`) + } else { + provider.connect() + setConnect(`connected`) + } + } + + useEffect(() => { + const ytext = ydoc.getText(`codemirror`) + + provider.awareness.setLocalStateField(`user`, { + name: `Anonymous ` + Math.floor(Math.random() * 100), + color: userColor.color, + colorLight: userColor.light, + }) + + const state = EditorState.create({ + doc: ytext.toString(), + extensions: [ + keymap.of([...yUndoManagerKeymap]), + basicSetup, + javascript(), + EditorView.lineWrapping, + yCollab(ytext, provider.awareness), + theme, + ], + }) + + const view = new EditorView({ state, parent: editor.current ?? undefined }) + + return () => { + view.destroy() + // editor.current.removeEventListener("input", log); + } + }) + + return ( +
+
toggle()}> + +
+
+
+ ) +} diff --git a/examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts b/examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts new file mode 100644 index 0000000000..54e3562d55 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts @@ -0,0 +1,29 @@ +export async function GET( + request: Request, + { params }: { params: { table: string } } +) { + const url = new URL(request.url) + const { table } = params + const originUrl = new URL(`http://localhost:3000/v1/shape/${table}`) + url.searchParams.forEach((value, key) => { + originUrl.searchParams.set(key, value) + }) + + // When proxying long-polling requests, content-encoding & content-length are added + // erroneously (saying the body is gzipped when it's not) so we'll just remove + // them to avoid content decoding errors in the browser. + // + // Similar-ish problem to https://github.com/wintercg/fetch/issues/23 + let resp = await fetch(originUrl.toString()) + if (resp.headers.get(`content-encoding`)) { + const headers = new Headers(resp.headers) + headers.delete(`content-encoding`) + headers.delete(`content-length`) + resp = new Response(resp.body, { + status: resp.status, + statusText: resp.statusText, + headers, + }) + } + return resp +} diff --git a/examples/nextjs-yjs-codemirror/app/style.css b/examples/nextjs-yjs-codemirror/app/style.css new file mode 100644 index 0000000000..c763f4d19f --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/style.css @@ -0,0 +1,52 @@ +body { + margin: 0; + font-family: "Helvetica Neue", Helvetica, sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + background: #1c1e20; + min-width: 360px; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", + monospace; +} + + +/* YJS awareness */ + +.remote-caret { + position: relative; + border-left: 2px solid black; + margin-left: -1px; + margin-right: -1px; + box-sizing: border-box; +} + +.remote-caret>div { + position: absolute; + top: -1.05em; + left: -2px; + font-size: .6em; + background-color: rgb(250, 129, 0); + font-family: serif; + font-style: normal; + font-weight: normal; + line-height: normal; + user-select: none; + color: white; + padding-left: 2px; + padding-right: 2px; + z-index: 3; + transition: opacity .3s ease-in-out; +} + +.remote-caret.hide-name>div { + transition-delay: .7s; + opacity: 0; +} + +.remote-caret:hover>div { + opacity: 1; + transition-delay: 0s; +} \ No newline at end of file diff --git a/examples/yjs-codemirror/src/y-electric.js b/examples/nextjs-yjs-codemirror/app/y-electric.js similarity index 71% rename from examples/yjs-codemirror/src/y-electric.js rename to examples/nextjs-yjs-codemirror/app/y-electric.js index 55ed67cc47..c9154bbc2e 100644 --- a/examples/yjs-codemirror/src/y-electric.js +++ b/examples/nextjs-yjs-codemirror/app/y-electric.js @@ -87,13 +87,18 @@ messageHandlers[messageAwareness] = ( * @param {ElectricProvider} provider */ const setupShapeStream = (provider) => { - if (provider.shouldConnect && provider.stream === null) { + if (provider.shouldConnect && provider.operationsStream === null) { provider.connecting = true provider.connected = false provider.synced = false - provider.stream = new ShapeStream({ - url: provider.url, + provider.operationsStream = new ShapeStream({ + url: provider.operationsUrl, + signal: new AbortController().signal, + }) + + provider.awarenessStrean = new ShapeStream({ + url: provider.awarenessUrl, signal: new AbortController().signal, }) @@ -113,7 +118,8 @@ const setupShapeStream = (provider) => { const handleSyncMessage = (messages) => { provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { - if (message[`key`]) { + // ignore DELETE operations + if (message[`key`] && message[`value`][`op`]) { const buf = Base64.toUint8Array(message[`value`][`op`]) readMessage(provider, buf, true) } @@ -125,25 +131,19 @@ const setupShapeStream = (provider) => { provider.emit(`connection-error`, [event, provider]) } - const unsubscribeSyncHandler = provider.stream.subscribe( + const unsubscribeSyncHandler = provider.operationsStream.subscribe( handleSyncMessage, handleError ) - if (provider.awareness.getLocalState() !== null) { - const encoderAwarenessState = encoding.createEncoder() - encoding.writeVarUint(encoderAwarenessState, messageAwareness) - encoding.writeVarUint8Array( - encoderAwarenessState, - awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ - provider.doc.clientID, - ]) - ) - // websocket.send(encoding.toUint8Array(encoderAwarenessState)) - } + const unsubscribeAwarenessHandler = provider.awarenessStrean.subscribe( + handleSyncMessage, + handleError + ) provider.closeHandler = (event) => { - provider.stream = null + provider.operationsStream = null + provider.awarenessStrean = null provider.connecting = false if (provider.connected) { provider.connected = false @@ -164,11 +164,12 @@ const setupShapeStream = (provider) => { } unsubscribeSyncHandler() + unsubscribeAwarenessHandler() provider.closeHandler = null provider.emit(`connection-close`, [event, provider]) } - const handleOnceUpToDate = () => { + const handleOperationsFirstSync = () => { provider.lastMessageReceived = time.getUnixTime() provider.connecting = false provider.connected = true @@ -178,13 +179,30 @@ const setupShapeStream = (provider) => { }, ]) - provider.pending - .splice(0) - .forEach((buf) => broadcastMessage(provider, buf)) + provider.pending.splice(0).forEach((buf) => sendOperation(provider, buf)) } - provider.stream.subscribeOnceToUpToDate( - () => handleOnceUpToDate(), + provider.operationsStream.subscribeOnceToUpToDate( + () => handleOperationsFirstSync(), + () => handleError() + ) + + const handleAwarenessFirstSync = () => { + if (provider.awareness.getLocalState() !== null) { + const encoderAwarenessState = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessState, messageAwareness) + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ + provider.doc.clientID, + ]) + ) + sendAwareness(provider, encoding.toUint8Array(encoderAwarenessState)) + } + } + + provider.awarenessStrean.subscribeOnceToUpToDate( + () => handleAwarenessFirstSync(), () => handleError() ) @@ -200,49 +218,35 @@ const setupShapeStream = (provider) => { * @param {ElectricProvider} provider * @param {Uint8Array} buf */ -const broadcastMessage = (provider, buf) => { - if (provider.connected && provider.stream !== null) { - const clientId = provider.doc.clientID +const sendOperation = (provider, buf) => { + if (provider.connected && provider.operationsStream !== null) { const name = provider.roomname const op = Base64.fromUint8Array(buf) - const mutation = { - action: `insert`, - schema: `public`, - tablename: `ydoc_updates`, - row: { name, op }, - } - const req = buildRequest(clientId, new Date().getTime(), [mutation]) - fetch(req) + fetch(`/api/operation`, { + method: `POST`, + body: JSON.stringify({ name, op }), + }) } } -function buildRequest(clientId, requestId, mutations) { - const url = `http://localhost:8080/` - return new Request(url, { - method: `POST`, - headers: { - "Content-Type": `application/json`, - "X-Electric-Request-Id": requestId, - "X-Electric-User-Id": clientId, // TODO: drop this - }, - body: JSON.stringify(mutations), - }) -} - /** - * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document. - * The document name is attached to the provided url. I.e. the following example - * creates a websocket connection to http://localhost:1234/my-document-name - * - * @example - * import * as Y from 'yjs' - * import { ElectricProvider } from 'y-websocket' - * const doc = new Y.Doc() - * const provider = new ElectricProvider('http://localhost:1234', 'my-document-name', doc) - * - * @extends {Observable} + * @param {ElectricProvider} provider + * @param {Uint8Array} buf */ +const sendAwareness = async (provider, buf) => { + if (provider.connected && provider.operationsStream !== null) { + const name = provider.roomname + const clientID = provider.doc.clientID + const op = Base64.fromUint8Array(buf) + + fetch(`/api/awareness`, { + method: `POST`, + body: JSON.stringify({ client: clientID, name, op }), + }) + } +} + export class ElectricProvider extends Observable { /** * @param {string} serverUrl @@ -290,14 +294,14 @@ export class ElectricProvider extends Observable { * @type {ShapeStream?} */ - this.stream = null + this.operationsStream = null this.pending = [] this.closeHandler = null /** - * Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel) + * Listens to Yjs updates and sends them to remote peers * @param {Uint8Array} update * @param {any} origin */ @@ -314,7 +318,7 @@ export class ElectricProvider extends Observable { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) syncProtocol.writeUpdate(encoder, update) - broadcastMessage(this, encoding.toUint8Array(encoder)) + sendOperation(this, encoding.toUint8Array(encoder)) } } } @@ -324,16 +328,19 @@ export class ElectricProvider extends Observable { * @param {any} changed * @param {any} _origin */ - this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { - const changedClients = added.concat(updated).concat(removed) - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) - ) - // broadcastMessage(this, encoding.toUint8Array(encoder)) + this._awarenessUpdateHandler = ({ added, updated, removed }, origin) => { + if (origin === `local`) { + const changedClients = added.concat(updated).concat(removed) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) + ) + sendAwareness(this, encoding.toUint8Array(encoder)) + } } + this._exitHandler = () => { awarenessProtocol.removeAwarenessStates( this.awareness, @@ -351,12 +358,18 @@ export class ElectricProvider extends Observable { } } - get url() { + get operationsUrl() { const params = { where: `name = '${this.roomname}'` } const encodedParams = url.encodeQueryParams(params) return this.serverUrl + `/v1/shape/ydoc_updates?` + encodedParams } + get awarenessUrl() { + const params = { where: `name = '${this.roomname}'` } + const encodedParams = url.encodeQueryParams(params) + return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams + } + /** * @type {boolean} */ @@ -389,7 +402,7 @@ export class ElectricProvider extends Observable { connect() { this.shouldConnect = true - if (!this.connected && this.stream === null) { + if (!this.connected && this.operationsStream === null) { setupShapeStream(this) } } diff --git a/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql new file mode 100644 index 0000000000..0cd0faf3dc --- /dev/null +++ b/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql @@ -0,0 +1,27 @@ +CREATE TABLE ydoc_updates( + id SERIAL PRIMARY KEY, + name TEXT, + op TEXT NOT NULL +); + +CREATE TABLE ydoc_awareness( + client TEXT, + name TEXT, + op TEXT NOT NULL, + updated TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (client, name) +); + +CREATE OR REPLACE FUNCTION delete_old_rows() +RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM ydoc_awareness + WHERE updated < NOW() - INTERVAL '2 minutes'; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER delete_old_rows_trigger +AFTER INSERT OR UPDATE ON ydoc_awareness +FOR EACH STATEMENT +EXECUTE FUNCTION delete_old_rows(); diff --git a/examples/nextjs-yjs-codemirror/index.html b/examples/nextjs-yjs-codemirror/index.html new file mode 100644 index 0000000000..7f2247de29 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/index.html @@ -0,0 +1,18 @@ + + + + + + + + Web Example - ElectricSQL + + + + + +
+ + + + \ No newline at end of file diff --git a/examples/nextjs-yjs-codemirror/package.json b/examples/nextjs-yjs-codemirror/package.json new file mode 100644 index 0000000000..c943469916 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/package.json @@ -0,0 +1,48 @@ +{ + "name": "@electric-examples/basic-example", + "private": true, + "version": "0.0.1", + "author": "ElectricSQL", + "license": "Apache-2.0", + "type": "module", + "scripts": { + "backend:up": "PROJECT_NAME=nextjs-basic-example pnpm -C ../../ run example-backend:up && pnpm db:migrate", + "backend:down": "PROJECT_NAME=nextjs-basic-example pnpm -C ../../ run example-backend:down", + "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", + "dev": "next dev --turbo -p 5173", + "build": "next build", + "start": "next start", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "stylecheck": "eslint . --quiet", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@codemirror/basic-setup": "^0.19.0", + "@codemirror/lang-javascript": "^0.19.0", + "@codemirror/state": "^0.19.0", + "@codemirror/view": "^0.19.0", + "@electric-sql/client": "workspace:*", + "@electric-sql/react": "workspace:*", + "abort-controller": "^3.0.0", + "js-base64": "^3.7.7", + "lib0": "^0.2.96", + "next": "^14.2.5", + "pg": "^8.12.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "y-codemirror.next": "^0.1.0", + "y-protocols": "1.0.6", + "yjs": "^13.6.18" + }, + "devDependencies": { + "@databases/pg-migrations": "^5.0.3", + "@types/pg": "^8.11.6", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.1", + "dotenv": "^16.4.5", + "eslint": "^8.57.0", + "typescript": "^5.5.3", + "vite": "^5.3.4" + } +} diff --git a/examples/nextjs-yjs-codemirror/public/favicon.ico b/examples/nextjs-yjs-codemirror/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..e55095b98323d3dd569e75945b47d612e02b620b GIT binary patch literal 1659 zcmXw32~<;88on=!frKpNJxJIR8d(Y=fdLIrlxPvehQ%o=ZNV5Ug31u2NIV1Py}AHu z6;q@rq6DNv1&^p$gSDC*sdaTbL*lx`(xzXgWV;9&!JbO!v|NNO!( ziI=jLg_BzLfq@3lUk@Ir!9YFfLK$oM65_>j=Eo7lO9xm{e!CW0p{uf5PR>4SK zLH-^i{Ct!x3?w%y8L6@C#Q~HP3i^gPgu`VA`cbNPvIG1m-xh%TYJ#qkzCMo7Sw;V1 z4Sk~o;kqF%Tte4(Y~O{HihSsH4Q*>G!gYsz7djlu1%^7%ucqa$M?5^h{W|DQ4J~&A z%;!7o%Yp8mggrfBKF_fv6XE(!iNy|%v7YZmx z@(_-jQ)V*Mdx9O{OV5zdH^dWeRyln04~8U$c=;gIbDYpoNv_}RBu#|9`7BWgGwvf+ zLh(8^1k|mQHl#srbx-)~mE{?W+ox|QO zu&;n}yol7imlY98I$z3M8A1H<5cA_mMv|D3vVyTbj{ezd$K9C@<=fc)K2zag+4DCk zFGxy-`Wh590FX8(#I8u)_3z1{><#c*WdA61rXh(Gq|>iUQ3ZPMDffvx$|@(nOl6RN+E$g#S4A=ZP zo@U9gy_u*Tq<3p}^l#zy#R#=x@-N9_V_6}labTV9WX9N)yjz>ql8efm{{n=1?coFH zM=CTvl&MAi4{IBb^}LNi-KvK7^Ocs-GQM&LUf_oxT;q<;2J`UdZL6NOK3P0y)V-s# zE8g$FPN^naTZR`6^07Z#Ec8D-W^d1mrZ`h_YUzlbyzI8kC9-Rq-#I9O1=U%TXi ztFK9;XpgprzDrw1tDh@6WAV=TU6u;o8Ixj*SN5p!)zsQ>e4`(}{Lh*)>2a+F6<>*I zf#0**@!@hbrClq}NxP!GM*9mj|A6l6=IbLnI*lIRn|cKPktM(|nIrd`Dck?{i}}ec zqnsD=&Hpjqw-CP$b;S*qTRw-iI6^}&oK^Kod%9ZR7nMnCQ0wI!?K8=#`sqc%xan*u zAxn96Z@=WU?a5b_V|wGwu1~^(mV+shkR$!z@x$$T(P*#UTY70|XdF3Y+uYtM%6lq! zs&7^+hDC!nK2>VIE;3KWOy>g4lkO9oIi*1>kAo3!!%Fq$&`Iv@gMg zwUjvh`XpUvkcWeK?QIY=w}3ePzmqiGfIJZU=i0md@(;1*^O>(iFm`cxHMpmQp|rPj zlA+4wo4J=1LzeO$ewmnWZt}pc$LN7|?j_kEl)g?U(1fYURVP5#K)EwbY_axAZ{13eVd)m2Gf<&Sm54VIA3slCitWUGh%Yq8 zpx6E!arMs%uMVg%VQa6m=cY&Uq1VZN*eU!L@x=^t=^L1#Zi|UJG6JHmb*nse8_`TC zXx1OmZ9xUlxmkafF3&at1(}5&x*}Tvbj~b<#Uf>a55;gevE%3nU|T;Svqq%?j6Eui z#%gN0oDpd_wk^+m^~fM-!x_%_DnhTRhw0tR7h$hFe@`&=(9}_FaknhbNFy_S)n^F~ zuWc^k8dL^$;AH5(e~eLBFX}kr1~d<-@JMJt9c!xt&GzT2wkg0;zSxKA{m8ZmOSkob zZ6j@|whD0nakbl7+z6!xv0caO=<2lH;6Vvi^hD|3mK7EB#Mq9vHq5n=( + + + diff --git a/examples/nextjs-yjs-codemirror/public/robots.txt b/examples/nextjs-yjs-codemirror/public/robots.txt new file mode 100644 index 0000000000..e9e57dc4d4 --- /dev/null +++ b/examples/nextjs-yjs-codemirror/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/examples/nextjs-yjs-codemirror/tsconfig.json b/examples/nextjs-yjs-codemirror/tsconfig.json new file mode 100644 index 0000000000..e06a4454ab --- /dev/null +++ b/examples/nextjs-yjs-codemirror/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "es5", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/examples/remix-basic/.eslintrc.cjs b/examples/remix-basic/.eslintrc.cjs index 8aeb375841..f3c2d7c6f2 100644 --- a/examples/remix-basic/.eslintrc.cjs +++ b/examples/remix-basic/.eslintrc.cjs @@ -37,5 +37,6 @@ module.exports = { `tsup.config.ts`, `vitest.config.ts`, `.eslintrc.js`, + `*.css`, ], }; diff --git a/examples/yjs-codemirror/.gitignore b/examples/yjs-codemirror/.gitignore deleted file mode 100644 index 9829edff9a..0000000000 --- a/examples/yjs-codemirror/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -dist -.env.local diff --git a/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql deleted file mode 100644 index a9e5aab189..0000000000 --- a/examples/yjs-codemirror/db/migrations/01-create_yjs_tables.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE ydoc_updates( - id SERIAL PRIMARY KEY, - name TEXT, - op TEXT NOT NULL -); - -CREATE TABLE ydoc_awareness( - id SERIAL, - client_id TEXT, - name TEXT, - op TEXT NOT NULL, - PRIMARY KEY (id, client_id, name) -); \ No newline at end of file diff --git a/examples/yjs-codemirror/index.html b/examples/yjs-codemirror/index.html deleted file mode 100644 index a1a50b0e07..0000000000 --- a/examples/yjs-codemirror/index.html +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - Yjs CodeMirror Example - - - - - - - -

-

- This is a demo of the Yjs ⇔ - CodeMirror binding: - y-codemirror. -

-

- The content of this editor is shared with every client that visits this - domain. -

- - - \ No newline at end of file diff --git a/examples/yjs-codemirror/package.json b/examples/yjs-codemirror/package.json deleted file mode 100644 index 73f72f2efc..0000000000 --- a/examples/yjs-codemirror/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@electric-examples/yjs-codemirror", - "private": true, - "version": "0.0.1", - "author": "ElectricSQL", - "license": "Apache-2.0", - "type": "module", - "scripts": { - "backend:up": "PROJECT_NAME=yjs-codemirror pnpm -C ../../ run example-backend:up && pnpm db:migrate", - "backend:down": "PROJECT_NAME=yjs-codemirror pnpm -C ../../ run example-backend:down", - "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", - "dev": "vite", - "build": "vite build", - "lint": "eslint . --ext js,ts,tsx --report-unused-disable-directives --max-warnings 0", - "preview": "vite preview", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@electric-sql/client": "workspace:*", - "abort-controller": "^3.0.0", - "js-base64": "^3.7.7", - "lib0": "^0.2.96", - "y-protocols": "^1.0.5", - "codemirror": "^5.64.0", - "y-codemirror": "^2.1.1", - "yjs": "^13.5.22" - - }, - "devDependencies": { - "@databases/pg-migrations": "^5.0.3", - "dotenv": "^16.4.5", - "eslint": "^8.57.0", - "vite": "^5.3.4" - } -} diff --git a/examples/yjs-codemirror/src/codemirror.css b/examples/yjs-codemirror/src/codemirror.css deleted file mode 100644 index f4d5718a78..0000000000 --- a/examples/yjs-codemirror/src/codemirror.css +++ /dev/null @@ -1,344 +0,0 @@ -/* BASICS */ - -.CodeMirror { - /* Set height, width, borders, and global font properties here */ - font-family: monospace; - height: 300px; - color: black; - direction: ltr; -} - -/* PADDING */ - -.CodeMirror-lines { - padding: 4px 0; /* Vertical padding around content */ -} -.CodeMirror pre.CodeMirror-line, -.CodeMirror pre.CodeMirror-line-like { - padding: 0 4px; /* Horizontal padding of content */ -} - -.CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { - background-color: white; /* The little square between H and V scrollbars */ -} - -/* GUTTER */ - -.CodeMirror-gutters { - border-right: 1px solid #ddd; - background-color: #f7f7f7; - white-space: nowrap; -} -.CodeMirror-linenumbers {} -.CodeMirror-linenumber { - padding: 0 3px 0 5px; - min-width: 20px; - text-align: right; - color: #999; - white-space: nowrap; -} - -.CodeMirror-guttermarker { color: black; } -.CodeMirror-guttermarker-subtle { color: #999; } - -/* CURSOR */ - -.CodeMirror-cursor { - border-left: 1px solid black; - border-right: none; - width: 0; -} -/* Shown when moving in bi-directional text */ -.CodeMirror div.CodeMirror-secondarycursor { - border-left: 1px solid silver; -} -.cm-fat-cursor .CodeMirror-cursor { - width: auto; - border: 0 !important; - background: #7e7; -} -.cm-fat-cursor div.CodeMirror-cursors { - z-index: 1; -} -.cm-fat-cursor .CodeMirror-line::selection, -.cm-fat-cursor .CodeMirror-line > span::selection, -.cm-fat-cursor .CodeMirror-line > span > span::selection { background: transparent; } -.cm-fat-cursor .CodeMirror-line::-moz-selection, -.cm-fat-cursor .CodeMirror-line > span::-moz-selection, -.cm-fat-cursor .CodeMirror-line > span > span::-moz-selection { background: transparent; } -.cm-fat-cursor { caret-color: transparent; } -@-moz-keyframes blink { - 0% {} - 50% { background-color: transparent; } - 100% {} -} -@-webkit-keyframes blink { - 0% {} - 50% { background-color: transparent; } - 100% {} -} -@keyframes blink { - 0% {} - 50% { background-color: transparent; } - 100% {} -} - -/* Can style cursor different in overwrite (non-insert) mode */ -.CodeMirror-overwrite .CodeMirror-cursor {} - -.cm-tab { display: inline-block; text-decoration: inherit; } - -.CodeMirror-rulers { - position: absolute; - left: 0; right: 0; top: -50px; bottom: 0; - overflow: hidden; -} -.CodeMirror-ruler { - border-left: 1px solid #ccc; - top: 0; bottom: 0; - position: absolute; -} - -/* DEFAULT THEME */ - -.cm-s-default .cm-header {color: blue;} -.cm-s-default .cm-quote {color: #090;} -.cm-negative {color: #d44;} -.cm-positive {color: #292;} -.cm-header, .cm-strong {font-weight: bold;} -.cm-em {font-style: italic;} -.cm-link {text-decoration: underline;} -.cm-strikethrough {text-decoration: line-through;} - -.cm-s-default .cm-keyword {color: #708;} -.cm-s-default .cm-atom {color: #219;} -.cm-s-default .cm-number {color: #164;} -.cm-s-default .cm-def {color: #00f;} -.cm-s-default .cm-variable, -.cm-s-default .cm-punctuation, -.cm-s-default .cm-property, -.cm-s-default .cm-operator {} -.cm-s-default .cm-variable-2 {color: #05a;} -.cm-s-default .cm-variable-3, .cm-s-default .cm-type {color: #085;} -.cm-s-default .cm-comment {color: #a50;} -.cm-s-default .cm-string {color: #a11;} -.cm-s-default .cm-string-2 {color: #f50;} -.cm-s-default .cm-meta {color: #555;} -.cm-s-default .cm-qualifier {color: #555;} -.cm-s-default .cm-builtin {color: #30a;} -.cm-s-default .cm-bracket {color: #997;} -.cm-s-default .cm-tag {color: #170;} -.cm-s-default .cm-attribute {color: #00c;} -.cm-s-default .cm-hr {color: #999;} -.cm-s-default .cm-link {color: #00c;} - -.cm-s-default .cm-error {color: #f00;} -.cm-invalidchar {color: #f00;} - -.CodeMirror-composing { border-bottom: 2px solid; } - -/* Default styles for common addons */ - -div.CodeMirror span.CodeMirror-matchingbracket {color: #0b0;} -div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #a22;} -.CodeMirror-matchingtag { background: rgba(255, 150, 0, .3); } -.CodeMirror-activeline-background {background: #e8f2ff;} - -/* STOP */ - -/* The rest of this file contains styles related to the mechanics of - the editor. You probably shouldn't touch them. */ - -.CodeMirror { - position: relative; - overflow: hidden; - background: white; -} - -.CodeMirror-scroll { - overflow: scroll !important; /* Things will break if this is overridden */ - /* 50px is the magic margin used to hide the element's real scrollbars */ - /* See overflow: hidden in .CodeMirror */ - margin-bottom: -50px; margin-right: -50px; - padding-bottom: 50px; - height: 100%; - outline: none; /* Prevent dragging from highlighting the element */ - position: relative; - z-index: 0; -} -.CodeMirror-sizer { - position: relative; - border-right: 50px solid transparent; -} - -/* The fake, visible scrollbars. Used to force redraw during scrolling - before actual scrolling happens, thus preventing shaking and - flickering artifacts. */ -.CodeMirror-vscrollbar, .CodeMirror-hscrollbar, .CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { - position: absolute; - z-index: 6; - display: none; - outline: none; -} -.CodeMirror-vscrollbar { - right: 0; top: 0; - overflow-x: hidden; - overflow-y: scroll; -} -.CodeMirror-hscrollbar { - bottom: 0; left: 0; - overflow-y: hidden; - overflow-x: scroll; -} -.CodeMirror-scrollbar-filler { - right: 0; bottom: 0; -} -.CodeMirror-gutter-filler { - left: 0; bottom: 0; -} - -.CodeMirror-gutters { - position: absolute; left: 0; top: 0; - min-height: 100%; - z-index: 3; -} -.CodeMirror-gutter { - white-space: normal; - height: 100%; - display: inline-block; - vertical-align: top; - margin-bottom: -50px; -} -.CodeMirror-gutter-wrapper { - position: absolute; - z-index: 4; - background: none !important; - border: none !important; -} -.CodeMirror-gutter-background { - position: absolute; - top: 0; bottom: 0; - z-index: 4; -} -.CodeMirror-gutter-elt { - position: absolute; - cursor: default; - z-index: 4; -} -.CodeMirror-gutter-wrapper ::selection { background-color: transparent } -.CodeMirror-gutter-wrapper ::-moz-selection { background-color: transparent } - -.CodeMirror-lines { - cursor: text; - min-height: 1px; /* prevents collapsing before first draw */ -} -.CodeMirror pre.CodeMirror-line, -.CodeMirror pre.CodeMirror-line-like { - /* Reset some styles that the rest of the page might have set */ - -moz-border-radius: 0; -webkit-border-radius: 0; border-radius: 0; - border-width: 0; - background: transparent; - font-family: inherit; - font-size: inherit; - margin: 0; - white-space: pre; - word-wrap: normal; - line-height: inherit; - color: inherit; - z-index: 2; - position: relative; - overflow: visible; - -webkit-tap-highlight-color: transparent; - -webkit-font-variant-ligatures: contextual; - font-variant-ligatures: contextual; -} -.CodeMirror-wrap pre.CodeMirror-line, -.CodeMirror-wrap pre.CodeMirror-line-like { - word-wrap: break-word; - white-space: pre-wrap; - word-break: normal; -} - -.CodeMirror-linebackground { - position: absolute; - left: 0; right: 0; top: 0; bottom: 0; - z-index: 0; -} - -.CodeMirror-linewidget { - position: relative; - z-index: 2; - padding: 0.1px; /* Force widget margins to stay inside of the container */ -} - -.CodeMirror-widget {} - -.CodeMirror-rtl pre { direction: rtl; } - -.CodeMirror-code { - outline: none; -} - -/* Force content-box sizing for the elements where we expect it */ -.CodeMirror-scroll, -.CodeMirror-sizer, -.CodeMirror-gutter, -.CodeMirror-gutters, -.CodeMirror-linenumber { - -moz-box-sizing: content-box; - box-sizing: content-box; -} - -.CodeMirror-measure { - position: absolute; - width: 100%; - height: 0; - overflow: hidden; - visibility: hidden; -} - -.CodeMirror-cursor { - position: absolute; - pointer-events: none; -} -.CodeMirror-measure pre { position: static; } - -div.CodeMirror-cursors { - visibility: hidden; - position: relative; - z-index: 3; -} -div.CodeMirror-dragcursors { - visibility: visible; -} - -.CodeMirror-focused div.CodeMirror-cursors { - visibility: visible; -} - -.CodeMirror-selected { background: #d9d9d9; } -.CodeMirror-focused .CodeMirror-selected { background: #d7d4f0; } -.CodeMirror-crosshair { cursor: crosshair; } -.CodeMirror-line::selection, .CodeMirror-line > span::selection, .CodeMirror-line > span > span::selection { background: #d7d4f0; } -.CodeMirror-line::-moz-selection, .CodeMirror-line > span::-moz-selection, .CodeMirror-line > span > span::-moz-selection { background: #d7d4f0; } - -.cm-searching { - background-color: #ffa; - background-color: rgba(255, 255, 0, .4); -} - -/* Used to force a border model for a node */ -.cm-force-border { padding-right: .1px; } - -@media print { - /* Hide the cursor when printing */ - .CodeMirror div.CodeMirror-cursors { - visibility: hidden; - } -} - -/* See issue #2901 */ -.cm-tab-wrap-hack:after { content: ''; } - -/* Help users use markselection to safely style text background */ -span.CodeMirror-selectedtext { background: none; } diff --git a/examples/yjs-codemirror/src/codemirror.html b/examples/yjs-codemirror/src/codemirror.html deleted file mode 100644 index 6f80c41b56..0000000000 --- a/examples/yjs-codemirror/src/codemirror.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - Yjs CodeMirror Example - - - - - - -

-

- This is a demo of the Yjs ⇔ - CodeMirror binding: - y-codemirror. -

-

- The content of this editor is shared with every client that visits this - domain. -

- - diff --git a/examples/yjs-codemirror/src/codemirror.js b/examples/yjs-codemirror/src/codemirror.js deleted file mode 100644 index ee15f70809..0000000000 --- a/examples/yjs-codemirror/src/codemirror.js +++ /dev/null @@ -1,42 +0,0 @@ -/* eslint-env browser */ - -// @ts-ignore -import CodeMirror from 'codemirror' -import * as Y from 'yjs' -import { ElectricProvider } from './y-electric.js' -import { CodemirrorBinding } from 'y-codemirror' -import 'codemirror/mode/javascript/javascript.js' - -window.addEventListener('load', () => { - const ydoc = new Y.Doc() - const provider = new ElectricProvider( - `http://localhost:3000/`, - 'codemirror-demo-2024/06', - ydoc - ) - const ytext = ydoc.getText('codemirror') - const editorContainer = document.createElement('div') - editorContainer.setAttribute('id', 'editor') - document.body.insertBefore(editorContainer, null) - - const editor = CodeMirror(editorContainer, { - mode: 'javascript', - lineNumbers: true - }) - - const binding = new CodemirrorBinding(ytext, editor, provider.awareness) - - const connectBtn = /** @type {HTMLElement} */ (document.getElementById('y-connect-btn')) - connectBtn.addEventListener('click', () => { - if (provider.shouldConnect) { - provider.disconnect() - connectBtn.textContent = 'Connect' - } else { - provider.connect() - connectBtn.textContent = 'Disconnect' - } - }) - - // @ts-ignore - window.example = { provider, ydoc, ytext, binding, Y } -}) From 8654612c4d3dc8308b1d0c49a45d7f8a0b22f2b6 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 13 Aug 2024 17:32:21 +0100 Subject: [PATCH 05/47] Code cleanup --- .../app/api/operation/route.ts | 2 +- examples/nextjs-yjs-codemirror/app/page.tsx | 26 +- .../nextjs-yjs-codemirror/app/y-electric.js | 248 ++++++------------ .../db/migrations/01-create_yjs_tables.sql | 2 +- examples/nextjs-yjs-codemirror/package.json | 4 +- 5 files changed, 92 insertions(+), 190 deletions(-) diff --git a/examples/nextjs-yjs-codemirror/app/api/operation/route.ts b/examples/nextjs-yjs-codemirror/app/api/operation/route.ts index 68ad7ba3f3..81c2ca490b 100644 --- a/examples/nextjs-yjs-codemirror/app/api/operation/route.ts +++ b/examples/nextjs-yjs-codemirror/app/api/operation/route.ts @@ -4,7 +4,7 @@ import { NextResponse } from "next/server" export async function POST(request: Request) { const body = await request.json() await db.query( - `INSERT INTO ydoc_updates (name, op) + `INSERT INTO ydoc_operations (name, op) VALUES ($1, $2)`, [body.name, body.op] ) diff --git a/examples/nextjs-yjs-codemirror/app/page.tsx b/examples/nextjs-yjs-codemirror/app/page.tsx index 14688b2ca4..e1c8eb660e 100644 --- a/examples/nextjs-yjs-codemirror/app/page.tsx +++ b/examples/nextjs-yjs-codemirror/app/page.tsx @@ -13,6 +13,8 @@ import { javascript } from "@codemirror/lang-javascript" import * as random from "lib0/random" +const room = `electric-demo` + const usercolors = [ { color: `#30bced`, light: `#30bced33` }, { color: `#6eeb83`, light: `#6eeb8333` }, @@ -39,13 +41,12 @@ const theme = EditorView.theme( }, { dark: true } ) - const ydoc = new Y.Doc() -const provider = new ElectricProvider( - `http://localhost:3000/`, - `electric-demo`, - ydoc -) +let provider: ElectricProvider | null = null + +if (typeof window !== `undefined`) { + provider = new ElectricProvider(`http://localhost:3000/`, room, ydoc) +} export default function Home() { const editor = useRef(null) @@ -54,16 +55,18 @@ export default function Home() { const toggle = () => { if (connect === `connected`) { - provider.disconnect() + provider?.disconnect() setConnect(`disconnected`) } else { - provider.connect() + provider?.connect() setConnect(`connected`) } } useEffect(() => { - const ytext = ydoc.getText(`codemirror`) + if (provider === null) return + + const ytext = ydoc.getText(room) provider.awareness.setLocalStateField(`user`, { name: `Anonymous ` + Math.floor(Math.random() * 100), @@ -85,10 +88,7 @@ export default function Home() { const view = new EditorView({ state, parent: editor.current ?? undefined }) - return () => { - view.destroy() - // editor.current.removeEventListener("input", log); - } + return () => view.destroy() }) return ( diff --git a/examples/nextjs-yjs-codemirror/app/y-electric.js b/examples/nextjs-yjs-codemirror/app/y-electric.js index c9154bbc2e..e9f77d35ef 100644 --- a/examples/nextjs-yjs-codemirror/app/y-electric.js +++ b/examples/nextjs-yjs-codemirror/app/y-electric.js @@ -1,11 +1,9 @@ /** - * @module provider/websocket + * @module provider/electric */ -/* eslint-env browser */ - -import * as Y from 'yjs' // eslint-disable-line import * as time from "lib0/time" +import { toBase64, fromBase64 } from "lib0/buffer" import * as encoding from "lib0/encoding" import * as decoding from "lib0/decoding" import * as syncProtocol from "y-protocols/sync" @@ -15,74 +13,10 @@ import * as url from "lib0/url" import * as env from "lib0/environment" export const messageSync = 0 -export const messageQueryAwareness = 3 export const messageAwareness = 1 import { ShapeStream } from "@electric-sql/client" -// Check if we can handle encoding another way -import { Base64 } from "js-base64" - -/** - * encoder, decoder, provider, emitSynced, messageType - * @type {Array} - */ -const messageHandlers = [] - -messageHandlers[messageSync] = ( - encoder, - decoder, - provider, - emitSynced, - _messageType -) => { - encoding.writeVarUint(encoder, messageSync) - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - provider.doc, - provider - ) - if ( - emitSynced && - syncMessageType === syncProtocol.messageYjsSyncStep2 && - !provider.synced - ) { - provider.synced = true - } -} - -messageHandlers[messageQueryAwareness] = ( - encoder, - _decoder, - provider, - _emitSynced, - _messageType -) => { - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate( - provider.awareness, - Array.from(provider.awareness.getStates().keys()) - ) - ) -} - -messageHandlers[messageAwareness] = ( - _encoder, - decoder, - provider, - _emitSynced, - _messageType -) => { - awarenessProtocol.applyAwarenessUpdate( - provider.awareness, - decoding.readVarUint8Array(decoder), - provider - ) -} - /** * @param {ElectricProvider} provider */ @@ -94,38 +28,51 @@ const setupShapeStream = (provider) => { provider.operationsStream = new ShapeStream({ url: provider.operationsUrl, - signal: new AbortController().signal, }) - provider.awarenessStrean = new ShapeStream({ + provider.awarenessStream = new ShapeStream({ url: provider.awarenessUrl, - signal: new AbortController().signal, }) - const readMessage = (provider, buf, emitSynced) => { - const decoder = decoding.createDecoder(buf) - const encoder = encoding.createEncoder() - const messageType = decoding.readVarUint(decoder) - const messageHandler = provider.messageHandlers[messageType] - if (/** @type {any} */ (messageHandler)) { - messageHandler(encoder, decoder, provider, emitSynced, messageType) - } else { - console.error(`Unable to compute message`) - } - return encoder + const handleMessages = (messages) => { + provider.lastMessageReceived = time.getUnixTime() + return messages + .filter((message) => message[`key`] && message[`value`][`op`]) + .map((message) => message[`value`][`op`]) + .map((operation) => { + const base64 = fromBase64(operation) + return decoding.createDecoder(base64) + }) } - const handleSyncMessage = (messages) => { - provider.lastMessageReceived = time.getUnixTime() - messages.forEach((message) => { - // ignore DELETE operations - if (message[`key`] && message[`value`][`op`]) { - const buf = Base64.toUint8Array(message[`value`][`op`]) - readMessage(provider, buf, true) + const handleSyncMessage = (messages) => + handleMessages(messages).forEach((decoder) => { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + provider.doc, + provider + ) + if ( + syncMessageType === syncProtocol.messageYjsSyncStep2 && + !provider.synced + ) { + provider.synced = true } }) - } + const handleAwarenessMessage = (messages) => + handleMessages(messages).forEach((decoder) => { + awarenessProtocol.applyAwarenessUpdate( + provider.awareness, + decoding.readVarUint8Array(decoder), + provider + ) + }) + + // TODO: need to improve error handling const handleError = (event) => { console.warn(`fetch shape error`, event) provider.emit(`connection-error`, [event, provider]) @@ -136,19 +83,19 @@ const setupShapeStream = (provider) => { handleError ) - const unsubscribeAwarenessHandler = provider.awarenessStrean.subscribe( - handleSyncMessage, + const unsubscribeAwarenessHandler = provider.awarenessStream.subscribe( + handleAwarenessMessage, handleError ) provider.closeHandler = (event) => { provider.operationsStream = null - provider.awarenessStrean = null + provider.awarenessStream = null provider.connecting = false if (provider.connected) { provider.connected = false provider.synced = false - // update awareness (all users except local left) + awarenessProtocol.removeAwarenessStates( provider.awareness, Array.from(provider.awareness.getStates().keys()).filter( @@ -156,11 +103,7 @@ const setupShapeStream = (provider) => { ), provider ) - provider.emit(`status`, [ - { - status: `disconnected`, - }, - ]) + provider.emit(`status`, [{ status: `disconnected` }]) } unsubscribeSyncHandler() @@ -173,13 +116,11 @@ const setupShapeStream = (provider) => { provider.lastMessageReceived = time.getUnixTime() provider.connecting = false provider.connected = true - provider.emit(`status`, [ - { - status: `connected`, - }, - ]) + provider.emit(`status`, [{ status: `connected` }]) - provider.pending.splice(0).forEach((buf) => sendOperation(provider, buf)) + provider.pending + .splice(0) + .forEach((update) => sendOperation(provider, update)) } provider.operationsStream.subscribeOnceToUpToDate( @@ -189,41 +130,33 @@ const setupShapeStream = (provider) => { const handleAwarenessFirstSync = () => { if (provider.awareness.getLocalState() !== null) { - const encoderAwarenessState = encoding.createEncoder() - encoding.writeVarUint(encoderAwarenessState, messageAwareness) - encoding.writeVarUint8Array( - encoderAwarenessState, - awarenessProtocol.encodeAwarenessUpdate(provider.awareness, [ - provider.doc.clientID, - ]) - ) - sendAwareness(provider, encoding.toUint8Array(encoderAwarenessState)) + sendAwareness(provider, [provider.doc.clientID]) } } - provider.awarenessStrean.subscribeOnceToUpToDate( + provider.awarenessStream.subscribeOnceToUpToDate( () => handleAwarenessFirstSync(), () => handleError() ) - provider.emit(`status`, [ - { - status: `connecting`, - }, - ]) + provider.emit(`status`, [{ status: `connecting` }]) } } /** * @param {ElectricProvider} provider - * @param {Uint8Array} buf + * @param {Uint8Array} op */ -const sendOperation = (provider, buf) => { - if (provider.connected && provider.operationsStream !== null) { +const sendOperation = async (provider, update) => { + if (!(provider.connected && provider.operationsStream !== null)) { + provider.pending.push(update) + } else { + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, update) + const op = toBase64(encoding.toUint8Array(encoder)) const name = provider.roomname - const op = Base64.fromUint8Array(buf) - fetch(`/api/operation`, { + await fetch(`/api/operation`, { method: `POST`, body: JSON.stringify({ name, op }), }) @@ -232,15 +165,21 @@ const sendOperation = (provider, buf) => { /** * @param {ElectricProvider} provider - * @param {Uint8Array} buf + * @param {Uint8Array} op */ -const sendAwareness = async (provider, buf) => { +const sendAwareness = async (provider, changedClients) => { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(provider.awareness, changedClients) + ) + const op = toBase64(encoding.toUint8Array(encoder)) + if (provider.connected && provider.operationsStream !== null) { const name = provider.roomname - const clientID = provider.doc.clientID - const op = Base64.fromUint8Array(buf) + const clientID = `${provider.doc.clientID}` - fetch(`/api/awareness`, { + await fetch(`/api/awareness`, { method: `POST`, body: JSON.stringify({ client: clientID, name, op }), }) @@ -256,8 +195,6 @@ export class ElectricProvider extends Observable { * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] * @param {Object} [opts.params] specify url parameters - * @param {Array} [opts.protocols] specify websocket protocols - * @param {number} [opts.maxBackoffTime] Maximum amount of time to wait before trying to reconnect (we try to reconnect using exponential backoff) */ constructor( serverUrl, @@ -266,78 +203,45 @@ export class ElectricProvider extends Observable { { connect = true, awareness = new awarenessProtocol.Awareness(doc) } = {} ) { super() - // ensure that url is always ends with / - while (serverUrl[serverUrl.length - 1] === `/`) { - serverUrl = serverUrl.slice(0, serverUrl.length - 1) - } + this.serverUrl = serverUrl this.roomname = roomname this.doc = doc this.awareness = awareness this.connected = false this.connecting = false - - this.messageHandlers = messageHandlers.slice() - /** - * @type {boolean} - */ this._synced = false this.lastMessageReceived = 0 - /** - * Whether to connect to other peers or not - * @type {boolean} - */ this.shouldConnect = connect - /** - * @type {ShapeStream?} - */ - this.operationsStream = null + this.awarenessStream = null this.pending = [] this.closeHandler = null /** - * Listens to Yjs updates and sends them to remote peers + * Listens to Yjs updates and sends to the backend * @param {Uint8Array} update * @param {any} origin */ this._updateHandler = (update, origin) => { - // TODO would be nice to skip updates that are already included if (origin !== this) { - if (!this.connected) { - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.writeUpdate(encoder, update) - - this.pending.push(encoding.toUint8Array(encoder)) - } else { - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.writeUpdate(encoder, update) - sendOperation(this, encoding.toUint8Array(encoder)) - } + sendOperation(this, update) } } this.doc.on(`update`, this._updateHandler) /** * @param {any} changed - * @param {any} _origin + * @param {any} origin */ this._awarenessUpdateHandler = ({ added, updated, removed }, origin) => { if (origin === `local`) { const changedClients = added.concat(updated).concat(removed) - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) - ) - sendAwareness(this, encoding.toUint8Array(encoder)) + sendAwareness(this, changedClients) } } @@ -361,7 +265,7 @@ export class ElectricProvider extends Observable { get operationsUrl() { const params = { where: `name = '${this.roomname}'` } const encodedParams = url.encodeQueryParams(params) - return this.serverUrl + `/v1/shape/ydoc_updates?` + encodedParams + return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams } get awarenessUrl() { diff --git a/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql index 0cd0faf3dc..b7f4cdaa79 100644 --- a/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql +++ b/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql @@ -1,4 +1,4 @@ -CREATE TABLE ydoc_updates( +CREATE TABLE ydoc_operations( id SERIAL PRIMARY KEY, name TEXT, op TEXT NOT NULL diff --git a/examples/nextjs-yjs-codemirror/package.json b/examples/nextjs-yjs-codemirror/package.json index c943469916..e16f5dba93 100644 --- a/examples/nextjs-yjs-codemirror/package.json +++ b/examples/nextjs-yjs-codemirror/package.json @@ -12,7 +12,7 @@ "dev": "next dev --turbo -p 5173", "build": "next build", "start": "next start", - "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "lint": "eslint . --ext js,ts,tsx --report-unused-disable-directives --max-warnings 0", "stylecheck": "eslint . --quiet", "typecheck": "tsc --noEmit" }, @@ -23,8 +23,6 @@ "@codemirror/view": "^0.19.0", "@electric-sql/client": "workspace:*", "@electric-sql/react": "workspace:*", - "abort-controller": "^3.0.0", - "js-base64": "^3.7.7", "lib0": "^0.2.96", "next": "^14.2.5", "pg": "^8.12.0", From ddeaccdf1f7f25db10a9b8eba98ba7ccd1d4b5c4 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 13 Aug 2024 17:32:29 +0100 Subject: [PATCH 06/47] Compaction endpoint --- .../app/api/compaction/route.ts | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 examples/nextjs-yjs-codemirror/app/api/compaction/route.ts diff --git a/examples/nextjs-yjs-codemirror/app/api/compaction/route.ts b/examples/nextjs-yjs-codemirror/app/api/compaction/route.ts new file mode 100644 index 0000000000..9652b3586a --- /dev/null +++ b/examples/nextjs-yjs-codemirror/app/api/compaction/route.ts @@ -0,0 +1,58 @@ +import { db } from "../../db" +import { NextRequest, NextResponse } from "next/server" + +import * as Y from "yjs" +import * as syncProtocol from "y-protocols/sync" + +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" + +import { toBase64, fromBase64 } from "lib0/buffer" + +const ydoc = new Y.Doc() + +export async function GET(request: NextRequest) { + const room = request.nextUrl.searchParams.get(`room`) + + if (!room) { + return NextResponse.json({ error: `room is required` }, { status: 400 }) + } + + const res = await db.query( + `SELECT id, op FROM ydoc_operations + WHERE name = $1 + ORDER BY id ASC + LIMIT 1000`, + [room] + ) + + const ytext = ydoc.getText(room) + res.rows.map(({ op }) => { + const buf = fromBase64(op) + const decoder = decoding.createDecoder(buf) + syncProtocol.readSyncMessage( + decoder, + encoding.createEncoder(), + ydoc, + `server` + ) + }) + + await db.query( + `DELETE FROM ydoc_operations + WHERE name = $1 AND id <= $2`, + [room, res.rows[res.rows.length - 1].id] + ) + + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + const encoded = toBase64(encoding.toUint8Array(encoder)) + + await db.query( + `INSERT INTO ydoc_operations (name, op) + VALUES ($1, $2)`, + [room, encoded] + ) + + return NextResponse.json({ text: ytext.toJSON() }) +} From 344d1cc04fabc00216999ccf6eaa3c3b7e53d6dd Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 13 Aug 2024 17:40:20 +0100 Subject: [PATCH 07/47] Moved project --- .../.eslintignore | 0 .../.eslintrc.cjs | 0 .../.gitignore | 0 .../.prettierrc | 0 .../README.md | 2 +- .../app/App.css | 0 .../app/Example.css | 0 .../app/api/awareness/route.ts | 0 .../app/api/compaction/route.ts | 0 .../app/api/operation/route.ts | 0 .../app/db.ts | 0 .../app/layout.tsx | 0 .../app/page.tsx | 0 .../app/shape-proxy/[...table]/route.ts | 0 .../app/style.css | 0 .../app/y-electric.js | 0 .../db/migrations/01-create_yjs_tables.sql | 0 .../index.html | 0 .../package.json | 6 +++--- .../public/favicon.ico | Bin .../public/logo.svg | 0 .../public/robots.txt | 0 .../tsconfig.json | 0 23 files changed, 4 insertions(+), 4 deletions(-) rename examples/{nextjs-yjs-codemirror => yjs-provider}/.eslintignore (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/.eslintrc.cjs (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/.gitignore (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/.prettierrc (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/README.md (91%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/App.css (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/Example.css (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/api/awareness/route.ts (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/api/compaction/route.ts (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/api/operation/route.ts (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/db.ts (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/layout.tsx (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/page.tsx (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/shape-proxy/[...table]/route.ts (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/style.css (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/app/y-electric.js (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/db/migrations/01-create_yjs_tables.sql (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/index.html (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/package.json (83%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/public/favicon.ico (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/public/logo.svg (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/public/robots.txt (100%) rename examples/{nextjs-yjs-codemirror => yjs-provider}/tsconfig.json (100%) diff --git a/examples/nextjs-yjs-codemirror/.eslintignore b/examples/yjs-provider/.eslintignore similarity index 100% rename from examples/nextjs-yjs-codemirror/.eslintignore rename to examples/yjs-provider/.eslintignore diff --git a/examples/nextjs-yjs-codemirror/.eslintrc.cjs b/examples/yjs-provider/.eslintrc.cjs similarity index 100% rename from examples/nextjs-yjs-codemirror/.eslintrc.cjs rename to examples/yjs-provider/.eslintrc.cjs diff --git a/examples/nextjs-yjs-codemirror/.gitignore b/examples/yjs-provider/.gitignore similarity index 100% rename from examples/nextjs-yjs-codemirror/.gitignore rename to examples/yjs-provider/.gitignore diff --git a/examples/nextjs-yjs-codemirror/.prettierrc b/examples/yjs-provider/.prettierrc similarity index 100% rename from examples/nextjs-yjs-codemirror/.prettierrc rename to examples/yjs-provider/.prettierrc diff --git a/examples/nextjs-yjs-codemirror/README.md b/examples/yjs-provider/README.md similarity index 91% rename from examples/nextjs-yjs-codemirror/README.md rename to examples/yjs-provider/README.md index 49a66ab337..5b80d41a6e 100644 --- a/examples/nextjs-yjs-codemirror/README.md +++ b/examples/yjs-provider/README.md @@ -1,4 +1,4 @@ -# Basic Remix example +# Yjs Electric provider example ## Setup diff --git a/examples/nextjs-yjs-codemirror/app/App.css b/examples/yjs-provider/app/App.css similarity index 100% rename from examples/nextjs-yjs-codemirror/app/App.css rename to examples/yjs-provider/app/App.css diff --git a/examples/nextjs-yjs-codemirror/app/Example.css b/examples/yjs-provider/app/Example.css similarity index 100% rename from examples/nextjs-yjs-codemirror/app/Example.css rename to examples/yjs-provider/app/Example.css diff --git a/examples/nextjs-yjs-codemirror/app/api/awareness/route.ts b/examples/yjs-provider/app/api/awareness/route.ts similarity index 100% rename from examples/nextjs-yjs-codemirror/app/api/awareness/route.ts rename to examples/yjs-provider/app/api/awareness/route.ts diff --git a/examples/nextjs-yjs-codemirror/app/api/compaction/route.ts b/examples/yjs-provider/app/api/compaction/route.ts similarity index 100% rename from examples/nextjs-yjs-codemirror/app/api/compaction/route.ts rename to examples/yjs-provider/app/api/compaction/route.ts diff --git a/examples/nextjs-yjs-codemirror/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts similarity index 100% rename from examples/nextjs-yjs-codemirror/app/api/operation/route.ts rename to examples/yjs-provider/app/api/operation/route.ts diff --git a/examples/nextjs-yjs-codemirror/app/db.ts b/examples/yjs-provider/app/db.ts similarity index 100% rename from examples/nextjs-yjs-codemirror/app/db.ts rename to examples/yjs-provider/app/db.ts diff --git a/examples/nextjs-yjs-codemirror/app/layout.tsx b/examples/yjs-provider/app/layout.tsx similarity index 100% rename from examples/nextjs-yjs-codemirror/app/layout.tsx rename to examples/yjs-provider/app/layout.tsx diff --git a/examples/nextjs-yjs-codemirror/app/page.tsx b/examples/yjs-provider/app/page.tsx similarity index 100% rename from examples/nextjs-yjs-codemirror/app/page.tsx rename to examples/yjs-provider/app/page.tsx diff --git a/examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts similarity index 100% rename from examples/nextjs-yjs-codemirror/app/shape-proxy/[...table]/route.ts rename to examples/yjs-provider/app/shape-proxy/[...table]/route.ts diff --git a/examples/nextjs-yjs-codemirror/app/style.css b/examples/yjs-provider/app/style.css similarity index 100% rename from examples/nextjs-yjs-codemirror/app/style.css rename to examples/yjs-provider/app/style.css diff --git a/examples/nextjs-yjs-codemirror/app/y-electric.js b/examples/yjs-provider/app/y-electric.js similarity index 100% rename from examples/nextjs-yjs-codemirror/app/y-electric.js rename to examples/yjs-provider/app/y-electric.js diff --git a/examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql similarity index 100% rename from examples/nextjs-yjs-codemirror/db/migrations/01-create_yjs_tables.sql rename to examples/yjs-provider/db/migrations/01-create_yjs_tables.sql diff --git a/examples/nextjs-yjs-codemirror/index.html b/examples/yjs-provider/index.html similarity index 100% rename from examples/nextjs-yjs-codemirror/index.html rename to examples/yjs-provider/index.html diff --git a/examples/nextjs-yjs-codemirror/package.json b/examples/yjs-provider/package.json similarity index 83% rename from examples/nextjs-yjs-codemirror/package.json rename to examples/yjs-provider/package.json index e16f5dba93..a274feb5b8 100644 --- a/examples/nextjs-yjs-codemirror/package.json +++ b/examples/yjs-provider/package.json @@ -1,13 +1,13 @@ { - "name": "@electric-examples/basic-example", + "name": "@electric-examples/yjs-provider", "private": true, "version": "0.0.1", "author": "ElectricSQL", "license": "Apache-2.0", "type": "module", "scripts": { - "backend:up": "PROJECT_NAME=nextjs-basic-example pnpm -C ../../ run example-backend:up && pnpm db:migrate", - "backend:down": "PROJECT_NAME=nextjs-basic-example pnpm -C ../../ run example-backend:down", + "backend:up": "PROJECT_NAME=yjs-provider pnpm -C ../../ run example-backend:up && pnpm db:migrate", + "backend:down": "PROJECT_NAME=yjs-provider pnpm -C ../../ run example-backend:down", "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", "dev": "next dev --turbo -p 5173", "build": "next build", diff --git a/examples/nextjs-yjs-codemirror/public/favicon.ico b/examples/yjs-provider/public/favicon.ico similarity index 100% rename from examples/nextjs-yjs-codemirror/public/favicon.ico rename to examples/yjs-provider/public/favicon.ico diff --git a/examples/nextjs-yjs-codemirror/public/logo.svg b/examples/yjs-provider/public/logo.svg similarity index 100% rename from examples/nextjs-yjs-codemirror/public/logo.svg rename to examples/yjs-provider/public/logo.svg diff --git a/examples/nextjs-yjs-codemirror/public/robots.txt b/examples/yjs-provider/public/robots.txt similarity index 100% rename from examples/nextjs-yjs-codemirror/public/robots.txt rename to examples/yjs-provider/public/robots.txt diff --git a/examples/nextjs-yjs-codemirror/tsconfig.json b/examples/yjs-provider/tsconfig.json similarity index 100% rename from examples/nextjs-yjs-codemirror/tsconfig.json rename to examples/yjs-provider/tsconfig.json From 77a9a4f2327f12e47844997bc24529825a5b093b Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 13 Aug 2024 18:01:14 +0100 Subject: [PATCH 08/47] revert wrong change --- examples/remix-basic/.eslintrc.cjs | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/remix-basic/.eslintrc.cjs b/examples/remix-basic/.eslintrc.cjs index f3c2d7c6f2..8aeb375841 100644 --- a/examples/remix-basic/.eslintrc.cjs +++ b/examples/remix-basic/.eslintrc.cjs @@ -37,6 +37,5 @@ module.exports = { `tsup.config.ts`, `vitest.config.ts`, `.eslintrc.js`, - `*.css`, ], }; From bbad0846fc100693f0fa85358b25cccbf0a7b55c Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 13 Aug 2024 18:30:10 +0100 Subject: [PATCH 09/47] Added idb persistence. Disabled until handling multitab and resume from shape --- examples/yjs-provider/app/page.tsx | 21 ++++++++++++++------- examples/yjs-provider/app/y-electric.js | 17 ++++++++++++----- examples/yjs-provider/package.json | 1 + 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index e1c8eb660e..78d2c06ed1 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -6,6 +6,8 @@ import { useEffect, useRef, useState } from "react" import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" +import { IndexeddbPersistence } from "y-indexeddb" +import * as awarenessProtocol from "y-protocols/awareness" import { EditorState, EditorView, basicSetup } from "@codemirror/basic-setup" import { keymap } from "@codemirror/view" @@ -42,10 +44,15 @@ const theme = EditorView.theme( { dark: true } ) const ydoc = new Y.Doc() -let provider: ElectricProvider | null = null +let network: ElectricProvider | null = null if (typeof window !== `undefined`) { - provider = new ElectricProvider(`http://localhost:3000/`, room, ydoc) + const opts = { + connect: true, + awareness: new awarenessProtocol.Awareness(ydoc), + // persistence: new IndexeddbPersistence(room, ydoc), + } + network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) } export default function Home() { @@ -55,20 +62,20 @@ export default function Home() { const toggle = () => { if (connect === `connected`) { - provider?.disconnect() + network?.disconnect() setConnect(`disconnected`) } else { - provider?.connect() + network?.connect() setConnect(`connected`) } } useEffect(() => { - if (provider === null) return + if (network === null) return const ytext = ydoc.getText(room) - provider.awareness.setLocalStateField(`user`, { + network.awareness.setLocalStateField(`user`, { name: `Anonymous ` + Math.floor(Math.random() * 100), color: userColor.color, colorLight: userColor.light, @@ -81,7 +88,7 @@ export default function Home() { basicSetup, javascript(), EditorView.lineWrapping, - yCollab(ytext, provider.awareness), + yCollab(ytext, network.awareness), theme, ], }) diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index e9f77d35ef..0af3a31154 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -194,13 +194,14 @@ export class ElectricProvider extends Observable { * @param {object} opts * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] + * @param {IndexeddbPersistence} [opts.persistence] * @param {Object} [opts.params] specify url parameters */ constructor( serverUrl, roomname, doc, - { connect = true, awareness = new awarenessProtocol.Awareness(doc) } = {} + { connect = false, awareness = null, persistence = null } = {} ) { super() @@ -222,6 +223,12 @@ export class ElectricProvider extends Observable { this.closeHandler = null + this.loaded = persistence === null + persistence?.on(`synced`, () => { + this.loaded = true + this.connect() + }) + /** * Listens to Yjs updates and sends to the backend * @param {Uint8Array} update @@ -255,9 +262,9 @@ export class ElectricProvider extends Observable { if (env.isNode && typeof process !== `undefined`) { process.on(`exit`, this._exitHandler) } - awareness.on(`update`, this._awarenessUpdateHandler) + awareness?.on(`update`, this._awarenessUpdateHandler) - if (connect) { + if (connect && this.loaded) { this.connect() } } @@ -294,7 +301,7 @@ export class ElectricProvider extends Observable { if (env.isNode && typeof process !== `undefined`) { process.off(`exit`, this._exitHandler) } - this.awareness.off(`update`, this._awarenessUpdateHandler) + this.awareness?.off(`update`, this._awarenessUpdateHandler) this.doc.off(`update`, this._updateHandler) super.destroy() } @@ -305,7 +312,7 @@ export class ElectricProvider extends Observable { } connect() { - this.shouldConnect = true + this.shouldConnect = true && this.loaded if (!this.connected && this.operationsStream === null) { setupShapeStream(this) } diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index a274feb5b8..8528c29660 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -29,6 +29,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "y-codemirror.next": "^0.1.0", + "y-indexeddb": "^9.0.12", "y-protocols": "1.0.6", "yjs": "^13.6.18" }, From 3ae2dc38018626b0e496d038d08bc7a49538185c Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 14 Aug 2024 13:39:25 +0100 Subject: [PATCH 10/47] Schema change --- .../yjs-provider/app/api/awareness/route.ts | 6 +++--- .../yjs-provider/app/api/operation/route.ts | 4 ++-- examples/yjs-provider/app/y-electric.js | 17 +++++++++-------- .../db/migrations/01-create_yjs_tables.sql | 6 +++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/yjs-provider/app/api/awareness/route.ts b/examples/yjs-provider/app/api/awareness/route.ts index 41a0b68a6e..d63eabf0dd 100644 --- a/examples/yjs-provider/app/api/awareness/route.ts +++ b/examples/yjs-provider/app/api/awareness/route.ts @@ -5,11 +5,11 @@ export async function POST(request: Request) { const body = await request.json() await db.query( - `INSERT INTO ydoc_awareness (client, name, op) + `INSERT INTO ydoc_awareness (client, room, op) VALUES ($1, $2, $3) - ON CONFLICT (client, name) + ON CONFLICT (client, room) DO UPDATE SET op = $3`, - [body.client, body.name, body.op] + [body.client, body.room, body.op] ) return NextResponse.json({}) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 81c2ca490b..5a170417fa 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -4,9 +4,9 @@ import { NextResponse } from "next/server" export async function POST(request: Request) { const body = await request.json() await db.query( - `INSERT INTO ydoc_operations (name, op) + `INSERT INTO ydoc_operations (room, op) VALUES ($1, $2)`, - [body.name, body.op] + [body.room, body.op] ) return NextResponse.json({}) } diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 0af3a31154..948ab25955 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -94,6 +94,7 @@ const setupShapeStream = (provider) => { provider.connecting = false if (provider.connected) { provider.connected = false + provider.synced = false awarenessProtocol.removeAwarenessStates( @@ -148,17 +149,17 @@ const setupShapeStream = (provider) => { * @param {Uint8Array} op */ const sendOperation = async (provider, update) => { - if (!(provider.connected && provider.operationsStream !== null)) { + if (!provider.connected) { provider.pending.push(update) } else { const encoder = encoding.createEncoder() syncProtocol.writeUpdate(encoder, update) const op = toBase64(encoding.toUint8Array(encoder)) - const name = provider.roomname + const room = provider.roomname await fetch(`/api/operation`, { method: `POST`, - body: JSON.stringify({ name, op }), + body: JSON.stringify({ room, op }), }) } } @@ -175,13 +176,13 @@ const sendAwareness = async (provider, changedClients) => { ) const op = toBase64(encoding.toUint8Array(encoder)) - if (provider.connected && provider.operationsStream !== null) { - const name = provider.roomname + if (provider.connected) { + const room = provider.roomname const clientID = `${provider.doc.clientID}` await fetch(`/api/awareness`, { method: `POST`, - body: JSON.stringify({ client: clientID, name, op }), + body: JSON.stringify({ client: clientID, room, op }), }) } } @@ -270,13 +271,13 @@ export class ElectricProvider extends Observable { } get operationsUrl() { - const params = { where: `name = '${this.roomname}'` } + const params = { where: `room = '${this.roomname}'` } const encodedParams = url.encodeQueryParams(params) return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams } get awarenessUrl() { - const params = { where: `name = '${this.roomname}'` } + const params = { where: `room = '${this.roomname}'` } const encodedParams = url.encodeQueryParams(params) return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams } diff --git a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql index b7f4cdaa79..cc142500f2 100644 --- a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql +++ b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql @@ -1,15 +1,15 @@ CREATE TABLE ydoc_operations( id SERIAL PRIMARY KEY, - name TEXT, + room TEXT, op TEXT NOT NULL ); CREATE TABLE ydoc_awareness( client TEXT, - name TEXT, + room TEXT, op TEXT NOT NULL, updated TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (client, name) + PRIMARY KEY (client, room) ); CREATE OR REPLACE FUNCTION delete_old_rows() From 06a7c53410ce87fc4f2d7f4cada96a868cf8708d Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 14 Aug 2024 13:40:36 +0100 Subject: [PATCH 11/47] avoid loading yjs in server --- examples/yjs-provider/app/page.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 78d2c06ed1..93ce8421b9 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -71,11 +71,11 @@ export default function Home() { } useEffect(() => { - if (network === null) return + if (typeof window === `undefined`) return const ytext = ydoc.getText(room) - network.awareness.setLocalStateField(`user`, { + network!.awareness.setLocalStateField(`user`, { name: `Anonymous ` + Math.floor(Math.random() * 100), color: userColor.color, colorLight: userColor.light, @@ -88,7 +88,7 @@ export default function Home() { basicSetup, javascript(), EditorView.lineWrapping, - yCollab(ytext, network.awareness), + yCollab(ytext, network!.awareness), theme, ], }) From d5b7d6bc8f404a78908007c9a0e7c0a2a22b0c26 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 14 Aug 2024 13:45:46 +0100 Subject: [PATCH 12/47] Compaction on the server and cleanup --- .../yjs-provider/app/api/operation/route.ts | 71 +++++++++++++++++++ examples/yjs-provider/app/page.tsx | 23 +++--- examples/yjs-provider/app/y-electric.js | 70 +++++++++++++++--- 3 files changed, 140 insertions(+), 24 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 5a170417fa..03d1072663 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,12 +1,83 @@ import { db } from "../../db" import { NextResponse } from "next/server" +// TODO: still loading yjs twice +import * as Y from "yjs" +import * as syncProtocol from "y-protocols/sync" + +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" + +import { toBase64, fromBase64 } from "lib0/buffer" + +const maxRowCount = 50 + export async function POST(request: Request) { const body = await request.json() + + const errorResponse = validateRequest(body) + if (errorResponse) { + return errorResponse + } + await db.query( `INSERT INTO ydoc_operations (room, op) VALUES ($1, $2)`, [body.room, body.op] ) + await maybeCompact(body.room) + return NextResponse.json({}) } + +function validateRequest({ room, op }: { room: string; op: string }) { + if (!room) { + return NextResponse.json({ error: `'room' is required` }, { status: 400 }) + } + + if (!op) { + return NextResponse.json({ error: `'op' is required` }, { status: 400 }) + } +} + +// naive implementation of compaction +async function maybeCompact(room: string) { + const ydoc = new Y.Doc() + + const res0 = await db.query( + `SELECT COUNT(*) as count FROM ydoc_operations`, + [] + ) + if (res0.rows[0].count < maxRowCount) { + return + } + + console.log(`compaction`) + const res1 = await db.query( + `SELECT id, op FROM ydoc_operations + WHERE room = $1 + ORDER BY id DESC`, + [room] + ) + res1.rows.map(({ op }) => { + const buf = fromBase64(op) + const decoder = decoding.createDecoder(buf) + syncProtocol.readSyncMessage( + decoder, + encoding.createEncoder(), + ydoc, + `server` + ) + }) + + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + const encoded = toBase64(encoding.toUint8Array(encoder)) + + await db.query(`TRUNCATE ydoc_operations`) + await db.query( + `INSERT INTO ydoc_operations (room, op) + VALUES ($1, $2)`, + [room, encoded] + ) +} diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 93ce8421b9..8fe5dbee89 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -24,8 +24,6 @@ const usercolors = [ { color: `#ecd444`, light: `#ecd44433` }, { color: `#ee6352`, light: `#ee635233` }, { color: `#9ac2c9`, light: `#9ac2c933` }, - { color: `#8acb88`, light: `#8acb8833` }, - { color: `#1be7ff`, light: `#1be7ff33` }, ] const userColor = usercolors[random.uint32() % usercolors.length] @@ -44,16 +42,13 @@ const theme = EditorView.theme( { dark: true } ) const ydoc = new Y.Doc() -let network: ElectricProvider | null = null -if (typeof window !== `undefined`) { - const opts = { - connect: true, - awareness: new awarenessProtocol.Awareness(ydoc), - // persistence: new IndexeddbPersistence(room, ydoc), - } - network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) +const opts = { + connect: true, + awareness: new awarenessProtocol.Awareness(ydoc), + persistence: new IndexeddbPersistence(room, ydoc), } +const network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) export default function Home() { const editor = useRef(null) @@ -71,12 +66,10 @@ export default function Home() { } useEffect(() => { - if (typeof window === `undefined`) return - const ytext = ydoc.getText(room) - network!.awareness.setLocalStateField(`user`, { - name: `Anonymous ` + Math.floor(Math.random() * 100), + network.awareness.setLocalStateField(`user`, { + name: userColor.color, color: userColor.color, colorLight: userColor.light, }) @@ -88,7 +81,7 @@ export default function Home() { basicSetup, javascript(), EditorView.lineWrapping, - yCollab(ytext, network!.awareness), + yCollab(ytext, network.awareness), theme, ], }) diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 948ab25955..5ae159e5e0 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -45,7 +45,25 @@ const setupShapeStream = (provider) => { }) } - const handleSyncMessage = (messages) => + // Should handle multiple clients + const updateShapeState = (name, offset, shapeId) => { + if (provider.persistence === null) { + return + } + provider.persistence.set(name, { offset, shape_id: shapeId }) + } + + const handleSyncMessage = (messages) => { + if (messages.length < 2) { + return + } + const { offset } = messages[messages.length - 2] + updateShapeState( + `operations_state`, + Number(offset.split(`_`)[0]), + provider.operationsStream.shapeId + ) + handleMessages(messages).forEach((decoder) => { const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) @@ -62,8 +80,19 @@ const setupShapeStream = (provider) => { provider.synced = true } }) + } + + const handleAwarenessMessage = (messages) => { + if (messages.length < 2) { + return + } + const { offset } = messages[messages.length - 2] + updateShapeState( + `awareness_state`, + Number(offset.split(`_`)[0]), + provider.awarenessStream.shapeId + ) - const handleAwarenessMessage = (messages) => handleMessages(messages).forEach((decoder) => { awarenessProtocol.applyAwarenessUpdate( provider.awareness, @@ -71,6 +100,7 @@ const setupShapeStream = (provider) => { provider ) }) + } // TODO: need to improve error handling const handleError = (event) => { @@ -94,7 +124,7 @@ const setupShapeStream = (provider) => { provider.connecting = false if (provider.connected) { provider.connected = false - + provider.synced = false awarenessProtocol.removeAwarenessStates( @@ -196,13 +226,13 @@ export class ElectricProvider extends Observable { * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] * @param {IndexeddbPersistence} [opts.persistence] - * @param {Object} [opts.params] specify url parameters + * @param {Object} [opts.resume] */ constructor( serverUrl, roomname, doc, - { connect = false, awareness = null, persistence = null } = {} + { connect = false, awareness = null, persistence = null, resume = {} } = {} ) { super() @@ -221,13 +251,27 @@ export class ElectricProvider extends Observable { this.awarenessStream = null this.pending = [] + this.resume = resume ?? {} this.closeHandler = null + this.persistence = persistence this.loaded = persistence === null + persistence?.on(`synced`, () => { - this.loaded = true - this.connect() + persistence + .get(`operations_state`) + .then((opsState) => { + this.resume.operations = opsState + return persistence.get(`awareness_state`) + }) + .then((awarenessState) => { + this.resume.awareness = awarenessState + }) + .then(() => { + this.loaded = true + this.connect() + }) }) /** @@ -271,14 +315,22 @@ export class ElectricProvider extends Observable { } get operationsUrl() { - const params = { where: `room = '${this.roomname}'` } + const params = { + where: `room = '${this.roomname}'`, + ...this.resume.operations, + } const encodedParams = url.encodeQueryParams(params) + console.log(params) return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams } get awarenessUrl() { - const params = { where: `room = '${this.roomname}'` } + const params = { + where: `room = '${this.roomname}'`, + ...this.resume.awareness, + } const encodedParams = url.encodeQueryParams(params) + console.log(params) return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams } From cc91d27c5e660b9c14f29d22f95d5784fae2f755 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 14 Aug 2024 17:18:44 +0100 Subject: [PATCH 13/47] Added Nextjs ESLint --- examples/yjs-provider/.eslintrc.cjs | 1 + examples/yjs-provider/package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/yjs-provider/.eslintrc.cjs b/examples/yjs-provider/.eslintrc.cjs index c5c99d0cd9..915fca4da9 100644 --- a/examples/yjs-provider/.eslintrc.cjs +++ b/examples/yjs-provider/.eslintrc.cjs @@ -8,6 +8,7 @@ module.exports = { `eslint:recommended`, `plugin:@typescript-eslint/recommended`, `plugin:prettier/recommended`, + 'plugin:@next/next/recommended', ], parserOptions: { ecmaVersion: 2022, diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 8528c29660..517a99000a 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -35,6 +35,7 @@ }, "devDependencies": { "@databases/pg-migrations": "^5.0.3", + "@next/eslint-plugin-next": "^14.2.5", "@types/pg": "^8.11.6", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", From 64e57774f9663186d14f6f38076f1556f4c05bb2 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 14 Aug 2024 18:13:56 +0100 Subject: [PATCH 14/47] Cherry picked broadcast impl from y-websocket --- examples/yjs-provider/.eslintrc.cjs | 1 + examples/yjs-provider/app/page.tsx | 27 ++- examples/yjs-provider/app/y-broadcast.js | 264 +++++++++++++++++++++++ 3 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 examples/yjs-provider/app/y-broadcast.js diff --git a/examples/yjs-provider/.eslintrc.cjs b/examples/yjs-provider/.eslintrc.cjs index 915fca4da9..1e2f4aa287 100644 --- a/examples/yjs-provider/.eslintrc.cjs +++ b/examples/yjs-provider/.eslintrc.cjs @@ -31,6 +31,7 @@ module.exports = { caughtErrorsIgnorePattern: `^_`, }, ], + "@next/next/no-img-element": "off" }, ignorePatterns: [ `**/node_modules/**`, diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 8fe5dbee89..5f2dd25e36 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -7,6 +7,7 @@ import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" import { IndexeddbPersistence } from "y-indexeddb" +import { BroadcastProvider } from "./y-broadcast" import * as awarenessProtocol from "y-protocols/awareness" import { EditorState, EditorView, basicSetup } from "@codemirror/basic-setup" @@ -42,13 +43,21 @@ const theme = EditorView.theme( { dark: true } ) const ydoc = new Y.Doc() +let network: ElectricProvider | null = null + +if (typeof window !== `undefined`) { + const awareness = new awarenessProtocol.Awareness(ydoc) + const opts = { + connect: true, + awareness, + persistence: new IndexeddbPersistence(room, ydoc), + } + network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) -const opts = { - connect: true, - awareness: new awarenessProtocol.Awareness(ydoc), - persistence: new IndexeddbPersistence(room, ydoc), + new BroadcastProvider(room, ydoc, { + awareness, + }) } -const network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) export default function Home() { const editor = useRef(null) @@ -66,9 +75,13 @@ export default function Home() { } useEffect(() => { + if (typeof window === `undefined`) { + return + } + const ytext = ydoc.getText(room) - network.awareness.setLocalStateField(`user`, { + network?.awareness.setLocalStateField(`user`, { name: userColor.color, color: userColor.color, colorLight: userColor.light, @@ -81,7 +94,7 @@ export default function Home() { basicSetup, javascript(), EditorView.lineWrapping, - yCollab(ytext, network.awareness), + yCollab(ytext, network?.awareness), theme, ], }) diff --git a/examples/yjs-provider/app/y-broadcast.js b/examples/yjs-provider/app/y-broadcast.js new file mode 100644 index 0000000000..5698ed62b3 --- /dev/null +++ b/examples/yjs-provider/app/y-broadcast.js @@ -0,0 +1,264 @@ +/** + * Extracted this from y-websocket + */ + +/* eslint-env browser */ + +import * as bc from "lib0/broadcastchannel" +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" +import * as syncProtocol from "y-protocols/sync" +import * as awarenessProtocol from "y-protocols/awareness" +import { Observable } from "lib0/observable" +export const messageSync = 0 +export const messageQueryAwareness = 3 +export const messageAwareness = 1 + +const messageHandlers = [] + +messageHandlers[messageSync] = ( + encoder, + decoder, + provider, + emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + provider.doc, + provider + ) + if ( + emitSynced && + syncMessageType === syncProtocol.messageYjsSyncStep2 && + !provider.synced + ) { + provider.synced = true + } +} + +messageHandlers[messageQueryAwareness] = ( + encoder, + _decoder, + provider, + _emitSynced, + _messageType +) => { + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate( + provider.awareness, + Array.from(provider.awareness.getStates().keys()) + ) + ) +} + +messageHandlers[messageAwareness] = ( + _encoder, + decoder, + provider, + _emitSynced, + _messageType +) => { + awarenessProtocol.applyAwarenessUpdate( + provider.awareness, + decoding.readVarUint8Array(decoder), + provider + ) +} + +/** + * @param {BroadcastProvider} provider + * @param {Uint8Array} buf + * @param {boolean} emitSynced + * @return {encoding.Encoder} + */ +const readMessage = (provider, buf, emitSynced) => { + const decoder = decoding.createDecoder(buf) + const encoder = encoding.createEncoder() + const messageType = decoding.readVarUint(decoder) + const messageHandler = provider.messageHandlers[messageType] + if (/** @type {any} */ (messageHandler)) { + messageHandler(encoder, decoder, provider, emitSynced, messageType) + } else { + console.error(`Unable to compute message`) + } + return encoder +} + +/** + * @param {BroadcastProvider} provider + */ + +/** + * @param {BroadcastProvider} provider + * @param {ArrayBuffer} buf + */ +const broadcastMessage = (provider, buf) => { + if (provider.bcconnected) { + bc.publish(provider.bcChannel, buf, provider) + } +} + +export class BroadcastProvider extends Observable { + constructor( + roomname, + doc, + { + connect = true, + disableBc = false, + awareness = new awarenessProtocol.Awareness(doc), + } = {} + ) { + super() + + this.bcChannel = roomname + this.awareness = awareness + this.roomname = roomname + this.doc = doc + this.bcconnected = false + this.disableBc = disableBc + this.messageHandlers = messageHandlers.slice() + + this._synced = false + this.wsLastMessageReceived = 0 + + this._bcSubscriber = (data, origin) => { + if (origin !== this) { + const encoder = readMessage(this, new Uint8Array(data), false) + if (encoding.length(encoder) > 1) { + bc.publish(this.bcChannel, encoding.toUint8Array(encoder), this) + } + } + } + + this._updateHandler = (update, origin) => { + if (origin !== this) { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeUpdate(encoder, update) + broadcastMessage(this, encoding.toUint8Array(encoder)) + } + } + this.doc.on(`update`, this._updateHandler) + + this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { + const changedClients = added.concat(updated).concat(removed) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) + ) + broadcastMessage(this, encoding.toUint8Array(encoder)) + } + this._exitHandler = () => { + awarenessProtocol.removeAwarenessStates( + this.awareness, + [doc.clientID], + `app closed` + ) + } + + awareness.on(`update`, this._awarenessUpdateHandler) + + if (connect) { + this.connect() + } + } + + /** + * @type {boolean} + */ + get synced() { + return this._synced + } + + set synced(state) { + if (this._synced !== state) { + this._synced = state + this.emit(`synced`, [state]) + this.emit(`sync`, [state]) + } + } + + destroy() { + this.disconnect() + this.awareness.off(`update`, this._awarenessUpdateHandler) + this.doc.off(`update`, this._updateHandler) + super.destroy() + } + + connectBc() { + if (this.disableBc) { + return + } + if (!this.bcconnected) { + bc.subscribe(this.bcChannel, this._bcSubscriber) + this.bcconnected = true + } + // send sync step1 to bc + // write sync step 1 + const encoderSync = encoding.createEncoder() + encoding.writeVarUint(encoderSync, messageSync) + syncProtocol.writeSyncStep1(encoderSync, this.doc) + bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this) + // broadcast local state + const encoderState = encoding.createEncoder() + encoding.writeVarUint(encoderState, messageSync) + syncProtocol.writeSyncStep2(encoderState, this.doc) + bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this) + // write queryAwareness + const encoderAwarenessQuery = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness) + bc.publish( + this.bcChannel, + encoding.toUint8Array(encoderAwarenessQuery), + this + ) + // broadcast local awareness state + const encoderAwarenessState = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessState, messageAwareness) + encoding.writeVarUint8Array( + encoderAwarenessState, + awarenessProtocol.encodeAwarenessUpdate(this.awareness, [ + this.doc.clientID, + ]) + ) + bc.publish( + this.bcChannel, + encoding.toUint8Array(encoderAwarenessState), + this + ) + } + + disconnectBc() { + // broadcast message with local awareness state set to null (indicating disconnect) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate( + this.awareness, + [this.doc.clientID], + new Map() + ) + ) + broadcastMessage(this, encoding.toUint8Array(encoder)) + if (this.bcconnected) { + bc.unsubscribe(this.bcChannel, this._bcSubscriber) + this.bcconnected = false + } + } + + disconnect() { + this.disconnectBc() + } + + connect() { + this.connectBc() + } +} From 1a657c5dc4d8581fd084a86a5c65e2e5c018c156 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 15 Aug 2024 01:33:01 +0100 Subject: [PATCH 15/47] Added connection pool More cleanup --- .../yjs-provider/app/api/awareness/route.ts | 20 ++-- .../yjs-provider/app/api/compaction/route.ts | 58 ----------- .../yjs-provider/app/api/operation/route.ts | 82 +++++++++------- examples/yjs-provider/app/db.ts | 11 +-- examples/yjs-provider/app/y-broadcast.js | 96 ++++--------------- examples/yjs-provider/app/y-electric.js | 32 ++----- 6 files changed, 86 insertions(+), 213 deletions(-) delete mode 100644 examples/yjs-provider/app/api/compaction/route.ts diff --git a/examples/yjs-provider/app/api/awareness/route.ts b/examples/yjs-provider/app/api/awareness/route.ts index d63eabf0dd..b6108b8d4c 100644 --- a/examples/yjs-provider/app/api/awareness/route.ts +++ b/examples/yjs-provider/app/api/awareness/route.ts @@ -1,16 +1,20 @@ -import { db } from "../../db" +import { pool } from "../../db" import { NextResponse } from "next/server" export async function POST(request: Request) { - const body = await request.json() + const db = await pool.connect() + try { + const body = await request.json() - await db.query( - `INSERT INTO ydoc_awareness (client, room, op) + await db.query( + `INSERT INTO ydoc_awareness (client, room, op) VALUES ($1, $2, $3) ON CONFLICT (client, room) DO UPDATE SET op = $3`, - [body.client, body.room, body.op] - ) - - return NextResponse.json({}) + [body.client, body.room, body.op] + ) + return NextResponse.json({}) + } finally { + db.release() + } } diff --git a/examples/yjs-provider/app/api/compaction/route.ts b/examples/yjs-provider/app/api/compaction/route.ts deleted file mode 100644 index 9652b3586a..0000000000 --- a/examples/yjs-provider/app/api/compaction/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { db } from "../../db" -import { NextRequest, NextResponse } from "next/server" - -import * as Y from "yjs" -import * as syncProtocol from "y-protocols/sync" - -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" - -import { toBase64, fromBase64 } from "lib0/buffer" - -const ydoc = new Y.Doc() - -export async function GET(request: NextRequest) { - const room = request.nextUrl.searchParams.get(`room`) - - if (!room) { - return NextResponse.json({ error: `room is required` }, { status: 400 }) - } - - const res = await db.query( - `SELECT id, op FROM ydoc_operations - WHERE name = $1 - ORDER BY id ASC - LIMIT 1000`, - [room] - ) - - const ytext = ydoc.getText(room) - res.rows.map(({ op }) => { - const buf = fromBase64(op) - const decoder = decoding.createDecoder(buf) - syncProtocol.readSyncMessage( - decoder, - encoding.createEncoder(), - ydoc, - `server` - ) - }) - - await db.query( - `DELETE FROM ydoc_operations - WHERE name = $1 AND id <= $2`, - [room, res.rows[res.rows.length - 1].id] - ) - - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - const encoded = toBase64(encoding.toUint8Array(encoder)) - - await db.query( - `INSERT INTO ydoc_operations (name, op) - VALUES ($1, $2)`, - [room, encoded] - ) - - return NextResponse.json({ text: ytext.toJSON() }) -} diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 03d1072663..bd58d49409 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,4 +1,4 @@ -import { db } from "../../db" +import { pool } from "../../db" import { NextResponse } from "next/server" // TODO: still loading yjs twice @@ -9,57 +9,57 @@ import * as encoding from "lib0/encoding" import * as decoding from "lib0/decoding" import { toBase64, fromBase64 } from "lib0/buffer" +import { PoolClient } from "pg" const maxRowCount = 50 export async function POST(request: Request) { - const body = await request.json() + const db = await pool.connect() - const errorResponse = validateRequest(body) - if (errorResponse) { - return errorResponse - } + try { + const body = await request.json() - await db.query( - `INSERT INTO ydoc_operations (room, op) - VALUES ($1, $2)`, - [body.room, body.op] - ) - await maybeCompact(body.room) + const errorResponse = validateRequest(body) + if (errorResponse) { + return errorResponse + } - return NextResponse.json({}) -} - -function validateRequest({ room, op }: { room: string; op: string }) { - if (!room) { - return NextResponse.json({ error: `'room' is required` }, { status: 400 }) - } + await db.query(`BEGIN`) + await db.query( + `INSERT INTO ydoc_operations (room, op) + VALUES ($1, $2)`, + [body.room, body.op] + ) + await maybeCompact(db, body.room) + await db.query(`COMMIT`) - if (!op) { - return NextResponse.json({ error: `'op' is required` }, { status: 400 }) + return NextResponse.json({}) + } catch (e) { + await db.query(`ROLLBACK`) + throw e + } finally { + db.release() } } // naive implementation of compaction -async function maybeCompact(room: string) { - const ydoc = new Y.Doc() - - const res0 = await db.query( - `SELECT COUNT(*) as count FROM ydoc_operations`, - [] - ) - if (res0.rows[0].count < maxRowCount) { - return - } - - console.log(`compaction`) - const res1 = await db.query( +async function maybeCompact(db: PoolClient, room: string) { + const res = await db.query( `SELECT id, op FROM ydoc_operations WHERE room = $1 ORDER BY id DESC`, [room] ) - res1.rows.map(({ op }) => { + + if (res.rows.length < maxRowCount) { + return + } + + console.log(`compaction`) + + const ydoc = new Y.Doc() + + res.rows.map(({ op }) => { const buf = fromBase64(op) const decoder = decoding.createDecoder(buf) syncProtocol.readSyncMessage( @@ -77,7 +77,17 @@ async function maybeCompact(room: string) { await db.query(`TRUNCATE ydoc_operations`) await db.query( `INSERT INTO ydoc_operations (room, op) - VALUES ($1, $2)`, + VALUES ($1, $2)`, [room, encoded] ) } + +function validateRequest({ room, op }: { room: string; op: string }) { + if (!room) { + return NextResponse.json({ error: `'room' is required` }, { status: 400 }) + } + + if (!op) { + return NextResponse.json({ error: `'op' is required` }, { status: 400 }) + } +} diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index 1571b1341c..6847116c08 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -1,14 +1,13 @@ -import pgPkg from "pg" -const { Client } = pgPkg +import { Pool } from "pg" -const db = new Client({ +console.log(`init pool`) +const pool = new Pool({ host: `localhost`, port: 54321, password: `password`, user: `postgres`, database: `electric`, + max: 1, }) -db.connect() - -export { db } +export { pool } diff --git a/examples/yjs-provider/app/y-broadcast.js b/examples/yjs-provider/app/y-broadcast.js index 5698ed62b3..b4b7a3c0b9 100644 --- a/examples/yjs-provider/app/y-broadcast.js +++ b/examples/yjs-provider/app/y-broadcast.js @@ -2,50 +2,26 @@ * Extracted this from y-websocket */ -/* eslint-env browser */ - import * as bc from "lib0/broadcastchannel" import * as encoding from "lib0/encoding" import * as decoding from "lib0/decoding" import * as syncProtocol from "y-protocols/sync" import * as awarenessProtocol from "y-protocols/awareness" import { Observable } from "lib0/observable" +import * as env from "lib0/environment" + export const messageSync = 0 export const messageQueryAwareness = 3 export const messageAwareness = 1 const messageHandlers = [] -messageHandlers[messageSync] = ( - encoder, - decoder, - provider, - emitSynced, - _messageType -) => { +messageHandlers[messageSync] = (encoder, decoder, provider) => { encoding.writeVarUint(encoder, messageSync) - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - provider.doc, - provider - ) - if ( - emitSynced && - syncMessageType === syncProtocol.messageYjsSyncStep2 && - !provider.synced - ) { - provider.synced = true - } + syncProtocol.readSyncMessage(decoder, encoder, provider.doc, provider) } -messageHandlers[messageQueryAwareness] = ( - encoder, - _decoder, - provider, - _emitSynced, - _messageType -) => { +messageHandlers[messageQueryAwareness] = (encoder, _decoder, provider) => { encoding.writeVarUint(encoder, messageAwareness) encoding.writeVarUint8Array( encoder, @@ -56,13 +32,7 @@ messageHandlers[messageQueryAwareness] = ( ) } -messageHandlers[messageAwareness] = ( - _encoder, - decoder, - provider, - _emitSynced, - _messageType -) => { +messageHandlers[messageAwareness] = (_encoder, decoder, provider) => { awarenessProtocol.applyAwarenessUpdate( provider.awareness, decoding.readVarUint8Array(decoder), @@ -70,33 +40,19 @@ messageHandlers[messageAwareness] = ( ) } -/** - * @param {BroadcastProvider} provider - * @param {Uint8Array} buf - * @param {boolean} emitSynced - * @return {encoding.Encoder} - */ -const readMessage = (provider, buf, emitSynced) => { +const readMessage = (provider, buf) => { const decoder = decoding.createDecoder(buf) const encoder = encoding.createEncoder() const messageType = decoding.readVarUint(decoder) const messageHandler = provider.messageHandlers[messageType] if (/** @type {any} */ (messageHandler)) { - messageHandler(encoder, decoder, provider, emitSynced, messageType) + messageHandler(encoder, decoder, provider, false, messageType) } else { console.error(`Unable to compute message`) } return encoder } -/** - * @param {BroadcastProvider} provider - */ - -/** - * @param {BroadcastProvider} provider - * @param {ArrayBuffer} buf - */ const broadcastMessage = (provider, buf) => { if (provider.bcconnected) { bc.publish(provider.bcChannel, buf, provider) @@ -107,11 +63,7 @@ export class BroadcastProvider extends Observable { constructor( roomname, doc, - { - connect = true, - disableBc = false, - awareness = new awarenessProtocol.Awareness(doc), - } = {} + { connect = true, awareness = new awarenessProtocol.Awareness(doc) } = {} ) { super() @@ -120,12 +72,8 @@ export class BroadcastProvider extends Observable { this.roomname = roomname this.doc = doc this.bcconnected = false - this.disableBc = disableBc this.messageHandlers = messageHandlers.slice() - this._synced = false - this.wsLastMessageReceived = 0 - this._bcSubscriber = (data, origin) => { if (origin !== this) { const encoder = readMessage(this, new Uint8Array(data), false) @@ -144,7 +92,7 @@ export class BroadcastProvider extends Observable { } } this.doc.on(`update`, this._updateHandler) - + this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { const changedClients = added.concat(updated).concat(removed) const encoder = encoding.createEncoder() @@ -162,6 +110,9 @@ export class BroadcastProvider extends Observable { `app closed` ) } + if (env.isNode && typeof process !== `undefined`) { + process.on(`exit`, this._exitHandler) + } awareness.on(`update`, this._awarenessUpdateHandler) @@ -170,32 +121,17 @@ export class BroadcastProvider extends Observable { } } - /** - * @type {boolean} - */ - get synced() { - return this._synced - } - - set synced(state) { - if (this._synced !== state) { - this._synced = state - this.emit(`synced`, [state]) - this.emit(`sync`, [state]) - } - } - destroy() { this.disconnect() + if (env.isNode && typeof process !== `undefined`) { + process.off(`exit`, this._exitHandler) + } this.awareness.off(`update`, this._awarenessUpdateHandler) this.doc.off(`update`, this._updateHandler) super.destroy() } connectBc() { - if (this.disableBc) { - return - } if (!this.bcconnected) { bc.subscribe(this.bcChannel, this._bcSubscriber) this.bcconnected = true diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 5ae159e5e0..51bfe148c6 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -45,12 +45,9 @@ const setupShapeStream = (provider) => { }) } - // Should handle multiple clients + // Should deduplicate persistence const updateShapeState = (name, offset, shapeId) => { - if (provider.persistence === null) { - return - } - provider.persistence.set(name, { offset, shape_id: shapeId }) + provider.persistence?.set(name, { offset, shape_id: shapeId }) } const handleSyncMessage = (messages) => { @@ -174,11 +171,7 @@ const setupShapeStream = (provider) => { } } -/** - * @param {ElectricProvider} provider - * @param {Uint8Array} op - */ -const sendOperation = async (provider, update) => { +const sendOperation = (provider, update) => { if (!provider.connected) { provider.pending.push(update) } else { @@ -187,18 +180,14 @@ const sendOperation = async (provider, update) => { const op = toBase64(encoding.toUint8Array(encoder)) const room = provider.roomname - await fetch(`/api/operation`, { + fetch(`/api/operation`, { method: `POST`, body: JSON.stringify({ room, op }), }) } } -/** - * @param {ElectricProvider} provider - * @param {Uint8Array} op - */ -const sendAwareness = async (provider, changedClients) => { +const sendAwareness = (provider, changedClients) => { const encoder = encoding.createEncoder() encoding.writeVarUint8Array( encoder, @@ -210,7 +199,7 @@ const sendAwareness = async (provider, changedClients) => { const room = provider.roomname const clientID = `${provider.doc.clientID}` - await fetch(`/api/awareness`, { + fetch(`/api/awareness`, { method: `POST`, body: JSON.stringify({ client: clientID, room, op }), }) @@ -274,13 +263,8 @@ export class ElectricProvider extends Observable { }) }) - /** - * Listens to Yjs updates and sends to the backend - * @param {Uint8Array} update - * @param {any} origin - */ this._updateHandler = (update, origin) => { - if (origin !== this) { + if (origin !== this && !origin.bcChannel) { sendOperation(this, update) } } @@ -320,7 +304,6 @@ export class ElectricProvider extends Observable { ...this.resume.operations, } const encodedParams = url.encodeQueryParams(params) - console.log(params) return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams } @@ -330,7 +313,6 @@ export class ElectricProvider extends Observable { ...this.resume.awareness, } const encodedParams = url.encodeQueryParams(params) - console.log(params) return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams } From 68e41a1cfcc823f27667dc44dc38adb6332ad9fc Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 19 Aug 2024 23:00:20 +0100 Subject: [PATCH 16/47] Updated codemirror deps --- examples/yjs-provider/app/page.tsx | 3 ++- examples/yjs-provider/package.json | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 5f2dd25e36..6aefc6e7fa 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -10,7 +10,8 @@ import { IndexeddbPersistence } from "y-indexeddb" import { BroadcastProvider } from "./y-broadcast" import * as awarenessProtocol from "y-protocols/awareness" -import { EditorState, EditorView, basicSetup } from "@codemirror/basic-setup" +import { EditorState } from "@codemirror/state" +import { EditorView, basicSetup } from "codemirror" import { keymap } from "@codemirror/view" import { javascript } from "@codemirror/lang-javascript" diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 517a99000a..4233217ced 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -17,18 +17,18 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@codemirror/basic-setup": "^0.19.0", - "@codemirror/lang-javascript": "^0.19.0", - "@codemirror/state": "^0.19.0", - "@codemirror/view": "^0.19.0", + "@codemirror/lang-javascript": "^6.2.2", + "@codemirror/state": "^6.4.1", + "@codemirror/view": "^6.32.0", "@electric-sql/client": "workspace:*", "@electric-sql/react": "workspace:*", + "codemirror": "^6.0.1", "lib0": "^0.2.96", "next": "^14.2.5", "pg": "^8.12.0", "react": "^18.3.1", "react-dom": "^18.3.1", - "y-codemirror.next": "^0.1.0", + "y-codemirror.next": "0.3.5", "y-indexeddb": "^9.0.12", "y-protocols": "1.0.6", "yjs": "^13.6.18" From 9ae41309ab40998c5dc782e7bf21f6576c18a31a Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 19 Aug 2024 23:10:24 +0100 Subject: [PATCH 17/47] use shape resume as ShapeOptions instead of building the URL --- examples/yjs-provider/app/y-electric.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 51bfe148c6..9026751ab1 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -28,10 +28,12 @@ const setupShapeStream = (provider) => { provider.operationsStream = new ShapeStream({ url: provider.operationsUrl, + ...provider.resume.operations, }) provider.awarenessStream = new ShapeStream({ url: provider.awarenessUrl, + ...provider.resume.awareness, }) const handleMessages = (messages) => { @@ -45,9 +47,10 @@ const setupShapeStream = (provider) => { }) } - // Should deduplicate persistence + // is it necessary to deduplicate persistence? + // only for performance const updateShapeState = (name, offset, shapeId) => { - provider.persistence?.set(name, { offset, shape_id: shapeId }) + provider.persistence?.set(name, { offset, shapeId }) } const handleSyncMessage = (messages) => { @@ -57,7 +60,7 @@ const setupShapeStream = (provider) => { const { offset } = messages[messages.length - 2] updateShapeState( `operations_state`, - Number(offset.split(`_`)[0]), + offset, provider.operationsStream.shapeId ) @@ -86,7 +89,7 @@ const setupShapeStream = (provider) => { const { offset } = messages[messages.length - 2] updateShapeState( `awareness_state`, - Number(offset.split(`_`)[0]), + offset, provider.awarenessStream.shapeId ) @@ -301,7 +304,6 @@ export class ElectricProvider extends Observable { get operationsUrl() { const params = { where: `room = '${this.roomname}'`, - ...this.resume.operations, } const encodedParams = url.encodeQueryParams(params) return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams @@ -310,7 +312,6 @@ export class ElectricProvider extends Observable { get awarenessUrl() { const params = { where: `room = '${this.roomname}'`, - ...this.resume.awareness, } const encodedParams = url.encodeQueryParams(params) return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams From 6f9b19f5d6184875892e511f8bcac42b2dca3b2e Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 19 Aug 2024 23:39:35 +0100 Subject: [PATCH 18/47] Removed all Electric swag --- examples/yjs-provider/app/App.css | 25 ------------- examples/yjs-provider/app/Example.css | 25 ------------- examples/yjs-provider/app/layout.tsx | 12 +------ examples/yjs-provider/app/page.tsx | 22 ++++-------- examples/yjs-provider/app/style.css | 52 --------------------------- examples/yjs-provider/index.html | 8 +---- 6 files changed, 8 insertions(+), 136 deletions(-) delete mode 100644 examples/yjs-provider/app/App.css delete mode 100644 examples/yjs-provider/app/Example.css delete mode 100644 examples/yjs-provider/app/style.css diff --git a/examples/yjs-provider/app/App.css b/examples/yjs-provider/app/App.css deleted file mode 100644 index 620047949a..0000000000 --- a/examples/yjs-provider/app/App.css +++ /dev/null @@ -1,25 +0,0 @@ -.App { - text-align: center; -} - -.App-logo { - height: min(160px, 30vmin); - pointer-events: none; - margin-top: min(30px, 5vmin); - margin-bottom: min(30px, 5vmin); -} - -.App-header { - background-color: #1c1e20; - min-height: 100vh; - display: flex; - flex-direction: column; - align-items: top; - justify-content: top; - font-size: calc(10px + 2vmin); - color: white; -} - -.App-link { - color: #61dafb; -} \ No newline at end of file diff --git a/examples/yjs-provider/app/Example.css b/examples/yjs-provider/app/Example.css deleted file mode 100644 index b6fb457c58..0000000000 --- a/examples/yjs-provider/app/Example.css +++ /dev/null @@ -1,25 +0,0 @@ -.controls { - margin-bottom: 1.5rem; -} - -.button { - display: inline-block; - line-height: 1.3; - text-align: center; - text-decoration: none; - vertical-align: middle; - cursor: pointer; - user-select: none; - width: calc(15vw + 100px); - margin-right: 0.5rem !important; - margin-left: 0.5rem !important; - border-radius: 32px; - text-shadow: 2px 6px 20px rgba(0, 0, 0, 0.4); - box-shadow: rgba(0, 0, 0, 0.5) 1px 2px 8px 0px; - background: #1e2123; - border: 2px solid #229089; - color: #f9fdff; - font-size: 16px; - font-weight: 500; - padding: 10px 18px; -} \ No newline at end of file diff --git a/examples/yjs-provider/app/layout.tsx b/examples/yjs-provider/app/layout.tsx index dded27eb6a..38b9f24675 100644 --- a/examples/yjs-provider/app/layout.tsx +++ b/examples/yjs-provider/app/layout.tsx @@ -1,6 +1,3 @@ -import "./style.css" -import "./App.css" - export const metadata = { title: `Next.js Forms Example`, description: `Example application with forms and Postgres.`, @@ -13,14 +10,7 @@ export default function RootLayout({ }) { return ( - -
-
- logo - {children} -
-
- + {children} ) } diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 6aefc6e7fa..a9163cd206 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -1,6 +1,5 @@ "use client" -import "./Example.css" import { useEffect, useRef, useState } from "react" import * as Y from "yjs" @@ -30,19 +29,6 @@ const usercolors = [ const userColor = usercolors[random.uint32() % usercolors.length] -const theme = EditorView.theme( - { - ".cm-content": { - minWidth: `400px`, - textAlign: `left`, - backgroundColor: `#223239`, - }, - ".cm-content .cm-gutter": { - minHeight: `200px`, - }, - }, - { dark: true } -) const ydoc = new Y.Doc() let network: ElectricProvider | null = null @@ -96,7 +82,6 @@ export default function Home() { javascript(), EditorView.lineWrapping, yCollab(ytext, network?.awareness), - theme, ], }) @@ -112,7 +97,12 @@ export default function Home() { {connect} -
+

+ This is a demo of Yjs shared + editor backed by Postgres using{` `} + Electric. +

+
) } diff --git a/examples/yjs-provider/app/style.css b/examples/yjs-provider/app/style.css deleted file mode 100644 index c763f4d19f..0000000000 --- a/examples/yjs-provider/app/style.css +++ /dev/null @@ -1,52 +0,0 @@ -body { - margin: 0; - font-family: "Helvetica Neue", Helvetica, sans-serif; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; - background: #1c1e20; - min-width: 360px; -} - -code { - font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", - monospace; -} - - -/* YJS awareness */ - -.remote-caret { - position: relative; - border-left: 2px solid black; - margin-left: -1px; - margin-right: -1px; - box-sizing: border-box; -} - -.remote-caret>div { - position: absolute; - top: -1.05em; - left: -2px; - font-size: .6em; - background-color: rgb(250, 129, 0); - font-family: serif; - font-style: normal; - font-weight: normal; - line-height: normal; - user-select: none; - color: white; - padding-left: 2px; - padding-right: 2px; - z-index: 3; - transition: opacity .3s ease-in-out; -} - -.remote-caret.hide-name>div { - transition-delay: .7s; - opacity: 0; -} - -.remote-caret:hover>div { - opacity: 1; - transition-delay: 0s; -} \ No newline at end of file diff --git a/examples/yjs-provider/index.html b/examples/yjs-provider/index.html index 7f2247de29..1c8f62ecc9 100644 --- a/examples/yjs-provider/index.html +++ b/examples/yjs-provider/index.html @@ -5,14 +5,8 @@ - Web Example - ElectricSQL + Yjs Electric provider - - -
- - - \ No newline at end of file From 3e3b327968327f192a6664f93996ae469ce37735 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 23 Aug 2024 09:04:51 +0100 Subject: [PATCH 19/47] Further cleanup --- .../yjs-provider/app/api/awareness/route.ts | 20 ---- .../yjs-provider/app/api/compaction/route.ts | 75 ++++++++++++ .../yjs-provider/app/api/operation/route.ts | 110 +++++++----------- examples/yjs-provider/app/db.ts | 1 - examples/yjs-provider/app/layout.tsx | 4 +- examples/yjs-provider/app/page.tsx | 3 +- examples/yjs-provider/app/y-broadcast.js | 2 +- examples/yjs-provider/app/y-electric.js | 8 +- .../db/migrations/01-create_yjs_tables.sql | 4 +- 9 files changed, 127 insertions(+), 100 deletions(-) delete mode 100644 examples/yjs-provider/app/api/awareness/route.ts create mode 100644 examples/yjs-provider/app/api/compaction/route.ts diff --git a/examples/yjs-provider/app/api/awareness/route.ts b/examples/yjs-provider/app/api/awareness/route.ts deleted file mode 100644 index b6108b8d4c..0000000000 --- a/examples/yjs-provider/app/api/awareness/route.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { pool } from "../../db" -import { NextResponse } from "next/server" - -export async function POST(request: Request) { - const db = await pool.connect() - try { - const body = await request.json() - - await db.query( - `INSERT INTO ydoc_awareness (client, room, op) - VALUES ($1, $2, $3) - ON CONFLICT (client, room) - DO UPDATE SET op = $3`, - [body.client, body.room, body.op] - ) - return NextResponse.json({}) - } finally { - db.release() - } -} diff --git a/examples/yjs-provider/app/api/compaction/route.ts b/examples/yjs-provider/app/api/compaction/route.ts new file mode 100644 index 0000000000..82ed3c3e8c --- /dev/null +++ b/examples/yjs-provider/app/api/compaction/route.ts @@ -0,0 +1,75 @@ +import { pool } from "../../db" +import { NextRequest, NextResponse } from "next/server" + +import * as Y from "yjs" +import * as syncProtocol from "y-protocols/sync" + +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" + +import { toBase64, fromBase64 } from "lib0/buffer" + +export async function GET(request: NextRequest) { + try { + const { room } = await getRequestParams(request) + + doCompation(room) + + return NextResponse.json({}) + } catch (e) { + const resp = e instanceof Error ? e.message : e + return NextResponse.json(resp, { status: 400 }) + } +} + +async function doCompation(room: string) { + const db = await pool.connect() + try { + await db.query(`BEGIN`) + const res = await db.query( + `DELETE FROM ydoc_operations + WHERE room = $1 + RETURNING *`, + [room] + ) + + const ydoc = new Y.Doc() + res.rows.map(({ op }) => { + const buf = fromBase64(op) + const decoder = decoding.createDecoder(buf) + syncProtocol.readSyncMessage( + decoder, + encoding.createEncoder(), + ydoc, + `server` + ) + }) + + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + const encoded = toBase64(encoding.toUint8Array(encoder)) + + await db.query( + `INSERT INTO ydoc_operations (room, op) + VALUES ($1, $2)`, + [room, encoded] + ) + await db.query(`COMMIT`) + } catch (e) { + await db.query(`ROLLBACK`) + throw e + } finally { + db.release() + } +} + +async function getRequestParams( + request: NextRequest +): Promise<{ room: string }> { + const room = await request.nextUrl.searchParams.get(`room`) + if (!room) { + throw new Error(`'room' is required`) + } + + return { room } +} diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index bd58d49409..8dabcf54ba 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,93 +1,63 @@ import { pool } from "../../db" import { NextResponse } from "next/server" -// TODO: still loading yjs twice -import * as Y from "yjs" -import * as syncProtocol from "y-protocols/sync" - -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" - -import { toBase64, fromBase64 } from "lib0/buffer" -import { PoolClient } from "pg" - -const maxRowCount = 50 - export async function POST(request: Request) { - const db = await pool.connect() - try { - const body = await request.json() + const { room, op, clientId } = await getRequestParams(request) - const errorResponse = validateRequest(body) - if (errorResponse) { - return errorResponse + if (!clientId) { + saveOperation(room, op) + } else { + saveAwarenessOperation(room, op, clientId) } - await db.query(`BEGIN`) - await db.query( - `INSERT INTO ydoc_operations (room, op) - VALUES ($1, $2)`, - [body.room, body.op] - ) - await maybeCompact(db, body.room) - await db.query(`COMMIT`) - return NextResponse.json({}) } catch (e) { - await db.query(`ROLLBACK`) - throw e - } finally { - db.release() + const resp = e instanceof Error ? e.message : e + return NextResponse.json(resp, { status: 400 }) } } -// naive implementation of compaction -async function maybeCompact(db: PoolClient, room: string) { - const res = await db.query( - `SELECT id, op FROM ydoc_operations - WHERE room = $1 - ORDER BY id DESC`, - [room] - ) - - if (res.rows.length < maxRowCount) { - return +async function saveOperation(room: string, op: string) { + const db = await pool.connect() + try { + await db.query(`INSERT INTO ydoc_operations (room, op) VALUES ($1, $2)`, [ + room, + op, + ]) + } finally { + db.release() } +} - console.log(`compaction`) - - const ydoc = new Y.Doc() - - res.rows.map(({ op }) => { - const buf = fromBase64(op) - const decoder = decoding.createDecoder(buf) - syncProtocol.readSyncMessage( - decoder, - encoding.createEncoder(), - ydoc, - `server` +async function saveAwarenessOperation( + room: string, + op: string, + clientId: string +) { + const db = await pool.connect() + try { + await db.query( + `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, $3) + ON CONFLICT (clientId, room) + DO UPDATE SET op = $3`, + [room, clientId, op] ) - }) - - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - const encoded = toBase64(encoding.toUint8Array(encoder)) - - await db.query(`TRUNCATE ydoc_operations`) - await db.query( - `INSERT INTO ydoc_operations (room, op) - VALUES ($1, $2)`, - [room, encoded] - ) + } finally { + db.release() + } } -function validateRequest({ room, op }: { room: string; op: string }) { +async function getRequestParams( + request: Request +): Promise<{ room: string; op: string; clientId?: string }> { + const { room, op, clientId } = await request.json() if (!room) { - return NextResponse.json({ error: `'room' is required` }, { status: 400 }) + throw new Error(`'room' is required`) } - if (!op) { - return NextResponse.json({ error: `'op' is required` }, { status: 400 }) + throw new Error(`'op' is required`) } + + return { room, op, clientId } } diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index 6847116c08..c778845f4a 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -1,6 +1,5 @@ import { Pool } from "pg" -console.log(`init pool`) const pool = new Pool({ host: `localhost`, port: 54321, diff --git a/examples/yjs-provider/app/layout.tsx b/examples/yjs-provider/app/layout.tsx index 38b9f24675..ddc3e573cc 100644 --- a/examples/yjs-provider/app/layout.tsx +++ b/examples/yjs-provider/app/layout.tsx @@ -1,6 +1,6 @@ export const metadata = { - title: `Next.js Forms Example`, - description: `Example application with forms and Postgres.`, + title: `Yjs <> Electric`, + description: `Yjs synching with Electric`, } export default function RootLayout({ diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index a9163cd206..8b750758f4 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -42,6 +42,7 @@ if (typeof window !== `undefined`) { network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) new BroadcastProvider(room, ydoc, { + connect: true, awareness, }) } @@ -99,7 +100,7 @@ export default function Home() {

This is a demo of Yjs shared - editor backed by Postgres using{` `} + editor synching with {` `} Electric.

diff --git a/examples/yjs-provider/app/y-broadcast.js b/examples/yjs-provider/app/y-broadcast.js index b4b7a3c0b9..382bf165a3 100644 --- a/examples/yjs-provider/app/y-broadcast.js +++ b/examples/yjs-provider/app/y-broadcast.js @@ -63,7 +63,7 @@ export class BroadcastProvider extends Observable { constructor( roomname, doc, - { connect = true, awareness = new awarenessProtocol.Awareness(doc) } = {} + { connect = false, awareness = new awarenessProtocol.Awareness(doc) } = {} ) { super() diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 9026751ab1..be9f0ea54f 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -200,11 +200,11 @@ const sendAwareness = (provider, changedClients) => { if (provider.connected) { const room = provider.roomname - const clientID = `${provider.doc.clientID}` + const clientId = `${provider.doc.clientID}` - fetch(`/api/awareness`, { + fetch(`/api/operation`, { method: `POST`, - body: JSON.stringify({ client: clientID, room, op }), + body: JSON.stringify({ clientId, room, op }), }) } } @@ -267,6 +267,8 @@ export class ElectricProvider extends Observable { }) this._updateHandler = (update, origin) => { + // prevent pushing operations that come from the + // broadcast provider, when it is being used if (origin !== this && !origin.bcChannel) { sendOperation(this, update) } diff --git a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql index cc142500f2..419c0a5dd3 100644 --- a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql +++ b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql @@ -5,11 +5,11 @@ CREATE TABLE ydoc_operations( ); CREATE TABLE ydoc_awareness( - client TEXT, + clientId TEXT, room TEXT, op TEXT NOT NULL, updated TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (client, room) + PRIMARY KEY (clientId, room) ); CREATE OR REPLACE FUNCTION delete_old_rows() From 3213f41748b82090f590d1d8978413f3e58554c5 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 26 Aug 2024 18:38:06 +0100 Subject: [PATCH 20/47] Fixes persistence issue --- examples/yjs-provider/app/y-electric.js | 88 ++++++++++++++++++------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index be9f0ea54f..4182fd6855 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -11,6 +11,7 @@ import * as awarenessProtocol from "y-protocols/awareness" import { Observable } from "lib0/observable" import * as url from "lib0/url" import * as env from "lib0/environment" +import * as Y from "yjs" export const messageSync = 0 export const messageAwareness = 1 @@ -47,8 +48,6 @@ const setupShapeStream = (provider) => { }) } - // is it necessary to deduplicate persistence? - // only for performance const updateShapeState = (name, offset, shapeId) => { provider.persistence?.set(name, { offset, shapeId }) } @@ -118,7 +117,7 @@ const setupShapeStream = (provider) => { handleError ) - provider.closeHandler = (event) => { + provider.disconnectHandler = (event) => { provider.operationsStream = null provider.awarenessStream = null provider.connecting = false @@ -134,12 +133,13 @@ const setupShapeStream = (provider) => { ), provider ) + provider.lastSyncedStateVector = Y.encodeStateVector(provider.doc) provider.emit(`status`, [{ status: `disconnected` }]) } unsubscribeSyncHandler() unsubscribeAwarenessHandler() - provider.closeHandler = null + provider.disconnectHandler = null provider.emit(`connection-close`, [event, provider]) } @@ -147,11 +147,21 @@ const setupShapeStream = (provider) => { provider.lastMessageReceived = time.getUnixTime() provider.connecting = false provider.connected = true - provider.emit(`status`, [{ status: `connected` }]) - provider.pending - .splice(0) - .forEach((update) => sendOperation(provider, update)) + if (provider.modifiedWhileOffline) { + const pendingUpdates = Y.encodeStateAsUpdate( + provider.doc, + provider.lastSyncedStateVector + ) + const encoderState = encoding.createEncoder() + syncProtocol.writeUpdate(encoderState, pendingUpdates) + + sendOperation(provider, pendingUpdates) + .then(() => clearLastSyncedStateVector(provider)) + .then(() => { + provider.emit(`status`, [{ status: `connected` }]) + }) + } } provider.operationsStream.subscribeOnceToUpToDate( @@ -174,20 +184,43 @@ const setupShapeStream = (provider) => { } } -const sendOperation = (provider, update) => { - if (!provider.connected) { - provider.pending.push(update) - } else { - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, update) - const op = toBase64(encoding.toUint8Array(encoder)) - const room = provider.roomname +const saveLastSyncedStateVector = (provider) => { + provider.modifiedWhileOffline = true + return provider.persistence?.set( + `last_synced_state_vector`, + provider.lastSyncedStateVector + ) +} - fetch(`/api/operation`, { - method: `POST`, - body: JSON.stringify({ room, op }), - }) +const clearLastSyncedStateVector = async (provider) => { + provider.lastSyncedStateVector = null + provider.modifiedWhileOffline = false + provider.persistence?.del(`last_synced_state_vector`) +} + +const sendOperation = async (provider, update) => { + // ignore updates that have no updates + // there is probably a better way of checking this + if (update.length <= 2) { + return + } + + if (!provider.connected) { + if (!provider.modifiedWhileOffline) { + return saveLastSyncedStateVector(provider, update) + } + return } + + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, update) + const op = toBase64(encoding.toUint8Array(encoder)) + const room = provider.roomname + + return fetch(`/api/operation`, { + method: `POST`, + body: JSON.stringify({ room, op }), + }) } const sendAwareness = (provider, changedClients) => { @@ -242,10 +275,11 @@ export class ElectricProvider extends Observable { this.operationsStream = null this.awarenessStream = null - this.pending = [] + this.lastSyncedStateVector = null + this.modifiedWhileOffline = false this.resume = resume ?? {} - this.closeHandler = null + this.disconnectHandler = null this.persistence = persistence this.loaded = persistence === null @@ -259,6 +293,11 @@ export class ElectricProvider extends Observable { }) .then((awarenessState) => { this.resume.awareness = awarenessState + return persistence.get(`last_synced_state_vector`) + }) + .then((lastSyncedStateVector) => { + this.lastSyncedStateVector = lastSyncedStateVector ?? null + this.modifiedWhileOffline = lastSyncedStateVector ?? false }) .then(() => { this.loaded = true @@ -267,8 +306,7 @@ export class ElectricProvider extends Observable { }) this._updateHandler = (update, origin) => { - // prevent pushing operations that come from the - // broadcast provider, when it is being used + // deduplicate events that come from broadcast provider if (origin !== this && !origin.bcChannel) { sendOperation(this, update) } @@ -346,7 +384,7 @@ export class ElectricProvider extends Observable { disconnect() { this.shouldConnect = false - this.closeHandler() + this.disconnectHandler() } connect() { From 180615ad262fc599f62541e16fa2d32e68fba230 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 28 Aug 2024 23:38:53 +0100 Subject: [PATCH 21/47] maitnain server-side ydoc for rendering client page with initial state of the object --- examples/yjs-provider/app/page-client.tsx | 122 ++++++++++++++++++++++ examples/yjs-provider/app/page.tsx | 112 ++------------------ examples/yjs-provider/app/shape.ts | 57 ++++++++++ examples/yjs-provider/app/y-electric.js | 2 +- 4 files changed, 186 insertions(+), 107 deletions(-) create mode 100644 examples/yjs-provider/app/page-client.tsx create mode 100644 examples/yjs-provider/app/shape.ts diff --git a/examples/yjs-provider/app/page-client.tsx b/examples/yjs-provider/app/page-client.tsx new file mode 100644 index 0000000000..b71fb3fc5b --- /dev/null +++ b/examples/yjs-provider/app/page-client.tsx @@ -0,0 +1,122 @@ +"use client" + +import { useEffect, useRef, useState } from "react" + +import * as Y from "yjs" +import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" +import { ElectricProvider } from "./y-electric" +import { IndexeddbPersistence } from "y-indexeddb" +import { BroadcastProvider } from "./y-broadcast" +import * as awarenessProtocol from "y-protocols/awareness" + +import { EditorState } from "@codemirror/state" +import { EditorView, basicSetup } from "codemirror" +import { keymap } from "@codemirror/view" +import { javascript } from "@codemirror/lang-javascript" + +import * as random from "lib0/random" +import * as decoding from "lib0/decoding" + +import { ShapeData } from "./shape" +import { fromBase64 } from "lib0/buffer" + +const room = `electric-demo` + +const usercolors = [ + { color: `#30bced`, light: `#30bced33` }, + { color: `#6eeb83`, light: `#6eeb8333` }, + { color: `#ffbc42`, light: `#ffbc4233` }, + { color: `#ecd444`, light: `#ecd44433` }, + { color: `#ee6352`, light: `#ee635233` }, + { color: `#9ac2c9`, light: `#9ac2c933` }, +] + +const userColor = usercolors[random.uint32() % usercolors.length] + +const ydoc = new Y.Doc() +let network: ElectricProvider | null = null + +export default function Home({ shapeData }: { shapeData: ShapeData }) { + const editor = useRef(null) + + const [connect, setConnect] = useState(`connected`) + + const toggle = () => { + if (connect === `connected`) { + network?.disconnect() + setConnect(`disconnected`) + } else { + network?.connect() + setConnect(`connected`) + } + } + + useEffect(() => { + if (typeof window === `undefined`) { + return + } + + if (typeof window !== `undefined` && network === null) { + const awareness = new awarenessProtocol.Awareness(ydoc) + + const { doc, offset, shapeId } = shapeData + + const decoder = decoding.createDecoder(fromBase64(doc)) + decoding.readVarUint(decoder) + Y.applyUpdate(ydoc, decoding.readVarUint8Array(decoder), `server`) + + const opts = { + connect: true, + awareness, + persistence: new IndexeddbPersistence(room, ydoc), + resume: { operations: { offset, shapeId } }, + } + + network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) + + new BroadcastProvider(room, ydoc, { + connect: true, + awareness, + }) + } + + const ytext = ydoc.getText(room) + + network?.awareness.setLocalStateField(`user`, { + name: userColor.color, + color: userColor.color, + colorLight: userColor.light, + }) + + const state = EditorState.create({ + doc: ytext.toString(), + extensions: [ + keymap.of([...yUndoManagerKeymap]), + basicSetup, + javascript(), + EditorView.lineWrapping, + yCollab(ytext, network?.awareness), + ], + }) + + const view = new EditorView({ state, parent: editor.current ?? undefined }) + + return () => view.destroy() + }) + + return ( +
+
toggle()}> + +
+

+ This is a demo of Yjs shared + editor synching with {` `} + Electric. +

+
+
+ ) +} diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 8b750758f4..497671fd69 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -1,109 +1,9 @@ -"use client" +"use server" -import { useEffect, useRef, useState } from "react" +import React from "react" +import Home from "./page-client" +import { getShapeData } from "./shape" -import * as Y from "yjs" -import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" -import { ElectricProvider } from "./y-electric" -import { IndexeddbPersistence } from "y-indexeddb" -import { BroadcastProvider } from "./y-broadcast" -import * as awarenessProtocol from "y-protocols/awareness" +const Page = async () => -import { EditorState } from "@codemirror/state" -import { EditorView, basicSetup } from "codemirror" -import { keymap } from "@codemirror/view" -import { javascript } from "@codemirror/lang-javascript" - -import * as random from "lib0/random" - -const room = `electric-demo` - -const usercolors = [ - { color: `#30bced`, light: `#30bced33` }, - { color: `#6eeb83`, light: `#6eeb8333` }, - { color: `#ffbc42`, light: `#ffbc4233` }, - { color: `#ecd444`, light: `#ecd44433` }, - { color: `#ee6352`, light: `#ee635233` }, - { color: `#9ac2c9`, light: `#9ac2c933` }, -] - -const userColor = usercolors[random.uint32() % usercolors.length] - -const ydoc = new Y.Doc() -let network: ElectricProvider | null = null - -if (typeof window !== `undefined`) { - const awareness = new awarenessProtocol.Awareness(ydoc) - const opts = { - connect: true, - awareness, - persistence: new IndexeddbPersistence(room, ydoc), - } - network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) - - new BroadcastProvider(room, ydoc, { - connect: true, - awareness, - }) -} - -export default function Home() { - const editor = useRef(null) - - const [connect, setConnect] = useState(`connected`) - - const toggle = () => { - if (connect === `connected`) { - network?.disconnect() - setConnect(`disconnected`) - } else { - network?.connect() - setConnect(`connected`) - } - } - - useEffect(() => { - if (typeof window === `undefined`) { - return - } - - const ytext = ydoc.getText(room) - - network?.awareness.setLocalStateField(`user`, { - name: userColor.color, - color: userColor.color, - colorLight: userColor.light, - }) - - const state = EditorState.create({ - doc: ytext.toString(), - extensions: [ - keymap.of([...yUndoManagerKeymap]), - basicSetup, - javascript(), - EditorView.lineWrapping, - yCollab(ytext, network?.awareness), - ], - }) - - const view = new EditorView({ state, parent: editor.current ?? undefined }) - - return () => view.destroy() - }) - - return ( -
-
toggle()}> - -
-

- This is a demo of Yjs shared - editor synching with {` `} - Electric. -

-
-
- ) -} +export default Page diff --git a/examples/yjs-provider/app/shape.ts b/examples/yjs-provider/app/shape.ts new file mode 100644 index 0000000000..6255182556 --- /dev/null +++ b/examples/yjs-provider/app/shape.ts @@ -0,0 +1,57 @@ +import { Offset, ShapeStream } from "@electric-sql/client" + +import * as Y from "yjs" +import * as syncProtocol from "y-protocols/sync" + +import * as url from "lib0/url" +import { fromBase64, toBase64 } from "lib0/buffer" +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" + +export type ShapeData = { + doc: string + offset: string + shapeId: string + // TODO: awareness +} + +const ydoc = new Y.Doc() + +let cached: string | null = null +let offset: Offset = "-1" + +const encodedParams = url.encodeQueryParams({ + where: `room = 'electric-demo'`, +}) + +const stream = new ShapeStream({ + url: `http://localhost:3000//v1/shape/ydoc_operations?` + encodedParams, +}) + +stream.subscribe((messages) => { + messages.map((message: any) => { + const op = fromBase64(message[`value`][`op`]) + syncProtocol.readSyncMessage( + decoding.createDecoder(op), + encoding.createEncoder(), + ydoc, + `server` + ) + offset = message[`offset`] + cached = null + }) +}) + +export function getShapeData(): ShapeData { + return { + doc: cached ?? (cached = getDocAsBase64()), + offset, + shapeId: stream["shapeId"]!, + } +} + +function getDocAsBase64() { + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + return toBase64(encoding.toUint8Array(encoder)) +} diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 4182fd6855..c9f883787c 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -251,7 +251,7 @@ export class ElectricProvider extends Observable { * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] * @param {IndexeddbPersistence} [opts.persistence] - * @param {Object} [opts.resume] + * @param {Object} [opts.resume] */ constructor( serverUrl, From c0e45630540a61c79c1e2a1701a7bef6fdfe0330 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sat, 9 Nov 2024 22:08:37 +0000 Subject: [PATCH 22/47] Changes after Electric breaking changes --- examples/yjs-provider/app/page-client.tsx | 18 ++++++++-------- examples/yjs-provider/app/shape.ts | 23 ++++++++++++-------- examples/yjs-provider/app/y-electric.js | 26 ++++++++++------------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/examples/yjs-provider/app/page-client.tsx b/examples/yjs-provider/app/page-client.tsx index b71fb3fc5b..cdd2a4c6e5 100644 --- a/examples/yjs-provider/app/page-client.tsx +++ b/examples/yjs-provider/app/page-client.tsx @@ -5,8 +5,8 @@ import { useEffect, useRef, useState } from "react" import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" -import { IndexeddbPersistence } from "y-indexeddb" -import { BroadcastProvider } from "./y-broadcast" +// import { IndexeddbPersistence } from "y-indexeddb" +// import { BroadcastProvider } from "./y-broadcast" import * as awarenessProtocol from "y-protocols/awareness" import { EditorState } from "@codemirror/state" @@ -59,7 +59,7 @@ export default function Home({ shapeData }: { shapeData: ShapeData }) { if (typeof window !== `undefined` && network === null) { const awareness = new awarenessProtocol.Awareness(ydoc) - const { doc, offset, shapeId } = shapeData + const { doc, offset, shapeHandle } = shapeData const decoder = decoding.createDecoder(fromBase64(doc)) decoding.readVarUint(decoder) @@ -68,16 +68,16 @@ export default function Home({ shapeData }: { shapeData: ShapeData }) { const opts = { connect: true, awareness, - persistence: new IndexeddbPersistence(room, ydoc), - resume: { operations: { offset, shapeId } }, + // persistence: new IndexeddbPersistence(room, ydoc), + resume: { operations: { offset, shapeHandle } }, } network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) - new BroadcastProvider(room, ydoc, { - connect: true, - awareness, - }) + // new BroadcastProvider(room, ydoc, { + // connect: true, + // awareness, + // }) } const ytext = ydoc.getText(room) diff --git a/examples/yjs-provider/app/shape.ts b/examples/yjs-provider/app/shape.ts index 6255182556..e567efb59e 100644 --- a/examples/yjs-provider/app/shape.ts +++ b/examples/yjs-provider/app/shape.ts @@ -1,4 +1,9 @@ -import { Offset, ShapeStream } from "@electric-sql/client" +import { + isChangeMessage, + isControlMessage, + Offset, + ShapeStream, +} from "@electric-sql/client" import * as Y from "yjs" import * as syncProtocol from "y-protocols/sync" @@ -11,8 +16,7 @@ import * as decoding from "lib0/decoding" export type ShapeData = { doc: string offset: string - shapeId: string - // TODO: awareness + shapeHandle: string } const ydoc = new Y.Doc() @@ -20,16 +24,17 @@ const ydoc = new Y.Doc() let cached: string | null = null let offset: Offset = "-1" -const encodedParams = url.encodeQueryParams({ - where: `room = 'electric-demo'`, -}) - const stream = new ShapeStream({ - url: `http://localhost:3000//v1/shape/ydoc_operations?` + encodedParams, + url: `http://localhost:3000/v1/shape/`, + table: `ydoc_operations`, + where: `room = 'electric-demo'`, }) stream.subscribe((messages) => { messages.map((message: any) => { + if (isControlMessage(message)) { + return + } const op = fromBase64(message[`value`][`op`]) syncProtocol.readSyncMessage( decoding.createDecoder(op), @@ -46,7 +51,7 @@ export function getShapeData(): ShapeData { return { doc: cached ?? (cached = getDocAsBase64()), offset, - shapeId: stream["shapeId"]!, + shapeHandle: stream.shapeHandle, } } diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index c9f883787c..5fbe0d6030 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -29,11 +29,15 @@ const setupShapeStream = (provider) => { provider.operationsStream = new ShapeStream({ url: provider.operationsUrl, + table: `ydoc_operations`, + where: `room = '${provider.roomname}'`, ...provider.resume.operations, }) provider.awarenessStream = new ShapeStream({ url: provider.awarenessUrl, + where: `room = '${provider.roomname}'`, + table: `ydoc_awareness`, ...provider.resume.awareness, }) @@ -48,8 +52,8 @@ const setupShapeStream = (provider) => { }) } - const updateShapeState = (name, offset, shapeId) => { - provider.persistence?.set(name, { offset, shapeId }) + const updateShapeState = (name, offset, shapeHandle) => { + provider.persistence?.set(name, { offset, shapeHandle }) } const handleSyncMessage = (messages) => { @@ -60,7 +64,7 @@ const setupShapeStream = (provider) => { updateShapeState( `operations_state`, offset, - provider.operationsStream.shapeId + provider.operationsStream.shapeHandle ) handleMessages(messages).forEach((decoder) => { @@ -89,7 +93,7 @@ const setupShapeStream = (provider) => { updateShapeState( `awareness_state`, offset, - provider.awarenessStream.shapeId + provider.awarenessStream.shapeHandle ) handleMessages(messages).forEach((decoder) => { @@ -251,7 +255,7 @@ export class ElectricProvider extends Observable { * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] * @param {IndexeddbPersistence} [opts.persistence] - * @param {Object} [opts.resume] + * @param {Object} [opts.resume] */ constructor( serverUrl, @@ -342,19 +346,11 @@ export class ElectricProvider extends Observable { } get operationsUrl() { - const params = { - where: `room = '${this.roomname}'`, - } - const encodedParams = url.encodeQueryParams(params) - return this.serverUrl + `/v1/shape/ydoc_operations?` + encodedParams + return this.serverUrl + `/v1/shape` } get awarenessUrl() { - const params = { - where: `room = '${this.roomname}'`, - } - const encodedParams = url.encodeQueryParams(params) - return this.serverUrl + `/v1/shape/ydoc_awareness?` + encodedParams + return this.serverUrl + `/v1/shape/` } /** From 9c4c8a1b23919839759c21be6113bbb2d5eb03a9 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 11 Nov 2024 12:52:17 +0000 Subject: [PATCH 23/47] use bytea instead of string use reducer on server to generate document state --- .../yjs-provider/app/api/operation/route.ts | 12 +-- examples/yjs-provider/app/page-client.tsx | 2 +- examples/yjs-provider/app/page.tsx | 4 +- examples/yjs-provider/app/reduce-stream.ts | 84 +++++++++++++++++++ examples/yjs-provider/app/shape.ts | 62 -------------- examples/yjs-provider/app/utils.js | 10 +++ examples/yjs-provider/app/y-electric.js | 15 ++-- examples/yjs-provider/app/ydoc-shape.ts | 59 +++++++++++++ .../db/migrations/01-create_yjs_tables.sql | 4 +- examples/yjs-provider/tsconfig.json | 2 +- 10 files changed, 172 insertions(+), 82 deletions(-) create mode 100644 examples/yjs-provider/app/reduce-stream.ts delete mode 100644 examples/yjs-provider/app/shape.ts create mode 100644 examples/yjs-provider/app/utils.js create mode 100644 examples/yjs-provider/app/ydoc-shape.ts diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 8dabcf54ba..c756ea423a 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -21,10 +21,10 @@ export async function POST(request: Request) { async function saveOperation(room: string, op: string) { const db = await pool.connect() try { - await db.query(`INSERT INTO ydoc_operations (room, op) VALUES ($1, $2)`, [ - room, - op, - ]) + await db.query( + `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, + [room, op] + ) } finally { db.release() } @@ -38,9 +38,9 @@ async function saveAwarenessOperation( const db = await pool.connect() try { await db.query( - `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, $3) + `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) - DO UPDATE SET op = $3`, + DO UPDATE SET op = decode($3, 'base64')`, [room, clientId, op] ) } finally { diff --git a/examples/yjs-provider/app/page-client.tsx b/examples/yjs-provider/app/page-client.tsx index cdd2a4c6e5..2096ce6df8 100644 --- a/examples/yjs-provider/app/page-client.tsx +++ b/examples/yjs-provider/app/page-client.tsx @@ -17,7 +17,7 @@ import { javascript } from "@codemirror/lang-javascript" import * as random from "lib0/random" import * as decoding from "lib0/decoding" -import { ShapeData } from "./shape" +import { ShapeData } from "./ydoc-shape" import { fromBase64 } from "lib0/buffer" const room = `electric-demo` diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 497671fd69..4c3b99f71f 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -2,8 +2,8 @@ import React from "react" import Home from "./page-client" -import { getShapeData } from "./shape" +import { getShapeData } from "./ydoc-shape" -const Page = async () => +const Page = async () => export default Page diff --git a/examples/yjs-provider/app/reduce-stream.ts b/examples/yjs-provider/app/reduce-stream.ts new file mode 100644 index 0000000000..49ef78a608 --- /dev/null +++ b/examples/yjs-provider/app/reduce-stream.ts @@ -0,0 +1,84 @@ +import { + ChangeMessage, + ControlMessage, + GetExtensions, + isControlMessage, + Message, + Publisher, + Row, + ShapeStream, + ShapeStreamInterface, + ShapeStreamOptions, +} from "@electric-sql/client/" + +// Reduce all rows for a shape into a single value. +// Batches all changes until a control message is received. +export type ReduceFunction, Y> = ( + acc: Y, + message: ChangeMessage +) => Y + +export class ReduceStream, Y> + extends Publisher<{ acc: Y }> + implements ShapeStreamInterface<{ acc: Y }> +{ + readonly #stream: ShapeStream + options: ShapeStreamOptions> + + readonly #callback: ReduceFunction + #accMessage: Partial> + #acc?: Y + + constructor(stream: ShapeStream, callback: ReduceFunction, init: Y) { + super() + this.#stream = stream + this.options = stream.options + this.#callback = callback + this.#accMessage = { value: { acc: init } } + this.#acc = init + + stream.subscribe((messages) => { + messages.map((message: Message) => { + const messages = [] + if (isControlMessage(message)) { + if (this.#acc) { + this.#accMessage.value = { acc: this.#acc! } + this.#accMessage.key = this.options.table! + messages.push(this.#accMessage as ChangeMessage<{ acc: Y }>) + this.#acc = undefined + } + messages.push(message) + this.publish(messages) + } else { + this.#acc = this.#callback(this.#acc!, message) + this.#accMessage = { ...message, value: { acc: this.#acc } } + } + }) + }) + } + + get shapeHandle(): string { + return this.#stream.shapeHandle + } + get isUpToDate(): boolean { + return this.#stream.isUpToDate + } + get error(): unknown { + return this.#stream.error + } + start(): Promise { + return this.#stream.start() + } + lastSyncedAt(): number | undefined { + return this.#stream.lastSyncedAt() + } + lastSynced(): number { + return this.#stream.lastSynced() + } + isConnected(): boolean { + return this.#stream.isConnected() + } + isLoading(): boolean { + return this.#stream.isLoading() + } +} diff --git a/examples/yjs-provider/app/shape.ts b/examples/yjs-provider/app/shape.ts deleted file mode 100644 index e567efb59e..0000000000 --- a/examples/yjs-provider/app/shape.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { - isChangeMessage, - isControlMessage, - Offset, - ShapeStream, -} from "@electric-sql/client" - -import * as Y from "yjs" -import * as syncProtocol from "y-protocols/sync" - -import * as url from "lib0/url" -import { fromBase64, toBase64 } from "lib0/buffer" -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" - -export type ShapeData = { - doc: string - offset: string - shapeHandle: string -} - -const ydoc = new Y.Doc() - -let cached: string | null = null -let offset: Offset = "-1" - -const stream = new ShapeStream({ - url: `http://localhost:3000/v1/shape/`, - table: `ydoc_operations`, - where: `room = 'electric-demo'`, -}) - -stream.subscribe((messages) => { - messages.map((message: any) => { - if (isControlMessage(message)) { - return - } - const op = fromBase64(message[`value`][`op`]) - syncProtocol.readSyncMessage( - decoding.createDecoder(op), - encoding.createEncoder(), - ydoc, - `server` - ) - offset = message[`offset`] - cached = null - }) -}) - -export function getShapeData(): ShapeData { - return { - doc: cached ?? (cached = getDocAsBase64()), - offset, - shapeHandle: stream.shapeHandle, - } -} - -function getDocAsBase64() { - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - return toBase64(encoding.toUint8Array(encoder)) -} diff --git a/examples/yjs-provider/app/utils.js b/examples/yjs-provider/app/utils.js new file mode 100644 index 0000000000..686aba9b68 --- /dev/null +++ b/examples/yjs-provider/app/utils.js @@ -0,0 +1,10 @@ +export const parser = { + bytea: (hexString) => { + const cleanHexString = hexString.startsWith("\\x") + ? hexString.slice(2) + : hexString + return new Uint8Array( + cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) + ) + }, + } \ No newline at end of file diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 5fbe0d6030..fb39a267d9 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -3,13 +3,12 @@ */ import * as time from "lib0/time" -import { toBase64, fromBase64 } from "lib0/buffer" +import { toBase64 } from "lib0/buffer" import * as encoding from "lib0/encoding" import * as decoding from "lib0/decoding" import * as syncProtocol from "y-protocols/sync" import * as awarenessProtocol from "y-protocols/awareness" import { Observable } from "lib0/observable" -import * as url from "lib0/url" import * as env from "lib0/environment" import * as Y from "yjs" @@ -17,6 +16,7 @@ export const messageSync = 0 export const messageAwareness = 1 import { ShapeStream } from "@electric-sql/client" +import { parser } from "./utils" /** * @param {ElectricProvider} provider @@ -31,14 +31,16 @@ const setupShapeStream = (provider) => { url: provider.operationsUrl, table: `ydoc_operations`, where: `room = '${provider.roomname}'`, - ...provider.resume.operations, + parser, + ...provider.resume.operations }) provider.awarenessStream = new ShapeStream({ url: provider.awarenessUrl, where: `room = '${provider.roomname}'`, table: `ydoc_awareness`, - ...provider.resume.awareness, + parser, + ...provider.resume.awareness }) const handleMessages = (messages) => { @@ -46,10 +48,7 @@ const setupShapeStream = (provider) => { return messages .filter((message) => message[`key`] && message[`value`][`op`]) .map((message) => message[`value`][`op`]) - .map((operation) => { - const base64 = fromBase64(operation) - return decoding.createDecoder(base64) - }) + .map(decoding.createDecoder) } const updateShapeState = (name, offset, shapeHandle) => { diff --git a/examples/yjs-provider/app/ydoc-shape.ts b/examples/yjs-provider/app/ydoc-shape.ts new file mode 100644 index 0000000000..f69633835b --- /dev/null +++ b/examples/yjs-provider/app/ydoc-shape.ts @@ -0,0 +1,59 @@ +import { Offset, Shape, ShapeStream } from "@electric-sql/client" + +import { parser } from "./utils" + +import * as Y from "yjs" +import * as syncProtocol from "y-protocols/sync" + +import { toBase64 } from "lib0/buffer" +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" +import { ReduceFunction, ReduceStream } from "./reduce-stream" + +export type ShapeData = { + doc: string + offset: string + shapeHandle: string +} + +let offset: Offset = "-1" +let room = `electric-demo` + +const stream = new ShapeStream<{ op: Uint8Array }>({ + url: `http://localhost:3000/v1/shape/`, + table: `ydoc_operations`, + where: `room = '${room}'`, + parser, +}) + +const reduceChangesToDoc: ReduceFunction<{ op: Uint8Array }, Y.Doc> = ( + acc, + message +) => { + syncProtocol.readSyncMessage( + decoding.createDecoder(message.value.op), + encoding.createEncoder(), + acc, + `server` + ) + offset = message[`offset`] + return acc +} + +const reduceStream = new ReduceStream(stream, reduceChangesToDoc, new Y.Doc()) +const shape = new Shape(reduceStream) + +export async function getShapeData(): Promise { + const doc = (await shape.value).get("ydoc_operations")!.acc + return { + doc: getDocAsBase64(doc), + offset, + shapeHandle: stream.shapeHandle, + } +} + +function getDocAsBase64(ydoc: Y.Doc) { + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + return toBase64(encoding.toUint8Array(encoder)) +} diff --git a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql index 419c0a5dd3..1e9d43e165 100644 --- a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql +++ b/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql @@ -1,13 +1,13 @@ CREATE TABLE ydoc_operations( id SERIAL PRIMARY KEY, room TEXT, - op TEXT NOT NULL + op BYTEA NOT NULL ); CREATE TABLE ydoc_awareness( clientId TEXT, room TEXT, - op TEXT NOT NULL, + op BYTEA NOT NULL, updated TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (clientId, room) ); diff --git a/examples/yjs-provider/tsconfig.json b/examples/yjs-provider/tsconfig.json index e06a4454ab..14d189328b 100644 --- a/examples/yjs-provider/tsconfig.json +++ b/examples/yjs-provider/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es5", + "target": "ES2015", "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, From aecdc00c7a0c212fb7ec12b3d75a6f77fff7c3f2 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 12 Nov 2024 01:18:32 +0000 Subject: [PATCH 24/47] Improved parsing --- examples/yjs-provider/app/reduce-stream.ts | 38 ++++--- examples/yjs-provider/app/utils.js | 30 ++++-- examples/yjs-provider/app/y-electric.js | 116 ++++++++++----------- examples/yjs-provider/app/ydoc-shape.ts | 66 ++++++------ 4 files changed, 138 insertions(+), 112 deletions(-) diff --git a/examples/yjs-provider/app/reduce-stream.ts b/examples/yjs-provider/app/reduce-stream.ts index 49ef78a608..9fb046bd29 100644 --- a/examples/yjs-provider/app/reduce-stream.ts +++ b/examples/yjs-provider/app/reduce-stream.ts @@ -1,6 +1,5 @@ import { ChangeMessage, - ControlMessage, GetExtensions, isControlMessage, Message, @@ -11,7 +10,7 @@ import { ShapeStreamOptions, } from "@electric-sql/client/" -// Reduce all rows for a shape into a single value. +// Reduce all changes for a shape into a single value. // Batches all changes until a control message is received. export type ReduceFunction, Y> = ( acc: Y, @@ -26,32 +25,32 @@ export class ReduceStream, Y> options: ShapeStreamOptions> readonly #callback: ReduceFunction - #accMessage: Partial> - #acc?: Y + #accMessage: ChangeMessage<{ acc: Y }> constructor(stream: ShapeStream, callback: ReduceFunction, init: Y) { super() this.#stream = stream this.options = stream.options this.#callback = callback - this.#accMessage = { value: { acc: init } } - this.#acc = init + this.#accMessage = getInitAccMessage(this.options.table!, init) stream.subscribe((messages) => { messages.map((message: Message) => { const messages = [] if (isControlMessage(message)) { - if (this.#acc) { - this.#accMessage.value = { acc: this.#acc! } - this.#accMessage.key = this.options.table! - messages.push(this.#accMessage as ChangeMessage<{ acc: Y }>) - this.#acc = undefined + if (message.headers.control === "up-to-date") { + messages.push(this.#accMessage) } messages.push(message) this.publish(messages) } else { - this.#acc = this.#callback(this.#acc!, message) - this.#accMessage = { ...message, value: { acc: this.#acc } } + const current = this.#accMessage.value.acc + const next = this.#callback(current, message) + this.#accMessage = { + ...message, + key: this.options.table!, + value: { acc: next }, + } } }) }) @@ -82,3 +81,16 @@ export class ReduceStream, Y> return this.#stream.isLoading() } } + +function getInitAccMessage( + key: string, + value: Y +): ChangeMessage<{ acc: Y }> { + return { + headers: { operation: "insert" }, + offset: "-1", + key: key, + value: { acc: value }, + } +} + diff --git a/examples/yjs-provider/app/utils.js b/examples/yjs-provider/app/utils.js index 686aba9b68..2392411e2c 100644 --- a/examples/yjs-provider/app/utils.js +++ b/examples/yjs-provider/app/utils.js @@ -1,10 +1,22 @@ -export const parser = { +import * as decoding from "lib0/decoding" + +const hexStringToUint8Array = (hexString) => { + const cleanHexString = hexString.startsWith("\\x") + ? hexString.slice(2) + : hexString + return new Uint8Array( + cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) + ) + } + +export const parseToUint8Array = { + bytea: hexStringToUint8Array, + } + +export const parseToDecoder = { bytea: (hexString) => { - const cleanHexString = hexString.startsWith("\\x") - ? hexString.slice(2) - : hexString - return new Uint8Array( - cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) - ) - }, - } \ No newline at end of file + const uint8Array = hexStringToUint8Array(hexString) + const decoder = decoding.createDecoder(uint8Array) + return decoding.createDecoder(uint8Array) + } +} \ No newline at end of file diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index fb39a267d9..05f714ea49 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -15,8 +15,8 @@ import * as Y from "yjs" export const messageSync = 0 export const messageAwareness = 1 -import { ShapeStream } from "@electric-sql/client" -import { parser } from "./utils" +import { isChangeMessage, ShapeStream } from "@electric-sql/client" +import { parseToDecoder as parser } from "./utils" /** * @param {ElectricProvider} provider @@ -35,27 +35,10 @@ const setupShapeStream = (provider) => { ...provider.resume.operations }) - provider.awarenessStream = new ShapeStream({ - url: provider.awarenessUrl, - where: `room = '${provider.roomname}'`, - table: `ydoc_awareness`, - parser, - ...provider.resume.awareness - }) - - const handleMessages = (messages) => { + // handle persistence + provider.operationsStream.subscribe((messages) => { provider.lastMessageReceived = time.getUnixTime() - return messages - .filter((message) => message[`key`] && message[`value`][`op`]) - .map((message) => message[`value`][`op`]) - .map(decoding.createDecoder) - } - - const updateShapeState = (name, offset, shapeHandle) => { - provider.persistence?.set(name, { offset, shapeHandle }) - } - const handleSyncMessage = (messages) => { if (messages.length < 2) { return } @@ -65,26 +48,18 @@ const setupShapeStream = (provider) => { offset, provider.operationsStream.shapeHandle ) + }) - handleMessages(messages).forEach((decoder) => { - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - provider.doc, - provider - ) - if ( - syncMessageType === syncProtocol.messageYjsSyncStep2 && - !provider.synced - ) { - provider.synced = true - } - }) - } + provider.awarenessStream = new ShapeStream({ + url: provider.awarenessUrl, + where: `room = '${provider.roomname}'`, + table: `ydoc_awareness`, + parser, + ...provider.resume.awareness + }) - const handleAwarenessMessage = (messages) => { + provider.awarenessStream.subscribe((messages) => { + provider.lastMessageReceived = time.getUnixTime() if (messages.length < 2) { return } @@ -92,32 +67,55 @@ const setupShapeStream = (provider) => { updateShapeState( `awareness_state`, offset, - provider.awarenessStream.shapeHandle + provider.operationsStream.shapeHandle ) + }) - handleMessages(messages).forEach((decoder) => { - awarenessProtocol.applyAwarenessUpdate( - provider.awareness, - decoding.readVarUint8Array(decoder), - provider - ) - }) + const updateShapeState = (name, offset, shapeHandle) => { + provider.persistence?.set(name, { offset, shapeHandle }) } - // TODO: need to improve error handling - const handleError = (event) => { - console.warn(`fetch shape error`, event) - provider.emit(`connection-error`, [event, provider]) + const handleSyncMessage = (messages) => { + messages.forEach((message) => { + if(isChangeMessage(message) && message.value.op) { + const decoder = message.value.op + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + provider.doc, + provider + ) + if ( + syncMessageType === syncProtocol.messageYjsSyncStep2 && + !provider.synced + ) { + provider.synced = true + } + } + }) } + const handleAwarenessMessage = (messages) => { + messages.forEach((message) => { + // sometimes buffer is empty + if(isChangeMessage(message) && message.value.op) { + const decoder = message.value.op + awarenessProtocol.applyAwarenessUpdate( + provider.awareness, + decoding.readVarUint8Array(decoder), + provider + ) + } + })} + const unsubscribeSyncHandler = provider.operationsStream.subscribe( - handleSyncMessage, - handleError + handleSyncMessage ) const unsubscribeAwarenessHandler = provider.awarenessStream.subscribe( - handleAwarenessMessage, - handleError + handleAwarenessMessage ) provider.disconnectHandler = (event) => { @@ -168,8 +166,7 @@ const setupShapeStream = (provider) => { } provider.operationsStream.subscribeOnceToUpToDate( - () => handleOperationsFirstSync(), - () => handleError() + handleOperationsFirstSync.bind(this) ) const handleAwarenessFirstSync = () => { @@ -179,8 +176,7 @@ const setupShapeStream = (provider) => { } provider.awarenessStream.subscribeOnceToUpToDate( - () => handleAwarenessFirstSync(), - () => handleError() + handleAwarenessFirstSync.bind(this) ) provider.emit(`status`, [{ status: `connecting` }]) @@ -202,7 +198,7 @@ const clearLastSyncedStateVector = async (provider) => { } const sendOperation = async (provider, update) => { - // ignore updates that have no updates + // ignore requests that have no changes // there is probably a better way of checking this if (update.length <= 2) { return diff --git a/examples/yjs-provider/app/ydoc-shape.ts b/examples/yjs-provider/app/ydoc-shape.ts index f69633835b..7e28d47d20 100644 --- a/examples/yjs-provider/app/ydoc-shape.ts +++ b/examples/yjs-provider/app/ydoc-shape.ts @@ -1,6 +1,11 @@ -import { Offset, Shape, ShapeStream } from "@electric-sql/client" +import { + Row, + Shape, + ShapeStream, + ShapeStreamOptions, +} from "@electric-sql/client" -import { parser } from "./utils" +import { parseToUint8Array as parser } from "./utils" import * as Y from "yjs" import * as syncProtocol from "y-protocols/sync" @@ -15,45 +20,46 @@ export type ShapeData = { offset: string shapeHandle: string } +type YOp = { op: Uint8Array } +type YDoc = { acc: Y.Doc } -let offset: Offset = "-1" -let room = `electric-demo` +function getYDocShape(stream: ShapeStream): Shape { + const reduceChangesToDoc: ReduceFunction = (acc, message) => { + syncProtocol.readSyncMessage( + decoding.createDecoder(message.value.op), + encoding.createEncoder(), + acc, + `server` + ) + return acc + } + + const reduceStream = new ReduceStream(stream, reduceChangesToDoc, new Y.Doc()) + return new Shape(reduceStream) +} -const stream = new ShapeStream<{ op: Uint8Array }>({ +function getDocAsBase64(ydoc: Y.Doc) { + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) + return toBase64(encoding.toUint8Array(encoder)) +} + +const options: ShapeStreamOptions = { url: `http://localhost:3000/v1/shape/`, table: `ydoc_operations`, - where: `room = '${room}'`, + where: `room = 'electric-demo'`, parser, -}) - -const reduceChangesToDoc: ReduceFunction<{ op: Uint8Array }, Y.Doc> = ( - acc, - message -) => { - syncProtocol.readSyncMessage( - decoding.createDecoder(message.value.op), - encoding.createEncoder(), - acc, - `server` - ) - offset = message[`offset`] - return acc } -const reduceStream = new ReduceStream(stream, reduceChangesToDoc, new Y.Doc()) -const shape = new Shape(reduceStream) +const stream = new ShapeStream(options) +const shape = getYDocShape(stream) -export async function getShapeData(): Promise { +export const getShapeData = async () => { const doc = (await shape.value).get("ydoc_operations")!.acc + console.log("offset", stream.lastOffset) return { doc: getDocAsBase64(doc), - offset, + offset: stream.lastOffset, shapeHandle: stream.shapeHandle, } } - -function getDocAsBase64(ydoc: Y.Doc) { - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - return toBase64(encoding.toUint8Array(encoder)) -} From cdad7812978b3c9223d15f51334d88246a79add6 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 12 Nov 2024 02:10:59 +0000 Subject: [PATCH 25/47] Removed code related with persistence and broadcasting --- examples/yjs-provider/app/page-client.tsx | 8 --- examples/yjs-provider/app/y-electric.js | 75 +++-------------------- 2 files changed, 8 insertions(+), 75 deletions(-) diff --git a/examples/yjs-provider/app/page-client.tsx b/examples/yjs-provider/app/page-client.tsx index 2096ce6df8..8ad4abbacb 100644 --- a/examples/yjs-provider/app/page-client.tsx +++ b/examples/yjs-provider/app/page-client.tsx @@ -5,8 +5,6 @@ import { useEffect, useRef, useState } from "react" import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" -// import { IndexeddbPersistence } from "y-indexeddb" -// import { BroadcastProvider } from "./y-broadcast" import * as awarenessProtocol from "y-protocols/awareness" import { EditorState } from "@codemirror/state" @@ -68,16 +66,10 @@ export default function Home({ shapeData }: { shapeData: ShapeData }) { const opts = { connect: true, awareness, - // persistence: new IndexeddbPersistence(room, ydoc), resume: { operations: { offset, shapeHandle } }, } network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) - - // new BroadcastProvider(room, ydoc, { - // connect: true, - // awareness, - // }) } const ytext = ydoc.getText(room) diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 05f714ea49..1d1f6858d3 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -35,21 +35,6 @@ const setupShapeStream = (provider) => { ...provider.resume.operations }) - // handle persistence - provider.operationsStream.subscribe((messages) => { - provider.lastMessageReceived = time.getUnixTime() - - if (messages.length < 2) { - return - } - const { offset } = messages[messages.length - 2] - updateShapeState( - `operations_state`, - offset, - provider.operationsStream.shapeHandle - ) - }) - provider.awarenessStream = new ShapeStream({ url: provider.awarenessUrl, where: `room = '${provider.roomname}'`, @@ -58,24 +43,9 @@ const setupShapeStream = (provider) => { ...provider.resume.awareness }) - provider.awarenessStream.subscribe((messages) => { - provider.lastMessageReceived = time.getUnixTime() - if (messages.length < 2) { - return - } - const { offset } = messages[messages.length - 2] - updateShapeState( - `awareness_state`, - offset, - provider.operationsStream.shapeHandle - ) - }) - - const updateShapeState = (name, offset, shapeHandle) => { - provider.persistence?.set(name, { offset, shapeHandle }) - } const handleSyncMessage = (messages) => { + provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { if(isChangeMessage(message) && message.value.op) { const decoder = message.value.op @@ -98,6 +68,7 @@ const setupShapeStream = (provider) => { } const handleAwarenessMessage = (messages) => { + provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { // sometimes buffer is empty if(isChangeMessage(message) && message.value.op) { @@ -184,17 +155,13 @@ const setupShapeStream = (provider) => { } const saveLastSyncedStateVector = (provider) => { - provider.modifiedWhileOffline = true - return provider.persistence?.set( - `last_synced_state_vector`, - provider.lastSyncedStateVector - ) + provider.modifiedWhileOffline = true } const clearLastSyncedStateVector = async (provider) => { provider.lastSyncedStateVector = null provider.modifiedWhileOffline = false - provider.persistence?.del(`last_synced_state_vector`) + } const sendOperation = async (provider, update) => { @@ -249,14 +216,13 @@ export class ElectricProvider extends Observable { * @param {object} opts * @param {boolean} [opts.connect] * @param {awarenessProtocol.Awareness} [opts.awareness] - * @param {IndexeddbPersistence} [opts.persistence] * @param {Object} [opts.resume] */ constructor( serverUrl, roomname, doc, - { connect = false, awareness = null, persistence = null, resume = {} } = {} + { connect = false, awareness = null, resume = {} } = {} ) { super() @@ -280,33 +246,8 @@ export class ElectricProvider extends Observable { this.disconnectHandler = null - this.persistence = persistence - this.loaded = persistence === null - - persistence?.on(`synced`, () => { - persistence - .get(`operations_state`) - .then((opsState) => { - this.resume.operations = opsState - return persistence.get(`awareness_state`) - }) - .then((awarenessState) => { - this.resume.awareness = awarenessState - return persistence.get(`last_synced_state_vector`) - }) - .then((lastSyncedStateVector) => { - this.lastSyncedStateVector = lastSyncedStateVector ?? null - this.modifiedWhileOffline = lastSyncedStateVector ?? false - }) - .then(() => { - this.loaded = true - this.connect() - }) - }) - this._updateHandler = (update, origin) => { - // deduplicate events that come from broadcast provider - if (origin !== this && !origin.bcChannel) { + if (origin !== this) { sendOperation(this, update) } } @@ -335,7 +276,7 @@ export class ElectricProvider extends Observable { } awareness?.on(`update`, this._awarenessUpdateHandler) - if (connect && this.loaded) { + if (connect) { this.connect() } } @@ -379,7 +320,7 @@ export class ElectricProvider extends Observable { } connect() { - this.shouldConnect = true && this.loaded + this.shouldConnect = true if (!this.connected && this.operationsStream === null) { setupShapeStream(this) } From e5c03bd834cc947ea1aa57594270d76c252452bb Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 12 Nov 2024 11:32:05 +0000 Subject: [PATCH 26/47] Cleanup --- .../{page-client.tsx => electric-editor.tsx} | 50 +++++++++++---- examples/yjs-provider/app/page.tsx | 11 +++- examples/yjs-provider/app/reduce-stream.ts | 9 ++- examples/yjs-provider/app/utils.js | 39 +++++++----- examples/yjs-provider/app/y-electric.js | 20 +++--- examples/yjs-provider/app/ydoc-shape.ts | 62 ++++++++++++------- 6 files changed, 124 insertions(+), 67 deletions(-) rename examples/yjs-provider/app/{page-client.tsx => electric-editor.tsx} (70%) diff --git a/examples/yjs-provider/app/page-client.tsx b/examples/yjs-provider/app/electric-editor.tsx similarity index 70% rename from examples/yjs-provider/app/page-client.tsx rename to examples/yjs-provider/app/electric-editor.tsx index 8ad4abbacb..b82dee1f25 100644 --- a/examples/yjs-provider/app/page-client.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef, useState } from "react" import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" -import * as awarenessProtocol from "y-protocols/awareness" +import { Awareness, applyAwarenessUpdate } from "y-protocols/awareness" import { EditorState } from "@codemirror/state" import { EditorView, basicSetup } from "codemirror" @@ -30,11 +30,16 @@ const usercolors = [ ] const userColor = usercolors[random.uint32() % usercolors.length] - const ydoc = new Y.Doc() let network: ElectricProvider | null = null -export default function Home({ shapeData }: { shapeData: ShapeData }) { +export default function ElectricEditor({ + docShape, + awarenessShape, +}: { + docShape: ShapeData + awarenessShape: ShapeData +}) { const editor = useRef(null) const [connect, setConnect] = useState(`connected`) @@ -55,21 +60,25 @@ export default function Home({ shapeData }: { shapeData: ShapeData }) { } if (typeof window !== `undefined` && network === null) { - const awareness = new awarenessProtocol.Awareness(ydoc) - - const { doc, offset, shapeHandle } = shapeData - - const decoder = decoding.createDecoder(fromBase64(doc)) - decoding.readVarUint(decoder) - Y.applyUpdate(ydoc, decoding.readVarUint8Array(decoder), `server`) + initDoc(ydoc, docShape.data) + const awareness = new Awareness(ydoc) + initAwareness(awareness, awarenessShape.data) const opts = { connect: true, awareness, - resume: { operations: { offset, shapeHandle } }, + resume: { + operations: docShape.resume, + awareness: awarenessShape.resume, + }, } - network = new ElectricProvider(`http://localhost:3000/`, room, ydoc, opts) + network = new ElectricProvider( + `${process.env.ELECTRIC_URL || `http://localhost:3000`}`, + room, + ydoc, + opts + ) } const ytext = ydoc.getText(room) @@ -112,3 +121,20 @@ export default function Home({ shapeData }: { shapeData: ShapeData }) { ) } + +const initDoc = (ydoc: Y.Doc, data: string) => { + const decoder = decoding.createDecoder(fromBase64(data)) + decoding.readVarUint(decoder) + Y.applyUpdate(ydoc, decoding.readVarUint8Array(decoder), `server`) +} + +const initAwareness = (awareness: Awareness, data: string) => { + for (const client of JSON.parse(data)) { + const decoder = decoding.createDecoder(fromBase64(client)) + applyAwarenessUpdate( + awareness, + decoding.readVarUint8Array(decoder), + `server` + ) + } +} diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 4c3b99f71f..6a46ff4b44 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -1,9 +1,14 @@ "use server" import React from "react" -import Home from "./page-client" -import { getShapeData } from "./ydoc-shape" +import ElectricEditor from "./electric-editor" +import { getAwarenessData, getDocData } from "./ydoc-shape" -const Page = async () => +const Page = async () => ( + +) export default Page diff --git a/examples/yjs-provider/app/reduce-stream.ts b/examples/yjs-provider/app/reduce-stream.ts index 9fb046bd29..7ef6a3a64f 100644 --- a/examples/yjs-provider/app/reduce-stream.ts +++ b/examples/yjs-provider/app/reduce-stream.ts @@ -8,7 +8,7 @@ import { ShapeStream, ShapeStreamInterface, ShapeStreamOptions, -} from "@electric-sql/client/" +} from "@electric-sql/client" // Reduce all changes for a shape into a single value. // Batches all changes until a control message is received. @@ -38,7 +38,7 @@ export class ReduceStream, Y> messages.map((message: Message) => { const messages = [] if (isControlMessage(message)) { - if (message.headers.control === "up-to-date") { + if (message.headers.control === `up-to-date`) { messages.push(this.#accMessage) } messages.push(message) @@ -87,10 +87,9 @@ function getInitAccMessage( value: Y ): ChangeMessage<{ acc: Y }> { return { - headers: { operation: "insert" }, - offset: "-1", + headers: { operation: `insert` }, + offset: `-1`, key: key, value: { acc: value }, } } - diff --git a/examples/yjs-provider/app/utils.js b/examples/yjs-provider/app/utils.js index 2392411e2c..049dcc94d2 100644 --- a/examples/yjs-provider/app/utils.js +++ b/examples/yjs-provider/app/utils.js @@ -1,22 +1,31 @@ +import { toBase64 } from "lib0/buffer" import * as decoding from "lib0/decoding" +export const room = `electric-demo` + const hexStringToUint8Array = (hexString) => { - const cleanHexString = hexString.startsWith("\\x") - ? hexString.slice(2) - : hexString - return new Uint8Array( - cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) - ) - } + const cleanHexString = hexString.startsWith(`\\x`) + ? hexString.slice(2) + : hexString + return new Uint8Array( + cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) + ) +} export const parseToUint8Array = { - bytea: hexStringToUint8Array, - } + bytea: hexStringToUint8Array, +} + +export const parseToBase64 = { + bytea: (hexString) => { + const uint8Array = hexStringToUint8Array(hexString) + return toBase64(uint8Array) + }, +} export const parseToDecoder = { - bytea: (hexString) => { - const uint8Array = hexStringToUint8Array(hexString) - const decoder = decoding.createDecoder(uint8Array) - return decoding.createDecoder(uint8Array) - } -} \ No newline at end of file + bytea: (hexString) => { + const uint8Array = hexStringToUint8Array(hexString) + return decoding.createDecoder(uint8Array) + }, +} diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index 1d1f6858d3..c95b50e8bb 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -32,7 +32,7 @@ const setupShapeStream = (provider) => { table: `ydoc_operations`, where: `room = '${provider.roomname}'`, parser, - ...provider.resume.operations + ...provider.resume.operations, }) provider.awarenessStream = new ShapeStream({ @@ -40,14 +40,13 @@ const setupShapeStream = (provider) => { where: `room = '${provider.roomname}'`, table: `ydoc_awareness`, parser, - ...provider.resume.awareness + ...provider.resume.awareness, }) - const handleSyncMessage = (messages) => { provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { - if(isChangeMessage(message) && message.value.op) { + if (isChangeMessage(message) && message.value.op) { const decoder = message.value.op const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) @@ -71,7 +70,7 @@ const setupShapeStream = (provider) => { provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { // sometimes buffer is empty - if(isChangeMessage(message) && message.value.op) { + if (isChangeMessage(message) && message.value.op) { const decoder = message.value.op awarenessProtocol.applyAwarenessUpdate( provider.awareness, @@ -79,11 +78,11 @@ const setupShapeStream = (provider) => { provider ) } - })} + }) + } - const unsubscribeSyncHandler = provider.operationsStream.subscribe( - handleSyncMessage - ) + const unsubscribeSyncHandler = + provider.operationsStream.subscribe(handleSyncMessage) const unsubscribeAwarenessHandler = provider.awarenessStream.subscribe( handleAwarenessMessage @@ -155,13 +154,12 @@ const setupShapeStream = (provider) => { } const saveLastSyncedStateVector = (provider) => { - provider.modifiedWhileOffline = true + provider.modifiedWhileOffline = true } const clearLastSyncedStateVector = async (provider) => { provider.lastSyncedStateVector = null provider.modifiedWhileOffline = false - } const sendOperation = async (provider, update) => { diff --git a/examples/yjs-provider/app/ydoc-shape.ts b/examples/yjs-provider/app/ydoc-shape.ts index 7e28d47d20..ea92b5239c 100644 --- a/examples/yjs-provider/app/ydoc-shape.ts +++ b/examples/yjs-provider/app/ydoc-shape.ts @@ -1,11 +1,6 @@ -import { - Row, - Shape, - ShapeStream, - ShapeStreamOptions, -} from "@electric-sql/client" +import { Shape, ShapeStream, ShapeStreamOptions } from "@electric-sql/client" -import { parseToUint8Array as parser } from "./utils" +import { parseToUint8Array as parser, parseToBase64, room } from "./utils" import * as Y from "yjs" import * as syncProtocol from "y-protocols/sync" @@ -16,9 +11,11 @@ import * as decoding from "lib0/decoding" import { ReduceFunction, ReduceStream } from "./reduce-stream" export type ShapeData = { - doc: string - offset: string - shapeHandle: string + data: string + resume: { + offset: string + shapeHandle: string + } } type YOp = { op: Uint8Array } type YDoc = { acc: Y.Doc } @@ -33,7 +30,6 @@ function getYDocShape(stream: ShapeStream): Shape { ) return acc } - const reduceStream = new ReduceStream(stream, reduceChangesToDoc, new Y.Doc()) return new Shape(reduceStream) } @@ -44,22 +40,46 @@ function getDocAsBase64(ydoc: Y.Doc) { return toBase64(encoding.toUint8Array(encoder)) } -const options: ShapeStreamOptions = { +const doc: ShapeStreamOptions = { url: `http://localhost:3000/v1/shape/`, table: `ydoc_operations`, - where: `room = 'electric-demo'`, + where: `room = '${room}'`, parser, } +const docStream = new ShapeStream(doc) +const docShape = getYDocShape(docStream) -const stream = new ShapeStream(options) -const shape = getYDocShape(stream) +export const getDocData = async () => { + const doc = (await docShape.value).get(`ydoc_operations`)!.acc + return { + data: getDocAsBase64(doc), + resume: { + offset: docStream.lastOffset, + shapeHandle: docStream.shapeHandle, + }, + } +} -export const getShapeData = async () => { - const doc = (await shape.value).get("ydoc_operations")!.acc - console.log("offset", stream.lastOffset) +const awareness: ShapeStreamOptions = { + url: `http://localhost:3000/v1/shape/`, + table: `ydoc_awareness`, + where: `room = '${room}'`, + parser: parseToBase64, +} +const awarenessStream = new ShapeStream(awareness) +const awarenessShape = new Shape(awarenessStream) + +export const getAwarenessData = async () => { + const clients = await awarenessShape.value + const data = [] + for (const client of clients.values()) { + data.push(client.op) + } return { - doc: getDocAsBase64(doc), - offset: stream.lastOffset, - shapeHandle: stream.shapeHandle, + data: JSON.stringify(data), + resume: { + offset: awarenessStream.lastOffset, + shapeHandle: awarenessStream.shapeHandle, + }, } } From 6f11529f2b674b74160a24004f76102742f99a69 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 12 Nov 2024 11:52:47 +0000 Subject: [PATCH 27/47] Fixed dynamic rendering issue --- examples/yjs-provider/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 4233217ced..8ec9527fa3 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -10,7 +10,7 @@ "backend:down": "PROJECT_NAME=yjs-provider pnpm -C ../../ run example-backend:down", "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", "dev": "next dev --turbo -p 5173", - "build": "next build", + "build": "next build --experimental-build-mode=compile", "start": "next start", "lint": "eslint . --ext js,ts,tsx --report-unused-disable-directives --max-warnings 0", "stylecheck": "eslint . --quiet", From f9faa581f2cc90852c600c6cd7332de9f58b5cb7 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 29 Nov 2024 18:09:28 +0000 Subject: [PATCH 28/47] added deployment stack and required changes --- examples/yjs-provider/app/db.ts | 12 +- examples/yjs-provider/app/electric-editor.tsx | 16 +-- examples/yjs-provider/app/page.tsx | 4 +- .../app/shape-proxy/[...table]/route.ts | 22 ++- examples/yjs-provider/app/y-electric.js | 1 + examples/yjs-provider/app/ydoc-shape.ts | 8 +- examples/yjs-provider/next.config.js | 0 examples/yjs-provider/package.json | 1 + examples/yjs-provider/sst-env.d.ts | 10 ++ examples/yjs-provider/sst.config.ts | 132 ++++++++++++++++++ 10 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 examples/yjs-provider/next.config.js create mode 100644 examples/yjs-provider/sst-env.d.ts create mode 100644 examples/yjs-provider/sst.config.ts diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index c778845f4a..cc3465876f 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -1,12 +1,10 @@ -import { Pool } from "pg" +import pgPkg from "pg" +const { Pool } = pgPkg + +console.log(`POOLED_DATABASE_URL: ${process.env.POOLED_DATABASE_URL}`) const pool = new Pool({ - host: `localhost`, - port: 54321, - password: `password`, - user: `postgres`, - database: `electric`, - max: 1, + connectionString: process.env.POOLED_DATABASE_URL, }) export { pool } diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index b82dee1f25..1848f8b5ff 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -37,8 +37,8 @@ export default function ElectricEditor({ docShape, awarenessShape, }: { - docShape: ShapeData - awarenessShape: ShapeData + docShape?: ShapeData + awarenessShape?: ShapeData }) { const editor = useRef(null) @@ -60,17 +60,17 @@ export default function ElectricEditor({ } if (typeof window !== `undefined` && network === null) { - initDoc(ydoc, docShape.data) + // initDoc(ydoc, docShape.data) const awareness = new Awareness(ydoc) - initAwareness(awareness, awarenessShape.data) + // initAwareness(awareness, awarenessShape.data) const opts = { connect: true, awareness, - resume: { - operations: docShape.resume, - awareness: awarenessShape.resume, - }, + // resume: { + // operations: docShape.resume, + // awareness: awarenessShape.resume, + // }, } network = new ElectricProvider( diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 6a46ff4b44..099d7cfda2 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -6,8 +6,8 @@ import { getAwarenessData, getDocData } from "./ydoc-shape" const Page = async () => ( ) diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts index 54e3562d55..0073117db9 100644 --- a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts +++ b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts @@ -3,12 +3,30 @@ export async function GET( { params }: { params: { table: string } } ) { const url = new URL(request.url) - const { table } = params - const originUrl = new URL(`http://localhost:3000/v1/shape/${table}`) + const originUrl = new URL( + process.env.ELECTRIC_URL + ? `${process.env.ELECTRIC_URL}/v1/shape` + : `http://localhost:3000/v1/shape` + ) + url.searchParams.forEach((value, key) => { originUrl.searchParams.set(key, value) }) + if (process.env.DATABASE_ID) { + originUrl.searchParams.set(`database_id`, process.env.DATABASE_ID) + } + + const headers = new Headers() + if (process.env.ELECTRIC_TOKEN) { + originUrl.searchParams.set(`token`, process.env.ELECTRIC_TOKEN) + } + + const newRequest = new Request(originUrl.toString(), { + method: `GET`, + headers, + }) + // When proxying long-polling requests, content-encoding & content-length are added // erroneously (saying the body is gzipped when it's not) so we'll just remove // them to avoid content decoding errors in the browser. diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js index c95b50e8bb..7fc09192d7 100644 --- a/examples/yjs-provider/app/y-electric.js +++ b/examples/yjs-provider/app/y-electric.js @@ -44,6 +44,7 @@ const setupShapeStream = (provider) => { }) const handleSyncMessage = (messages) => { + console.log(messages) provider.lastMessageReceived = time.getUnixTime() messages.forEach((message) => { if (isChangeMessage(message) && message.value.op) { diff --git a/examples/yjs-provider/app/ydoc-shape.ts b/examples/yjs-provider/app/ydoc-shape.ts index ea92b5239c..e858bbd3ed 100644 --- a/examples/yjs-provider/app/ydoc-shape.ts +++ b/examples/yjs-provider/app/ydoc-shape.ts @@ -39,9 +39,13 @@ function getDocAsBase64(ydoc: Y.Doc) { syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) return toBase64(encoding.toUint8Array(encoder)) } +const url = process.env.ELECTRIC_URL + ? `${process.env.ELECTRIC_URL}/v1/shape` + : `http://localhost:3000/v1/shape/` + const doc: ShapeStreamOptions = { - url: `http://localhost:3000/v1/shape/`, + url, table: `ydoc_operations`, where: `room = '${room}'`, parser, @@ -61,7 +65,7 @@ export const getDocData = async () => { } const awareness: ShapeStreamOptions = { - url: `http://localhost:3000/v1/shape/`, + url, table: `ydoc_awareness`, where: `room = '${room}'`, parser: parseToBase64, diff --git a/examples/yjs-provider/next.config.js b/examples/yjs-provider/next.config.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 8ec9527fa3..5c32b2ccaa 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -28,6 +28,7 @@ "pg": "^8.12.0", "react": "^18.3.1", "react-dom": "^18.3.1", + "sst": "^3.3.35", "y-codemirror.next": "0.3.5", "y-indexeddb": "^9.0.12", "y-protocols": "1.0.6", diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs-provider/sst-env.d.ts new file mode 100644 index 0000000000..e973cf25d9 --- /dev/null +++ b/examples/yjs-provider/sst-env.d.ts @@ -0,0 +1,10 @@ +/* This file is auto-generated by SST. Do not edit. */ +/* tslint:disable */ +/* eslint-disable */ +/* deno-fmt-ignore-file */ +import "sst" +export {} +declare module "sst" { + export interface Resource { + } +} diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts new file mode 100644 index 0000000000..9690e02e7b --- /dev/null +++ b/examples/yjs-provider/sst.config.ts @@ -0,0 +1,132 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// + +import { execSync } from "child_process" + +export default $config({ + app(input) { + return { + name: `yjs`, + removal: input?.stage === `production` ? `retain` : `remove`, + home: `aws`, + providers: { + cloudflare: `5.42.0`, + aws: { + version: `6.57.0`, + }, + neon: `0.6.3`, + }, + } + }, + async run() { + const project = neon.getProjectOutput({ id: process.env.NEON_PROJECT_ID! }) + const base = { + projectId: project.id, + branchId: project.defaultBranchId, + } + + const db = new neon.Database(`yjs`, { + ...base, + ownerName: `neondb_owner`, + name: + $app.stage === `Production` ? `yjs-production` : `yjs-${$app.stage}`, + }) + + const databaseUri = getNeonDbUri(project, db) + const pooledDatabaseUri = getNeonDbUri(project, db, true) + try { + databaseUri.apply(applyMigrations) + + const electricInfo = databaseUri.apply((uri) => + addDatabaseToElectric(uri) + ) + + const website = deployApp(electricInfo, databaseUri, pooledDatabaseUri) + return { + databaseUri, + pooledUri: pooledDatabaseUri, + database_id: electricInfo.id, + electric_token: electricInfo.token, + website: website.url, + } + } catch (e) { + console.error(`Failed to deploy yjs example stack`, e) + } + }, +}) + +function applyMigrations(uri: string) { + execSync(`pnpm exec pg-migrations apply --directory ./db/migrations`, { + env: { + ...process.env, + DATABASE_URL: uri, + }, + }) +} + +function deployApp( + electricInfo: $util.Output<{ id: string; token: string }>, + uri: $util.Output, + pooledUri: $util.Output +) { + return new sst.aws.Nextjs(`yjs`, { + environment: { + ELECTRIC_URL: process.env.ELECTRIC_API!, + ELECTRIC_TOKEN: electricInfo.token, + DATABASE_ID: electricInfo.id, + DATABASE_URL: uri, + POOLED_DATABASE_URL: pooledUri, + }, + domain: { + name: `yjs${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + dns: sst.cloudflare.dns(), + }, + }) +} + +function getNeonDbUri( + project: $util.Output, + db: neon.Database, + pool: boolean = false +) { + const passwordOutput = neon.getBranchRolePasswordOutput({ + projectId: project.id, + branchId: project.defaultBranchId, + roleName: db.ownerName, + }) + + return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require`.apply( + (v) => { + if (pool) { + return v.replace( + process.env.NEON_PROJECT_ID!, + process.env.NEON_PROJECT_ID + `-pooler` + ) + } + return v + } + ) +} + +async function addDatabaseToElectric( + uri: string +): Promise<{ id: string; token: string }> { + const adminApi = process.env.ELECTRIC_ADMIN_API + + const result = await fetch(`${adminApi}/v1/databases`, { + method: `PUT`, + headers: { "Content-Type": `application/json` }, + body: JSON.stringify({ + database_url: uri, + region: `us-east-1`, + }), + }) + + if (!result.ok) { + throw new Error( + `Could not add database to Electric (${result.status}): ${await result.text()}` + ) + } + + return await result.json() +} From 9cea54c7fd2b1adb7bb3c466fa771b1cbb6dd92d Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 2 Dec 2024 23:08:19 +0000 Subject: [PATCH 29/47] Converted provider to typescript --- .../nextjs-example/app/shape-proxy/route.ts | 2 +- .../yjs-provider/app/api/compaction/route.ts | 75 ---- .../yjs-provider/app/api/operation/route.ts | 26 +- examples/yjs-provider/app/db.ts | 6 +- examples/yjs-provider/app/electric-editor.tsx | 74 +--- examples/yjs-provider/app/page.tsx | 8 +- examples/yjs-provider/app/reduce-stream.ts | 95 ----- .../app/shape-proxy/[...table]/route.ts | 9 +- .../yjs-provider/app/{utils.js => utils.ts} | 10 +- examples/yjs-provider/app/y-broadcast.js | 200 ----------- examples/yjs-provider/app/y-electric.js | 327 ----------------- examples/yjs-provider/app/y-electric.ts | 336 ++++++++++++++++++ examples/yjs-provider/app/ydoc-shape.ts | 89 ----- examples/yjs-provider/next.config.js | 0 examples/yjs-provider/next.config.mjs | 12 + 15 files changed, 389 insertions(+), 880 deletions(-) delete mode 100644 examples/yjs-provider/app/api/compaction/route.ts delete mode 100644 examples/yjs-provider/app/reduce-stream.ts rename examples/yjs-provider/app/{utils.js => utils.ts} (72%) delete mode 100644 examples/yjs-provider/app/y-broadcast.js delete mode 100644 examples/yjs-provider/app/y-electric.js create mode 100644 examples/yjs-provider/app/y-electric.ts delete mode 100644 examples/yjs-provider/app/ydoc-shape.ts delete mode 100644 examples/yjs-provider/next.config.js create mode 100644 examples/yjs-provider/next.config.mjs diff --git a/examples/nextjs-example/app/shape-proxy/route.ts b/examples/nextjs-example/app/shape-proxy/route.ts index aadfae4d3a..0e66e0506f 100644 --- a/examples/nextjs-example/app/shape-proxy/route.ts +++ b/examples/nextjs-example/app/shape-proxy/route.ts @@ -41,4 +41,4 @@ export async function GET(request: Request) { }) } return resp -} +} \ No newline at end of file diff --git a/examples/yjs-provider/app/api/compaction/route.ts b/examples/yjs-provider/app/api/compaction/route.ts deleted file mode 100644 index 82ed3c3e8c..0000000000 --- a/examples/yjs-provider/app/api/compaction/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { pool } from "../../db" -import { NextRequest, NextResponse } from "next/server" - -import * as Y from "yjs" -import * as syncProtocol from "y-protocols/sync" - -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" - -import { toBase64, fromBase64 } from "lib0/buffer" - -export async function GET(request: NextRequest) { - try { - const { room } = await getRequestParams(request) - - doCompation(room) - - return NextResponse.json({}) - } catch (e) { - const resp = e instanceof Error ? e.message : e - return NextResponse.json(resp, { status: 400 }) - } -} - -async function doCompation(room: string) { - const db = await pool.connect() - try { - await db.query(`BEGIN`) - const res = await db.query( - `DELETE FROM ydoc_operations - WHERE room = $1 - RETURNING *`, - [room] - ) - - const ydoc = new Y.Doc() - res.rows.map(({ op }) => { - const buf = fromBase64(op) - const decoder = decoding.createDecoder(buf) - syncProtocol.readSyncMessage( - decoder, - encoding.createEncoder(), - ydoc, - `server` - ) - }) - - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - const encoded = toBase64(encoding.toUint8Array(encoder)) - - await db.query( - `INSERT INTO ydoc_operations (room, op) - VALUES ($1, $2)`, - [room, encoded] - ) - await db.query(`COMMIT`) - } catch (e) { - await db.query(`ROLLBACK`) - throw e - } finally { - db.release() - } -} - -async function getRequestParams( - request: NextRequest -): Promise<{ room: string }> { - const room = await request.nextUrl.searchParams.get(`room`) - if (!room) { - throw new Error(`'room' is required`) - } - - return { room } -} diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index c756ea423a..b4b48e902c 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -19,15 +19,10 @@ export async function POST(request: Request) { } async function saveOperation(room: string, op: string) { - const db = await pool.connect() - try { - await db.query( - `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, - [room, op] - ) - } finally { - db.release() - } + pool.query( + `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, + [room, op] + ) } async function saveAwarenessOperation( @@ -35,17 +30,12 @@ async function saveAwarenessOperation( op: string, clientId: string ) { - const db = await pool.connect() - try { - await db.query( - `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) + await pool.query( + `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) DO UPDATE SET op = decode($3, 'base64')`, - [room, clientId, op] - ) - } finally { - db.release() - } + [room, clientId, op] + ) } async function getRequestParams( diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index cc3465876f..9e522b7a3f 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -1,10 +1,12 @@ import pgPkg from "pg" const { Pool } = pgPkg -console.log(`POOLED_DATABASE_URL: ${process.env.POOLED_DATABASE_URL}`) +const connectionString = + process.env.POOLED_DATABASE_URL || + `postgresql://postgres:password@localhost:54321/electric` const pool = new Pool({ - connectionString: process.env.POOLED_DATABASE_URL, + connectionString, }) export { pool } diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index 1848f8b5ff..f9fad1ae9a 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef, useState } from "react" import * as Y from "yjs" import { yCollab, yUndoManagerKeymap } from "y-codemirror.next" import { ElectricProvider } from "./y-electric" -import { Awareness, applyAwarenessUpdate } from "y-protocols/awareness" +import { Awareness } from "y-protocols/awareness" import { EditorState } from "@codemirror/state" import { EditorView, basicSetup } from "codemirror" @@ -13,10 +13,6 @@ import { keymap } from "@codemirror/view" import { javascript } from "@codemirror/lang-javascript" import * as random from "lib0/random" -import * as decoding from "lib0/decoding" - -import { ShapeData } from "./ydoc-shape" -import { fromBase64 } from "lib0/buffer" const room = `electric-demo` @@ -31,15 +27,10 @@ const usercolors = [ const userColor = usercolors[random.uint32() % usercolors.length] const ydoc = new Y.Doc() -let network: ElectricProvider | null = null - -export default function ElectricEditor({ - docShape, - awarenessShape, -}: { - docShape?: ShapeData - awarenessShape?: ShapeData -}) { +let network: ElectricProvider | undefined +let awareness: Awareness | undefined + +export default function ElectricEditor() { const editor = useRef(null) const [connect, setConnect] = useState(`connected`) @@ -59,36 +50,26 @@ export default function ElectricEditor({ return } - if (typeof window !== `undefined` && network === null) { - // initDoc(ydoc, docShape.data) - const awareness = new Awareness(ydoc) - // initAwareness(awareness, awarenessShape.data) - - const opts = { - connect: true, - awareness, - // resume: { - // operations: docShape.resume, - // awareness: awarenessShape.resume, - // }, - } - + if (typeof window !== `undefined` && !network) { + awareness = new Awareness(ydoc) network = new ElectricProvider( - `${process.env.ELECTRIC_URL || `http://localhost:3000`}`, + new URL(`/shape-proxy`, window?.location.origin).href, room, ydoc, - opts + { + connect: true, + awareness, + } ) + awareness?.setLocalStateField(`user`, { + name: userColor.color, + color: userColor.color, + colorLight: userColor.light, + }) } const ytext = ydoc.getText(room) - network?.awareness.setLocalStateField(`user`, { - name: userColor.color, - color: userColor.color, - colorLight: userColor.light, - }) - const state = EditorState.create({ doc: ytext.toString(), extensions: [ @@ -96,7 +77,7 @@ export default function ElectricEditor({ basicSetup, javascript(), EditorView.lineWrapping, - yCollab(ytext, network?.awareness), + yCollab(ytext, awareness), ], }) @@ -114,27 +95,10 @@ export default function ElectricEditor({

This is a demo of Yjs shared - editor synching with {` `} + editor syncing with {` `} Electric.

) } - -const initDoc = (ydoc: Y.Doc, data: string) => { - const decoder = decoding.createDecoder(fromBase64(data)) - decoding.readVarUint(decoder) - Y.applyUpdate(ydoc, decoding.readVarUint8Array(decoder), `server`) -} - -const initAwareness = (awareness: Awareness, data: string) => { - for (const client of JSON.parse(data)) { - const decoder = decoding.createDecoder(fromBase64(client)) - applyAwarenessUpdate( - awareness, - decoding.readVarUint8Array(decoder), - `server` - ) - } -} diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs-provider/app/page.tsx index 099d7cfda2..b31810d8f9 100644 --- a/examples/yjs-provider/app/page.tsx +++ b/examples/yjs-provider/app/page.tsx @@ -2,13 +2,7 @@ import React from "react" import ElectricEditor from "./electric-editor" -import { getAwarenessData, getDocData } from "./ydoc-shape" -const Page = async () => ( - -) +const Page = async () => export default Page diff --git a/examples/yjs-provider/app/reduce-stream.ts b/examples/yjs-provider/app/reduce-stream.ts deleted file mode 100644 index 7ef6a3a64f..0000000000 --- a/examples/yjs-provider/app/reduce-stream.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { - ChangeMessage, - GetExtensions, - isControlMessage, - Message, - Publisher, - Row, - ShapeStream, - ShapeStreamInterface, - ShapeStreamOptions, -} from "@electric-sql/client" - -// Reduce all changes for a shape into a single value. -// Batches all changes until a control message is received. -export type ReduceFunction, Y> = ( - acc: Y, - message: ChangeMessage -) => Y - -export class ReduceStream, Y> - extends Publisher<{ acc: Y }> - implements ShapeStreamInterface<{ acc: Y }> -{ - readonly #stream: ShapeStream - options: ShapeStreamOptions> - - readonly #callback: ReduceFunction - #accMessage: ChangeMessage<{ acc: Y }> - - constructor(stream: ShapeStream, callback: ReduceFunction, init: Y) { - super() - this.#stream = stream - this.options = stream.options - this.#callback = callback - this.#accMessage = getInitAccMessage(this.options.table!, init) - - stream.subscribe((messages) => { - messages.map((message: Message) => { - const messages = [] - if (isControlMessage(message)) { - if (message.headers.control === `up-to-date`) { - messages.push(this.#accMessage) - } - messages.push(message) - this.publish(messages) - } else { - const current = this.#accMessage.value.acc - const next = this.#callback(current, message) - this.#accMessage = { - ...message, - key: this.options.table!, - value: { acc: next }, - } - } - }) - }) - } - - get shapeHandle(): string { - return this.#stream.shapeHandle - } - get isUpToDate(): boolean { - return this.#stream.isUpToDate - } - get error(): unknown { - return this.#stream.error - } - start(): Promise { - return this.#stream.start() - } - lastSyncedAt(): number | undefined { - return this.#stream.lastSyncedAt() - } - lastSynced(): number { - return this.#stream.lastSynced() - } - isConnected(): boolean { - return this.#stream.isConnected() - } - isLoading(): boolean { - return this.#stream.isLoading() - } -} - -function getInitAccMessage( - key: string, - value: Y -): ChangeMessage<{ acc: Y }> { - return { - headers: { operation: `insert` }, - offset: `-1`, - key: key, - value: { acc: value }, - } -} diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts index 0073117db9..dd74d5142d 100644 --- a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts +++ b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts @@ -1,7 +1,4 @@ -export async function GET( - request: Request, - { params }: { params: { table: string } } -) { +export async function GET(request: Request) { const url = new URL(request.url) const originUrl = new URL( process.env.ELECTRIC_URL @@ -22,6 +19,8 @@ export async function GET( originUrl.searchParams.set(`token`, process.env.ELECTRIC_TOKEN) } + console.log(originUrl.toString()) + const newRequest = new Request(originUrl.toString(), { method: `GET`, headers, @@ -32,7 +31,7 @@ export async function GET( // them to avoid content decoding errors in the browser. // // Similar-ish problem to https://github.com/wintercg/fetch/issues/23 - let resp = await fetch(originUrl.toString()) + let resp = await fetch(newRequest) if (resp.headers.get(`content-encoding`)) { const headers = new Headers(resp.headers) headers.delete(`content-encoding`) diff --git a/examples/yjs-provider/app/utils.js b/examples/yjs-provider/app/utils.ts similarity index 72% rename from examples/yjs-provider/app/utils.js rename to examples/yjs-provider/app/utils.ts index 049dcc94d2..cc7bce63ed 100644 --- a/examples/yjs-provider/app/utils.js +++ b/examples/yjs-provider/app/utils.ts @@ -1,14 +1,12 @@ import { toBase64 } from "lib0/buffer" import * as decoding from "lib0/decoding" -export const room = `electric-demo` - -const hexStringToUint8Array = (hexString) => { +const hexStringToUint8Array = (hexString: string) => { const cleanHexString = hexString.startsWith(`\\x`) ? hexString.slice(2) : hexString return new Uint8Array( - cleanHexString.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)) + cleanHexString.match(/.{1,2}/g)!.map((byte) => parseInt(byte, 16)) ) } @@ -17,14 +15,14 @@ export const parseToUint8Array = { } export const parseToBase64 = { - bytea: (hexString) => { + bytea: (hexString: string) => { const uint8Array = hexStringToUint8Array(hexString) return toBase64(uint8Array) }, } export const parseToDecoder = { - bytea: (hexString) => { + bytea: (hexString: string) => { const uint8Array = hexStringToUint8Array(hexString) return decoding.createDecoder(uint8Array) }, diff --git a/examples/yjs-provider/app/y-broadcast.js b/examples/yjs-provider/app/y-broadcast.js deleted file mode 100644 index 382bf165a3..0000000000 --- a/examples/yjs-provider/app/y-broadcast.js +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Extracted this from y-websocket - */ - -import * as bc from "lib0/broadcastchannel" -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" -import * as syncProtocol from "y-protocols/sync" -import * as awarenessProtocol from "y-protocols/awareness" -import { Observable } from "lib0/observable" -import * as env from "lib0/environment" - -export const messageSync = 0 -export const messageQueryAwareness = 3 -export const messageAwareness = 1 - -const messageHandlers = [] - -messageHandlers[messageSync] = (encoder, decoder, provider) => { - encoding.writeVarUint(encoder, messageSync) - syncProtocol.readSyncMessage(decoder, encoder, provider.doc, provider) -} - -messageHandlers[messageQueryAwareness] = (encoder, _decoder, provider) => { - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate( - provider.awareness, - Array.from(provider.awareness.getStates().keys()) - ) - ) -} - -messageHandlers[messageAwareness] = (_encoder, decoder, provider) => { - awarenessProtocol.applyAwarenessUpdate( - provider.awareness, - decoding.readVarUint8Array(decoder), - provider - ) -} - -const readMessage = (provider, buf) => { - const decoder = decoding.createDecoder(buf) - const encoder = encoding.createEncoder() - const messageType = decoding.readVarUint(decoder) - const messageHandler = provider.messageHandlers[messageType] - if (/** @type {any} */ (messageHandler)) { - messageHandler(encoder, decoder, provider, false, messageType) - } else { - console.error(`Unable to compute message`) - } - return encoder -} - -const broadcastMessage = (provider, buf) => { - if (provider.bcconnected) { - bc.publish(provider.bcChannel, buf, provider) - } -} - -export class BroadcastProvider extends Observable { - constructor( - roomname, - doc, - { connect = false, awareness = new awarenessProtocol.Awareness(doc) } = {} - ) { - super() - - this.bcChannel = roomname - this.awareness = awareness - this.roomname = roomname - this.doc = doc - this.bcconnected = false - this.messageHandlers = messageHandlers.slice() - - this._bcSubscriber = (data, origin) => { - if (origin !== this) { - const encoder = readMessage(this, new Uint8Array(data), false) - if (encoding.length(encoder) > 1) { - bc.publish(this.bcChannel, encoding.toUint8Array(encoder), this) - } - } - } - - this._updateHandler = (update, origin) => { - if (origin !== this) { - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.writeUpdate(encoder, update) - broadcastMessage(this, encoding.toUint8Array(encoder)) - } - } - this.doc.on(`update`, this._updateHandler) - - this._awarenessUpdateHandler = ({ added, updated, removed }, _origin) => { - const changedClients = added.concat(updated).concat(removed) - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients) - ) - broadcastMessage(this, encoding.toUint8Array(encoder)) - } - this._exitHandler = () => { - awarenessProtocol.removeAwarenessStates( - this.awareness, - [doc.clientID], - `app closed` - ) - } - if (env.isNode && typeof process !== `undefined`) { - process.on(`exit`, this._exitHandler) - } - - awareness.on(`update`, this._awarenessUpdateHandler) - - if (connect) { - this.connect() - } - } - - destroy() { - this.disconnect() - if (env.isNode && typeof process !== `undefined`) { - process.off(`exit`, this._exitHandler) - } - this.awareness.off(`update`, this._awarenessUpdateHandler) - this.doc.off(`update`, this._updateHandler) - super.destroy() - } - - connectBc() { - if (!this.bcconnected) { - bc.subscribe(this.bcChannel, this._bcSubscriber) - this.bcconnected = true - } - // send sync step1 to bc - // write sync step 1 - const encoderSync = encoding.createEncoder() - encoding.writeVarUint(encoderSync, messageSync) - syncProtocol.writeSyncStep1(encoderSync, this.doc) - bc.publish(this.bcChannel, encoding.toUint8Array(encoderSync), this) - // broadcast local state - const encoderState = encoding.createEncoder() - encoding.writeVarUint(encoderState, messageSync) - syncProtocol.writeSyncStep2(encoderState, this.doc) - bc.publish(this.bcChannel, encoding.toUint8Array(encoderState), this) - // write queryAwareness - const encoderAwarenessQuery = encoding.createEncoder() - encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness) - bc.publish( - this.bcChannel, - encoding.toUint8Array(encoderAwarenessQuery), - this - ) - // broadcast local awareness state - const encoderAwarenessState = encoding.createEncoder() - encoding.writeVarUint(encoderAwarenessState, messageAwareness) - encoding.writeVarUint8Array( - encoderAwarenessState, - awarenessProtocol.encodeAwarenessUpdate(this.awareness, [ - this.doc.clientID, - ]) - ) - bc.publish( - this.bcChannel, - encoding.toUint8Array(encoderAwarenessState), - this - ) - } - - disconnectBc() { - // broadcast message with local awareness state set to null (indicating disconnect) - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageAwareness) - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate( - this.awareness, - [this.doc.clientID], - new Map() - ) - ) - broadcastMessage(this, encoding.toUint8Array(encoder)) - if (this.bcconnected) { - bc.unsubscribe(this.bcChannel, this._bcSubscriber) - this.bcconnected = false - } - } - - disconnect() { - this.disconnectBc() - } - - connect() { - this.connectBc() - } -} diff --git a/examples/yjs-provider/app/y-electric.js b/examples/yjs-provider/app/y-electric.js deleted file mode 100644 index 7fc09192d7..0000000000 --- a/examples/yjs-provider/app/y-electric.js +++ /dev/null @@ -1,327 +0,0 @@ -/** - * @module provider/electric - */ - -import * as time from "lib0/time" -import { toBase64 } from "lib0/buffer" -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" -import * as syncProtocol from "y-protocols/sync" -import * as awarenessProtocol from "y-protocols/awareness" -import { Observable } from "lib0/observable" -import * as env from "lib0/environment" -import * as Y from "yjs" - -export const messageSync = 0 -export const messageAwareness = 1 - -import { isChangeMessage, ShapeStream } from "@electric-sql/client" -import { parseToDecoder as parser } from "./utils" - -/** - * @param {ElectricProvider} provider - */ -const setupShapeStream = (provider) => { - if (provider.shouldConnect && provider.operationsStream === null) { - provider.connecting = true - provider.connected = false - provider.synced = false - - provider.operationsStream = new ShapeStream({ - url: provider.operationsUrl, - table: `ydoc_operations`, - where: `room = '${provider.roomname}'`, - parser, - ...provider.resume.operations, - }) - - provider.awarenessStream = new ShapeStream({ - url: provider.awarenessUrl, - where: `room = '${provider.roomname}'`, - table: `ydoc_awareness`, - parser, - ...provider.resume.awareness, - }) - - const handleSyncMessage = (messages) => { - console.log(messages) - provider.lastMessageReceived = time.getUnixTime() - messages.forEach((message) => { - if (isChangeMessage(message) && message.value.op) { - const decoder = message.value.op - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - provider.doc, - provider - ) - if ( - syncMessageType === syncProtocol.messageYjsSyncStep2 && - !provider.synced - ) { - provider.synced = true - } - } - }) - } - - const handleAwarenessMessage = (messages) => { - provider.lastMessageReceived = time.getUnixTime() - messages.forEach((message) => { - // sometimes buffer is empty - if (isChangeMessage(message) && message.value.op) { - const decoder = message.value.op - awarenessProtocol.applyAwarenessUpdate( - provider.awareness, - decoding.readVarUint8Array(decoder), - provider - ) - } - }) - } - - const unsubscribeSyncHandler = - provider.operationsStream.subscribe(handleSyncMessage) - - const unsubscribeAwarenessHandler = provider.awarenessStream.subscribe( - handleAwarenessMessage - ) - - provider.disconnectHandler = (event) => { - provider.operationsStream = null - provider.awarenessStream = null - provider.connecting = false - if (provider.connected) { - provider.connected = false - - provider.synced = false - - awarenessProtocol.removeAwarenessStates( - provider.awareness, - Array.from(provider.awareness.getStates().keys()).filter( - (client) => client !== provider.doc.clientID - ), - provider - ) - provider.lastSyncedStateVector = Y.encodeStateVector(provider.doc) - provider.emit(`status`, [{ status: `disconnected` }]) - } - - unsubscribeSyncHandler() - unsubscribeAwarenessHandler() - provider.disconnectHandler = null - provider.emit(`connection-close`, [event, provider]) - } - - const handleOperationsFirstSync = () => { - provider.lastMessageReceived = time.getUnixTime() - provider.connecting = false - provider.connected = true - - if (provider.modifiedWhileOffline) { - const pendingUpdates = Y.encodeStateAsUpdate( - provider.doc, - provider.lastSyncedStateVector - ) - const encoderState = encoding.createEncoder() - syncProtocol.writeUpdate(encoderState, pendingUpdates) - - sendOperation(provider, pendingUpdates) - .then(() => clearLastSyncedStateVector(provider)) - .then(() => { - provider.emit(`status`, [{ status: `connected` }]) - }) - } - } - - provider.operationsStream.subscribeOnceToUpToDate( - handleOperationsFirstSync.bind(this) - ) - - const handleAwarenessFirstSync = () => { - if (provider.awareness.getLocalState() !== null) { - sendAwareness(provider, [provider.doc.clientID]) - } - } - - provider.awarenessStream.subscribeOnceToUpToDate( - handleAwarenessFirstSync.bind(this) - ) - - provider.emit(`status`, [{ status: `connecting` }]) - } -} - -const saveLastSyncedStateVector = (provider) => { - provider.modifiedWhileOffline = true -} - -const clearLastSyncedStateVector = async (provider) => { - provider.lastSyncedStateVector = null - provider.modifiedWhileOffline = false -} - -const sendOperation = async (provider, update) => { - // ignore requests that have no changes - // there is probably a better way of checking this - if (update.length <= 2) { - return - } - - if (!provider.connected) { - if (!provider.modifiedWhileOffline) { - return saveLastSyncedStateVector(provider, update) - } - return - } - - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, update) - const op = toBase64(encoding.toUint8Array(encoder)) - const room = provider.roomname - - return fetch(`/api/operation`, { - method: `POST`, - body: JSON.stringify({ room, op }), - }) -} - -const sendAwareness = (provider, changedClients) => { - const encoder = encoding.createEncoder() - encoding.writeVarUint8Array( - encoder, - awarenessProtocol.encodeAwarenessUpdate(provider.awareness, changedClients) - ) - const op = toBase64(encoding.toUint8Array(encoder)) - - if (provider.connected) { - const room = provider.roomname - const clientId = `${provider.doc.clientID}` - - fetch(`/api/operation`, { - method: `POST`, - body: JSON.stringify({ clientId, room, op }), - }) - } -} - -export class ElectricProvider extends Observable { - /** - * @param {string} serverUrl - * @param {string} roomname - * @param {Y.Doc} doc - * @param {object} opts - * @param {boolean} [opts.connect] - * @param {awarenessProtocol.Awareness} [opts.awareness] - * @param {Object} [opts.resume] - */ - constructor( - serverUrl, - roomname, - doc, - { connect = false, awareness = null, resume = {} } = {} - ) { - super() - - this.serverUrl = serverUrl - this.roomname = roomname - this.doc = doc - this.awareness = awareness - this.connected = false - this.connecting = false - this._synced = false - - this.lastMessageReceived = 0 - this.shouldConnect = connect - - this.operationsStream = null - this.awarenessStream = null - - this.lastSyncedStateVector = null - this.modifiedWhileOffline = false - this.resume = resume ?? {} - - this.disconnectHandler = null - - this._updateHandler = (update, origin) => { - if (origin !== this) { - sendOperation(this, update) - } - } - this.doc.on(`update`, this._updateHandler) - - /** - * @param {any} changed - * @param {any} origin - */ - this._awarenessUpdateHandler = ({ added, updated, removed }, origin) => { - if (origin === `local`) { - const changedClients = added.concat(updated).concat(removed) - sendAwareness(this, changedClients) - } - } - - this._exitHandler = () => { - awarenessProtocol.removeAwarenessStates( - this.awareness, - [doc.clientID], - `app closed` - ) - } - if (env.isNode && typeof process !== `undefined`) { - process.on(`exit`, this._exitHandler) - } - awareness?.on(`update`, this._awarenessUpdateHandler) - - if (connect) { - this.connect() - } - } - - get operationsUrl() { - return this.serverUrl + `/v1/shape` - } - - get awarenessUrl() { - return this.serverUrl + `/v1/shape/` - } - - /** - * @type {boolean} - */ - get synced() { - return this._synced - } - - set synced(state) { - if (this._synced !== state) { - this._synced = state - this.emit(`synced`, [state]) - this.emit(`sync`, [state]) - } - } - - destroy() { - this.disconnect() - if (env.isNode && typeof process !== `undefined`) { - process.off(`exit`, this._exitHandler) - } - this.awareness?.off(`update`, this._awarenessUpdateHandler) - this.doc.off(`update`, this._updateHandler) - super.destroy() - } - - disconnect() { - this.shouldConnect = false - this.disconnectHandler() - } - - connect() { - this.shouldConnect = true - if (!this.connected && this.operationsStream === null) { - setupShapeStream(this) - } - } -} diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts new file mode 100644 index 0000000000..41ab620e37 --- /dev/null +++ b/examples/yjs-provider/app/y-electric.ts @@ -0,0 +1,336 @@ +import { toBase64 } from "lib0/buffer" +import * as encoding from "lib0/encoding" +import * as decoding from "lib0/decoding" +import * as syncProtocol from "y-protocols/sync" +import * as awarenessProtocol from "y-protocols/awareness" +import { ObservableV2 } from "lib0/observable" +import * as env from "lib0/environment" +import * as Y from "yjs" +import { + FetchError, + isChangeMessage, + Message, + ShapeStream, +} from "@electric-sql/client" +import { parseToDecoder } from "./utils" + +type OperationMessage = { + op: decoding.Decoder +} + +type AwarenessMessage = { + op: decoding.Decoder + clientId: string + room: string +} + +type ObservableProvider = { + sync: (state: boolean) => void + synced: (state: boolean) => void + status: (status: { + status: `connecting` | `connected` | `disconnected` + }) => void + // eslint-disable-next-line quotes + "connection-close": () => void +} + +const messageSync = 0 + +export class ElectricProvider extends ObservableV2 { + private serverUrl: string + private roomName: string + private doc: Y.Doc + public awareness?: awarenessProtocol.Awareness + + private operationsStream?: ShapeStream + private awarenessStream?: ShapeStream + + private shouldConnect: boolean + private connected: boolean + private _synced: boolean + + private modifiedWhileOffline: boolean + private lastSyncedStateVector?: Uint8Array + + private updateHandler: (update: Uint8Array, origin: unknown) => void + private awarenessUpdateHandler?: ( + changed: { added: number[]; updated: number[]; removed: number[] }, + origin: string + ) => void + private disconnectHandler?: () => void + private exitHandler?: () => void + + constructor( + serverUrl: string, + roomName: string, + doc: Y.Doc, + options: { awareness?: awarenessProtocol.Awareness; connect?: boolean } + ) { + super() + + this.serverUrl = serverUrl + this.roomName = roomName + + this.doc = doc + this.awareness = options.awareness + + this.connected = false + this._synced = false + this.shouldConnect = options.connect ?? false + + this.modifiedWhileOffline = false + + this.updateHandler = (update: Uint8Array, origin: unknown) => { + if (origin !== this) { + this.sendOperation(update) + } + } + this.doc.on(`update`, this.updateHandler) + + if (this.awareness) { + this.awarenessUpdateHandler = ({ added, updated, removed }, origin) => { + if (origin === `local`) { + const changedClients = added.concat(updated).concat(removed) + this.sendAwareness(changedClients) + } + } + this.awareness.on(`update`, this.awarenessUpdateHandler) + } + + if (env.isNode && typeof process !== `undefined`) { + this.exitHandler = () => { + if (this.awareness) { + awarenessProtocol.removeAwarenessStates( + this.awareness, + [doc.clientID], + `app closed` + ) + } + process.on(`exit`, () => this.exitHandler!()) + } + } + + if (options.connect) { + this.connect() + } + } + + private get operationsUrl() { + return this.serverUrl + `/v1/shape` + } + + private get awarenessUrl() { + return this.serverUrl + `/v1/shape/` + } + + get synced() { + return this._synced + } + + set synced(state) { + if (this._synced !== state) { + this._synced = state + this.emit(`synced`, [state]) + this.emit(`sync`, [state]) + } + } + + destroy() { + this.disconnect() + this.doc.off(`update`, this.updateHandler) + this.awareness?.off(`update`, this.awarenessUpdateHandler!) + if (env.isNode && typeof process !== `undefined`) { + process.off(`exit`, this.exitHandler!) + } + super.destroy() + } + + disconnect() { + this.shouldConnect = false + if (this.disconnectHandler) { + this.disconnectHandler() + } + } + + connect() { + this.shouldConnect = true + if (!this.connected && !this.operationsStream) { + this.setupShapeStream() + } + } + + private sendOperation(update: Uint8Array) { + if (update.length <= 2) { + throw Error( + `Shouldn't be trying to send operations without pending operations` + ) + } + + if (!this.connected) { + this.modifiedWhileOffline = true + return Promise.resolve() + } + + const encoder = encoding.createEncoder() + syncProtocol.writeUpdate(encoder, update) + const op = toBase64(encoding.toUint8Array(encoder)) + const room = this.roomName + + return fetch(`/api/operation`, { + method: `POST`, + body: JSON.stringify({ room, op }), + }) + } + + private sendAwareness(changedClients: number[]) { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array( + encoder, + awarenessProtocol.encodeAwarenessUpdate(this.awareness!, changedClients) + ) + const op = toBase64(encoding.toUint8Array(encoder)) + + if (this.connected) { + const room = this.roomName + const clientId = `${this.doc.clientID}` + + return fetch(`/api/operation`, { + method: `POST`, + body: JSON.stringify({ clientId, room, op }), + }) + } + } + + private setupShapeStream() { + if (this.shouldConnect && !this.operationsStream) { + this.connected = false + this.synced = false + + this.operationsStream = new ShapeStream({ + url: this.operationsUrl, + table: `ydoc_operations`, + where: `room = '${this.roomName}'`, + parser: parseToDecoder, + subscribe: true, + }) + + this.awarenessStream = new ShapeStream({ + url: this.awarenessUrl, + where: `room = '${this.roomName}'`, + table: `ydoc_awareness`, + parser: parseToDecoder, + }) + + const errorHandler = (e: FetchError | Error) => { + throw e + } + + const handleSyncMessage = (messages: Message[]) => { + messages.forEach((message) => { + if (isChangeMessage(message) && message.value.op) { + const decoder = message.value.op + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage( + decoder, + encoder, + this.doc, + this + ) + if ( + syncMessageType === syncProtocol.messageYjsSyncStep2 && + !this.synced + ) { + this.synced = true + } + } + }) + } + + const unsubscribeSyncHandler = this.operationsStream.subscribe( + handleSyncMessage, + errorHandler + ) + + const handleAwarenessMessage = ( + messages: Message[] + ) => { + messages.forEach((message) => { + if (isChangeMessage(message) && message.value.op) { + const decoder = message.value.op + awarenessProtocol.applyAwarenessUpdate( + this.awareness!, + decoding.readVarUint8Array(decoder), + this + ) + } + }) + } + + const unsubscribeAwarenessHandler = this.awarenessStream.subscribe( + handleAwarenessMessage, + errorHandler + ) + + this.disconnectHandler = () => { + this.operationsStream = undefined + this.awarenessStream = undefined + + if (this.connected) { + this.connected = false + + this.synced = false + + if (this.awareness) { + awarenessProtocol.removeAwarenessStates( + this.awareness, + Array.from(this.awareness.getStates().keys()).filter( + (client) => client !== this.doc.clientID + ), + this + ) + } + this.lastSyncedStateVector = Y.encodeStateVector(this.doc) + this.emit(`status`, [{ status: `disconnected` }]) + } + + unsubscribeSyncHandler() + unsubscribeAwarenessHandler() + this.disconnectHandler = undefined + this.emit(`connection-close`, []) + } + + // send pending changes + const unsubscribeOps = this.operationsStream!.subscribe(() => { + this.connected = true + + if (this.modifiedWhileOffline) { + const pendingUpdates = Y.encodeStateAsUpdate( + this.doc, + this.lastSyncedStateVector + ) + const encoderState = encoding.createEncoder() + syncProtocol.writeUpdate(encoderState, pendingUpdates) + + this.sendOperation(pendingUpdates).then(() => { + this.lastSyncedStateVector = undefined + this.modifiedWhileOffline = false + this.emit(`status`, [{ status: `connected` }]) + }) + } + unsubscribeOps() + }) + + if (this.awarenessStream) { + const unsubscribeAwareness = this.awarenessStream.subscribe(() => { + if (this.awareness!.getLocalState() !== null) { + this.sendAwareness([this.doc.clientID]) + } + unsubscribeAwareness() + }) + } + + this.emit(`status`, [{ status: `connecting` }]) + } + } +} diff --git a/examples/yjs-provider/app/ydoc-shape.ts b/examples/yjs-provider/app/ydoc-shape.ts deleted file mode 100644 index e858bbd3ed..0000000000 --- a/examples/yjs-provider/app/ydoc-shape.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Shape, ShapeStream, ShapeStreamOptions } from "@electric-sql/client" - -import { parseToUint8Array as parser, parseToBase64, room } from "./utils" - -import * as Y from "yjs" -import * as syncProtocol from "y-protocols/sync" - -import { toBase64 } from "lib0/buffer" -import * as encoding from "lib0/encoding" -import * as decoding from "lib0/decoding" -import { ReduceFunction, ReduceStream } from "./reduce-stream" - -export type ShapeData = { - data: string - resume: { - offset: string - shapeHandle: string - } -} -type YOp = { op: Uint8Array } -type YDoc = { acc: Y.Doc } - -function getYDocShape(stream: ShapeStream): Shape { - const reduceChangesToDoc: ReduceFunction = (acc, message) => { - syncProtocol.readSyncMessage( - decoding.createDecoder(message.value.op), - encoding.createEncoder(), - acc, - `server` - ) - return acc - } - const reduceStream = new ReduceStream(stream, reduceChangesToDoc, new Y.Doc()) - return new Shape(reduceStream) -} - -function getDocAsBase64(ydoc: Y.Doc) { - const encoder = encoding.createEncoder() - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(ydoc)) - return toBase64(encoding.toUint8Array(encoder)) -} -const url = process.env.ELECTRIC_URL - ? `${process.env.ELECTRIC_URL}/v1/shape` - : `http://localhost:3000/v1/shape/` - - -const doc: ShapeStreamOptions = { - url, - table: `ydoc_operations`, - where: `room = '${room}'`, - parser, -} -const docStream = new ShapeStream(doc) -const docShape = getYDocShape(docStream) - -export const getDocData = async () => { - const doc = (await docShape.value).get(`ydoc_operations`)!.acc - return { - data: getDocAsBase64(doc), - resume: { - offset: docStream.lastOffset, - shapeHandle: docStream.shapeHandle, - }, - } -} - -const awareness: ShapeStreamOptions = { - url, - table: `ydoc_awareness`, - where: `room = '${room}'`, - parser: parseToBase64, -} -const awarenessStream = new ShapeStream(awareness) -const awarenessShape = new Shape(awarenessStream) - -export const getAwarenessData = async () => { - const clients = await awarenessShape.value - const data = [] - for (const client of clients.values()) { - data.push(client.op) - } - return { - data: JSON.stringify(data), - resume: { - offset: awarenessStream.lastOffset, - shapeHandle: awarenessStream.shapeHandle, - }, - } -} diff --git a/examples/yjs-provider/next.config.js b/examples/yjs-provider/next.config.js deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/yjs-provider/next.config.mjs b/examples/yjs-provider/next.config.mjs new file mode 100644 index 0000000000..a25265cd27 --- /dev/null +++ b/examples/yjs-provider/next.config.mjs @@ -0,0 +1,12 @@ +/** + * @type {import('next').NextConfig} + */ +const nextConfig = { + // logging: { + // fetches: { + // fullUrl: true, + // hmrRefreshes: true, + // }, + // }, + } + export default nextConfig \ No newline at end of file From 718f9a89071dbf1918e25d79a5f67920aa86ccae Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 3 Dec 2024 00:02:37 +0000 Subject: [PATCH 30/47] removed unused logo --- examples/yjs-provider/public/logo.svg | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 examples/yjs-provider/public/logo.svg diff --git a/examples/yjs-provider/public/logo.svg b/examples/yjs-provider/public/logo.svg deleted file mode 100644 index f5ec440a63..0000000000 --- a/examples/yjs-provider/public/logo.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - From 9c72588040459387614fda13749bc4655158b1c6 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 3 Dec 2024 23:15:37 +0000 Subject: [PATCH 31/47] sst config with lambda/neon and standalone servers --- .dockerignore | 3 + examples/yjs-provider/Dockerfile | 31 ++++++++++ .../yjs-provider/app/api/operation/route.ts | 35 ++++++++--- examples/yjs-provider/app/db.ts | 3 +- .../app/shape-proxy/[...table]/route.ts | 6 +- examples/yjs-provider/app/y-electric.ts | 2 +- examples/yjs-provider/next.config.mjs | 11 +--- examples/yjs-provider/package.json | 5 +- examples/yjs-provider/sst-env.d.ts | 12 ++++ examples/yjs-provider/sst.config.ts | 61 +++++++++++++------ 10 files changed, 130 insertions(+), 39 deletions(-) create mode 100644 .dockerignore create mode 100644 examples/yjs-provider/Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..d98fe17111 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ + +# sst +.sst \ No newline at end of file diff --git a/examples/yjs-provider/Dockerfile b/examples/yjs-provider/Dockerfile new file mode 100644 index 0000000000..a1f6b8a52e --- /dev/null +++ b/examples/yjs-provider/Dockerfile @@ -0,0 +1,31 @@ +FROM node:lts-alpine AS base + +# Stage 1: Install dependencies +FROM base AS deps +WORKDIR /app + +RUN npm install -g pnpm + +COPY pnpm-*.yaml ./ +COPY package.json ./ +COPY tsconfig.build.json ./ +COPY packages/typescript-client packages/typescript-client/ +COPY packages/react-hooks packages/react-hooks/ +COPY examples/yjs-provider/ examples/yjs-provider/ + +# Install dependencies +RUN pnpm install --frozen-lockfile +RUN pnpm run -r build + + +# Need to make production image more clean +FROM node:lts-alpine AS prod +WORKDIR /app + +ENV NODE_ENV=production +COPY --from=deps /app/ ./ + +WORKDIR /app/examples/yjs-provider/ + +EXPOSE 3000 +CMD ["npm", "run", "start"] \ No newline at end of file diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index b4b48e902c..071f05d3fc 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,5 +1,10 @@ import { pool } from "../../db" import { NextResponse } from "next/server" +import { neon } from "@neondatabase/serverless" + +const sql = process.env.POOLED_DATABASE_URL + ? neon(process.env.POOLED_DATABASE_URL) + : undefined export async function POST(request: Request) { try { @@ -19,10 +24,16 @@ export async function POST(request: Request) { } async function saveOperation(room: string, op: string) { - pool.query( - `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, - [room, op] - ) + if (sql) { + await sql` + INSERT INTO ydoc_operations (room, op) VALUES (${room}, decode(${op}, 'base64')) + ` + } else { + await pool!.query( + `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, + [room, op] + ) + } } async function saveAwarenessOperation( @@ -30,12 +41,20 @@ async function saveAwarenessOperation( op: string, clientId: string ) { - await pool.query( - `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) + if (sql) { + await sql` + INSERT INTO ydoc_awareness (room, clientId, op) VALUES (${room}, ${clientId}, decode(${op}, 'base64')) + ON CONFLICT (clientId, room) + DO UPDATE SET op = decode(${op}, 'base64') + ` + } else { + await pool!.query( + `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) DO UPDATE SET op = decode($3, 'base64')`, - [room, clientId, op] - ) + [room, clientId, op] + ) + } } async function getRequestParams( diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index 9e522b7a3f..bb1f6cfdc7 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -2,7 +2,8 @@ import pgPkg from "pg" const { Pool } = pgPkg const connectionString = - process.env.POOLED_DATABASE_URL || + // process.env.POOLED_DATABASE_URL || disabled so that we use neon client for pooled connection + process.env.DATABASE_URL || `postgresql://postgres:password@localhost:54321/electric` const pool = new Pool({ diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts index dd74d5142d..4df989a263 100644 --- a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts +++ b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts @@ -2,8 +2,8 @@ export async function GET(request: Request) { const url = new URL(request.url) const originUrl = new URL( process.env.ELECTRIC_URL - ? `${process.env.ELECTRIC_URL}/v1/shape` - : `http://localhost:3000/v1/shape` + ? `${process.env.ELECTRIC_URL}/v1/shape/` + : `http://localhost:3000/v1/shape/` ) url.searchParams.forEach((value, key) => { @@ -19,6 +19,8 @@ export async function GET(request: Request) { originUrl.searchParams.set(`token`, process.env.ELECTRIC_TOKEN) } + console.log("database_id", process.env.DATABASE_ID) + console.log("electric_token", process.env.ELECTRIC_TOKEN) console.log(originUrl.toString()) const newRequest = new Request(originUrl.toString(), { diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 41ab620e37..4b5d312667 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -120,7 +120,7 @@ export class ElectricProvider extends ObservableV2 { } private get awarenessUrl() { - return this.serverUrl + `/v1/shape/` + return this.serverUrl + `/v1/shape` } get synced() { diff --git a/examples/yjs-provider/next.config.mjs b/examples/yjs-provider/next.config.mjs index a25265cd27..e559ceb6ca 100644 --- a/examples/yjs-provider/next.config.mjs +++ b/examples/yjs-provider/next.config.mjs @@ -2,11 +2,6 @@ * @type {import('next').NextConfig} */ const nextConfig = { - // logging: { - // fetches: { - // fullUrl: true, - // hmrRefreshes: true, - // }, - // }, - } - export default nextConfig \ No newline at end of file +} + +export default nextConfig \ No newline at end of file diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 5c32b2ccaa..30965d7c05 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -22,10 +22,11 @@ "@codemirror/view": "^6.32.0", "@electric-sql/client": "workspace:*", "@electric-sql/react": "workspace:*", + "@neondatabase/serverless": "^0.10.4", "codemirror": "^6.0.1", "lib0": "^0.2.96", - "next": "^14.2.5", - "pg": "^8.12.0", + "next": "^14.2.9", + "pg": "^8.13.1", "react": "^18.3.1", "react-dom": "^18.3.1", "sst": "^3.3.35", diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs-provider/sst-env.d.ts index e973cf25d9..bd7b4723ec 100644 --- a/examples/yjs-provider/sst-env.d.ts +++ b/examples/yjs-provider/sst-env.d.ts @@ -6,5 +6,17 @@ import "sst" export {} declare module "sst" { export interface Resource { + "yjs": { + "type": "sst.aws.Nextjs" + "url": string + } + "yjs-service-vbalegas": { + "service": string + "type": "sst.aws.Service" + "url": string + } + "yjs-vpc-vbalegas": { + "type": "sst.aws.Vpc" + } } } diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index 9690e02e7b..5c9ad420ee 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -33,7 +33,6 @@ export default $config({ }) const databaseUri = getNeonDbUri(project, db) - const pooledDatabaseUri = getNeonDbUri(project, db, true) try { databaseUri.apply(applyMigrations) @@ -41,13 +40,19 @@ export default $config({ addDatabaseToElectric(uri) ) - const website = deployApp(electricInfo, databaseUri, pooledDatabaseUri) + const website = deployServerlessApp( + electricInfo, + databaseUri, + databaseUri + ) + + const server = deployApp(electricInfo, databaseUri) return { databaseUri, - pooledUri: pooledDatabaseUri, database_id: electricInfo.id, electric_token: electricInfo.token, website: website.url, + server: server.url, } } catch (e) { console.error(`Failed to deploy yjs example stack`, e) @@ -65,6 +70,39 @@ function applyMigrations(uri: string) { } function deployApp( + { id, token }: $util.Output<{ id: string; token: string }>, + uri: $util.Output +) { + const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`) + const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc }) + + const service = cluster.addService(`yjs-service-${$app.stage}`, { + loadBalancer: { + ports: [{ listen: "443/https", forward: "3000/http" }], + domain: { + name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + dns: sst.cloudflare.dns(), + }, + }, + environment: { + ELECTRIC_URL: process.env.ELECTRIC_API!, + DATABASE_URL: uri, + DATABASE_ID: id, + ELECTRIC_TOKEN: token, + }, + image: { + context: "../..", + dockerfile: "Dockerfile", + }, + dev: { + command: "npm run dev", + }, + }) + + return service +} + +function deployServerlessApp( electricInfo: $util.Output<{ id: string; token: string }>, uri: $util.Output, pooledUri: $util.Output @@ -75,7 +113,7 @@ function deployApp( ELECTRIC_TOKEN: electricInfo.token, DATABASE_ID: electricInfo.id, DATABASE_URL: uri, - POOLED_DATABASE_URL: pooledUri, + // POOLED_DATABASE_URL: TODO }, domain: { name: `yjs${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, @@ -86,8 +124,7 @@ function deployApp( function getNeonDbUri( project: $util.Output, - db: neon.Database, - pool: boolean = false + db: neon.Database ) { const passwordOutput = neon.getBranchRolePasswordOutput({ projectId: project.id, @@ -95,17 +132,7 @@ function getNeonDbUri( roleName: db.ownerName, }) - return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require`.apply( - (v) => { - if (pool) { - return v.replace( - process.env.NEON_PROJECT_ID!, - process.env.NEON_PROJECT_ID + `-pooler` - ) - } - return v - } - ) + return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require` } async function addDatabaseToElectric( From 6d943f90ae879790a4c942ee4d06e0941f4a99a0 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 3 Dec 2024 23:50:55 +0000 Subject: [PATCH 32/47] emit sync event when up-to-date --- .../yjs-provider/app/shape-proxy/[...table]/route.ts | 4 ---- examples/yjs-provider/app/y-electric.ts | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts index 4df989a263..d389389091 100644 --- a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts +++ b/examples/yjs-provider/app/shape-proxy/[...table]/route.ts @@ -19,10 +19,6 @@ export async function GET(request: Request) { originUrl.searchParams.set(`token`, process.env.ELECTRIC_TOKEN) } - console.log("database_id", process.env.DATABASE_ID) - console.log("electric_token", process.env.ELECTRIC_TOKEN) - console.log(originUrl.toString()) - const newRequest = new Request(originUrl.toString(), { method: `GET`, headers, diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 4b5d312667..057af90578 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -9,6 +9,7 @@ import * as Y from "yjs" import { FetchError, isChangeMessage, + isControlMessage, Message, ShapeStream, } from "@electric-sql/client" @@ -237,12 +238,11 @@ export class ElectricProvider extends ObservableV2 { this.doc, this ) - if ( - syncMessageType === syncProtocol.messageYjsSyncStep2 && - !this.synced - ) { - this.synced = true - } + } else if ( + isControlMessage(message) && + message.headers.control === "up-to-date" + ) { + this.synced = true } }) } From f8a3e15a5230a4fbcbdff290bdc071a4bb96432a Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 4 Dec 2024 23:47:35 +0000 Subject: [PATCH 33/47] Missing awaits --- examples/yjs-provider/app/api/operation/route.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 071f05d3fc..bc15cf2ae6 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -11,9 +11,9 @@ export async function POST(request: Request) { const { room, op, clientId } = await getRequestParams(request) if (!clientId) { - saveOperation(room, op) + await saveOperation(room, op) } else { - saveAwarenessOperation(room, op, clientId) + await saveAwarenessOperation(room, op, clientId) } return NextResponse.json({}) From 1a5da59e213ca82a075c6cdda6df385c3fe179e1 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 5 Dec 2024 08:03:42 +0000 Subject: [PATCH 34/47] SST looking good --- .../yjs-provider/app/api/operation/route.ts | 52 ++++---- examples/yjs-provider/app/db.ts | 2 +- examples/yjs-provider/sst.config.ts | 118 ++++++++++-------- 3 files changed, 91 insertions(+), 81 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index bc15cf2ae6..cb45a58cf0 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,60 +1,50 @@ +import { Pool } from "pg" import { pool } from "../../db" import { NextResponse } from "next/server" -import { neon } from "@neondatabase/serverless" - -const sql = process.env.POOLED_DATABASE_URL - ? neon(process.env.POOLED_DATABASE_URL) - : undefined export async function POST(request: Request) { + let connected = false try { const { room, op, clientId } = await getRequestParams(request) + await pool.connect() + connected = true if (!clientId) { - await saveOperation(room, op) + await saveOperation(room, op, pool) } else { - await saveAwarenessOperation(room, op, clientId) + await saveAwarenessOperation(room, op, clientId, pool) } return NextResponse.json({}) } catch (e) { const resp = e instanceof Error ? e.message : e return NextResponse.json(resp, { status: 400 }) + } finally { + if (connected) { + pool.end() + } } } -async function saveOperation(room: string, op: string) { - if (sql) { - await sql` - INSERT INTO ydoc_operations (room, op) VALUES (${room}, decode(${op}, 'base64')) - ` - } else { - await pool!.query( - `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, - [room, op] - ) - } +async function saveOperation(room: string, op: string, connection: Pool) { + await connection.query( + `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, + [room, op] + ) } async function saveAwarenessOperation( room: string, op: string, - clientId: string + clientId: string, + connection: Pool ) { - if (sql) { - await sql` - INSERT INTO ydoc_awareness (room, clientId, op) VALUES (${room}, ${clientId}, decode(${op}, 'base64')) - ON CONFLICT (clientId, room) - DO UPDATE SET op = decode(${op}, 'base64') - ` - } else { - await pool!.query( - `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) + await connection.query( + `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) DO UPDATE SET op = decode($3, 'base64')`, - [room, clientId, op] - ) - } + [room, clientId, op] + ) } async function getRequestParams( diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts index bb1f6cfdc7..f89e98b35e 100644 --- a/examples/yjs-provider/app/db.ts +++ b/examples/yjs-provider/app/db.ts @@ -2,7 +2,7 @@ import pgPkg from "pg" const { Pool } = pgPkg const connectionString = - // process.env.POOLED_DATABASE_URL || disabled so that we use neon client for pooled connection + process.env.POOLED_DATABASE_URL || process.env.DATABASE_URL || `postgresql://postgres:password@localhost:54321/electric` diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index 5c9ad420ee..d5af202da6 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -32,7 +32,21 @@ export default $config({ $app.stage === `Production` ? `yjs-production` : `yjs-${$app.stage}`, }) - const databaseUri = getNeonDbUri(project, db) + // const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`, { bastion: true }) + + // const rds = new sst.aws.Postgres(`yjs-${$app.stage}Database`, { + // vpc, + // // proxy: true, + // transform: { + // instance: { + // publiclyAccessible: true, + // }, + // }, + // }) + // const databaseUri = getRdsDbUri(rds) + + const databaseUri = getNeonDbUri(project, db, false) + const databasePooledUri = getNeonDbUri(project, db, true) try { databaseUri.apply(applyMigrations) @@ -43,20 +57,10 @@ export default $config({ const website = deployServerlessApp( electricInfo, databaseUri, - databaseUri + databasePooledUri ) - - const server = deployApp(electricInfo, databaseUri) - return { - databaseUri, - database_id: electricInfo.id, - electric_token: electricInfo.token, - website: website.url, - server: server.url, - } - } catch (e) { - console.error(`Failed to deploy yjs example stack`, e) - } + return { url: website.url, databaseUri, databasePooledUri } + } catch (e) {} }, }) @@ -69,38 +73,38 @@ function applyMigrations(uri: string) { }) } -function deployApp( - { id, token }: $util.Output<{ id: string; token: string }>, - uri: $util.Output -) { - const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`) - const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc }) - - const service = cluster.addService(`yjs-service-${$app.stage}`, { - loadBalancer: { - ports: [{ listen: "443/https", forward: "3000/http" }], - domain: { - name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, - dns: sst.cloudflare.dns(), - }, - }, - environment: { - ELECTRIC_URL: process.env.ELECTRIC_API!, - DATABASE_URL: uri, - DATABASE_ID: id, - ELECTRIC_TOKEN: token, - }, - image: { - context: "../..", - dockerfile: "Dockerfile", - }, - dev: { - command: "npm run dev", - }, - }) - - return service -} +// function deployApp( +// { id, token }: $util.Output<{ id: string; token: string }>, +// uri: $util.Output, +// vpc: sst.aws.Vpc +// ) { +// const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc }) + +// const service = cluster.addService(`yjs-service-${$app.stage}`, { +// loadBalancer: { +// ports: [{ listen: "443/https", forward: "3000/http" }], +// domain: { +// name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, +// dns: sst.cloudflare.dns(), +// }, +// }, +// environment: { +// ELECTRIC_URL: process.env.ELECTRIC_API!, +// DATABASE_URL: uri, +// DATABASE_ID: id, +// ELECTRIC_TOKEN: token, +// }, +// image: { +// context: "../..", +// dockerfile: "Dockerfile", +// }, +// dev: { +// command: "npm run dev", +// }, +// }) + +// return service +// } function deployServerlessApp( electricInfo: $util.Output<{ id: string; token: string }>, @@ -113,7 +117,7 @@ function deployServerlessApp( ELECTRIC_TOKEN: electricInfo.token, DATABASE_ID: electricInfo.id, DATABASE_URL: uri, - // POOLED_DATABASE_URL: TODO + POOLED_DATABASE_URL: pooledUri, }, domain: { name: `yjs${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, @@ -124,7 +128,8 @@ function deployServerlessApp( function getNeonDbUri( project: $util.Output, - db: neon.Database + db: neon.Database, + pooled: boolean ) { const passwordOutput = neon.getBranchRolePasswordOutput({ projectId: project.id, @@ -132,7 +137,22 @@ function getNeonDbUri( roleName: db.ownerName, }) - return $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${project.databaseHost}/${db.name}?sslmode=require` + const endpoint = neon.getBranchEndpointsOutput({ + projectId: project.id, + branchId: project.defaultBranchId, + }) + + const databaseHost = pooled + ? endpoint.endpoints?.apply((endpoints) => + endpoints![0].host.replace( + endpoints![0].id, + endpoints![0].id + "-pooled" + ) + ) + : project.databaseHost + + const url = $interpolate`postgresql://${passwordOutput.roleName}:${passwordOutput.password}@${databaseHost}/${db.name}?sslmode=require` + return url } async function addDatabaseToElectric( From fbec8d679e911e3fcced70593d885b9bda181a4a Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 5 Dec 2024 14:42:07 +0000 Subject: [PATCH 35/47] Open a single connection instead of a pool on write; fixes to sst --- .../yjs-provider/app/api/operation/route.ts | 29 +++++++++++-------- examples/yjs-provider/app/db.ts | 13 --------- examples/yjs-provider/package.json | 1 - examples/yjs-provider/sst-env.d.ts | 8 ----- examples/yjs-provider/sst.config.ts | 4 +-- 5 files changed, 19 insertions(+), 36 deletions(-) delete mode 100644 examples/yjs-provider/app/db.ts diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index cb45a58cf0..de243cf221 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,18 +1,25 @@ -import { Pool } from "pg" -import { pool } from "../../db" +import { Client } from "pg" import { NextResponse } from "next/server" +// TODO: need to use a connection pool for non-serverless deployments + +const connectionString = + process.env.POOLED_DATABASE_URL || + process.env.DATABASE_URL || + `postgresql://postgres:password@localhost:54321/electric` + export async function POST(request: Request) { - let connected = false + const client = new Client({ + connectionString, + }) try { const { room, op, clientId } = await getRequestParams(request) - await pool.connect() - connected = true + await client.connect() if (!clientId) { - await saveOperation(room, op, pool) + await saveOperation(room, op, client) } else { - await saveAwarenessOperation(room, op, clientId, pool) + await saveAwarenessOperation(room, op, clientId, client) } return NextResponse.json({}) @@ -20,13 +27,11 @@ export async function POST(request: Request) { const resp = e instanceof Error ? e.message : e return NextResponse.json(resp, { status: 400 }) } finally { - if (connected) { - pool.end() - } + client.end() } } -async function saveOperation(room: string, op: string, connection: Pool) { +async function saveOperation(room: string, op: string, connection: Client) { await connection.query( `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, [room, op] @@ -37,7 +42,7 @@ async function saveAwarenessOperation( room: string, op: string, clientId: string, - connection: Pool + connection: Client ) { await connection.query( `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) diff --git a/examples/yjs-provider/app/db.ts b/examples/yjs-provider/app/db.ts deleted file mode 100644 index f89e98b35e..0000000000 --- a/examples/yjs-provider/app/db.ts +++ /dev/null @@ -1,13 +0,0 @@ -import pgPkg from "pg" -const { Pool } = pgPkg - -const connectionString = - process.env.POOLED_DATABASE_URL || - process.env.DATABASE_URL || - `postgresql://postgres:password@localhost:54321/electric` - -const pool = new Pool({ - connectionString, -}) - -export { pool } diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 30965d7c05..db804bd688 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -22,7 +22,6 @@ "@codemirror/view": "^6.32.0", "@electric-sql/client": "workspace:*", "@electric-sql/react": "workspace:*", - "@neondatabase/serverless": "^0.10.4", "codemirror": "^6.0.1", "lib0": "^0.2.96", "next": "^14.2.9", diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs-provider/sst-env.d.ts index bd7b4723ec..427d35880f 100644 --- a/examples/yjs-provider/sst-env.d.ts +++ b/examples/yjs-provider/sst-env.d.ts @@ -10,13 +10,5 @@ declare module "sst" { "type": "sst.aws.Nextjs" "url": string } - "yjs-service-vbalegas": { - "service": string - "type": "sst.aws.Service" - "url": string - } - "yjs-vpc-vbalegas": { - "type": "sst.aws.Vpc" - } } } diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index d5af202da6..ef3828b634 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -120,7 +120,7 @@ function deployServerlessApp( POOLED_DATABASE_URL: pooledUri, }, domain: { - name: `yjs${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + name: `yjs${$app.stage === `Production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, dns: sst.cloudflare.dns(), }, }) @@ -146,7 +146,7 @@ function getNeonDbUri( ? endpoint.endpoints?.apply((endpoints) => endpoints![0].host.replace( endpoints![0].id, - endpoints![0].id + "-pooled" + endpoints![0].id + "-pooler" ) ) : project.databaseHost From 985ad8431109262192b35874b055cf75c3871d56 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 6 Dec 2024 13:08:11 +0000 Subject: [PATCH 36/47] Hybrid implementation with server and serverless --- .../yjs-provider/app/api/operation/route.ts | 63 ++++++++----- examples/yjs-provider/package.json | 1 + examples/yjs-provider/sst.config.ts | 89 +++++++++---------- 3 files changed, 81 insertions(+), 72 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index de243cf221..ef08ddf26d 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -1,55 +1,53 @@ -import { Client } from "pg" +import { Pool } from "pg" import { NextResponse } from "next/server" +import { neon } from "@neondatabase/serverless" -// TODO: need to use a connection pool for non-serverless deployments +// hybrid implementation for connection pool and serverless const connectionString = process.env.POOLED_DATABASE_URL || process.env.DATABASE_URL || `postgresql://postgres:password@localhost:54321/electric` +const sql = process.env.POOLED_DATABASE_URL ? neon(connectionString) : undefined + +const pool = process.env.DATABASE_URL + ? new Pool({ connectionString: process.env.DATABASE_URL }) + : undefined +let connected = false + export async function POST(request: Request) { - const client = new Client({ - connectionString, - }) + console.log(`Received request: ${request}`) try { const { room, op, clientId } = await getRequestParams(request) - await client.connect() - if (!clientId) { - await saveOperation(room, op, client) + await saveOperation(room, op) } else { - await saveAwarenessOperation(room, op, clientId, client) + await saveAwarenessOperation(room, op, clientId) } - return NextResponse.json({}) } catch (e) { const resp = e instanceof Error ? e.message : e return NextResponse.json(resp, { status: 400 }) - } finally { - client.end() } } -async function saveOperation(room: string, op: string, connection: Client) { - await connection.query( - `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))`, - [room, op] - ) +async function saveOperation(room: string, op: string) { + const q = `INSERT INTO ydoc_operations (room, op) VALUES ($1, decode($2, 'base64'))` + const params = [room, op] + await runQuery(q, params) } async function saveAwarenessOperation( room: string, op: string, - clientId: string, - connection: Client + clientId: string ) { - await connection.query( - `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) + const q = `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) - DO UPDATE SET op = decode($3, 'base64')`, - [room, clientId, op] - ) + DO UPDATE SET op = decode($3, 'base64')` + const params = [room, clientId, op] + await runQuery(q, params) } async function getRequestParams( @@ -65,3 +63,20 @@ async function getRequestParams( return { room, op, clientId } } + +async function runQuery(q: string, params: string[]) { + console.log(`Running query: ${q} with params: ${params}`, pool, sql) + if (pool) { + if (pool && !connected) { + await pool.connect() + connected = true + } + + await pool.query(q, params) + } + if (sql) { + await sql(q, params) + } else { + throw new Error(`No database driver provided`) + } +} diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index db804bd688..30965d7c05 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -22,6 +22,7 @@ "@codemirror/view": "^6.32.0", "@electric-sql/client": "workspace:*", "@electric-sql/react": "workspace:*", + "@neondatabase/serverless": "^0.10.4", "codemirror": "^6.0.1", "lib0": "^0.2.96", "next": "^14.2.9", diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index ef3828b634..fda852b1be 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -32,19 +32,6 @@ export default $config({ $app.stage === `Production` ? `yjs-production` : `yjs-${$app.stage}`, }) - // const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`, { bastion: true }) - - // const rds = new sst.aws.Postgres(`yjs-${$app.stage}Database`, { - // vpc, - // // proxy: true, - // transform: { - // instance: { - // publiclyAccessible: true, - // }, - // }, - // }) - // const databaseUri = getRdsDbUri(rds) - const databaseUri = getNeonDbUri(project, db, false) const databasePooledUri = getNeonDbUri(project, db, true) try { @@ -54,12 +41,20 @@ export default $config({ addDatabaseToElectric(uri) ) - const website = deployServerlessApp( + const serverless = deployServerlessApp( electricInfo, databaseUri, databasePooledUri ) - return { url: website.url, databaseUri, databasePooledUri } + + const website = deployAppServer(electricInfo, databasePooledUri) + + return { + server_url: website.url, + serverless_url: serverless.url, + databaseUri, + databasePooledUri, + } } catch (e) {} }, }) @@ -73,38 +68,37 @@ function applyMigrations(uri: string) { }) } -// function deployApp( -// { id, token }: $util.Output<{ id: string; token: string }>, -// uri: $util.Output, -// vpc: sst.aws.Vpc -// ) { -// const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc }) - -// const service = cluster.addService(`yjs-service-${$app.stage}`, { -// loadBalancer: { -// ports: [{ listen: "443/https", forward: "3000/http" }], -// domain: { -// name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, -// dns: sst.cloudflare.dns(), -// }, -// }, -// environment: { -// ELECTRIC_URL: process.env.ELECTRIC_API!, -// DATABASE_URL: uri, -// DATABASE_ID: id, -// ELECTRIC_TOKEN: token, -// }, -// image: { -// context: "../..", -// dockerfile: "Dockerfile", -// }, -// dev: { -// command: "npm run dev", -// }, -// }) - -// return service -// } +function deployAppServer( + { id, token }: $util.Output<{ id: string; token: string }>, + uri: $util.Output +) { + const vpc = new sst.aws.Vpc(`yjs-vpc-${$app.stage}`, { bastion: true }) + const cluster = new sst.aws.Cluster(`yjs-cluster-${$app.stage}`, { vpc }) + const service = cluster.addService(`yjs-service-${$app.stage}`, { + loadBalancer: { + ports: [{ listen: "443/https", forward: "3000/http" }], + domain: { + name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + dns: sst.cloudflare.dns(), + }, + }, + environment: { + ELECTRIC_URL: process.env.ELECTRIC_API!, + DATABASE_URL: uri, + DATABASE_ID: id, + ELECTRIC_TOKEN: token, + }, + image: { + context: "../..", + dockerfile: "Dockerfile", + }, + dev: { + command: "npm run dev", + }, + }) + + return service +} function deployServerlessApp( electricInfo: $util.Output<{ id: string; token: string }>, @@ -116,7 +110,6 @@ function deployServerlessApp( ELECTRIC_URL: process.env.ELECTRIC_API!, ELECTRIC_TOKEN: electricInfo.token, DATABASE_ID: electricInfo.id, - DATABASE_URL: uri, POOLED_DATABASE_URL: pooledUri, }, domain: { From 824ff8f329aa585fa5f2d400f4fb80fba0ef0b4f Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sun, 8 Dec 2024 00:48:54 +0000 Subject: [PATCH 37/47] Fix writes in dev mode --- .../yjs-provider/app/api/operation/route.ts | 15 +++++++-------- examples/yjs-provider/sst.config.ts | 17 ++++++----------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index ef08ddf26d..7e9e4ae8c8 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -5,19 +5,18 @@ import { neon } from "@neondatabase/serverless" // hybrid implementation for connection pool and serverless const connectionString = - process.env.POOLED_DATABASE_URL || + process.env.NEON_DATABASE_URL || process.env.DATABASE_URL || `postgresql://postgres:password@localhost:54321/electric` -const sql = process.env.POOLED_DATABASE_URL ? neon(connectionString) : undefined +const sql = process.env.NEON_DATABASE_URL ? neon(connectionString) : undefined -const pool = process.env.DATABASE_URL - ? new Pool({ connectionString: process.env.DATABASE_URL }) +const pool = !process.env.NEON_DATABASE_URL + ? new Pool({ connectionString }) : undefined let connected = false export async function POST(request: Request) { - console.log(`Received request: ${request}`) try { const { room, op, clientId } = await getRequestParams(request) if (!clientId) { @@ -27,6 +26,7 @@ export async function POST(request: Request) { } return NextResponse.json({}) } catch (e) { + connected = false const resp = e instanceof Error ? e.message : e return NextResponse.json(resp, { status: 400 }) } @@ -65,7 +65,7 @@ async function getRequestParams( } async function runQuery(q: string, params: string[]) { - console.log(`Running query: ${q} with params: ${params}`, pool, sql) + console.log(`Running query: ${q} with params: ${params}`) if (pool) { if (pool && !connected) { await pool.connect() @@ -73,8 +73,7 @@ async function runQuery(q: string, params: string[]) { } await pool.query(q, params) - } - if (sql) { + } else if (sql) { await sql(q, params) } else { throw new Error(`No database driver provided`) diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index fda852b1be..3b50da22c9 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -33,7 +33,7 @@ export default $config({ }) const databaseUri = getNeonDbUri(project, db, false) - const databasePooledUri = getNeonDbUri(project, db, true) + const pooledUri = getNeonDbUri(project, db, true) try { databaseUri.apply(applyMigrations) @@ -41,19 +41,15 @@ export default $config({ addDatabaseToElectric(uri) ) - const serverless = deployServerlessApp( - electricInfo, - databaseUri, - databasePooledUri - ) + const serverless = deployServerlessApp(electricInfo, pooledUri) - const website = deployAppServer(electricInfo, databasePooledUri) + const website = deployAppServer(electricInfo, databaseUri) return { server_url: website.url, serverless_url: serverless.url, databaseUri, - databasePooledUri, + databasePooledUri: pooledUri, } } catch (e) {} }, @@ -102,15 +98,14 @@ function deployAppServer( function deployServerlessApp( electricInfo: $util.Output<{ id: string; token: string }>, - uri: $util.Output, - pooledUri: $util.Output + uri: $util.Output ) { return new sst.aws.Nextjs(`yjs`, { environment: { ELECTRIC_URL: process.env.ELECTRIC_API!, ELECTRIC_TOKEN: electricInfo.token, DATABASE_ID: electricInfo.id, - POOLED_DATABASE_URL: pooledUri, + NEON_DATABASE_URL: uri, }, domain: { name: `yjs${$app.stage === `Production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, From a2f2ecbdc3b069ceb94fc2cacae294e791f0b794 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sun, 8 Dec 2024 00:49:11 +0000 Subject: [PATCH 38/47] Added idx db persistence. Needs more work --- examples/yjs-provider/app/electric-editor.tsx | 2 + examples/yjs-provider/app/y-electric.ts | 102 ++++++++++++++++-- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index f9fad1ae9a..d9d9f77541 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -13,6 +13,7 @@ import { keymap } from "@codemirror/view" import { javascript } from "@codemirror/lang-javascript" import * as random from "lib0/random" +import { IndexeddbPersistence } from "y-indexeddb" const room = `electric-demo` @@ -59,6 +60,7 @@ export default function ElectricEditor() { { connect: true, awareness, + persistence: new IndexeddbPersistence(room, ydoc), } ) awareness?.setLocalStateField(`user`, { diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 057af90578..2ef6fc5cda 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -11,9 +11,11 @@ import { isChangeMessage, isControlMessage, Message, + Offset, ShapeStream, } from "@electric-sql/client" import { parseToDecoder } from "./utils" +import { IndexeddbPersistence } from "y-indexeddb" type OperationMessage = { op: decoding.Decoder @@ -35,6 +37,11 @@ type ObservableProvider = { "connection-close": () => void } +// Awareness TODOs: +// Notify other users of departure +// Reload awareness state on reconnection +// Don't apply state changes older than ping period + const messageSync = 0 export class ElectricProvider extends ObservableV2 { @@ -61,11 +68,22 @@ export class ElectricProvider extends ObservableV2 { private disconnectHandler?: () => void private exitHandler?: () => void + private persistence?: IndexeddbPersistence + private loaded: boolean + private resume: { + operations?: { offset: Offset; handle: string } + awareness?: { offset: Offset; handle: string } + } = {} + constructor( serverUrl: string, roomName: string, doc: Y.Doc, - options: { awareness?: awarenessProtocol.Awareness; connect?: boolean } + options: { + awareness?: awarenessProtocol.Awareness + connect?: boolean + persistence?: IndexeddbPersistence + } // TODO: make it generic, we can load it outside the provider ) { super() @@ -81,6 +99,9 @@ export class ElectricProvider extends ObservableV2 { this.modifiedWhileOffline = false + this.persistence = options.persistence + this.loaded = this.persistence === undefined + this.updateHandler = (update: Uint8Array, origin: unknown) => { if (origin !== this) { this.sendOperation(update) @@ -111,7 +132,9 @@ export class ElectricProvider extends ObservableV2 { } } - if (options.connect) { + if (!this.loaded) { + this.loadState() + } else if (options.connect) { this.connect() } } @@ -136,6 +159,41 @@ export class ElectricProvider extends ObservableV2 { } } + async loadState() { + if (this.persistence) { + const operationsHandle = await this.persistence.get(`operation_handle`) + const operationsOffset = await this.persistence.get(`operation_offset`) + + const awarenessHandle = await this.persistence.get(`awareness_handle`) + const awarenessOffset = await this.persistence.get(`awareness_offset`) + + // TODO: fix not loading changes from other users + const lastSyncedStateVector = await this.persistence.get( + `last_synced_state_vector` + ) + + this.lastSyncedStateVector = lastSyncedStateVector + this.modifiedWhileOffline = this.lastSyncedStateVector !== undefined + + this.resume = { + operations: { + handle: operationsHandle, + offset: operationsOffset, + }, + // TODO: we want the last pings of users, so it's more complicated + awareness: { + handle: awarenessHandle, + offset: awarenessOffset, + }, + } + + this.loaded = true + if (this.shouldConnect) { + this.connect() + } + } + } + destroy() { this.disconnect() this.doc.off(`update`, this.updateHandler) @@ -207,12 +265,15 @@ export class ElectricProvider extends ObservableV2 { this.connected = false this.synced = false + console.log(`Setting up shape stream ${JSON.stringify(this.resume)}`) + this.operationsStream = new ShapeStream({ url: this.operationsUrl, table: `ydoc_operations`, where: `room = '${this.roomName}'`, parser: parseToDecoder, subscribe: true, + ...this.resume.operations, }) this.awarenessStream = new ShapeStream({ @@ -220,29 +281,42 @@ export class ElectricProvider extends ObservableV2 { where: `room = '${this.roomName}'`, table: `ydoc_awareness`, parser: parseToDecoder, + ...this.resume.awareness, }) const errorHandler = (e: FetchError | Error) => { throw e } + // we probably want to extract this code + // save state per user + const updateShapeState = ( + name: `operation` | `awareness`, + offset: Offset, + handle: string + ) => { + this.persistence?.set(`${name}_offset`, offset) + this.persistence?.set(`${name}_handle`, handle) + } + const handleSyncMessage = (messages: Message[]) => { messages.forEach((message) => { if (isChangeMessage(message) && message.value.op) { const decoder = message.value.op const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, messageSync) - const syncMessageType = syncProtocol.readSyncMessage( - decoder, - encoder, - this.doc, - this - ) + syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) } else if ( isControlMessage(message) && message.headers.control === "up-to-date" ) { this.synced = true + + updateShapeState( + `operation`, + this.operationsStream!.lastOffset, + this.operationsStream!.shapeHandle + ) } }) } @@ -265,6 +339,12 @@ export class ElectricProvider extends ObservableV2 { ) } }) + + updateShapeState( + `awareness`, + this.awarenessStream!.lastOffset, + this.awarenessStream!.shapeHandle + ) } const unsubscribeAwarenessHandler = this.awarenessStream.subscribe( @@ -289,8 +369,13 @@ export class ElectricProvider extends ObservableV2 { ), this ) + this.sendAwareness([this.doc.clientID]) } this.lastSyncedStateVector = Y.encodeStateVector(this.doc) + this.persistence?.set( + `last_synced_state_vector`, + this.lastSyncedStateVector + ) this.emit(`status`, [{ status: `disconnected` }]) } @@ -315,6 +400,7 @@ export class ElectricProvider extends ObservableV2 { this.sendOperation(pendingUpdates).then(() => { this.lastSyncedStateVector = undefined this.modifiedWhileOffline = false + this.persistence?.del(`last_synced_state_vector`) this.emit(`status`, [{ status: `connected` }]) }) } From 9aa5e4cc233df90e4091eeb8f68d080a88531caa Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sun, 8 Dec 2024 23:00:27 +0000 Subject: [PATCH 39/47] Addressed review; improvements to awareness handling. --- .../yjs-provider/app/api/operation/route.ts | 7 - examples/yjs-provider/app/electric-editor.tsx | 61 +++---- examples/yjs-provider/app/y-electric.ts | 149 ++++++++---------- 3 files changed, 96 insertions(+), 121 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 7e9e4ae8c8..369b978ceb 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -14,7 +14,6 @@ const sql = process.env.NEON_DATABASE_URL ? neon(connectionString) : undefined const pool = !process.env.NEON_DATABASE_URL ? new Pool({ connectionString }) : undefined -let connected = false export async function POST(request: Request) { try { @@ -26,7 +25,6 @@ export async function POST(request: Request) { } return NextResponse.json({}) } catch (e) { - connected = false const resp = e instanceof Error ? e.message : e return NextResponse.json(resp, { status: 400 }) } @@ -67,11 +65,6 @@ async function getRequestParams( async function runQuery(q: string, params: string[]) { console.log(`Running query: ${q} with params: ${params}`) if (pool) { - if (pool && !connected) { - await pool.connect() - connected = true - } - await pool.query(q, params) } else if (sql) { await sql(q, params) diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index d9d9f77541..7445769674 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -28,22 +28,44 @@ const usercolors = [ const userColor = usercolors[random.uint32() % usercolors.length] const ydoc = new Y.Doc() -let network: ElectricProvider | undefined -let awareness: Awareness | undefined + +const isServer = typeof window === `undefined` + +const awareness = !isServer ? new Awareness(ydoc) : undefined +awareness?.setLocalStateField(`user`, { + name: userColor.color, + color: userColor.color, + colorLight: userColor.light, +}) + +const network = !isServer + ? new ElectricProvider( + new URL(`/shape-proxy`, window?.location.origin).href, + room, + ydoc, + { + connect: true, + awareness, + persistence: new IndexeddbPersistence(room, ydoc), + } + ) + : undefined export default function ElectricEditor() { const editor = useRef(null) - const [connect, setConnect] = useState(`connected`) + const [connectivityStatus, setConnectivityStatus] = useState< + `connected` | `disconnected` + >(`connected`) const toggle = () => { - if (connect === `connected`) { - network?.disconnect() - setConnect(`disconnected`) - } else { - network?.connect() - setConnect(`connected`) + if (!network) { + return } + const toggleStatus = + connectivityStatus === `connected` ? `disconnected` : `connected` + setConnectivityStatus(toggleStatus) + toggleStatus === "connected" ? network.connect() : network.disconnect() } useEffect(() => { @@ -51,25 +73,6 @@ export default function ElectricEditor() { return } - if (typeof window !== `undefined` && !network) { - awareness = new Awareness(ydoc) - network = new ElectricProvider( - new URL(`/shape-proxy`, window?.location.origin).href, - room, - ydoc, - { - connect: true, - awareness, - persistence: new IndexeddbPersistence(room, ydoc), - } - ) - awareness?.setLocalStateField(`user`, { - name: userColor.color, - color: userColor.color, - colorLight: userColor.light, - }) - } - const ytext = ydoc.getText(room) const state = EditorState.create({ @@ -92,7 +95,7 @@ export default function ElectricEditor() {
toggle()}>

diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 2ef6fc5cda..389da0fe80 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -7,7 +7,6 @@ import { ObservableV2 } from "lib0/observable" import * as env from "lib0/environment" import * as Y from "yjs" import { - FetchError, isChangeMessage, isControlMessage, Message, @@ -37,15 +36,10 @@ type ObservableProvider = { "connection-close": () => void } -// Awareness TODOs: -// Notify other users of departure -// Reload awareness state on reconnection -// Don't apply state changes older than ping period - const messageSync = 0 export class ElectricProvider extends ObservableV2 { - private serverUrl: string + private baseUrl: string private roomName: string private doc: Y.Doc public awareness?: awarenessProtocol.Awareness @@ -65,7 +59,7 @@ export class ElectricProvider extends ObservableV2 { changed: { added: number[]; updated: number[]; removed: number[] }, origin: string ) => void - private disconnectHandler?: () => void + private disconnectShapeHandler?: () => void private exitHandler?: () => void private persistence?: IndexeddbPersistence @@ -75,6 +69,8 @@ export class ElectricProvider extends ObservableV2 { awareness?: { offset: Offset; handle: string } } = {} + private awarenessState: Record | null = null + constructor( serverUrl: string, roomName: string, @@ -87,7 +83,7 @@ export class ElectricProvider extends ObservableV2 { ) { super() - this.serverUrl = serverUrl + this.baseUrl = serverUrl + `/v1/shape` this.roomName = roomName this.doc = doc @@ -121,14 +117,7 @@ export class ElectricProvider extends ObservableV2 { if (env.isNode && typeof process !== `undefined`) { this.exitHandler = () => { - if (this.awareness) { - awarenessProtocol.removeAwarenessStates( - this.awareness, - [doc.clientID], - `app closed` - ) - } - process.on(`exit`, () => this.exitHandler!()) + process.on(`exit`, () => this.destroy()) } } @@ -139,14 +128,6 @@ export class ElectricProvider extends ObservableV2 { } } - private get operationsUrl() { - return this.serverUrl + `/v1/shape` - } - - private get awarenessUrl() { - return this.serverUrl + `/v1/shape` - } - get synced() { return this._synced } @@ -206,8 +187,27 @@ export class ElectricProvider extends ObservableV2 { disconnect() { this.shouldConnect = false - if (this.disconnectHandler) { - this.disconnectHandler() + + if (this.awareness) { + this.awarenessState = this.awareness.getLocalState() + + awarenessProtocol.removeAwarenessStates( + this.awareness, + Array.from(this.awareness.getStates().keys()).filter( + (client) => client !== this.doc.clientID + ), + this + ) + + awarenessProtocol.removeAwarenessStates( + this.awareness, + [this.doc.clientID], + `local` + ) + } + + if (this.disconnectShapeHandler) { + this.disconnectShapeHandler() } } @@ -216,6 +216,10 @@ export class ElectricProvider extends ObservableV2 { if (!this.connected && !this.operationsStream) { this.setupShapeStream() } + + if (this.awareness && this.awarenessState !== null) { + this.awareness.setLocalState(this.awarenessState) + } } private sendOperation(update: Uint8Array) { @@ -268,7 +272,7 @@ export class ElectricProvider extends ObservableV2 { console.log(`Setting up shape stream ${JSON.stringify(this.resume)}`) this.operationsStream = new ShapeStream({ - url: this.operationsUrl, + url: this.baseUrl, table: `ydoc_operations`, where: `room = '${this.roomName}'`, parser: parseToDecoder, @@ -277,24 +281,21 @@ export class ElectricProvider extends ObservableV2 { }) this.awarenessStream = new ShapeStream({ - url: this.awarenessUrl, + url: this.baseUrl, where: `room = '${this.roomName}'`, table: `ydoc_awareness`, parser: parseToDecoder, ...this.resume.awareness, }) - const errorHandler = (e: FetchError | Error) => { - throw e - } - // we probably want to extract this code // save state per user const updateShapeState = ( - name: `operation` | `awareness`, + name: `operations` | `awareness`, offset: Offset, handle: string ) => { + this.resume[name] = { offset, handle } this.persistence?.set(`${name}_offset`, offset) this.persistence?.set(`${name}_handle`, handle) } @@ -313,7 +314,7 @@ export class ElectricProvider extends ObservableV2 { this.synced = true updateShapeState( - `operation`, + `operations`, this.operationsStream!.lastOffset, this.operationsStream!.shapeHandle ) @@ -321,10 +322,8 @@ export class ElectricProvider extends ObservableV2 { }) } - const unsubscribeSyncHandler = this.operationsStream.subscribe( - handleSyncMessage, - errorHandler - ) + const unsubscribeSyncHandler = + this.operationsStream.subscribe(handleSyncMessage) const handleAwarenessMessage = ( messages: Message[] @@ -348,73 +347,53 @@ export class ElectricProvider extends ObservableV2 { } const unsubscribeAwarenessHandler = this.awarenessStream.subscribe( - handleAwarenessMessage, - errorHandler + handleAwarenessMessage ) - this.disconnectHandler = () => { + this.disconnectShapeHandler = () => { this.operationsStream = undefined this.awarenessStream = undefined if (this.connected) { - this.connected = false - - this.synced = false - - if (this.awareness) { - awarenessProtocol.removeAwarenessStates( - this.awareness, - Array.from(this.awareness.getStates().keys()).filter( - (client) => client !== this.doc.clientID - ), - this - ) - this.sendAwareness([this.doc.clientID]) - } this.lastSyncedStateVector = Y.encodeStateVector(this.doc) this.persistence?.set( `last_synced_state_vector`, this.lastSyncedStateVector ) + + this.connected = false + this.synced = false this.emit(`status`, [{ status: `disconnected` }]) } unsubscribeSyncHandler() unsubscribeAwarenessHandler() - this.disconnectHandler = undefined + this.disconnectShapeHandler = undefined this.emit(`connection-close`, []) } - // send pending changes - const unsubscribeOps = this.operationsStream!.subscribe(() => { - this.connected = true + const pushLocalChangesUnsubscribe = this.operationsStream!.subscribe( + () => { + this.connected = true - if (this.modifiedWhileOffline) { - const pendingUpdates = Y.encodeStateAsUpdate( - this.doc, - this.lastSyncedStateVector - ) - const encoderState = encoding.createEncoder() - syncProtocol.writeUpdate(encoderState, pendingUpdates) - - this.sendOperation(pendingUpdates).then(() => { - this.lastSyncedStateVector = undefined - this.modifiedWhileOffline = false - this.persistence?.del(`last_synced_state_vector`) - this.emit(`status`, [{ status: `connected` }]) - }) - } - unsubscribeOps() - }) - - if (this.awarenessStream) { - const unsubscribeAwareness = this.awarenessStream.subscribe(() => { - if (this.awareness!.getLocalState() !== null) { - this.sendAwareness([this.doc.clientID]) + if (this.modifiedWhileOffline) { + const pendingUpdates = Y.encodeStateAsUpdate( + this.doc, + this.lastSyncedStateVector + ) + const encoderState = encoding.createEncoder() + syncProtocol.writeUpdate(encoderState, pendingUpdates) + + this.sendOperation(pendingUpdates).then(() => { + this.lastSyncedStateVector = undefined + this.modifiedWhileOffline = false + this.persistence?.del(`last_synced_state_vector`) + this.emit(`status`, [{ status: `connected` }]) + }) } - unsubscribeAwareness() - }) - } + pushLocalChangesUnsubscribe() + } + ) this.emit(`status`, [{ status: `connecting` }]) } From 03844c670d55174c8e3da07997c921067de083a4 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Sun, 8 Dec 2024 23:54:00 +0000 Subject: [PATCH 40/47] Lazy decode and ignore late awareness messages --- .../yjs-provider/app/api/operation/route.ts | 5 ++--- examples/yjs-provider/app/utils.ts | 13 +++++++++++++ examples/yjs-provider/app/y-electric.ts | 17 +++++++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs-provider/app/api/operation/route.ts index 369b978ceb..3d562538e2 100644 --- a/examples/yjs-provider/app/api/operation/route.ts +++ b/examples/yjs-provider/app/api/operation/route.ts @@ -2,7 +2,7 @@ import { Pool } from "pg" import { NextResponse } from "next/server" import { neon } from "@neondatabase/serverless" -// hybrid implementation for connection pool and serverless +// hybrid implementation for connection pool and serverless with neon const connectionString = process.env.NEON_DATABASE_URL || @@ -43,7 +43,7 @@ async function saveAwarenessOperation( ) { const q = `INSERT INTO ydoc_awareness (room, clientId, op) VALUES ($1, $2, decode($3, 'base64')) ON CONFLICT (clientId, room) - DO UPDATE SET op = decode($3, 'base64')` + DO UPDATE SET op = decode($3, 'base64'), updated = now()` const params = [room, clientId, op] await runQuery(q, params) } @@ -63,7 +63,6 @@ async function getRequestParams( } async function runQuery(q: string, params: string[]) { - console.log(`Running query: ${q} with params: ${params}`) if (pool) { await pool.query(q, params) } else if (sql) { diff --git a/examples/yjs-provider/app/utils.ts b/examples/yjs-provider/app/utils.ts index cc7bce63ed..780dd50921 100644 --- a/examples/yjs-provider/app/utils.ts +++ b/examples/yjs-provider/app/utils.ts @@ -27,3 +27,16 @@ export const parseToDecoder = { return decoding.createDecoder(uint8Array) }, } + +export const parseToDecoderLazy = { + bytea: (hexString: string) => () => { + const uint8Array = hexStringToUint8Array(hexString) + return decoding.createDecoder(uint8Array) + }, +} + +export const paserToTimestamptz = { + timestamptz: (timestamp: string) => { + return new Date(timestamp) + }, +} diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 389da0fe80..2e80fcfdab 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -13,7 +13,7 @@ import { Offset, ShapeStream, } from "@electric-sql/client" -import { parseToDecoder } from "./utils" +import { parseToDecoder, parseToDecoderLazy, paserToTimestamptz } from "./utils" import { IndexeddbPersistence } from "y-indexeddb" type OperationMessage = { @@ -21,9 +21,10 @@ type OperationMessage = { } type AwarenessMessage = { - op: decoding.Decoder + op: () => decoding.Decoder clientId: string room: string + updated: Date } type ObservableProvider = { @@ -36,6 +37,9 @@ type ObservableProvider = { "connection-close": () => void } +// from yjs docs, need to check if is configurable +const awarenessPingPeriod = 30000 //ms + const messageSync = 0 export class ElectricProvider extends ObservableV2 { @@ -284,7 +288,7 @@ export class ElectricProvider extends ObservableV2 { url: this.baseUrl, where: `room = '${this.roomName}'`, table: `ydoc_awareness`, - parser: parseToDecoder, + parser: { ...parseToDecoderLazy, ...paserToTimestamptz }, ...this.resume.awareness, }) @@ -328,9 +332,14 @@ export class ElectricProvider extends ObservableV2 { const handleAwarenessMessage = ( messages: Message[] ) => { + const minTime = new Date(Date.now() - awarenessPingPeriod) messages.forEach((message) => { if (isChangeMessage(message) && message.value.op) { - const decoder = message.value.op + if (message.value.updated < minTime) { + return + } + + const decoder = message.value.op() awarenessProtocol.applyAwarenessUpdate( this.awareness!, decoding.readVarUint8Array(decoder), From c6311334d24729228c8f8394f4381ec90ab54359 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 00:03:27 +0000 Subject: [PATCH 41/47] lint; removed experimental build options; persistence bug --- examples/yjs-provider/app/electric-editor.tsx | 2 +- examples/yjs-provider/app/y-electric.ts | 9 +++++---- examples/yjs-provider/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index 7445769674..5988f7f830 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -65,7 +65,7 @@ export default function ElectricEditor() { const toggleStatus = connectivityStatus === `connected` ? `disconnected` : `connected` setConnectivityStatus(toggleStatus) - toggleStatus === "connected" ? network.connect() : network.disconnect() + toggleStatus === `connected` ? network.connect() : network.disconnect() } useEffect(() => { diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 2e80fcfdab..472b276173 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -73,7 +73,7 @@ export class ElectricProvider extends ObservableV2 { awareness?: { offset: Offset; handle: string } } = {} - private awarenessState: Record | null = null + private awarenessState: Record | null = null constructor( serverUrl: string, @@ -146,8 +146,8 @@ export class ElectricProvider extends ObservableV2 { async loadState() { if (this.persistence) { - const operationsHandle = await this.persistence.get(`operation_handle`) - const operationsOffset = await this.persistence.get(`operation_offset`) + const operationsHandle = await this.persistence.get(`operations_handle`) + const operationsOffset = await this.persistence.get(`operations_offset`) const awarenessHandle = await this.persistence.get(`awareness_handle`) const awarenessOffset = await this.persistence.get(`awareness_offset`) @@ -223,6 +223,7 @@ export class ElectricProvider extends ObservableV2 { if (this.awareness && this.awarenessState !== null) { this.awareness.setLocalState(this.awarenessState) + this.awarenessState = null } } @@ -313,7 +314,7 @@ export class ElectricProvider extends ObservableV2 { syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) } else if ( isControlMessage(message) && - message.headers.control === "up-to-date" + message.headers.control === `up-to-date` ) { this.synced = true diff --git a/examples/yjs-provider/package.json b/examples/yjs-provider/package.json index 30965d7c05..3475e8b5d2 100644 --- a/examples/yjs-provider/package.json +++ b/examples/yjs-provider/package.json @@ -10,7 +10,7 @@ "backend:down": "PROJECT_NAME=yjs-provider pnpm -C ../../ run example-backend:down", "db:migrate": "dotenv -e ../../.env.dev -- pnpm exec pg-migrations apply --directory ./db/migrations", "dev": "next dev --turbo -p 5173", - "build": "next build --experimental-build-mode=compile", + "build": "next build", "start": "next start", "lint": "eslint . --ext js,ts,tsx --report-unused-disable-directives --max-warnings 0", "stylecheck": "eslint . --quiet", From 34786d9961d6337cb4b61ed7794bfc4446a5e12a Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 00:20:31 +0000 Subject: [PATCH 42/47] some cleanup --- examples/yjs-provider/app/y-electric.ts | 70 +++++++++++-------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs-provider/app/y-electric.ts index 472b276173..39a1175f90 100644 --- a/examples/yjs-provider/app/y-electric.ts +++ b/examples/yjs-provider/app/y-electric.ts @@ -126,7 +126,7 @@ export class ElectricProvider extends ObservableV2 { } if (!this.loaded) { - this.loadState() + this.loadSyncState() } else if (options.connect) { this.connect() } @@ -144,38 +144,39 @@ export class ElectricProvider extends ObservableV2 { } } - async loadState() { - if (this.persistence) { - const operationsHandle = await this.persistence.get(`operations_handle`) - const operationsOffset = await this.persistence.get(`operations_offset`) + async loadSyncState() { + if (!this.persistence) { + throw Error(`Can't load sync state without persistence backend`) + } + const operationsHandle = await this.persistence.get(`operations_handle`) + const operationsOffset = await this.persistence.get(`operations_offset`) - const awarenessHandle = await this.persistence.get(`awareness_handle`) - const awarenessOffset = await this.persistence.get(`awareness_offset`) + const awarenessHandle = await this.persistence.get(`awareness_handle`) + const awarenessOffset = await this.persistence.get(`awareness_offset`) - // TODO: fix not loading changes from other users - const lastSyncedStateVector = await this.persistence.get( - `last_synced_state_vector` - ) + const lastSyncedStateVector = await this.persistence.get( + `last_synced_state_vector` + ) - this.lastSyncedStateVector = lastSyncedStateVector - this.modifiedWhileOffline = this.lastSyncedStateVector !== undefined - - this.resume = { - operations: { - handle: operationsHandle, - offset: operationsOffset, - }, - // TODO: we want the last pings of users, so it's more complicated - awareness: { - handle: awarenessHandle, - offset: awarenessOffset, - }, - } + this.lastSyncedStateVector = lastSyncedStateVector + this.modifiedWhileOffline = this.lastSyncedStateVector !== undefined - this.loaded = true - if (this.shouldConnect) { - this.connect() - } + this.resume = { + operations: { + handle: operationsHandle, + offset: operationsOffset, + }, + + // TODO: we might miss some awareness updates since last pings + awareness: { + handle: awarenessHandle, + offset: awarenessOffset, + }, + } + + this.loaded = true + if (this.shouldConnect) { + this.connect() } } @@ -192,7 +193,7 @@ export class ElectricProvider extends ObservableV2 { disconnect() { this.shouldConnect = false - if (this.awareness) { + if (this.awareness && this.connected) { this.awarenessState = this.awareness.getLocalState() awarenessProtocol.removeAwarenessStates( @@ -203,6 +204,7 @@ export class ElectricProvider extends ObservableV2 { this ) + // try to notify other clients that we are disconnected awarenessProtocol.removeAwarenessStates( this.awareness, [this.doc.clientID], @@ -228,12 +230,6 @@ export class ElectricProvider extends ObservableV2 { } private sendOperation(update: Uint8Array) { - if (update.length <= 2) { - throw Error( - `Shouldn't be trying to send operations without pending operations` - ) - } - if (!this.connected) { this.modifiedWhileOffline = true return Promise.resolve() @@ -293,8 +289,6 @@ export class ElectricProvider extends ObservableV2 { ...this.resume.awareness, }) - // we probably want to extract this code - // save state per user const updateShapeState = ( name: `operations` | `awareness`, offset: Offset, From ac4f0fcb05db2271c9008eee5f017b7c89d585c1 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 12:08:11 +0000 Subject: [PATCH 43/47] last few bits --- examples/yjs-provider/README.md | 30 +++++++----- examples/yjs-provider/app/electric-editor.tsx | 12 +++-- examples/yjs-provider/sst-env.d.ts | 9 +++- examples/yjs-provider/sst.config.ts | 47 ++++++++++--------- 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/examples/yjs-provider/README.md b/examples/yjs-provider/README.md index 5b80d41a6e..4c943b7ac2 100644 --- a/examples/yjs-provider/README.md +++ b/examples/yjs-provider/README.md @@ -1,22 +1,28 @@ -# Yjs Electric provider example +# Yjs Electric provider -## Setup +This example showcases a multiplayer [Codemirror](https://codemirror.net/) editor with [YJS](https://github.com/yjs/yjs) and [ElectricSQL](https://electric-sql.com/). All data is synchronized through [Postgres](https://www.postgresql.org/), eliminating the need for additional real-time infrastructure. -1. Make sure you've installed all dependencies for the monorepo and built packages +Y-Electric is a [YJS connection provider](https://docs.yjs.dev/ecosystem/connection-provider) that comes with offline support, integrates with [database providers](https://docs.yjs.dev/ecosystem/database-provider) and also handles [Presence/Awareness](https://docs.yjs.dev/api/about-awareness) data. It works with the entire YJS ecosystem and with you existing apps too! -From the root directory: +> We're releasing The Y-Electric backend as a package soon! -- `pnpm i` -- `pnpm run -r build` +## How to run -2. Start the docker containers +Make sure you've installed all dependencies for the monorepo and built the packages (from the monorepo root directory): -`pnpm run backend:up` +```shell +pnpm install +pnpm run -r build +``` -3. Start the dev server +Start the docker containers (in this directory): -`pnpm run dev` +```shell +pnpm backend:up +``` -4. When done, tear down the backend containers so you can run other examples +Start the dev server: -`pnpm run backend:down` +```shell +pnpm dev +``` \ No newline at end of file diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs-provider/app/electric-editor.tsx index 5988f7f830..f280e1a8c8 100644 --- a/examples/yjs-provider/app/electric-editor.tsx +++ b/examples/yjs-provider/app/electric-editor.tsx @@ -99,9 +99,15 @@ export default function ElectricEditor() {

- This is a demo of Yjs shared - editor syncing with {` `} - Electric. + This is a demo of Yjs using + {` `} + {` `} + Electric for + syncing. +

+

+ The content of this editor is shared with every client that visits this + domain.

diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs-provider/sst-env.d.ts index 427d35880f..95ab3fbb9b 100644 --- a/examples/yjs-provider/sst-env.d.ts +++ b/examples/yjs-provider/sst-env.d.ts @@ -6,9 +6,14 @@ import "sst" export {} declare module "sst" { export interface Resource { - "yjs": { - "type": "sst.aws.Nextjs" + "yjs-service-production": { + "service": string + "type": "sst.aws.Service" "url": string } + "yjs-vpc-production": { + "bastion": string + "type": "sst.aws.Vpc" + } } } diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index 3b50da22c9..624080b703 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -3,11 +3,14 @@ import { execSync } from "child_process" +const isProduction = () => $app.stage.toLocaleLowerCase() === `production` + export default $config({ app(input) { return { name: `yjs`, - removal: input?.stage === `production` ? `retain` : `remove`, + removal: + input?.stage.toLocaleLowerCase() === `production` ? `retain` : `remove`, home: `aws`, providers: { cloudflare: `5.42.0`, @@ -19,39 +22,41 @@ export default $config({ } }, async run() { - const project = neon.getProjectOutput({ id: process.env.NEON_PROJECT_ID! }) - const base = { - projectId: project.id, - branchId: project.defaultBranchId, - } + try { + const project = neon.getProjectOutput({ + id: process.env.NEON_PROJECT_ID!, + }) + const base = { + projectId: project.id, + branchId: project.defaultBranchId, + } - const db = new neon.Database(`yjs`, { - ...base, - ownerName: `neondb_owner`, - name: - $app.stage === `Production` ? `yjs-production` : `yjs-${$app.stage}`, - }) + const db = new neon.Database(`yjs-db`, { + ...base, + ownerName: `neondb_owner`, + name: isProduction() ? `yjs` : `yjs-${$app.stage}`, + }) - const databaseUri = getNeonDbUri(project, db, false) - const pooledUri = getNeonDbUri(project, db, true) - try { + const databaseUri = getNeonDbUri(project, db, false) + const pooledUri = getNeonDbUri(project, db, true) databaseUri.apply(applyMigrations) const electricInfo = databaseUri.apply((uri) => addDatabaseToElectric(uri) ) - const serverless = deployServerlessApp(electricInfo, pooledUri) - + // const serverless = deployServerlessApp(electricInfo, pooledUri) const website = deployAppServer(electricInfo, databaseUri) return { + // serverless_url: serverless.url, server_url: website.url, - serverless_url: serverless.url, databaseUri, databasePooledUri: pooledUri, } - } catch (e) {} + } catch (e) { + console.error(e) + } }, }) @@ -74,7 +79,7 @@ function deployAppServer( loadBalancer: { ports: [{ listen: "443/https", forward: "3000/http" }], domain: { - name: `yjs-server-${$app.stage === `production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + name: `yjs-server${isProduction() ? `` : `-${$app.stage}`}.examples.electric-sql.com`, dns: sst.cloudflare.dns(), }, }, @@ -108,7 +113,7 @@ function deployServerlessApp( NEON_DATABASE_URL: uri, }, domain: { - name: `yjs${$app.stage === `Production` ? `` : `-stage-${$app.stage}`}.electric-sql.com`, + name: `yjs${isProduction() ? `` : `-stage-${$app.stage}`}.examples.electric-sql.com`, dns: sst.cloudflare.dns(), }, }) From dc4e5aa4631b29b463f8338567f668b130f21fae Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 12:14:59 +0000 Subject: [PATCH 44/47] final domain --- examples/yjs-provider/sst.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs-provider/sst.config.ts index 624080b703..f4d9bfc3bb 100644 --- a/examples/yjs-provider/sst.config.ts +++ b/examples/yjs-provider/sst.config.ts @@ -79,7 +79,7 @@ function deployAppServer( loadBalancer: { ports: [{ listen: "443/https", forward: "3000/http" }], domain: { - name: `yjs-server${isProduction() ? `` : `-${$app.stage}`}.examples.electric-sql.com`, + name: `yjs${isProduction() ? `` : `-${$app.stage}`}.examples.electric-sql.com`, dns: sst.cloudflare.dns(), }, }, From 6bf8121e3151bc212f9ab55293c7d2eae8dec698 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 12:27:55 +0000 Subject: [PATCH 45/47] Move to folder --- examples/{yjs-provider => yjs}/.eslintignore | 0 examples/{yjs-provider => yjs}/.eslintrc.cjs | 0 examples/{yjs-provider => yjs}/.gitignore | 0 examples/{yjs-provider => yjs}/.prettierrc | 0 examples/{yjs-provider => yjs}/Dockerfile | 0 examples/{yjs-provider => yjs}/README.md | 0 .../app/api/operation/route.ts | 0 .../{yjs-provider => yjs}/app/electric-editor.tsx | 0 examples/{yjs-provider => yjs}/app/layout.tsx | 0 examples/{yjs-provider => yjs}/app/page.tsx | 0 .../app/shape-proxy/[...table]/route.ts | 0 examples/{yjs-provider => yjs}/app/utils.ts | 0 examples/{yjs-provider => yjs}/app/y-electric.ts | 0 .../db/migrations/01-create_yjs_tables.sql | 0 examples/{yjs-provider => yjs}/index.html | 0 examples/{yjs-provider => yjs}/next.config.mjs | 0 examples/{yjs-provider => yjs}/package.json | 0 examples/{yjs-provider => yjs}/public/favicon.ico | Bin examples/{yjs-provider => yjs}/public/robots.txt | 0 examples/{yjs-provider => yjs}/sst-env.d.ts | 0 examples/{yjs-provider => yjs}/sst.config.ts | 0 examples/{yjs-provider => yjs}/tsconfig.json | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename examples/{yjs-provider => yjs}/.eslintignore (100%) rename examples/{yjs-provider => yjs}/.eslintrc.cjs (100%) rename examples/{yjs-provider => yjs}/.gitignore (100%) rename examples/{yjs-provider => yjs}/.prettierrc (100%) rename examples/{yjs-provider => yjs}/Dockerfile (100%) rename examples/{yjs-provider => yjs}/README.md (100%) rename examples/{yjs-provider => yjs}/app/api/operation/route.ts (100%) rename examples/{yjs-provider => yjs}/app/electric-editor.tsx (100%) rename examples/{yjs-provider => yjs}/app/layout.tsx (100%) rename examples/{yjs-provider => yjs}/app/page.tsx (100%) rename examples/{yjs-provider => yjs}/app/shape-proxy/[...table]/route.ts (100%) rename examples/{yjs-provider => yjs}/app/utils.ts (100%) rename examples/{yjs-provider => yjs}/app/y-electric.ts (100%) rename examples/{yjs-provider => yjs}/db/migrations/01-create_yjs_tables.sql (100%) rename examples/{yjs-provider => yjs}/index.html (100%) rename examples/{yjs-provider => yjs}/next.config.mjs (100%) rename examples/{yjs-provider => yjs}/package.json (100%) rename examples/{yjs-provider => yjs}/public/favicon.ico (100%) rename examples/{yjs-provider => yjs}/public/robots.txt (100%) rename examples/{yjs-provider => yjs}/sst-env.d.ts (100%) rename examples/{yjs-provider => yjs}/sst.config.ts (100%) rename examples/{yjs-provider => yjs}/tsconfig.json (100%) diff --git a/examples/yjs-provider/.eslintignore b/examples/yjs/.eslintignore similarity index 100% rename from examples/yjs-provider/.eslintignore rename to examples/yjs/.eslintignore diff --git a/examples/yjs-provider/.eslintrc.cjs b/examples/yjs/.eslintrc.cjs similarity index 100% rename from examples/yjs-provider/.eslintrc.cjs rename to examples/yjs/.eslintrc.cjs diff --git a/examples/yjs-provider/.gitignore b/examples/yjs/.gitignore similarity index 100% rename from examples/yjs-provider/.gitignore rename to examples/yjs/.gitignore diff --git a/examples/yjs-provider/.prettierrc b/examples/yjs/.prettierrc similarity index 100% rename from examples/yjs-provider/.prettierrc rename to examples/yjs/.prettierrc diff --git a/examples/yjs-provider/Dockerfile b/examples/yjs/Dockerfile similarity index 100% rename from examples/yjs-provider/Dockerfile rename to examples/yjs/Dockerfile diff --git a/examples/yjs-provider/README.md b/examples/yjs/README.md similarity index 100% rename from examples/yjs-provider/README.md rename to examples/yjs/README.md diff --git a/examples/yjs-provider/app/api/operation/route.ts b/examples/yjs/app/api/operation/route.ts similarity index 100% rename from examples/yjs-provider/app/api/operation/route.ts rename to examples/yjs/app/api/operation/route.ts diff --git a/examples/yjs-provider/app/electric-editor.tsx b/examples/yjs/app/electric-editor.tsx similarity index 100% rename from examples/yjs-provider/app/electric-editor.tsx rename to examples/yjs/app/electric-editor.tsx diff --git a/examples/yjs-provider/app/layout.tsx b/examples/yjs/app/layout.tsx similarity index 100% rename from examples/yjs-provider/app/layout.tsx rename to examples/yjs/app/layout.tsx diff --git a/examples/yjs-provider/app/page.tsx b/examples/yjs/app/page.tsx similarity index 100% rename from examples/yjs-provider/app/page.tsx rename to examples/yjs/app/page.tsx diff --git a/examples/yjs-provider/app/shape-proxy/[...table]/route.ts b/examples/yjs/app/shape-proxy/[...table]/route.ts similarity index 100% rename from examples/yjs-provider/app/shape-proxy/[...table]/route.ts rename to examples/yjs/app/shape-proxy/[...table]/route.ts diff --git a/examples/yjs-provider/app/utils.ts b/examples/yjs/app/utils.ts similarity index 100% rename from examples/yjs-provider/app/utils.ts rename to examples/yjs/app/utils.ts diff --git a/examples/yjs-provider/app/y-electric.ts b/examples/yjs/app/y-electric.ts similarity index 100% rename from examples/yjs-provider/app/y-electric.ts rename to examples/yjs/app/y-electric.ts diff --git a/examples/yjs-provider/db/migrations/01-create_yjs_tables.sql b/examples/yjs/db/migrations/01-create_yjs_tables.sql similarity index 100% rename from examples/yjs-provider/db/migrations/01-create_yjs_tables.sql rename to examples/yjs/db/migrations/01-create_yjs_tables.sql diff --git a/examples/yjs-provider/index.html b/examples/yjs/index.html similarity index 100% rename from examples/yjs-provider/index.html rename to examples/yjs/index.html diff --git a/examples/yjs-provider/next.config.mjs b/examples/yjs/next.config.mjs similarity index 100% rename from examples/yjs-provider/next.config.mjs rename to examples/yjs/next.config.mjs diff --git a/examples/yjs-provider/package.json b/examples/yjs/package.json similarity index 100% rename from examples/yjs-provider/package.json rename to examples/yjs/package.json diff --git a/examples/yjs-provider/public/favicon.ico b/examples/yjs/public/favicon.ico similarity index 100% rename from examples/yjs-provider/public/favicon.ico rename to examples/yjs/public/favicon.ico diff --git a/examples/yjs-provider/public/robots.txt b/examples/yjs/public/robots.txt similarity index 100% rename from examples/yjs-provider/public/robots.txt rename to examples/yjs/public/robots.txt diff --git a/examples/yjs-provider/sst-env.d.ts b/examples/yjs/sst-env.d.ts similarity index 100% rename from examples/yjs-provider/sst-env.d.ts rename to examples/yjs/sst-env.d.ts diff --git a/examples/yjs-provider/sst.config.ts b/examples/yjs/sst.config.ts similarity index 100% rename from examples/yjs-provider/sst.config.ts rename to examples/yjs/sst.config.ts diff --git a/examples/yjs-provider/tsconfig.json b/examples/yjs/tsconfig.json similarity index 100% rename from examples/yjs-provider/tsconfig.json rename to examples/yjs/tsconfig.json From 8db19a332e3c6d85dca051898d2fa38738f940d7 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 12:47:00 +0000 Subject: [PATCH 46/47] changes after rebase --- examples/yjs/.eslintignore | 1 + examples/yjs/app/y-electric.ts | 12 ++++++++---- examples/yjs/tsconfig.json | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/yjs/.eslintignore b/examples/yjs/.eslintignore index e32a3e1834..362a28a121 100644 --- a/examples/yjs/.eslintignore +++ b/examples/yjs/.eslintignore @@ -1 +1,2 @@ /build/** +sst.config.ts diff --git a/examples/yjs/app/y-electric.ts b/examples/yjs/app/y-electric.ts index 39a1175f90..50ec608ad4 100644 --- a/examples/yjs/app/y-electric.ts +++ b/examples/yjs/app/y-electric.ts @@ -274,8 +274,10 @@ export class ElectricProvider extends ObservableV2 { this.operationsStream = new ShapeStream({ url: this.baseUrl, - table: `ydoc_operations`, - where: `room = '${this.roomName}'`, + params: { + table: `ydoc_operations`, + where: `room = '${this.roomName}'`, + }, parser: parseToDecoder, subscribe: true, ...this.resume.operations, @@ -283,8 +285,10 @@ export class ElectricProvider extends ObservableV2 { this.awarenessStream = new ShapeStream({ url: this.baseUrl, - where: `room = '${this.roomName}'`, - table: `ydoc_awareness`, + params: { + where: `room = '${this.roomName}'`, + table: `ydoc_awareness`, + }, parser: { ...parseToDecoderLazy, ...paserToTimestamptz }, ...this.resume.awareness, }) diff --git a/examples/yjs/tsconfig.json b/examples/yjs/tsconfig.json index 14d189328b..01460806e7 100644 --- a/examples/yjs/tsconfig.json +++ b/examples/yjs/tsconfig.json @@ -24,5 +24,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "sst.config.ts"] } From 88b49207d665565fd79117ea36afb1b93411f38b Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 9 Dec 2024 13:10:47 +0000 Subject: [PATCH 47/47] rebase conflict --- pnpm-lock.yaml | 400 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 395 insertions(+), 5 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d0f179b37..31fcf9ff79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -497,7 +497,7 @@ importers: version: 0.3.4 tsup: specifier: ^8.0.1 - version: 8.3.5(@swc/core@1.9.1(@swc/helpers@0.5.5))(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) + version: 8.3.5(@swc/core@1.9.1)(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) tsx: specifier: ^4.19.1 version: 4.19.2 @@ -815,6 +815,91 @@ importers: specifier: ^0.21.0 version: 0.21.0(vite@5.4.10(@types/node@20.17.6)(terser@5.36.0))(workbox-build@7.3.0(@types/babel__core@7.20.5))(workbox-window@7.3.0) + examples/yjs: + dependencies: + '@codemirror/lang-javascript': + specifier: ^6.2.2 + version: 6.2.2 + '@codemirror/state': + specifier: ^6.4.1 + version: 6.5.0 + '@codemirror/view': + specifier: ^6.32.0 + version: 6.35.2 + '@electric-sql/client': + specifier: workspace:* + version: link:../../packages/typescript-client + '@electric-sql/react': + specifier: workspace:* + version: link:../../packages/react-hooks + '@neondatabase/serverless': + specifier: ^0.10.4 + version: 0.10.4 + codemirror: + specifier: ^6.0.1 + version: 6.0.1(@lezer/common@1.2.3) + lib0: + specifier: ^0.2.96 + version: 0.2.99 + next: + specifier: ^14.2.9 + version: 14.2.17(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + pg: + specifier: ^8.13.1 + version: 8.13.1 + react: + specifier: ^18.3.1 + version: 18.3.1 + react-dom: + specifier: ^18.3.1 + version: 18.3.1(react@18.3.1) + sst: + specifier: ^3.3.35 + version: 3.3.59 + y-codemirror.next: + specifier: 0.3.5 + version: 0.3.5(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(yjs@13.6.20) + y-indexeddb: + specifier: ^9.0.12 + version: 9.0.12(yjs@13.6.20) + y-protocols: + specifier: 1.0.6 + version: 1.0.6(yjs@13.6.20) + yjs: + specifier: ^13.6.18 + version: 13.6.20 + devDependencies: + '@databases/pg-migrations': + specifier: ^5.0.3 + version: 5.0.3(typescript@5.6.3) + '@next/eslint-plugin-next': + specifier: ^14.2.5 + version: 14.2.20 + '@types/pg': + specifier: ^8.11.6 + version: 8.11.10 + '@types/react': + specifier: ^18.3.3 + version: 18.3.12 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.1 + '@vitejs/plugin-react': + specifier: ^4.3.1 + version: 4.3.3(vite@5.4.10(@types/node@20.17.6)(terser@5.36.0)) + dotenv: + specifier: ^16.4.5 + version: 16.4.5 + eslint: + specifier: ^8.57.0 + version: 8.57.1 + typescript: + specifier: ^5.5.3 + version: 5.6.3 + vite: + specifier: ^5.3.4 + version: 5.4.10(@types/node@20.17.6)(terser@5.36.0) + packages/elixir-client: {} packages/experimental: @@ -865,7 +950,7 @@ importers: version: 0.3.4 tsup: specifier: ^8.0.1 - version: 8.3.5(@swc/core@1.9.1(@swc/helpers@0.5.5))(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) + version: 8.3.5(@swc/core@1.9.1)(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) typescript: specifier: ^5.5.2 version: 5.6.3 @@ -941,7 +1026,7 @@ importers: version: 0.3.4 tsup: specifier: ^8.0.1 - version: 8.3.5(@swc/core@1.9.1(@swc/helpers@0.5.5))(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) + version: 8.3.5(@swc/core@1.9.1)(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) typescript: specifier: ^5.5.2 version: 5.6.3 @@ -1001,7 +1086,7 @@ importers: version: 0.3.4 tsup: specifier: ^8.0.1 - version: 8.3.5(@swc/core@1.9.1(@swc/helpers@0.5.5))(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) + version: 8.3.5(@swc/core@1.9.1)(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0) typescript: specifier: ^5.5.2 version: 5.6.3 @@ -1727,6 +1812,35 @@ packages: '@changesets/write@0.3.2': resolution: {integrity: sha512-kDxDrPNpUgsjDbWBvUo27PzKX4gqeKOlhibaOXDJA6kuBisGqNHv/HwGJrAu8U/dSf8ZEFIeHIPtvSlZI1kULw==} + '@codemirror/autocomplete@6.18.3': + resolution: {integrity: sha512-1dNIOmiM0z4BIBwxmxEfA1yoxh1MF/6KPBbh20a5vphGV0ictKlgQsbJs6D6SkR6iJpGbpwRsa6PFMNlg9T9pQ==} + peerDependencies: + '@codemirror/language': ^6.0.0 + '@codemirror/state': ^6.0.0 + '@codemirror/view': ^6.0.0 + '@lezer/common': ^1.0.0 + + '@codemirror/commands@6.7.1': + resolution: {integrity: sha512-llTrboQYw5H4THfhN4U3qCnSZ1SOJ60ohhz+SzU0ADGtwlc533DtklQP0vSFaQuCPDn3BPpOd1GbbnUtwNjsrw==} + + '@codemirror/lang-javascript@6.2.2': + resolution: {integrity: sha512-VGQfY+FCc285AhWuwjYxQyUQcYurWlxdKYT4bqwr3Twnd5wP5WSeu52t4tvvuWmljT4EmgEgZCqSieokhtY8hg==} + + '@codemirror/language@6.10.6': + resolution: {integrity: sha512-KrsbdCnxEztLVbB5PycWXFxas4EOyk/fPAfruSOnDDppevQgid2XZ+KbJ9u+fDikP/e7MW7HPBTvTb8JlZK9vA==} + + '@codemirror/lint@6.8.4': + resolution: {integrity: sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==} + + '@codemirror/search@6.5.8': + resolution: {integrity: sha512-PoWtZvo7c1XFeZWmmyaOp2G0XVbOnm+fJzvghqGAktBW3cufwJUWvSCcNG0ppXiBEM05mZu6RhMtXPv2hpllig==} + + '@codemirror/state@6.5.0': + resolution: {integrity: sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==} + + '@codemirror/view@6.35.2': + resolution: {integrity: sha512-u04R04XFCYCNaHoNRr37WUUAfnxKPwPdqV+370NiO6i85qB1J/qCD/WbbMJsyJfRWhXIJXAe2BG/oTzAggqv4A==} + '@databases/connection-pool@1.1.0': resolution: {integrity: sha512-/12/SNgl0V77mJTo5SX3yGPz4c9XGQwAlCfA0vlfs/0HcaErNpYXpmhj0StET07w6TmTJTnaUgX2EPcQK9ez5A==} @@ -2646,18 +2760,39 @@ packages: '@jspm/core@2.1.0': resolution: {integrity: sha512-3sRl+pkyFY/kLmHl0cgHiFp2xEqErA8N3ECjMs7serSUBmoJ70lBa0PG5t0IM6WJgdZNyyI0R8YFfi5wM8+mzg==} + '@lezer/common@1.2.3': + resolution: {integrity: sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==} + + '@lezer/highlight@1.2.1': + resolution: {integrity: sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==} + + '@lezer/javascript@1.4.21': + resolution: {integrity: sha512-lL+1fcuxWYPURMM/oFZLEDm0XuLN128QPV+VuGtKpeaOGdcl9F2LYC3nh1S9LkPqx9M0mndZFdXCipNAZpzIkQ==} + + '@lezer/lr@1.4.2': + resolution: {integrity: sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==} + '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} + '@marijn/find-cluster-break@1.0.0': + resolution: {integrity: sha512-0YSzy7M9mBiK+h1m33rD8vZOfaO8leG6CY3+Q+1Lig86snkc8OAHQVAdndmnXMWJlVIH6S7fSZVVcjLcq6OH1A==} + '@mdx-js/mdx@2.3.0': resolution: {integrity: sha512-jLuwRlz8DQfQNiUCJR50Y09CGPq3fLtmtUQfVrj79E0JWu3dvsVcxVIcfhR5h0iXu+/z++zDrYeiJqifRynJkA==} + '@neondatabase/serverless@0.10.4': + resolution: {integrity: sha512-2nZuh3VUO9voBauuh+IGYRhGU/MskWHt1IuZvHcJw6GLjDgtqj/KViKo7SIrLdGLdot7vFbiRRw+BgEy3wT9HA==} + '@next/env@14.2.17': resolution: {integrity: sha512-MCgO7VHxXo8sYR/0z+sk9fGyJJU636JyRmkjc7ZJY8Hurl8df35qG5hoAh5KMs75FLjhlEo9bb2LGe89Y/scDA==} + '@next/eslint-plugin-next@14.2.20': + resolution: {integrity: sha512-T0JRi706KLbvR1Uc46t56VtawbhR/igdBagzOrA7G+vv4rvjwnlu/Y4/Iq6X9TDVj5UZjyot4lUdkNd3V2kLhw==} + '@next/swc-darwin-arm64@14.2.17': resolution: {integrity: sha512-WiOf5nElPknrhRMTipXYTJcUz7+8IAjOYw3vXzj3BYRcVY0hRHKWgTgQ5439EvzQyHEko77XK+yN9x9OJ0oOog==} engines: {node: '>= 10'} @@ -4110,6 +4245,9 @@ packages: '@types/pg@8.11.10': resolution: {integrity: sha512-LczQUW4dbOQzsH2RQ5qoeJ6qJPdrcM/DcMLoqWQkMLMsq83J5lAX3LXjdkWdpscFy67JSOWDnh7Ny/sPFykmkg==} + '@types/pg@8.11.6': + resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} + '@types/prop-types@15.7.13': resolution: {integrity: sha512-hCZTSvwbzWGvhqxp/RqVqwU999pBf2vp7hzIjiYOsl8wqOmUxkQ6ddw1cV3l8811+kdUFus/q4d1Y3E3SyEifA==} @@ -4829,6 +4967,9 @@ packages: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} + codemirror@6.0.1: + resolution: {integrity: sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -5784,6 +5925,11 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} + glob@10.3.10: + resolution: {integrity: sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true @@ -6188,6 +6334,13 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isomorphic.js@0.2.5: + resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==} + + jackspeak@2.3.6: + resolution: {integrity: sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==} + engines: {node: '>=14'} + jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} @@ -6327,6 +6480,11 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lib0@0.2.99: + resolution: {integrity: sha512-vwztYuUf1uf/1zQxfzRfO5yzfNKhTtgOByCruuiQQxWQXnPb8Itaube5ylofcV0oM0aKal9Mv+S1s1Ky0UYP1w==} + engines: {node: '>=16'} + hasBin: true + lilconfig@2.1.0: resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} engines: {node: '>=10'} @@ -8108,31 +8266,68 @@ packages: resolution: {integrity: sha512-MGrFH9Z4NP9Iyhqn16sDtBpRRNJ0Y2hNa6D65h736fVSaPCHr4DM4sWUNvVaSuC+0OBGhwsrydQwmgfg5LncqQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + sst-darwin-arm64@3.3.59: + resolution: {integrity: sha512-v8E3pg3JK7LDFOD0ot5N+HMNOA6MDHczx1M1SsJnMcNZPLTFH/Dn1B3roUaTP55+8jLKtqSJhnp9kLtTPHR1GQ==} + cpu: [arm64] + os: [darwin] + sst-darwin-arm64@3.3.7: resolution: {integrity: sha512-2CQh78YIdvrRpO8enZ/Jx51JsUSFtk564u9w4ldcu5SsMMDY1ocdw5p/XIwBy1eKeRtrXLizd35sYbtSfSy6sw==} cpu: [arm64] os: [darwin] + sst-darwin-x64@3.3.59: + resolution: {integrity: sha512-SOOvHPxdxXwcMEwfdDXFiaMqZTdy1UcTL74omKGAn3i0pdoDN+FK2KPa865gNd1dzyYg0P6QXcUu9hI92g9Cyg==} + cpu: [x64] + os: [darwin] + sst-darwin-x64@3.3.7: resolution: {integrity: sha512-+hiDT3+am+CBO3xBy8yl3bmFeTjGXUT/+7V6NFOV2yxlRP3A8J65nEjWdzPTU/u7hRl+leE8EBu14j0grt/7/A==} cpu: [x64] os: [darwin] + sst-linux-arm64@3.3.59: + resolution: {integrity: sha512-C7RjmuO0RR6xhmz0EZDcL6FhPCInwj5odehJZkbrvW0SObdoBLZLeDF0Or/7DKKh9VuSes/dpN1UvuUtMVE5Cg==} + cpu: [arm64] + os: [linux] + sst-linux-arm64@3.3.7: resolution: {integrity: sha512-dYolpXAjq0S8QjL8sTKzcRpPNgZDeMcJ9PHnt/8GpdqxNxEpGlNF9gMl2cB7mleJyJYBNMPvi4YEeCGtcazmeQ==} cpu: [arm64] os: [linux] + sst-linux-x64@3.3.59: + resolution: {integrity: sha512-7XZaYHl7Uun9q6koMEIICvw/+BX18hqiVmBjaC2p6J6x5tw+z7kfpK3x+yD26HLHoFNhpIBLb+6Nqj7XDyCa/Q==} + cpu: [x64] + os: [linux] + sst-linux-x64@3.3.7: resolution: {integrity: sha512-K2vPOZ5DS8mJmE4QtffgZN5Nem1MIBhoVozNtZ0NoufeKHbFz0Hyw9wbqxYSbs2MOoVNKvG8qwcX99ojVXTFKw==} cpu: [x64] os: [linux] + sst-linux-x86@3.3.59: + resolution: {integrity: sha512-GA8dZw6ty12ZwvpdVXkKBtdNwNuGF8vIcOKSzAQ6eC2UmywHlvQNgz2fiWMPVz94QLnNHCHAZlL6XAR7fz7n4Q==} + cpu: [x86] + os: [linux] + sst-linux-x86@3.3.7: resolution: {integrity: sha512-4rXj54+UJd+HLmrhCHQ0k9AOkugHZhhh6sCUnkUNChJr5ei62pRscUQ7ge8/jywvfzHZGZw3eXXJWCCsjilXFA==} cpu: [x86] os: [linux] + sst@3.3.59: + resolution: {integrity: sha512-WzKYWMf41n/TMEVp54tEITvmZQ0iZHf2whevpqqS85dAGP6w74NbEkwPNAB5oU2RlA2sj8GgMF76NdctsYQ0JA==} + hasBin: true + peerDependencies: + hono: 4.x + valibot: 0.30.x + peerDependenciesMeta: + hono: + optional: true + valibot: + optional: true + sst@3.3.7: resolution: {integrity: sha512-qIJPQnGeIHarWZoUvphwi6R1nu6Pccd3Q2Qy9ltBLs4Z47TkSdwBNeqCBhgAzWA0eLDwStTXliexyQCcNM6gDQ==} hasBin: true @@ -8239,6 +8434,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + style-mod@4.1.2: + resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==} + style-to-object@0.4.4: resolution: {integrity: sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==} @@ -9100,6 +9298,25 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + y-codemirror.next@0.3.5: + resolution: {integrity: sha512-VluNu3e5HfEXybnypnsGwKAj+fKLd4iAnR7JuX1Sfyydmn1jCBS5wwEL/uS04Ch2ib0DnMAOF6ZRR/8kK3wyGw==} + peerDependencies: + '@codemirror/state': ^6.0.0 + '@codemirror/view': ^6.0.0 + yjs: ^13.5.6 + + y-indexeddb@9.0.12: + resolution: {integrity: sha512-9oCFRSPPzBK7/w5vOkJBaVCQZKHXB/v6SIT+WYhnJxlEC61juqG0hBrAf+y3gmSMLFLwICNH9nQ53uscuse6Hg==} + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + peerDependencies: + yjs: ^13.0.0 + + y-protocols@1.0.6: + resolution: {integrity: sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==} + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + peerDependencies: + yjs: ^13.0.0 + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -9141,6 +9358,10 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yjs@13.6.20: + resolution: {integrity: sha512-Z2YZI+SYqK7XdWlloI3lhMiKnCdFCVC4PchpdO+mCYwtiTwncjUbnRK9R1JmkNfdmHyDXuWN3ibJAt0wsqTbLQ==} + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -10141,6 +10362,61 @@ snapshots: human-id: 1.0.2 prettier: 2.8.8 + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3)': + dependencies: + '@codemirror/language': 6.10.6 + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + '@lezer/common': 1.2.3 + + '@codemirror/commands@6.7.1': + dependencies: + '@codemirror/language': 6.10.6 + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + '@lezer/common': 1.2.3 + + '@codemirror/lang-javascript@6.2.2': + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/language': 6.10.6 + '@codemirror/lint': 6.8.4 + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + '@lezer/common': 1.2.3 + '@lezer/javascript': 1.4.21 + + '@codemirror/language@6.10.6': + dependencies: + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + '@lezer/common': 1.2.3 + '@lezer/highlight': 1.2.1 + '@lezer/lr': 1.4.2 + style-mod: 4.1.2 + + '@codemirror/lint@6.8.4': + dependencies: + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + crelt: 1.0.6 + + '@codemirror/search@6.5.8': + dependencies: + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + crelt: 1.0.6 + + '@codemirror/state@6.5.0': + dependencies: + '@marijn/find-cluster-break': 1.0.0 + + '@codemirror/view@6.35.2': + dependencies: + '@codemirror/state': 6.5.0 + style-mod: 4.1.2 + w3c-keyname: 2.2.8 + '@databases/connection-pool@1.1.0': dependencies: '@databases/queue': 1.0.1 @@ -10766,6 +11042,22 @@ snapshots: '@jspm/core@2.1.0': {} + '@lezer/common@1.2.3': {} + + '@lezer/highlight@1.2.1': + dependencies: + '@lezer/common': 1.2.3 + + '@lezer/javascript@1.4.21': + dependencies: + '@lezer/common': 1.2.3 + '@lezer/highlight': 1.2.1 + '@lezer/lr': 1.4.2 + + '@lezer/lr@1.4.2': + dependencies: + '@lezer/common': 1.2.3 + '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.26.0 @@ -10782,6 +11074,8 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@marijn/find-cluster-break@1.0.0': {} + '@mdx-js/mdx@2.3.0': dependencies: '@types/estree-jsx': 1.0.5 @@ -10804,8 +11098,16 @@ snapshots: transitivePeerDependencies: - supports-color + '@neondatabase/serverless@0.10.4': + dependencies: + '@types/pg': 8.11.6 + '@next/env@14.2.17': {} + '@next/eslint-plugin-next@14.2.20': + dependencies: + glob: 10.3.10 + '@next/swc-darwin-arm64@14.2.17': optional: true @@ -12430,6 +12732,12 @@ snapshots: pg-protocol: 1.7.0 pg-types: 4.0.2 + '@types/pg@8.11.6': + dependencies: + '@types/node': 20.17.6 + pg-protocol: 1.7.0 + pg-types: 4.0.2 + '@types/prop-types@15.7.13': {} '@types/react-beautiful-dnd@13.1.8': @@ -13333,6 +13641,18 @@ snapshots: cluster-key-slot@1.1.2: {} + codemirror@6.0.1(@lezer/common@1.2.3): + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/commands': 6.7.1 + '@codemirror/language': 6.10.6 + '@codemirror/lint': 6.8.4 + '@codemirror/search': 6.5.8 + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + transitivePeerDependencies: + - '@lezer/common' + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -14475,6 +14795,14 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.3.10: + dependencies: + foreground-child: 3.3.0 + jackspeak: 2.3.6 + minimatch: 9.0.5 + minipass: 7.1.2 + path-scurry: 1.11.1 + glob@10.4.5: dependencies: foreground-child: 3.3.0 @@ -14886,6 +15214,14 @@ snapshots: isexe@2.0.0: {} + isomorphic.js@0.2.5: {} + + jackspeak@2.3.6: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jackspeak@3.4.3: dependencies: '@isaacs/cliui': 8.0.2 @@ -15058,6 +15394,10 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + lib0@0.2.99: + dependencies: + isomorphic.js: 0.2.5 + lilconfig@2.1.0: {} lilconfig@3.1.2: {} @@ -17115,21 +17455,48 @@ snapshots: dependencies: minipass: 7.1.2 + sst-darwin-arm64@3.3.59: + optional: true + sst-darwin-arm64@3.3.7: optional: true + sst-darwin-x64@3.3.59: + optional: true + sst-darwin-x64@3.3.7: optional: true + sst-linux-arm64@3.3.59: + optional: true + sst-linux-arm64@3.3.7: optional: true + sst-linux-x64@3.3.59: + optional: true + sst-linux-x64@3.3.7: optional: true + sst-linux-x86@3.3.59: + optional: true + sst-linux-x86@3.3.7: optional: true + sst@3.3.59: + dependencies: + aws4fetch: 1.0.20 + jose: 5.2.3 + openid-client: 5.6.4 + optionalDependencies: + sst-darwin-arm64: 3.3.59 + sst-darwin-x64: 3.3.59 + sst-linux-arm64: 3.3.59 + sst-linux-x64: 3.3.59 + sst-linux-x86: 3.3.59 + sst@3.3.7: dependencies: aws4fetch: 1.0.20 @@ -17247,6 +17614,8 @@ snapshots: strip-json-comments@3.1.1: {} + style-mod@4.1.2: {} + style-to-object@0.4.4: dependencies: inline-style-parser: 0.1.1 @@ -17506,7 +17875,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.3.5(@swc/core@1.9.1(@swc/helpers@0.5.5))(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0): + tsup@8.3.5(@swc/core@1.9.1)(jiti@1.21.6)(postcss@8.4.47)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.6.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.0) cac: 6.7.14 @@ -18250,6 +18619,23 @@ snapshots: xtend@4.0.2: {} + y-codemirror.next@0.3.5(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(yjs@13.6.20): + dependencies: + '@codemirror/state': 6.5.0 + '@codemirror/view': 6.35.2 + lib0: 0.2.99 + yjs: 13.6.20 + + y-indexeddb@9.0.12(yjs@13.6.20): + dependencies: + lib0: 0.2.99 + yjs: 13.6.20 + + y-protocols@1.0.6(yjs@13.6.20): + dependencies: + lib0: 0.2.99 + yjs: 13.6.20 + y18n@5.0.8: {} yallist@2.1.2: {} @@ -18288,6 +18674,10 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yjs@13.6.20: + dependencies: + lib0: 0.2.99 + yocto-queue@0.1.0: {} yoctocolors-cjs@2.1.2: {}