From b6954374155e69d25b6c1a556b07fada820e66c5 Mon Sep 17 00:00:00 2001 From: Chris Mounce Date: Thu, 25 Sep 2025 23:55:30 -0700 Subject: [PATCH] Tear out old code and turn on compaction --- src/components/App.tsx | 2 +- src/components/Chooser.tsx | 2 +- src/components/Debug.tsx | 10 ++++- src/components/Editor.tsx | 21 +++++----- src/persistence.ts | 73 ++++++++++++++++++----------------- src/sync.ts | 79 ++------------------------------------ 6 files changed, 63 insertions(+), 124 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index bd07cc7..7f933e3 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -34,7 +34,7 @@ function Layout(props: any) { const navigate = useNavigate(); const [fileId, setFileId] = createSignal(null); const [numUpdates, setNumUpdates] = createSignal(null); - const docsMap = getDocsMap(defaultVault.doc); + const docsMap = getDocsMap(defaultVault); const title = () => { const id = fileId(); diff --git a/src/components/Chooser.tsx b/src/components/Chooser.tsx index 6eafbbf..1142e84 100644 --- a/src/components/Chooser.tsx +++ b/src/components/Chooser.tsx @@ -12,7 +12,7 @@ function Chooser() { const { setFileId } = useNavbar(); onMount(() => setFileId(null)); - const docsMap = getDocsMap(defaultVault.doc); + const docsMap = getDocsMap(defaultVault); const computeDocsList = () => { const result: DocInfo[] = []; diff --git a/src/components/Debug.tsx b/src/components/Debug.tsx index 551e68b..0240dd5 100644 --- a/src/components/Debug.tsx +++ b/src/components/Debug.tsx @@ -3,11 +3,17 @@ import { createSignal, For } from 'solid-js'; const [getDebugLogs, setDebugLogs] = createSignal([]); export function debugLog(message: string, ...args: any[]) { + console.log(message, ...args); const components = [message, ...args.map((x) => JSON.stringify(x))]; const line = components.join(' '); - setDebugLogs([...getDebugLogs(), line]); + + const allLines = [...getDebugLogs(), line]; + if (allLines.length > 1000) { + allLines.splice(0, allLines.length - 1000); + } + setDebugLogs(allLines); } export default function DebugView() { - return {(logLine) =>

{logLine}

}; + return {(logLine) =>
{logLine}
}; } diff --git a/src/components/Editor.tsx b/src/components/Editor.tsx index 474157c..9a95ab0 100644 --- a/src/components/Editor.tsx +++ b/src/components/Editor.tsx @@ -13,10 +13,10 @@ import { onMount, Show, } from 'solid-js'; -import { LocalDocument } from '../sync'; import { useNavbar } from './App'; import { useParams } from '@solidjs/router'; import _ from 'lodash'; +import { closeDoc, getDocStats, openDoc } from '../persistence'; function Editor() { const { id: fileId } = useParams(); @@ -27,25 +27,25 @@ function Editor() { let editorDiv; let editorView: EditorView; - const [syncedDoc] = createResource(async () => LocalDocument.load(fileId)); + const [syncedDoc] = createResource(async () => openDoc(fileId)); createEffect(() => { const doc = syncedDoc(); if (!doc) { return; } - const ydoc = doc.doc; + const getNumRecords = () => getDocStats(doc).numRecords; const refreshNumUpdates = _.debounce(() => { - setNumUpdates(doc.numUpdates); + setNumUpdates(getNumRecords()); }, 100); - setNumUpdates(doc.numUpdates); - doc.doc.on('update', () => { + setNumUpdates(getNumRecords()); + doc.on('update', () => { setTimeout(refreshNumUpdates, 1500); }); const state = EditorState.create({ - doc: ydoc.getText().toString(), + doc: doc.getText().toString(), extensions: [ EditorView.lineWrapping, EditorView.contentAttributes.of({ @@ -56,7 +56,7 @@ function Editor() { keymap.of([...defaultKeymap, ...historyKeymap]), scrollPastEnd(), syntaxHighlighting(defaultHighlightStyle, { fallback: true }), - yCollab(ydoc.getText(), null), + yCollab(doc.getText(), null), ], }); editorView = new EditorView({ state, parent: editorDiv }); @@ -65,7 +65,10 @@ function Editor() { onCleanup(() => { setNumUpdates(null); editorView?.destroy(); - syncedDoc()?.finish(); + const doc = syncedDoc(); + if (doc) { + closeDoc(doc); + } }); return ( diff --git a/src/persistence.ts b/src/persistence.ts index 5ebc989..e08ad4c 100644 --- a/src/persistence.ts +++ b/src/persistence.ts @@ -2,71 +2,65 @@ import * as Y from 'yjs'; import { docsDb, LOCAL_DELTA_STORE } from './sync'; import { assert } from './util'; import _ from 'lodash'; +import { debugLog } from './components/Debug'; -const COMPACTION_THRESHOLD = 200; +const COMPACTION_THRESHOLD = 50; export async function openDoc(id: string): Promise { // Return existing doc if we still have it open const previousState = idToState.get(id); if (previousState) { previousState.refCount += 1; - return previousState.ydoc; + return previousState.savedDoc.ydoc; } - const state = await initialLoad(id); + const state: State = { + refCount: 1, + savedDoc: new SavedDoc(id), + }; idToState.set(id, state); - docToState.set(state.ydoc, state); + docToId.set(state.savedDoc.ydoc, id); + await state.savedDoc.connect(); + debugLog(`Opened doc ${id}`); - return state.ydoc; + return state.savedDoc.ydoc; +} + +export function getDocStats(doc: Y.Doc) { + const id = docToId.get(doc); + assert(id, 'Document is not already open'); + const state = idToState.get(id)!; + return state.savedDoc.getInfo(); } export function closeDoc(doc: Y.Doc) { - const state = docToState.get(doc); - assert(state, 'Document is not already open'); + const id = docToId.get(doc); + assert(id, 'Document is not already open'); + const state = idToState.get(id)!; state.refCount -= 1; if (state.refCount <= 0) { - idToState.delete(state.id); - docToState.delete(doc); - doc.destroy(); + idToState.delete(id); + docToId.delete(doc); + state.savedDoc.flush().then(() => doc.destroy()); } + debugLog(`Closed doc ${id} (refcount ${state.refCount})`); } +const docToId = new Map(); const idToState = new Map(); -const docToState = new Map(); type State = { - // TODO: Move more of this into class - id: string; - deltaBuffer: Uint8Array[]; - numRecords: number; refCount: number; - ydoc: Y.Doc; + savedDoc: SavedDoc; }; -async function initialLoad(id: string): Promise { - const ydoc = new Y.Doc(); - const records = await docsDb.getAll( - LOCAL_DELTA_STORE, - IDBKeyRange.bound([id, -Infinity], [id, Infinity]) - ); - Y.applyUpdate(ydoc, Y.mergeUpdates(records)); - return { - id, - deltaBuffer: [], - numRecords: records.length, - refCount: 1, - ydoc, - }; -} - -// @ts-ignore -class BufferedWriter { +class SavedDoc { + readonly docId: string; + readonly ydoc: Y.Doc; private deltaBuffer: Uint8Array[]; - private docId: string; private numRecords: number; private debouncedFlush: () => void; - private ydoc: Y.Doc; constructor(docId: string) { this.deltaBuffer = []; @@ -82,6 +76,10 @@ class BufferedWriter { this.ydoc = new Y.Doc(); } + public getInfo(): { numRecords: number } { + return { numRecords: this.numRecords }; + } + public async connect(): Promise { const records = await docsDb.getAll( LOCAL_DELTA_STORE, @@ -129,6 +127,9 @@ class BufferedWriter { if (this.numRecords < COMPACTION_THRESHOLD) { return; } + debugLog( + `Running compaction on doc ${this.docId}, ${this.numRecords} records...` + ); const tx = docsDb.transaction(LOCAL_DELTA_STORE, 'readwrite'); // Make sure we're not missing anything by pulling in all changes. diff --git a/src/sync.ts b/src/sync.ts index 7d91124..db37d95 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -3,7 +3,7 @@ import * as Y from 'yjs'; import { openDB, type DBSchema } from 'idb'; import { assert, generateId } from './util'; import { getDocsMap, getVaultMap, type DocMap } from './vault'; -import { debugLog } from './components/Debug'; +import { openDoc } from './persistence'; const DB_NAME = 'synced-docs'; export const LOCAL_DELTA_STORE = 'local-updates'; @@ -35,76 +35,6 @@ export const docsDb = await openDB(DB_NAME, 3, { }, }); -export class LocalDocument { - readonly id: string; - readonly doc: Y.Doc; - private updates: Uint8Array[]; - private totalUpdates: number; - - private constructor(id: string, ydoc: Y.Doc) { - this.id = id; - this.doc = ydoc; - this.updates = []; - this.totalUpdates = 0; - - const flush = _.debounce(() => this.flush(), 1000); - this.doc.on('update', (update: Uint8Array) => { - this.updates.push(update); - flush(); - }); - } - - get numUpdates(): number { - return this.totalUpdates; - } - - public static async load(id: string): Promise { - // Build Y.Doc from records on disk - const startMs = performance.now(); - const updates: Uint8Array[] = await docsDb.getAll( - LOCAL_DELTA_STORE, - IDBKeyRange.bound([id, -Infinity], [id, Infinity]) - ); - const loadedMs = performance.now(); - const mergedUpdate = Y.mergeUpdates(updates); - const mergedMs = performance.now(); - const ydoc = new Y.Doc(); - Y.applyUpdate(ydoc, mergedUpdate); - const appliedMs = performance.now(); - - // Build LocalDocument instance - const result = new LocalDocument(id, ydoc); - result.totalUpdates = updates.length; - - // Emit debug logs - const timings = [loadedMs - startMs, mergedMs - loadedMs, appliedMs - mergedMs]; - const sizes = _.sortBy(updates.map((x) => x.length)); - const quartiles = _.range(5).map((x) => { - const i = Math.round((sizes.length - 1) * (x / 4)); - return sizes[i]; - }); - debugLog(`Doc ${id}: load/merge/apply timings in ms`, timings); - debugLog(`${updates.length} updates, quartiles in bytes`, quartiles); - return result; - } - - public async flush(): Promise { - const batch = this.updates; - this.updates = []; - if (batch.length > 0) { - const update = Y.mergeUpdates(batch); - const key: LocalDeltaKey = [this.id, new Date().valueOf()]; - await docsDb.add(LOCAL_DELTA_STORE, update, key); - this.totalUpdates += 1; - } - } - - public async finish(): Promise { - await this.flush(); - this.doc.destroy(); - } -} - async function getOrCreateVaultDoc() { const records = await docsDb.getAll(VAULT_STORE); if (records.length > 1) { @@ -120,8 +50,8 @@ async function getOrCreateVaultDoc() { if (!id) { throw new Error("Vault record didn't have id"); } - const vault = await LocalDocument.load(id); - const vaultMap = getVaultMap(vault.doc); + const vault = await openDoc(id); + const vaultMap = getVaultMap(vault); if (!vaultMap.has('id')) { vaultMap.set('id', id); } @@ -141,9 +71,8 @@ async function getOrCreateVaultDoc() { } export const defaultVault = await getOrCreateVaultDoc(); -// console.log(defaultVault.doc.getMap().toJSON()) -const docsMap = getDocsMap(defaultVault.doc); +const docsMap = getDocsMap(defaultVault); if (docsMap.size === 0) { console.log('Running vault migration...'); for (const oldKey of ['foo', 'bar', 'baz']) {