Tear out old code and turn on compaction

src/components/App.tsx

@@ -34,7 +34,7 @@ function Layout(props: any) {
   const navigate = useNavigate();
   const [fileId, setFileId] = createSignal<string | null>(null);
   const [numUpdates, setNumUpdates] = createSignal<number | null>(null);
-  const docsMap = getDocsMap(defaultVault.doc);
+  const docsMap = getDocsMap(defaultVault);
 
   const title = () => {
     const id = fileId();

src/components/Chooser.tsx

@@ -12,7 +12,7 @@ function Chooser() {
   const { setFileId } = useNavbar();
   onMount(() => setFileId(null));
 
-  const docsMap = getDocsMap(defaultVault.doc);
+  const docsMap = getDocsMap(defaultVault);
 
   const computeDocsList = () => {
     const result: DocInfo[] = [];

src/components/Debug.tsx

@@ -3,11 +3,17 @@ import { createSignal, For } from 'solid-js';
 const [getDebugLogs, setDebugLogs] = createSignal<string[]>([]);
 
 export function debugLog(message: string, ...args: any[]) {
+  console.log(message, ...args);
   const components = [message, ...args.map((x) => JSON.stringify(x))];
   const line = components.join(' ');
-  setDebugLogs([...getDebugLogs(), line]);
+
+  const allLines = [...getDebugLogs(), line];
+  if (allLines.length > 1000) {
+    allLines.splice(0, allLines.length - 1000);
+  }
+  setDebugLogs(allLines);
 }
 
 export default function DebugView() {
-  return <For each={getDebugLogs()}>{(logLine) => <p>{logLine}</p>}</For>;
+  return <For each={getDebugLogs()}>{(logLine) => <div>{logLine}</div>}</For>;
 }

src/components/Editor.tsx

@@ -13,10 +13,10 @@ import {
   onMount,
   Show,
 } from 'solid-js';
-import { LocalDocument } from '../sync';
 import { useNavbar } from './App';
 import { useParams } from '@solidjs/router';
 import _ from 'lodash';
+import { closeDoc, getDocStats, openDoc } from '../persistence';
 
 function Editor() {
   const { id: fileId } = useParams();
@@ -27,25 +27,25 @@ function Editor() {
 
   let editorDiv;
   let editorView: EditorView;
-  const [syncedDoc] = createResource(async () => LocalDocument.load(fileId));
+  const [syncedDoc] = createResource(async () => openDoc(fileId));
 
   createEffect(() => {
     const doc = syncedDoc();
     if (!doc) {
       return;
     }
-    const ydoc = doc.doc;
 
+    const getNumRecords = () => getDocStats(doc).numRecords;
     const refreshNumUpdates = _.debounce(() => {
-      setNumUpdates(doc.numUpdates);
+      setNumUpdates(getNumRecords());
     }, 100);
-    setNumUpdates(doc.numUpdates);
+    setNumUpdates(getNumRecords());
-    doc.doc.on('update', () => {
+    doc.on('update', () => {
       setTimeout(refreshNumUpdates, 1500);
     });
 
     const state = EditorState.create({
-      doc: ydoc.getText().toString(),
+      doc: doc.getText().toString(),
       extensions: [
         EditorView.lineWrapping,
         EditorView.contentAttributes.of({
@@ -56,7 +56,7 @@ function Editor() {
         keymap.of([...defaultKeymap, ...historyKeymap]),
         scrollPastEnd(),
         syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
-        yCollab(ydoc.getText(), null),
+        yCollab(doc.getText(), null),
       ],
     });
     editorView = new EditorView({ state, parent: editorDiv });
@@ -65,7 +65,10 @@ function Editor() {
   onCleanup(() => {
     setNumUpdates(null);
     editorView?.destroy();
-    syncedDoc()?.finish();
+    const doc = syncedDoc();
+    if (doc) {
+      closeDoc(doc);
+    }
   });
 
   return (

src/persistence.ts

@@ -2,71 +2,65 @@ import * as Y from 'yjs';
 import { docsDb, LOCAL_DELTA_STORE } from './sync';
 import { assert } from './util';
 import _ from 'lodash';
+import { debugLog } from './components/Debug';
 
-const COMPACTION_THRESHOLD = 200;
+const COMPACTION_THRESHOLD = 50;
 
 export async function openDoc(id: string): Promise<Y.Doc> {
   // Return existing doc if we still have it open
   const previousState = idToState.get(id);
   if (previousState) {
     previousState.refCount += 1;
-    return previousState.ydoc;
+    return previousState.savedDoc.ydoc;
   }
 
-  const state = await initialLoad(id);
+  const state: State = {
+    refCount: 1,
+    savedDoc: new SavedDoc(id),
+  };
   idToState.set(id, state);
-  docToState.set(state.ydoc, state);
+  docToId.set(state.savedDoc.ydoc, id);
+  await state.savedDoc.connect();
+  debugLog(`Opened doc ${id}`);
 
-  return state.ydoc;
+  return state.savedDoc.ydoc;
+}
+
+export function getDocStats(doc: Y.Doc) {
+  const id = docToId.get(doc);
+  assert(id, 'Document is not already open');
+  const state = idToState.get(id)!;
+  return state.savedDoc.getInfo();
 }
 
 export function closeDoc(doc: Y.Doc) {
-  const state = docToState.get(doc);
+  const id = docToId.get(doc);
-  assert(state, 'Document is not already open');
+  assert(id, 'Document is not already open');
+  const state = idToState.get(id)!;
 
   state.refCount -= 1;
   if (state.refCount <= 0) {
-    idToState.delete(state.id);
+    idToState.delete(id);
-    docToState.delete(doc);
+    docToId.delete(doc);
-    doc.destroy();
+    state.savedDoc.flush().then(() => doc.destroy());
   }
+  debugLog(`Closed doc ${id} (refcount ${state.refCount})`);
 }
 
+const docToId = new Map<Y.Doc, string>();
 const idToState = new Map<string, State>();
-const docToState = new Map<Y.Doc, State>();
 
 type State = {
-  // TODO: Move more of this into class
-  id: string;
-  deltaBuffer: Uint8Array[];
-  numRecords: number;
   refCount: number;
-  ydoc: Y.Doc;
+  savedDoc: SavedDoc;
 };
 
-async function initialLoad(id: string): Promise<State> {
+class SavedDoc {
-  const ydoc = new Y.Doc();
+  readonly docId: string;
-  const records = await docsDb.getAll(
+  readonly ydoc: Y.Doc;
-    LOCAL_DELTA_STORE,
-    IDBKeyRange.bound([id, -Infinity], [id, Infinity])
-  );
-  Y.applyUpdate(ydoc, Y.mergeUpdates(records));
-  return {
-    id,
-    deltaBuffer: [],
-    numRecords: records.length,
-    refCount: 1,
-    ydoc,
-  };
-}
-
-// @ts-ignore
-class BufferedWriter {
   private deltaBuffer: Uint8Array[];
-  private docId: string;
   private numRecords: number;
   private debouncedFlush: () => void;
-  private ydoc: Y.Doc;
 
   constructor(docId: string) {
     this.deltaBuffer = [];
@@ -82,6 +76,10 @@ class BufferedWriter {
     this.ydoc = new Y.Doc();
   }
 
+  public getInfo(): { numRecords: number } {
+    return { numRecords: this.numRecords };
+  }
+
   public async connect(): Promise<Y.Doc> {
     const records = await docsDb.getAll(
       LOCAL_DELTA_STORE,
@@ -129,6 +127,9 @@ class BufferedWriter {
     if (this.numRecords < COMPACTION_THRESHOLD) {
       return;
     }
+    debugLog(
+      `Running compaction on doc ${this.docId}, ${this.numRecords} records...`
+    );
     const tx = docsDb.transaction(LOCAL_DELTA_STORE, 'readwrite');
 
     // Make sure we're not missing anything by pulling in all changes.

src/sync.ts

@@ -3,7 +3,7 @@ import * as Y from 'yjs';
 import { openDB, type DBSchema } from 'idb';
 import { assert, generateId } from './util';
 import { getDocsMap, getVaultMap, type DocMap } from './vault';
-import { debugLog } from './components/Debug';
+import { openDoc } from './persistence';
 
 const DB_NAME = 'synced-docs';
 export const LOCAL_DELTA_STORE = 'local-updates';
@@ -35,76 +35,6 @@ export const docsDb = await openDB<DocsDatabase>(DB_NAME, 3, {
   },
 });
 
-export class LocalDocument {
-  readonly id: string;
-  readonly doc: Y.Doc;
-  private updates: Uint8Array[];
-  private totalUpdates: number;
-
-  private constructor(id: string, ydoc: Y.Doc) {
-    this.id = id;
-    this.doc = ydoc;
-    this.updates = [];
-    this.totalUpdates = 0;
-
-    const flush = _.debounce(() => this.flush(), 1000);
-    this.doc.on('update', (update: Uint8Array) => {
-      this.updates.push(update);
-      flush();
-    });
-  }
-
-  get numUpdates(): number {
-    return this.totalUpdates;
-  }
-
-  public static async load(id: string): Promise<LocalDocument> {
-    // Build Y.Doc from records on disk
-    const startMs = performance.now();
-    const updates: Uint8Array[] = await docsDb.getAll(
-      LOCAL_DELTA_STORE,
-      IDBKeyRange.bound([id, -Infinity], [id, Infinity])
-    );
-    const loadedMs = performance.now();
-    const mergedUpdate = Y.mergeUpdates(updates);
-    const mergedMs = performance.now();
-    const ydoc = new Y.Doc();
-    Y.applyUpdate(ydoc, mergedUpdate);
-    const appliedMs = performance.now();
-
-    // Build LocalDocument instance
-    const result = new LocalDocument(id, ydoc);
-    result.totalUpdates = updates.length;
-
-    // Emit debug logs
-    const timings = [loadedMs - startMs, mergedMs - loadedMs, appliedMs - mergedMs];
-    const sizes = _.sortBy(updates.map((x) => x.length));
-    const quartiles = _.range(5).map((x) => {
-      const i = Math.round((sizes.length - 1) * (x / 4));
-      return sizes[i];
-    });
-    debugLog(`Doc ${id}: load/merge/apply timings in ms`, timings);
-    debugLog(`${updates.length} updates, quartiles in bytes`, quartiles);
-    return result;
-  }
-
-  public async flush(): Promise<void> {
-    const batch = this.updates;
-    this.updates = [];
-    if (batch.length > 0) {
-      const update = Y.mergeUpdates(batch);
-      const key: LocalDeltaKey = [this.id, new Date().valueOf()];
-      await docsDb.add(LOCAL_DELTA_STORE, update, key);
-      this.totalUpdates += 1;
-    }
-  }
-
-  public async finish(): Promise<void> {
-    await this.flush();
-    this.doc.destroy();
-  }
-}
-
 async function getOrCreateVaultDoc() {
   const records = await docsDb.getAll(VAULT_STORE);
   if (records.length > 1) {
@@ -120,8 +50,8 @@ async function getOrCreateVaultDoc() {
   if (!id) {
     throw new Error("Vault record didn't have id");
   }
-  const vault = await LocalDocument.load(id);
+  const vault = await openDoc(id);
-  const vaultMap = getVaultMap(vault.doc);
+  const vaultMap = getVaultMap(vault);
   if (!vaultMap.has('id')) {
     vaultMap.set('id', id);
   }
@@ -141,9 +71,8 @@ async function getOrCreateVaultDoc() {
 }
 
 export const defaultVault = await getOrCreateVaultDoc();
-// console.log(defaultVault.doc.getMap().toJSON())
 
-const docsMap = getDocsMap(defaultVault.doc);
+const docsMap = getDocsMap(defaultVault);
 if (docsMap.size === 0) {
   console.log('Running vault migration...');
   for (const oldKey of ['foo', 'bar', 'baz']) {