cmounce/clementine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tear out old code and turn on compaction

Browse Source
This commit is contained in:
Chris Mounce
2025-09-25 23:55:30 -07:00
parent c53f596d1d
commit b695437415
6 changed files with 63 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/App.tsx
+1 -1
View File
@@ -34,7 +34,7 @@ function Layout(props: any) {
const navigate = useNavigate();
const [fileId, setFileId] = createSignal<string | null>(null);
const [numUpdates, setNumUpdates] = createSignal<number | null>(null);
const docsMap = getDocsMap(defaultVault.doc);
const docsMap = getDocsMap(defaultVault);
const title = () => {
const id = fileId();
src/components/Chooser.tsx
+1 -1
View File
@@ -12,7 +12,7 @@ function Chooser() {
const { setFileId } = useNavbar();
onMount(() => setFileId(null));
const docsMap = getDocsMap(defaultVault.doc);
const docsMap = getDocsMap(defaultVault);
const computeDocsList = () => {
const result: DocInfo[] = [];
src/components/Debug.tsx
+8 -2
View File
@@ -3,11 +3,17 @@ import { createSignal, For } from 'solid-js';
const [getDebugLogs, setDebugLogs] = createSignal<string[]>([]);
export function debugLog(message: string, ...args: any[]) {
console.log(message, ...args);
const components = [message, ...args.map((x) => JSON.stringify(x))];
const line = components.join(' ');
setDebugLogs([...getDebugLogs(), line]);
const allLines = [...getDebugLogs(), line];
if (allLines.length > 1000) {
allLines.splice(0, allLines.length - 1000);
}
setDebugLogs(allLines);
}
export default function DebugView() {
return <For each={getDebugLogs()}>{(logLine) => <p>{logLine}</p>}</For>;
return <For each={getDebugLogs()}>{(logLine) => <div>{logLine}</div>}</For>;
}
src/components/Editor.tsx
+12 -9
View File
@@ -13,10 +13,10 @@ import {
onMount,
Show,
} from 'solid-js';
import { LocalDocument } from '../sync';
import { useNavbar } from './App';
import { useParams } from '@solidjs/router';
import _ from 'lodash';
import { closeDoc, getDocStats, openDoc } from '../persistence';
function Editor() {
const { id: fileId } = useParams();
@@ -27,25 +27,25 @@ function Editor() {
let editorDiv;
let editorView: EditorView;
const [syncedDoc] = createResource(async () => LocalDocument.load(fileId));
const [syncedDoc] = createResource(async () => openDoc(fileId));
createEffect(() => {
const doc = syncedDoc();
if (!doc) {
return;
}
const ydoc = doc.doc;
const getNumRecords = () => getDocStats(doc).numRecords;
const refreshNumUpdates = _.debounce(() => {
setNumUpdates(doc.numUpdates);
setNumUpdates(getNumRecords());
}, 100);
setNumUpdates(doc.numUpdates);
doc.doc.on('update', () => {
setNumUpdates(getNumRecords());
doc.on('update', () => {
setTimeout(refreshNumUpdates, 1500);
});
const state = EditorState.create({
doc: ydoc.getText().toString(),
doc: doc.getText().toString(),
extensions: [
EditorView.lineWrapping,
EditorView.contentAttributes.of({
@@ -56,7 +56,7 @@ function Editor() {
keymap.of([...defaultKeymap, ...historyKeymap]),
scrollPastEnd(),
syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
yCollab(ydoc.getText(), null),
yCollab(doc.getText(), null),
],
});
editorView = new EditorView({ state, parent: editorDiv });
@@ -65,7 +65,10 @@ function Editor() {
onCleanup(() => {
setNumUpdates(null);
editorView?.destroy();
syncedDoc()?.finish();
const doc = syncedDoc();
if (doc) {
closeDoc(doc);
}
});
return (
src/persistence.ts
+37 -36
View File
@@ -2,71 +2,65 @@ import * as Y from 'yjs';
import { docsDb, LOCAL_DELTA_STORE } from './sync';
import { assert } from './util';
import _ from 'lodash';
import { debugLog } from './components/Debug';
const COMPACTION_THRESHOLD = 200;
const COMPACTION_THRESHOLD = 50;
export async function openDoc(id: string): Promise<Y.Doc> {
// Return existing doc if we still have it open
const previousState = idToState.get(id);
if (previousState) {
previousState.refCount += 1;
return previousState.ydoc;
return previousState.savedDoc.ydoc;
}
const state = await initialLoad(id);
const state: State = {
refCount: 1,
savedDoc: new SavedDoc(id),
};
idToState.set(id, state);
docToState.set(state.ydoc, state);
docToId.set(state.savedDoc.ydoc, id);
await state.savedDoc.connect();
debugLog(`Opened doc ${id}`);
return state.ydoc;
return state.savedDoc.ydoc;
}
export function getDocStats(doc: Y.Doc) {
const id = docToId.get(doc);
assert(id, 'Document is not already open');
const state = idToState.get(id)!;
return state.savedDoc.getInfo();
}
export function closeDoc(doc: Y.Doc) {
const state = docToState.get(doc);
assert(state, 'Document is not already open');
const id = docToId.get(doc);
assert(id, 'Document is not already open');
const state = idToState.get(id)!;
state.refCount -= 1;
if (state.refCount <= 0) {
idToState.delete(state.id);
docToState.delete(doc);
doc.destroy();
idToState.delete(id);
docToId.delete(doc);
state.savedDoc.flush().then(() => doc.destroy());
}
debugLog(`Closed doc ${id} (refcount ${state.refCount})`);
}
const docToId = new Map<Y.Doc, string>();
const idToState = new Map<string, State>();
const docToState = new Map<Y.Doc, State>();
type State = {
// TODO: Move more of this into class
id: string;
deltaBuffer: Uint8Array[];
numRecords: number;
refCount: number;
ydoc: Y.Doc;
savedDoc: SavedDoc;
};
async function initialLoad(id: string): Promise<State> {
const ydoc = new Y.Doc();
const records = await docsDb.getAll(
LOCAL_DELTA_STORE,
IDBKeyRange.bound([id, -Infinity], [id, Infinity])
);
Y.applyUpdate(ydoc, Y.mergeUpdates(records));
return {
id,
deltaBuffer: [],
numRecords: records.length,
refCount: 1,
ydoc,
};
}
// @ts-ignore
class BufferedWriter {
class SavedDoc {
readonly docId: string;
readonly ydoc: Y.Doc;
private deltaBuffer: Uint8Array[];
private docId: string;
private numRecords: number;
private debouncedFlush: () => void;
private ydoc: Y.Doc;
constructor(docId: string) {
this.deltaBuffer = [];
@@ -82,6 +76,10 @@ class BufferedWriter {
this.ydoc = new Y.Doc();
}
public getInfo(): { numRecords: number } {
return { numRecords: this.numRecords };
}
public async connect(): Promise<Y.Doc> {
const records = await docsDb.getAll(
LOCAL_DELTA_STORE,
@@ -129,6 +127,9 @@ class BufferedWriter {
if (this.numRecords < COMPACTION_THRESHOLD) {
return;
}
debugLog(
`Running compaction on doc ${this.docId}, ${this.numRecords} records...`
);
const tx = docsDb.transaction(LOCAL_DELTA_STORE, 'readwrite');
// Make sure we're not missing anything by pulling in all changes.
src/sync.ts
+4 -75
View File
@@ -3,7 +3,7 @@ import * as Y from 'yjs';
import { openDB, type DBSchema } from 'idb';
import { assert, generateId } from './util';
import { getDocsMap, getVaultMap, type DocMap } from './vault';
import { debugLog } from './components/Debug';
import { openDoc } from './persistence';
const DB_NAME = 'synced-docs';
export const LOCAL_DELTA_STORE = 'local-updates';
@@ -35,76 +35,6 @@ export const docsDb = await openDB<DocsDatabase>(DB_NAME, 3, {
},
});
export class LocalDocument {
readonly id: string;
readonly doc: Y.Doc;
private updates: Uint8Array[];
private totalUpdates: number;
private constructor(id: string, ydoc: Y.Doc) {
this.id = id;
this.doc = ydoc;
this.updates = [];
this.totalUpdates = 0;
const flush = _.debounce(() => this.flush(), 1000);
this.doc.on('update', (update: Uint8Array) => {
this.updates.push(update);
flush();
});
}
get numUpdates(): number {
return this.totalUpdates;
}
public static async load(id: string): Promise<LocalDocument> {
// Build Y.Doc from records on disk
const startMs = performance.now();
const updates: Uint8Array[] = await docsDb.getAll(
LOCAL_DELTA_STORE,
IDBKeyRange.bound([id, -Infinity], [id, Infinity])
);
const loadedMs = performance.now();
const mergedUpdate = Y.mergeUpdates(updates);
const mergedMs = performance.now();
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, mergedUpdate);
const appliedMs = performance.now();
// Build LocalDocument instance
const result = new LocalDocument(id, ydoc);
result.totalUpdates = updates.length;
// Emit debug logs
const timings = [loadedMs - startMs, mergedMs - loadedMs, appliedMs - mergedMs];
const sizes = _.sortBy(updates.map((x) => x.length));
const quartiles = _.range(5).map((x) => {
const i = Math.round((sizes.length - 1) * (x / 4));
return sizes[i];
});
debugLog(`Doc ${id}: load/merge/apply timings in ms`, timings);
debugLog(`${updates.length} updates, quartiles in bytes`, quartiles);
return result;
}
public async flush(): Promise<void> {
const batch = this.updates;
this.updates = [];
if (batch.length > 0) {
const update = Y.mergeUpdates(batch);
const key: LocalDeltaKey = [this.id, new Date().valueOf()];
await docsDb.add(LOCAL_DELTA_STORE, update, key);
this.totalUpdates += 1;
}
}
public async finish(): Promise<void> {
await this.flush();
this.doc.destroy();
}
}
async function getOrCreateVaultDoc() {
const records = await docsDb.getAll(VAULT_STORE);
if (records.length > 1) {
@@ -120,8 +50,8 @@ async function getOrCreateVaultDoc() {
if (!id) {
throw new Error("Vault record didn't have id");
}
const vault = await LocalDocument.load(id);
const vaultMap = getVaultMap(vault.doc);
const vault = await openDoc(id);
const vaultMap = getVaultMap(vault);
if (!vaultMap.has('id')) {
vaultMap.set('id', id);
}
@@ -141,9 +71,8 @@ async function getOrCreateVaultDoc() {
}
export const defaultVault = await getOrCreateVaultDoc();
// console.log(defaultVault.doc.getMap().toJSON())
const docsMap = getDocsMap(defaultVault.doc);
const docsMap = getDocsMap(defaultVault);
if (docsMap.size === 0) {
console.log('Running vault migration...');
for (const oldKey of ['foo', 'bar', 'baz']) {