diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts index 990e0f906c6..773c575772e 100644 --- a/apps/sim/lib/collab-doc/converter.ts +++ b/apps/sim/lib/collab-doc/converter.ts @@ -17,6 +17,7 @@ import { parseMarkdownToDoc, serializeDocToMarkdown, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { COLLAB_DOC_FIELD } from './normalize' /** * Server-side conversion between a file's markdown and its collaborative Yjs document. @@ -36,13 +37,6 @@ import { * 'server-only'` marker because this repo does not use that package. */ -/** - * The Yjs `XmlFragment` name TipTap's Collaboration extension binds to. The client configures - * `Collaboration.configure({ document })` with no explicit `field`, so it uses TipTap's default, - * `'default'`. The server MUST target the same fragment or the client would sync an empty document. - */ -const COLLAB_DOC_FIELD = 'default' - let cachedSchema: Schema | null = null /** The shared ProseMirror schema, built headlessly from the exact client extension set. */ diff --git a/apps/sim/lib/collab-doc/normalize.test.ts b/apps/sim/lib/collab-doc/normalize.test.ts new file mode 100644 index 00000000000..f57bad15c72 --- /dev/null +++ b/apps/sim/lib/collab-doc/normalize.test.ts @@ -0,0 +1,127 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import * as Y from 'yjs' +import { COLLAB_DOC_FIELD, stripEmptyTopLevelParagraphs } from './normalize' + +/** Build a top-level element with the given tag and optional text content. */ +function element(tag: string, text?: string): Y.XmlElement { + const el = new Y.XmlElement(tag) + if (text !== undefined) el.insert(0, [new Y.XmlText(text)]) + return el +} + +/** Recursively concatenate the visible text of a Yjs XML node. */ +function textOf(node: Y.XmlElement | Y.XmlText | Y.XmlHook): string { + if (node instanceof Y.XmlText) return node.toString() + if (node instanceof Y.XmlElement) { + let text = '' + for (let i = 0; i < node.length; i++) text += textOf(node.get(i)) + return text + } + return '' +} + +/** The ordered list of top-level `[tag, text]` pairs currently in a doc's body fragment. */ +function structure(doc: Y.Doc): Array<[string, string]> { + const fragment = doc.getXmlFragment(COLLAB_DOC_FIELD) + const out: Array<[string, string]> = [] + for (let i = 0; i < fragment.length; i++) { + const node = fragment.get(i) + out.push([node instanceof Y.XmlElement ? node.nodeName! : 'text', textOf(node)]) + } + return out +} + +describe('stripEmptyTopLevelParagraphs', () => { + it('removes interior empty paragraphs while preserving content and order (production repro)', () => { + // Mirrors the persisted snapshot for random_data.md: a description paragraph, TWO consecutive empty + // paragraphs (the reported "two spaces"), then a bullet list, then another interior empty paragraph. + const doc = new Y.Doc() + const fragment = doc.getXmlFragment(COLLAB_DOC_FIELD) + fragment.insert(0, [ + element('paragraph', 'A small collection of sample data.'), + element('paragraph'), + element('paragraph'), + element('bulletList', 'list'), + element('paragraph'), + element('paragraph', 'trailing content'), + ]) + + expect(stripEmptyTopLevelParagraphs(doc)).toBe(true) + expect(structure(doc)).toEqual([ + ['paragraph', 'A small collection of sample data.'], + ['bulletList', 'list'], + ['paragraph', 'trailing content'], + ]) + doc.destroy() + }) + + it('is idempotent — a second pass finds nothing to remove', () => { + const doc = new Y.Doc() + doc + .getXmlFragment(COLLAB_DOC_FIELD) + .insert(0, [element('paragraph'), element('paragraph', 'body')]) + + expect(stripEmptyTopLevelParagraphs(doc)).toBe(true) + expect(stripEmptyTopLevelParagraphs(doc)).toBe(false) + expect(structure(doc)).toEqual([['paragraph', 'body']]) + doc.destroy() + }) + + it('returns false and mutates nothing when there are no top-level empty paragraphs', () => { + const doc = new Y.Doc() + doc + .getXmlFragment(COLLAB_DOC_FIELD) + .insert(0, [element('heading', 'Title'), element('paragraph', 'body')]) + + expect(stripEmptyTopLevelParagraphs(doc)).toBe(false) + expect(structure(doc)).toEqual([ + ['heading', 'Title'], + ['paragraph', 'body'], + ]) + doc.destroy() + }) + + it('leaves an empty paragraph nested inside another block untouched (only top-level is stripped)', () => { + const doc = new Y.Doc() + const listItem = new Y.XmlElement('listItem') + listItem.insert(0, [new Y.XmlElement('paragraph')]) // an empty paragraph BELOW the fragment root + const list = new Y.XmlElement('bulletList') + list.insert(0, [listItem]) + doc.getXmlFragment(COLLAB_DOC_FIELD).insert(0, [list]) + + expect(stripEmptyTopLevelParagraphs(doc)).toBe(false) + const nestedList = doc.getXmlFragment(COLLAB_DOC_FIELD).get(0) as Y.XmlElement + const nestedItem = nestedList.get(0) as Y.XmlElement + expect(nestedItem.get(0)).toBeInstanceOf(Y.XmlElement) + expect((nestedItem.get(0) as Y.XmlElement).nodeName).toBe('paragraph') + doc.destroy() + }) + + it('survives an encode/decode round-trip preserving CRDT ids and the config map (seed-repair path)', () => { + const original = new Y.Doc() + original + .getXmlFragment(COLLAB_DOC_FIELD) + .insert(0, [element('paragraph', 'kept'), element('paragraph')]) + original.getMap('config').set('initialContentLoaded', true) + original.getMap('config').set('frontmatter', 'title: x') + const before = Y.encodeStateAsUpdate(original) + original.destroy() + + // Repair exactly as normalizeSeedUpdate does: apply → strip → re-encode. + const repair = new Y.Doc() + Y.applyUpdate(repair, before) + expect(stripEmptyTopLevelParagraphs(repair)).toBe(true) + const after = Y.encodeStateAsUpdate(repair) + repair.destroy() + + const seeded = new Y.Doc() + Y.applyUpdate(seeded, after) + expect(structure(seeded)).toEqual([['paragraph', 'kept']]) + expect(seeded.getMap('config').get('initialContentLoaded')).toBe(true) + expect(seeded.getMap('config').get('frontmatter')).toBe('title: x') + seeded.destroy() + }) +}) diff --git a/apps/sim/lib/collab-doc/normalize.ts b/apps/sim/lib/collab-doc/normalize.ts new file mode 100644 index 00000000000..a82075f79be --- /dev/null +++ b/apps/sim/lib/collab-doc/normalize.ts @@ -0,0 +1,43 @@ +import * as Y from 'yjs' + +/** + * The Yjs `XmlFragment` name TipTap's Collaboration extension binds to (its default `field`). The + * client configures `Collaboration.configure({ document })` with no explicit `field`, so it uses + * TipTap's default, `'default'`. Server-side conversion, seeding, and persistence MUST target the same + * fragment or the client would sync an empty document — so this is the single canonical source consumed + * by both bundles (it imports only `yjs`, making it safe from client and server alike). + */ +export const COLLAB_DOC_FIELD = 'default' + +/** + * Remove every top-level empty paragraph (a `paragraph` element with no children) from a collaborative + * document's body fragment, returning whether it deleted any. + * + * The markdown parse pipeline strips these from EVERY parse target (see `stripEmptyParagraphs` in + * `markdown-parse.ts`): in markdown a run of blank lines between blocks is insignificant, so the static + * placeholder, the download, and every standard renderer show no interior blank. A cached Yjs snapshot, + * however, is a raw CRDT binary that bypasses that parse — so it can preserve an empty-paragraph node the + * re-parse would have dropped. When a warm room seeds from such a snapshot, the empty paragraph surfaces + * as a stray blank line appearing once the doc settles, diverging from the placeholder that was shown + * first. Enforcing the same no-top-level-empty-paragraph invariant on the Yjs side keeps the live + * collaborative doc rendering identically to the markdown re-parse. + * + * Idempotent, and only TOP-LEVEL paragraphs are touched — blank lines that carry meaning inside a + * construct (e.g. a loose list) live below the fragment root and are left alone. Runs its own Yjs + * transaction so the deletions commit atomically, iterating the fragment back-to-front so a deletion + * never shifts a not-yet-checked index. + */ +export function stripEmptyTopLevelParagraphs(doc: Y.Doc): boolean { + const fragment = doc.getXmlFragment(COLLAB_DOC_FIELD) + let removed = false + doc.transact(() => { + for (let i = fragment.length - 1; i >= 0; i--) { + const node = fragment.get(i) + if (node instanceof Y.XmlElement && node.nodeName === 'paragraph' && node.length === 0) { + fragment.delete(i, 1) + removed = true + } + } + }) + return removed +} diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index e07b292d628..7ba4edd9cfa 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -8,6 +8,7 @@ import { } from '@/lib/uploads/contexts/workspace' import { hashMarkdown, saveCollabDocState } from './collab-state' import { yDocToFileMarkdown } from './converter' +import { stripEmptyTopLevelParagraphs } from './normalize' const logger = createLogger('FileDocPersist') @@ -68,8 +69,14 @@ export async function persistFileDoc( const ydoc = new Y.Doc() let markdownBuffer: Buffer + // The Yjs snapshot cached below (`saveCollabDocState`) seeds a later cold room open directly, so it + // must never carry structure the markdown re-parse would strip — a top-level empty paragraph left in + // the snapshot resurfaces as a stray blank line when that warm doc settles, diverging from the static + // placeholder. Normalize it out here so the cached binary matches the durable markdown by construction. + let cachedDocState = docState try { Y.applyUpdate(ydoc, docState) + if (stripEmptyTopLevelParagraphs(ydoc)) cachedDocState = Y.encodeStateAsUpdate(ydoc) markdownBuffer = Buffer.from(yDocToFileMarkdown(ydoc), 'utf-8') } finally { ydoc.destroy() @@ -93,7 +100,7 @@ export async function persistFileDoc( // Cache the Yjs binary (tagged with the exact markdown just written) so a later cold room open loads // it directly instead of re-converting. Best-effort — the markdown is the durable source of truth. try { - await saveCollabDocState(fileId, docState, hashMarkdown(markdownBuffer)) + await saveCollabDocState(fileId, cachedDocState, hashMarkdown(markdownBuffer)) } catch (error) { logger.warn(`Failed to cache collab doc state for file ${fileId}`, { error: getErrorMessage(error), diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts index ece65549bf0..d0c7fa73f4d 100644 --- a/apps/sim/lib/collab-doc/seed.ts +++ b/apps/sim/lib/collab-doc/seed.ts @@ -6,6 +6,7 @@ import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contex import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { hashMarkdown, loadFreshCollabDocState } from './collab-state' import { markdownToYDoc } from './converter' +import { stripEmptyTopLevelParagraphs } from './normalize' const logger = createLogger('FileDocSeed') @@ -26,6 +27,24 @@ export interface FileDocSeed { version: number } +/** + * Repair a cached Yjs snapshot before it seeds a room: strip any top-level empty paragraphs the markdown + * re-parse would drop (see {@link stripEmptyTopLevelParagraphs}), so a warm seed renders identically to + * the static placeholder and never surfaces a stray blank line once the doc settles. Returns the original + * bytes untouched when the snapshot is already clean (the common case) — no re-encode cost — and a fresh + * encode (preserving the CRDT's client ids, only adding tombstones for the removed empties) when it + * repaired a legacy snapshot baked before this normalization existed. + */ +function normalizeSeedUpdate(cached: Uint8Array): Uint8Array { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, cached) + return stripEmptyTopLevelParagraphs(doc) ? Y.encodeStateAsUpdate(doc) : cached + } finally { + doc.destroy() + } +} + /** * Build the server-side seed for a file's collaborative document: load the file's current markdown * and convert it — through the exact client engine (see {@link markdownToYDoc}) — into a Yjs update. @@ -63,7 +82,7 @@ export async function buildFileDocSeed( // block the cold open — symmetric with persist's best-effort cache write. try { const cached = await loadFreshCollabDocState(fileId, hashMarkdown(buffer)) - if (cached) return { update: cached, version } + if (cached) return { update: normalizeSeedUpdate(cached), version } } catch (error) { logger.warn(`Failed to read cached collab doc state for file ${fileId}`, { error: getErrorMessage(error),