From f9e1ff7b7c6fec53f086f0a8686ec4857bbcf62e Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 3 Dec 2023 23:31:05 +0100 Subject: [PATCH 01/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?bility=20to=20print=20file=20to=20human-readable=20string?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 16 ++++++++++++++-- src/json-crdt/file/PatchLog.ts | 20 +++++++++++++++++--- src/json-crdt/file/__tests__/File.spec.ts | 19 +++++++++++++++++++ src/json-crdt/model/Model.ts | 2 +- 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 src/json-crdt/file/__tests__/File.spec.ts diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 8a66dfad41..5468c90068 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -6,10 +6,12 @@ import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/E import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; +import {printTree} from "../../util/print/printTree"; import type * as types from "./types"; +import type {Printable} from "../../util/print/types"; -export class File { - public static fromModel(model: Model): File { +export class File implements Printable { + public static fromModel(model: Model): File { return new File(model, PatchLog.fromModel(model)); } @@ -89,4 +91,14 @@ export class File { history, ]; } + + // ---------------------------------------------------------------- Printable + + public toString(tab?: string) { + return `file` + printTree(tab, [ + (tab) => this.model.toString(tab), + () => '', + (tab) => this.history.toString(tab), + ]); + } } diff --git a/src/json-crdt/file/PatchLog.ts b/src/json-crdt/file/PatchLog.ts index 7aa07cf7b8..59dd18875d 100644 --- a/src/json-crdt/file/PatchLog.ts +++ b/src/json-crdt/file/PatchLog.ts @@ -1,9 +1,11 @@ -import {ITimestampStruct, Patch, ServerClockVector, compare} from "../../json-crdt-patch"; +import {ITimestampStruct, Patch, compare} from "../../json-crdt-patch"; +import {printTree} from "../../util/print/printTree"; import {AvlMap} from "../../util/trees/avl/AvlMap"; import {Model} from "../model"; +import type {Printable} from "../../util/print/types"; -export class PatchLog { - public static fromModel (model: Model): PatchLog { +export class PatchLog implements Printable { + public static fromModel(model: Model): PatchLog { const start = new Model(model.clock.clone()); const log = new PatchLog(start); if (model.api.builder.patch.ops.length) { @@ -22,4 +24,16 @@ export class PatchLog { if (!id) return; this.patches.set(id, patch); } + + // ---------------------------------------------------------------- Printable + + public toString(tab?: string) { + const log: Patch[] = []; + this.patches.forEach(({v}) => log.push(v)); + return `log` + printTree(tab, [ + (tab) => this.start.toString(tab), + () => '', + (tab) => 'history' + printTree(tab, log.map((patch, i) => (tab) => `${i}: ${patch.toString(tab)}`)), + ]); + } } diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts new file mode 100644 index 0000000000..7b3bd30f29 --- /dev/null +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -0,0 +1,19 @@ +import {s} from '../../../json-crdt-patch'; +import {Model} from '../../model'; +import {File} from '../File'; + +test('can create File from new model', () => { + const model = Model.withServerClock() + .setSchema(s.obj({ + foo: s.str('bar'), + })); + const file = File.fromModel(model); + expect(file.history.start.view()).toBe(undefined); + expect(file.model.view()).toEqual({ + foo: 'bar', + }); + expect(file.history.start.clock.sid).toBe(file.model.clock.sid); +}); + +test.todo('patches are flushed and stored in memory'); +test.todo('can replay history'); diff --git a/src/json-crdt/model/Model.ts b/src/json-crdt/model/Model.ts index c5876f1a05..ee0c321c70 100644 --- a/src/json-crdt/model/Model.ts +++ b/src/json-crdt/model/Model.ts @@ -21,7 +21,7 @@ export const UNDEFINED = new ConNode(ORIGIN, undefined); * In instance of Model class represents the underlying data structure, * i.e. model, of the JSON CRDT document. */ -export class Model implements Printable { +export class Model> implements Printable { /** * Create a CRDT model which uses logical clock. Logical clock assigns a * logical timestamp to every node and operation. Logical timestamp consists From 76bb74814f3e893dd7fa63ecf3caf87a559f4fe2 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 3 Dec 2023 23:31:37 +0100 Subject: [PATCH 02/16] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 40 +++++++---------------- src/json-crdt/file/PatchLog.ts | 30 ++++++++++------- src/json-crdt/file/__tests__/File.spec.ts | 7 ++-- src/json-crdt/file/types.ts | 17 +++------- 4 files changed, 38 insertions(+), 56 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 5468c90068..414b68038b 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -1,31 +1,25 @@ -import {Model} from "../model"; -import {PatchLog} from "./PatchLog"; -import {FileModelEncoding} from "./constants"; +import {Model} from '../model'; +import {PatchLog} from './PatchLog'; +import {FileModelEncoding} from './constants'; import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder'; import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder'; import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; -import {printTree} from "../../util/print/printTree"; -import type * as types from "./types"; -import type {Printable} from "../../util/print/types"; +import {printTree} from '../../util/print/printTree'; +import type * as types from './types'; +import type {Printable} from '../../util/print/types'; export class File implements Printable { public static fromModel(model: Model): File { return new File(model, PatchLog.fromModel(model)); } - constructor( - public readonly model: Model, - public readonly history: PatchLog, - ) {} + constructor(public readonly model: Model, public readonly history: PatchLog) {} public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { const view = this.model.view(); - const metadata: types.FileMetadata = [ - {}, - FileModelEncoding.SidecarBinary, - ]; + const metadata: types.FileMetadata = [{}, FileModelEncoding.SidecarBinary]; let model: Uint8Array | unknown | null = null; const modelFormat = params.model ?? 'sidecar'; switch (modelFormat) { @@ -54,10 +48,7 @@ export class File implements Printable { default: throw new Error(`Invalid model format: ${modelFormat}`); } - const history: types.FileWriteSequenceHistory = [ - null, - [], - ]; + const history: types.FileWriteSequenceHistory = [null, []]; const patchFormat = params.history ?? 'binary'; switch (patchFormat) { case 'binary': { @@ -84,21 +75,12 @@ export class File implements Printable { default: throw new Error(`Invalid history format: ${patchFormat}`); } - return [ - view, - metadata, - model, - history, - ]; + return [view, metadata, model, history]; } // ---------------------------------------------------------------- Printable public toString(tab?: string) { - return `file` + printTree(tab, [ - (tab) => this.model.toString(tab), - () => '', - (tab) => this.history.toString(tab), - ]); + return `file` + printTree(tab, [(tab) => this.model.toString(tab), () => '', (tab) => this.history.toString(tab)]); } } diff --git a/src/json-crdt/file/PatchLog.ts b/src/json-crdt/file/PatchLog.ts index 59dd18875d..3266b873ff 100644 --- a/src/json-crdt/file/PatchLog.ts +++ b/src/json-crdt/file/PatchLog.ts @@ -1,8 +1,8 @@ -import {ITimestampStruct, Patch, compare} from "../../json-crdt-patch"; -import {printTree} from "../../util/print/printTree"; -import {AvlMap} from "../../util/trees/avl/AvlMap"; -import {Model} from "../model"; -import type {Printable} from "../../util/print/types"; +import {ITimestampStruct, Patch, compare} from '../../json-crdt-patch'; +import {printTree} from '../../util/print/printTree'; +import {AvlMap} from '../../util/trees/avl/AvlMap'; +import {Model} from '../model'; +import type {Printable} from '../../util/print/types'; export class PatchLog implements Printable { public static fromModel(model: Model): PatchLog { @@ -17,7 +17,7 @@ export class PatchLog implements Printable { public readonly patches = new AvlMap(compare); - constructor (public readonly start: Model) {} + constructor(public readonly start: Model) {} public push(patch: Patch): void { const id = patch.getId(); @@ -30,10 +30,18 @@ export class PatchLog implements Printable { public toString(tab?: string) { const log: Patch[] = []; this.patches.forEach(({v}) => log.push(v)); - return `log` + printTree(tab, [ - (tab) => this.start.toString(tab), - () => '', - (tab) => 'history' + printTree(tab, log.map((patch, i) => (tab) => `${i}: ${patch.toString(tab)}`)), - ]); + return ( + `log` + + printTree(tab, [ + (tab) => this.start.toString(tab), + () => '', + (tab) => + 'history' + + printTree( + tab, + log.map((patch, i) => (tab) => `${i}: ${patch.toString(tab)}`), + ), + ]) + ); } } diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 7b3bd30f29..ecef897bd7 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -3,10 +3,11 @@ import {Model} from '../../model'; import {File} from '../File'; test('can create File from new model', () => { - const model = Model.withServerClock() - .setSchema(s.obj({ + const model = Model.withServerClock().setSchema( + s.obj({ foo: s.str('bar'), - })); + }), + ); const file = File.fromModel(model); expect(file.history.start.view()).toBe(undefined); expect(file.model.view()).toEqual({ diff --git a/src/json-crdt/file/types.ts b/src/json-crdt/file/types.ts index 32b4450530..e7127bd74d 100644 --- a/src/json-crdt/file/types.ts +++ b/src/json-crdt/file/types.ts @@ -1,9 +1,6 @@ -import type {FileModelEncoding} from "./constants"; +import type {FileModelEncoding} from './constants'; -export type FileMetadata = [ - map: {}, - modelFormat: FileModelEncoding, -]; +export type FileMetadata = [map: {}, modelFormat: FileModelEncoding]; export type FileWriteSequence = [ view: unknown | null, @@ -12,15 +9,9 @@ export type FileWriteSequence = [ history: FileWriteSequenceHistory, ]; -export type FileWriteSequenceHistory = [ - model: Uint8Array | unknown | null, - patches: Array, -]; +export type FileWriteSequenceHistory = [model: Uint8Array | unknown | null, patches: Array]; -export type FileReadSequence = [ - ...FileWriteSequence, - ...frontier: Array, -]; +export type FileReadSequence = [...FileWriteSequence, ...frontier: Array]; export interface FileSerializeParams { noView?: boolean; From ea2693acc673b948b0ae39399ebbadf3945b1957 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 10:18:03 +0100 Subject: [PATCH 03/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20.?= =?UTF-8?q?toBinary()=20for=20File?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 23 ++++++++++++++++++ src/json-crdt/file/__tests__/File.spec.ts | 29 +++++++++++++++++++++++ src/json-crdt/file/types.ts | 4 ++++ 3 files changed, 56 insertions(+) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 414b68038b..d3ab9c4477 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -6,6 +6,9 @@ import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/E import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; import {encode as encodeVerbose} from '../../json-crdt-patch/codec/verbose/encode'; +import {Writer} from '../../util/buffers/Writer'; +import {CborEncoder} from '../../json-pack/cbor/CborEncoder'; +import {JsonEncoder} from '../../json-pack/json/JsonEncoder'; import {printTree} from '../../util/print/printTree'; import type * as types from './types'; import type {Printable} from '../../util/print/types'; @@ -78,6 +81,26 @@ export class File implements Printable { return [view, metadata, model, history]; } + public toBinary(params: types.FileEncodingParams): Uint8Array { + const sequence = this.serialize(params); + const writer = new Writer(16 * 1024); + switch (params.format) { + case 'ndjson': { + const json = new JsonEncoder(writer); + for (const component of sequence) { + json.writeAny(component); + json.writer.u8('\n'.charCodeAt(0)); + } + return json.writer.flush(); + } + case 'seq.cbor': { + const cbor = new CborEncoder(writer); + for (const component of sequence) cbor.writeAny(component); + return cbor.writer.flush(); + } + } + } + // ---------------------------------------------------------------- Printable public toString(tab?: string) { diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index ecef897bd7..1326325722 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -1,6 +1,15 @@ import {s} from '../../../json-crdt-patch'; import {Model} from '../../model'; import {File} from '../File'; +import {JsonDecoder} from '../../../json-pack/json/JsonDecoder'; +import {CborDecoder} from '../../../json-pack/cbor/CborDecoder'; + +const setup = (view: unknown) => { + const model = Model.withServerClock(); + model.api.root(view); + const file = File.fromModel(model); + return {model, file}; +}; test('can create File from new model', () => { const model = Model.withServerClock().setSchema( @@ -18,3 +27,23 @@ test('can create File from new model', () => { test.todo('patches are flushed and stored in memory'); test.todo('can replay history'); + +describe('.toBinary()', () => { + describe('can read first value as view', () => { + test('.ndjson', () => { + const {file} = setup({foo: 'bar'}); + const blob = file.toBinary({format: 'ndjson', model: 'compact', history: 'compact'}); + const decoder = new JsonDecoder(); + const view = decoder.read(blob); + expect(view).toEqual({foo: 'bar'}); + }); + + test('.seq.cbor', () => { + const {file} = setup({foo: 'bar'}); + const blob = file.toBinary({format: 'seq.cbor'}); + const decoder = new CborDecoder(); + const view = decoder.read(blob); + expect(view).toEqual({foo: 'bar'}); + }); + }); +}); diff --git a/src/json-crdt/file/types.ts b/src/json-crdt/file/types.ts index e7127bd74d..eb0891cdaf 100644 --- a/src/json-crdt/file/types.ts +++ b/src/json-crdt/file/types.ts @@ -18,3 +18,7 @@ export interface FileSerializeParams { model?: 'sidecar' | 'binary' | 'compact' | 'verbose'; history?: 'binary' | 'compact' | 'verbose'; } + +export interface FileEncodingParams extends FileSerializeParams { + format: 'ndjson' | 'seq.cbor'; +} From e1d9c039f20d09e00ffd964cb952e30c30ecc1f7 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 12:00:24 +0100 Subject: [PATCH 04/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20impleme?= =?UTF-8?q?nt=20file=20encoding=20and=20decoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 65 ++++++++++++++++++++--- src/json-crdt/file/PatchLog.ts | 6 +++ src/json-crdt/file/__tests__/File.spec.ts | 26 ++++++++- src/json-crdt/file/types.ts | 4 +- src/json-crdt/file/util.ts | 52 ++++++++++++++++++ 5 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 src/json-crdt/file/util.ts diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index d3ab9c4477..b6b11b03f2 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -2,6 +2,7 @@ import {Model} from '../model'; import {PatchLog} from './PatchLog'; import {FileModelEncoding} from './constants'; import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder'; +import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder'; import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder'; import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder'; import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode'; @@ -10,15 +11,55 @@ import {Writer} from '../../util/buffers/Writer'; import {CborEncoder} from '../../json-pack/cbor/CborEncoder'; import {JsonEncoder} from '../../json-pack/json/JsonEncoder'; import {printTree} from '../../util/print/printTree'; +import {decodeModel, decodeNdjsonComponents, decodePatch, decodeSeqCborComponents} from './util'; import type * as types from './types'; import type {Printable} from '../../util/print/types'; export class File implements Printable { + public static unserialize(components: types.FileReadSequence): File { + const [view, metadata, model, history, ...frontier] = components; + const modelFormat = metadata[1]; + let decodedModel: Model | null = null; + if (model && modelFormat !== FileModelEncoding.None) { + const isSidecar = modelFormat === FileModelEncoding.SidecarBinary; + if (isSidecar) { + const decoder = new SidecarDecoder(); + if (!(model instanceof Uint8Array)) throw new Error('NOT_BLOB'); + decodedModel = decoder.decode(view, model); + } else { + decodedModel = decodeModel(model); + } + } + let log: PatchLog | null = null; + if (history) { + const [start, patches] = history; + if (start) { + const startModel = decodeModel(start); + log = new PatchLog(startModel); + for (const patch of patches) log.push(decodePatch(patch)); + } + } + if (!log) throw new Error('NO_HISTORY'); + if (!decodedModel) decodedModel = log.replayToEnd(); + const file = new File(decodedModel, log); + return file; + } + + public static fromNdjson(blob: Uint8Array): File { + const components = decodeNdjsonComponents(blob); + return File.unserialize(components as types.FileReadSequence); + } + + public static fromSeqCbor(blob: Uint8Array): File { + const components = decodeSeqCborComponents(blob); + return File.unserialize(components as types.FileReadSequence); + } + public static fromModel(model: Model): File { return new File(model, PatchLog.fromModel(model)); } - constructor(public readonly model: Model, public readonly history: PatchLog) {} + constructor(public readonly model: Model, public readonly log: PatchLog) {} public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { const view = this.model.view(); @@ -48,6 +89,11 @@ export class File implements Printable { model = new StructuralEncoderVerbose().encode(this.model); break; } + case 'none': { + metadata[1] = FileModelEncoding.None; + model = null; + break; + } default: throw new Error(`Invalid model format: ${modelFormat}`); } @@ -55,26 +101,29 @@ export class File implements Printable { const patchFormat = params.history ?? 'binary'; switch (patchFormat) { case 'binary': { - history[0] = this.history.start.toBinary(); - this.history.patches.forEach(({v}) => { + history[0] = this.log.start.toBinary(); + this.log.patches.forEach(({v}) => { history[1].push(v.toBinary()); }); break; } case 'compact': { - history[0] = new StructuralEncoderCompact().encode(this.history.start); - this.history.patches.forEach(({v}) => { + history[0] = new StructuralEncoderCompact().encode(this.log.start); + this.log.patches.forEach(({v}) => { history[1].push(encodeCompact(v)); }); break; } case 'verbose': { - history[0] = new StructuralEncoderVerbose().encode(this.history.start); - this.history.patches.forEach(({v}) => { + history[0] = new StructuralEncoderVerbose().encode(this.log.start); + this.log.patches.forEach(({v}) => { history[1].push(encodeVerbose(v)); }); break; } + case 'none': { + break; + } default: throw new Error(`Invalid history format: ${patchFormat}`); } @@ -104,6 +153,6 @@ export class File implements Printable { // ---------------------------------------------------------------- Printable public toString(tab?: string) { - return `file` + printTree(tab, [(tab) => this.model.toString(tab), () => '', (tab) => this.history.toString(tab)]); + return `file` + printTree(tab, [(tab) => this.model.toString(tab), () => '', (tab) => this.log.toString(tab)]); } } diff --git a/src/json-crdt/file/PatchLog.ts b/src/json-crdt/file/PatchLog.ts index 3266b873ff..cd746a0002 100644 --- a/src/json-crdt/file/PatchLog.ts +++ b/src/json-crdt/file/PatchLog.ts @@ -25,6 +25,12 @@ export class PatchLog implements Printable { this.patches.set(id, patch); } + public replayToEnd(): Model { + const model = this.start.clone(); + this.patches.forEach(({v}) => model.applyPatch(v)); + return model; + } + // ---------------------------------------------------------------- Printable public toString(tab?: string) { diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 1326325722..294e06be67 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -18,11 +18,11 @@ test('can create File from new model', () => { }), ); const file = File.fromModel(model); - expect(file.history.start.view()).toBe(undefined); + expect(file.log.start.view()).toBe(undefined); expect(file.model.view()).toEqual({ foo: 'bar', }); - expect(file.history.start.clock.sid).toBe(file.model.clock.sid); + expect(file.log.start.clock.sid).toBe(file.model.clock.sid); }); test.todo('patches are flushed and stored in memory'); @@ -46,4 +46,26 @@ describe('.toBinary()', () => { expect(view).toEqual({foo: 'bar'}); }); }); + + describe('can decode from blob', () => { + test('.ndjson', () => { + const {file} = setup({foo: 'bar'}); + const blob = file.toBinary({format: 'ndjson', model: 'compact', history: 'compact'}); + const file2 = File.fromNdjson(blob); + expect(file2.model.view()).toEqual({foo: 'bar'}); + expect(file2.model !== file.model).toBe(true); + expect(file.log.start.view()).toEqual(undefined); + expect(file.log.replayToEnd().view()).toEqual({foo: 'bar'}); + }); + + test('.seq.cbor', () => { + const {file} = setup({foo: 'bar'}); + const blob = file.toBinary({format: 'seq.cbor', model: 'binary', history: 'binary'}); + const file2 = File.fromSeqCbor(blob); + expect(file2.model.view()).toEqual({foo: 'bar'}); + expect(file2.model !== file.model).toBe(true); + expect(file.log.start.view()).toEqual(undefined); + expect(file.log.replayToEnd().view()).toEqual({foo: 'bar'}); + }); + }); }); diff --git a/src/json-crdt/file/types.ts b/src/json-crdt/file/types.ts index eb0891cdaf..5d2f6cc8e3 100644 --- a/src/json-crdt/file/types.ts +++ b/src/json-crdt/file/types.ts @@ -15,8 +15,8 @@ export type FileReadSequence = [...FileWriteSequence, ...frontier: Array { + const decoder = new JsonDecoder(); + const reader = decoder.reader; + reader.reset(blob); + const components: unknown[] = []; + while (reader.x < blob.length) { + components.push(decoder.readAny()); + const nl = reader.u8(); + if (nl !== '\n'.charCodeAt(0)) throw new Error('NDJSON_UNEXPECTED_NEWLINE'); + } + return components; +}; + +export const decodeSeqCborComponents = (blob: Uint8Array): unknown[] => { + const decoder = new CborDecoder(); + const reader = decoder.reader; + reader.reset(blob); + const components: unknown[] = []; + while (reader.x < blob.length) components.push(decoder.val()); + return components; +}; + +export const decodeModel = (serialized: unknown): Model => { + if (!serialized) throw new Error('NO_MODEL'); + if (serialized instanceof Uint8Array) return Model.fromBinary(serialized); + if (Array.isArray(serialized)) + return new StructuralDecoderCompact().decode(serialized); + if (typeof serialized === 'object') + return new StructuralDecoderVerbose().decode(serialized); + throw new Error('UNKNOWN_MODEL'); +}; + +export const decodePatch = (serialized: unknown): Patch => { + if (!serialized) throw new Error('NO_MODEL'); + if (serialized instanceof Uint8Array) return Patch.fromBinary(serialized); + if (Array.isArray(serialized)) return decodeCompact(serialized); + if (typeof serialized === 'object') return decodeVerbose(serialized); + throw new Error('UNKNOWN_MODEL'); +}; From e26edd56626536d8f02b6547a5a9ec58af0f9d76 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 12:03:08 +0100 Subject: [PATCH 05/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20improve?= =?UTF-8?q?=20encoding=20constants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 8 ++------ src/json-crdt/file/constants.ts | 5 +---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index b6b11b03f2..a7545c760c 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -20,7 +20,7 @@ export class File implements Printable { const [view, metadata, model, history, ...frontier] = components; const modelFormat = metadata[1]; let decodedModel: Model | null = null; - if (model && modelFormat !== FileModelEncoding.None) { + if (model) { const isSidecar = modelFormat === FileModelEncoding.SidecarBinary; if (isSidecar) { const decoder = new SidecarDecoder(); @@ -63,7 +63,7 @@ export class File implements Printable { public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { const view = this.model.view(); - const metadata: types.FileMetadata = [{}, FileModelEncoding.SidecarBinary]; + const metadata: types.FileMetadata = [{}, FileModelEncoding.Auto]; let model: Uint8Array | unknown | null = null; const modelFormat = params.model ?? 'sidecar'; switch (modelFormat) { @@ -75,22 +75,18 @@ export class File implements Printable { break; } case 'binary': { - metadata[1] = FileModelEncoding.StructuralBinary; model = this.model.toBinary(); break; } case 'compact': { - metadata[1] = FileModelEncoding.StructuralCompact; model = new StructuralEncoderCompact().encode(this.model); break; } case 'verbose': { - metadata[1] = FileModelEncoding.StructuralVerbose; model = new StructuralEncoderVerbose().encode(this.model); break; } case 'none': { - metadata[1] = FileModelEncoding.None; model = null; break; } diff --git a/src/json-crdt/file/constants.ts b/src/json-crdt/file/constants.ts index 3ccde424af..912083d9dd 100644 --- a/src/json-crdt/file/constants.ts +++ b/src/json-crdt/file/constants.ts @@ -1,7 +1,4 @@ export const enum FileModelEncoding { - None = 0, + Auto = 0, SidecarBinary = 1, - StructuralBinary = 5, - StructuralCompact = 6, - StructuralVerbose = 7, } From cb9362d953fa0408c26642f43608428a7cbb6841 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 12:13:45 +0100 Subject: [PATCH 06/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?bility=20to=20encode=20file=20without=20model=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 5 ++-- src/json-crdt/file/__tests__/File.spec.ts | 33 +++++++++++++++++++++++ src/util/trees/avl/AvlMap.ts | 9 +++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index a7545c760c..ce23272204 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -62,7 +62,8 @@ export class File implements Printable { constructor(public readonly model: Model, public readonly log: PatchLog) {} public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { - const view = this.model.view(); + if (params.noView && (params.model === 'sidecar')) + throw new Error('SIDECAR_MODEL_WITHOUT_VIEW'); const metadata: types.FileMetadata = [{}, FileModelEncoding.Auto]; let model: Uint8Array | unknown | null = null; const modelFormat = params.model ?? 'sidecar'; @@ -123,7 +124,7 @@ export class File implements Printable { default: throw new Error(`Invalid history format: ${patchFormat}`); } - return [view, metadata, model, history]; + return [params.noView ? null : this.model.view(), metadata, model, history]; } public toBinary(params: types.FileEncodingParams): Uint8Array { diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 294e06be67..283f022fce 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -3,6 +3,7 @@ import {Model} from '../../model'; import {File} from '../File'; import {JsonDecoder} from '../../../json-pack/json/JsonDecoder'; import {CborDecoder} from '../../../json-pack/cbor/CborDecoder'; +import {FileEncodingParams} from '../types'; const setup = (view: unknown) => { const model = Model.withServerClock(); @@ -68,4 +69,36 @@ describe('.toBinary()', () => { expect(file.log.replayToEnd().view()).toEqual({foo: 'bar'}); }); }); + + const assertEncoding = (file: File, params: FileEncodingParams) => { + const blob = file.toBinary(params); + // if (params.format === 'ndjson') console.log(Buffer.from(blob).toString('utf8')) + const file2 = params.format === 'seq.cbor' ? File.fromSeqCbor(blob) : File.fromNdjson(blob); + expect(file2.model.view()).toEqual(file.model.view()); + expect(file2.model !== file.model).toBe(true); + expect(file2.log.start.view()).toEqual(undefined); + expect(file2.log.replayToEnd().view()).toEqual(file.model.view()); + expect(file2.log.patches.size()).toBe(file.log.patches.size()); + }; + + describe('can encode/decode all format combinations', () => { + const formats: FileEncodingParams['format'][] = ['ndjson', 'seq.cbor']; + const modelFormats: FileEncodingParams['model'][] = ['sidecar', 'binary', 'compact', 'verbose']; + const historyFormats: FileEncodingParams['history'][] = ['binary', 'compact', 'verbose']; + const noViews = [true, false]; + for (const format of formats) { + for (const model of modelFormats) { + for (const history of historyFormats) { + for (const noView of noViews) { + if (noView && (model === 'sidecar')) continue; + const params = {format, model, history, noView}; + test(JSON.stringify(params), () => { + const {file} = setup({foo: 'bar'}); + assertEncoding(file, params); + }); + } + } + } + } + }); }); diff --git a/src/util/trees/avl/AvlMap.ts b/src/util/trees/avl/AvlMap.ts index d0b3751f12..8babe77ee7 100644 --- a/src/util/trees/avl/AvlMap.ts +++ b/src/util/trees/avl/AvlMap.ts @@ -72,6 +72,15 @@ export class AvlMap implements Printable { return !!this.find(k); } + public size(): number { + const root = this.root; + if (!root) return 0; + let curr = first(root); + let size = 1; + while ((curr = next(curr as HeadlessNode) as AvlNode | undefined)) size++; + return size; + } + public getOrNextLower(k: K): AvlNode | undefined { return (findOrNextLower(this.root, k, this.comparator) as AvlNode) || undefined; } From 2bb8b9010044a395d3a5cc936640446d19bdc74a Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 12:14:18 +0100 Subject: [PATCH 07/16] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 3 +-- src/json-crdt/file/__tests__/File.spec.ts | 2 +- src/json-crdt/file/util.ts | 6 ++---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index ce23272204..9eb1757b83 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -62,8 +62,7 @@ export class File implements Printable { constructor(public readonly model: Model, public readonly log: PatchLog) {} public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { - if (params.noView && (params.model === 'sidecar')) - throw new Error('SIDECAR_MODEL_WITHOUT_VIEW'); + if (params.noView && params.model === 'sidecar') throw new Error('SIDECAR_MODEL_WITHOUT_VIEW'); const metadata: types.FileMetadata = [{}, FileModelEncoding.Auto]; let model: Uint8Array | unknown | null = null; const modelFormat = params.model ?? 'sidecar'; diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 283f022fce..58d986d1e9 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -90,7 +90,7 @@ describe('.toBinary()', () => { for (const model of modelFormats) { for (const history of historyFormats) { for (const noView of noViews) { - if (noView && (model === 'sidecar')) continue; + if (noView && model === 'sidecar') continue; const params = {format, model, history, noView}; test(JSON.stringify(params), () => { const {file} = setup({foo: 'bar'}); diff --git a/src/json-crdt/file/util.ts b/src/json-crdt/file/util.ts index 9fc29b329f..c268f98a97 100644 --- a/src/json-crdt/file/util.ts +++ b/src/json-crdt/file/util.ts @@ -36,10 +36,8 @@ export const decodeSeqCborComponents = (blob: Uint8Array): unknown[] => { export const decodeModel = (serialized: unknown): Model => { if (!serialized) throw new Error('NO_MODEL'); if (serialized instanceof Uint8Array) return Model.fromBinary(serialized); - if (Array.isArray(serialized)) - return new StructuralDecoderCompact().decode(serialized); - if (typeof serialized === 'object') - return new StructuralDecoderVerbose().decode(serialized); + if (Array.isArray(serialized)) return new StructuralDecoderCompact().decode(serialized); + if (typeof serialized === 'object') return new StructuralDecoderVerbose().decode(serialized); throw new Error('UNKNOWN_MODEL'); }; From 018a907fc08261fc710b432647f8907db04aea25 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 12:46:07 +0100 Subject: [PATCH 08/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20unseria?= =?UTF-8?q?lize=20frontier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt-patch/clock/clock.ts | 13 +++++++++++-- src/json-crdt/file/File.ts | 7 +++++++ src/json-crdt/file/__tests__/File.spec.ts | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/json-crdt-patch/clock/clock.ts b/src/json-crdt-patch/clock/clock.ts index 8fa440c6b3..d3e4b0591c 100644 --- a/src/json-crdt-patch/clock/clock.ts +++ b/src/json-crdt-patch/clock/clock.ts @@ -196,10 +196,10 @@ export class ClockVector extends LogicalClock implements IClockVector { } /** - * Returns a human-readable string representation of the vector clock. + * Returns a human-readable string representation of the clock vector. * * @param tab String to use for indentation. - * @returns Human-readable string representation of the vector clock. + * @returns Human-readable string representation of the clock vector. */ public toString(tab: string = ''): string { const last = this.peers.size; @@ -236,4 +236,13 @@ export class ServerClockVector extends LogicalClock implements IClockVector { public fork(): ServerClockVector { return new ServerClockVector(SESSION.SERVER, this.time); } + + /** + * Returns a human-readable string representation of the clock vector. + * + * @returns Human-readable string representation of the clock vector. + */ + public toString(): string { + return `clock ${this.sid}.${this.time}`; + } } diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 9eb1757b83..34552684c3 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -41,6 +41,13 @@ export class File implements Printable { } if (!log) throw new Error('NO_HISTORY'); if (!decodedModel) decodedModel = log.replayToEnd(); + if (frontier.length) { + for (const patch of frontier) { + const patchDecoded = decodePatch(patch); + decodedModel.applyPatch(patchDecoded); + log.push(patchDecoded); + } + } const file = new File(decodedModel, log); return file; } diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 58d986d1e9..2f6137f0cb 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -102,3 +102,20 @@ describe('.toBinary()', () => { } }); }); + +describe('.unserialize()', () => { + test('applies frontier', () => { + const {file, model} = setup({foo: 'bar'}); + const clone = model.clone(); + clone.api.obj([]).set({ + xyz: 123, + }); + const serialized = file.serialize({ + history: 'binary', + }); + serialized.push(clone.api.flush().toBinary()); + expect(file.model.view()).toEqual({foo: 'bar'}); + const file2 = File.unserialize(serialized); + expect(file2.model.view()).toEqual({foo: 'bar', xyz: 123}); + }); +}); From 404f9cbb7bb043ee0c51f2e77b053cd40136a46e Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 13:34:01 +0100 Subject: [PATCH 09/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?bility=20to=20replay=20to=20specific=20patch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/PatchLog.ts | 15 ++++++++--- src/json-crdt/file/__tests__/PatchLog.spec.ts | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 src/json-crdt/file/__tests__/PatchLog.spec.ts diff --git a/src/json-crdt/file/PatchLog.ts b/src/json-crdt/file/PatchLog.ts index cd746a0002..e74929e6f5 100644 --- a/src/json-crdt/file/PatchLog.ts +++ b/src/json-crdt/file/PatchLog.ts @@ -3,6 +3,7 @@ import {printTree} from '../../util/print/printTree'; import {AvlMap} from '../../util/trees/avl/AvlMap'; import {Model} from '../model'; import type {Printable} from '../../util/print/types'; +import {first, next} from '../../util/trees/util'; export class PatchLog implements Printable { public static fromModel(model: Model): PatchLog { @@ -26,9 +27,17 @@ export class PatchLog implements Printable { } public replayToEnd(): Model { - const model = this.start.clone(); - this.patches.forEach(({v}) => model.applyPatch(v)); - return model; + const clone = this.start.clone(); + for (let node = first(this.patches.root); node; node = next(node)) + clone.applyPatch(node.v); + return clone; + } + + public replayTo(ts: ITimestampStruct): Model { + const clone = this.start.clone(); + for (let node = first(this.patches.root); node && (compare(ts, node.k) >= 0); node = next(node)) + clone.applyPatch(node.v); + return clone; } // ---------------------------------------------------------------- Printable diff --git a/src/json-crdt/file/__tests__/PatchLog.spec.ts b/src/json-crdt/file/__tests__/PatchLog.spec.ts new file mode 100644 index 0000000000..ce1ac0d769 --- /dev/null +++ b/src/json-crdt/file/__tests__/PatchLog.spec.ts @@ -0,0 +1,27 @@ +import {Model} from '../../model'; +import {File} from '../File'; + +const setup = (view: unknown) => { + const model = Model.withServerClock(); + model.api.root(view); + const file = File.fromModel(model); + return {model, file}; +}; + +test('can replay to specific patch', () => { + const {model, file} = setup({foo: 'bar'}); + model.api.obj([]).set({x: 1}); + const patch1 = model.api.flush(); + model.api.obj([]).set({y: 2}); + const patch2 = model.api.flush(); + file.log.push(patch1); + file.log.push(patch2); + const model2 = file.log.replayToEnd(); + const model3 = file.log.replayTo(patch1.getId()!); + const model4 = file.log.replayTo(patch2.getId()!); + expect(model.view()).toEqual({foo: 'bar', x: 1, y: 2}); + expect(file.log.start.view()).toEqual({foo: 'bar'}); + expect(model2.view()).toEqual({foo: 'bar', x: 1, y: 2}); + expect(model3.view()).toEqual({foo: 'bar', x: 1}); + expect(model4.view()).toEqual({foo: 'bar', x: 1, y: 2}); +}); From 891b2594a5dd24afa9e6b95525a8cafb5f4ff58b Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 13:36:52 +0100 Subject: [PATCH 10/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20F?= =?UTF-8?q?ile.apply()=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 8 ++++++++ src/json-crdt/file/__tests__/PatchLog.spec.ts | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 34552684c3..551e1ff2a5 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -12,6 +12,7 @@ import {CborEncoder} from '../../json-pack/cbor/CborEncoder'; import {JsonEncoder} from '../../json-pack/json/JsonEncoder'; import {printTree} from '../../util/print/printTree'; import {decodeModel, decodeNdjsonComponents, decodePatch, decodeSeqCborComponents} from './util'; +import {Patch} from '../../json-crdt-patch'; import type * as types from './types'; import type {Printable} from '../../util/print/types'; @@ -68,6 +69,13 @@ export class File implements Printable { constructor(public readonly model: Model, public readonly log: PatchLog) {} + public apply(patch: Patch): void { + const id = patch.getId(); + if (!id) return; + this.model.applyPatch(patch); + this.log.push(patch); + } + public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { if (params.noView && params.model === 'sidecar') throw new Error('SIDECAR_MODEL_WITHOUT_VIEW'); const metadata: types.FileMetadata = [{}, FileModelEncoding.Auto]; diff --git a/src/json-crdt/file/__tests__/PatchLog.spec.ts b/src/json-crdt/file/__tests__/PatchLog.spec.ts index ce1ac0d769..c15e43c5c6 100644 --- a/src/json-crdt/file/__tests__/PatchLog.spec.ts +++ b/src/json-crdt/file/__tests__/PatchLog.spec.ts @@ -9,18 +9,20 @@ const setup = (view: unknown) => { }; test('can replay to specific patch', () => { - const {model, file} = setup({foo: 'bar'}); + const {file} = setup({foo: 'bar'}); + const model = file.model.clone(); model.api.obj([]).set({x: 1}); const patch1 = model.api.flush(); model.api.obj([]).set({y: 2}); const patch2 = model.api.flush(); - file.log.push(patch1); - file.log.push(patch2); + file.apply(patch1); + file.apply(patch2); const model2 = file.log.replayToEnd(); const model3 = file.log.replayTo(patch1.getId()!); const model4 = file.log.replayTo(patch2.getId()!); expect(model.view()).toEqual({foo: 'bar', x: 1, y: 2}); - expect(file.log.start.view()).toEqual({foo: 'bar'}); + expect(file.model.view()).toEqual({foo: 'bar', x: 1, y: 2}); + expect(file.log.start.view()).toEqual(undefined); expect(model2.view()).toEqual({foo: 'bar', x: 1, y: 2}); expect(model3.view()).toEqual({foo: 'bar', x: 1}); expect(model4.view()).toEqual({foo: 'bar', x: 1, y: 2}); From d0036aab1f943ed5210cdbd9ad9dcbc56d62d91c Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 4 Dec 2023 18:15:24 +0100 Subject: [PATCH 11/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?bility=20for=20file=20to=20sync=20to=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 16 +++++++++++++++ src/json-crdt/file/__tests__/File.spec.ts | 25 +++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 551e1ff2a5..53579673d5 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -76,6 +76,22 @@ export class File implements Printable { this.log.push(patch); } + public sync(): (() => void) { + const {model, log} = this; + const api = model.api; + const onPatchUnsubscribe = api.onPatch.listen((patch) => { + log.push(patch); + }); + const onLocalChangeUnsubscribe = api.onLocalChange.listen(() => { + const patch = api.flush(); + if (patch.ops.length) this.log.push(patch); + }); + return () => { + onPatchUnsubscribe(); + onLocalChangeUnsubscribe(); + }; + } + public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence { if (params.noView && params.model === 'sidecar') throw new Error('SIDECAR_MODEL_WITHOUT_VIEW'); const metadata: types.FileMetadata = [{}, FileModelEncoding.Auto]; diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 2f6137f0cb..6c3a98d2f2 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -119,3 +119,28 @@ describe('.unserialize()', () => { expect(file2.model.view()).toEqual({foo: 'bar', xyz: 123}); }); }); + +describe('.sync()', () => { + test('keeps track of local changes', async () => { + const {file, model} = setup({foo: 'bar'}); + file.sync(); + model.api.obj([]).set({x: 1}); + await Promise.resolve(); + expect(file.model.view()).toEqual({foo: 'bar', x: 1}); + expect(file.log.replayToEnd().view()).toEqual({foo: 'bar', x: 1}); + }); + + test('keeps track of remote changes', async () => { + const {file, model} = setup({foo: 'bar'}); + const clone = model.clone(); + file.sync(); + clone.api.obj([]).set({x: 1}); + expect(clone.view()).toEqual({foo: 'bar', x: 1}); + expect(file.model.view()).toEqual({foo: 'bar'}); + const patch = clone.api.flush(); + file.model.applyPatch(patch); + await Promise.resolve(); + expect(file.model.view()).toEqual({foo: 'bar', x: 1}); + expect(file.log.replayToEnd().view()).toEqual({foo: 'bar', x: 1}); + }); +}); From 6ae8a4622c25a0de065df0a3d5daa8efd4dfe9e8 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Dec 2023 00:49:33 +0100 Subject: [PATCH 12/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20buffer?= =?UTF-8?q?=20local=20transactions=20and=20allow=20to=20create=20transacti?= =?UTF-8?q?ons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 16 +++++++++++----- src/json-crdt/file/__tests__/File.spec.ts | 11 +++++++++++ src/json-crdt/model/api/ModelApi.ts | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 53579673d5..f395a0f209 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -79,16 +79,22 @@ export class File implements Printable { public sync(): (() => void) { const {model, log} = this; const api = model.api; + const drain = () => { + if (!api.builder.patch.ops.length) return; + const patch = api.flush(); + this.log.push(patch); + }; const onPatchUnsubscribe = api.onPatch.listen((patch) => { log.push(patch); }); - const onLocalChangeUnsubscribe = api.onLocalChange.listen(() => { - const patch = api.flush(); - if (patch.ops.length) this.log.push(patch); - }); + const onLocalChangesUnsubscribe = api.onLocalChanges.listen(drain); + const onBeforeTransactionUnsubscribe = api.onBeforeTransaction.listen(drain); + const onTransactionUnsubscribe = api.onTransaction.listen(drain); return () => { onPatchUnsubscribe(); - onLocalChangeUnsubscribe(); + onLocalChangesUnsubscribe(); + onBeforeTransactionUnsubscribe(); + onTransactionUnsubscribe(); }; } diff --git a/src/json-crdt/file/__tests__/File.spec.ts b/src/json-crdt/file/__tests__/File.spec.ts index 6c3a98d2f2..66222b0eda 100644 --- a/src/json-crdt/file/__tests__/File.spec.ts +++ b/src/json-crdt/file/__tests__/File.spec.ts @@ -130,6 +130,17 @@ describe('.sync()', () => { expect(file.log.replayToEnd().view()).toEqual({foo: 'bar', x: 1}); }); + test('processes local transactions', async () => { + const {file, model} = setup({foo: 'bar'}); + file.sync(); + const logLength = file.log.patches.size(); + model.api.transaction(() => { + model.api.obj([]).set({x: 1}); + model.api.obj([]).set({y: 2}); + }); + expect(file.log.patches.size()).toBe(logLength + 1); + }); + test('keeps track of remote changes', async () => { const {file, model} = setup({foo: 'bar'}); const clone = model.clone(); diff --git a/src/json-crdt/model/api/ModelApi.ts b/src/json-crdt/model/api/ModelApi.ts index 97b52b1060..37f87d6224 100644 --- a/src/json-crdt/model/api/ModelApi.ts +++ b/src/json-crdt/model/api/ModelApi.ts @@ -39,6 +39,9 @@ export class ModelApi implements SyncStore(); /** Emitted after local changes through `model.api` are applied. */ public readonly onLocalChange = new FanOut(); + /** Emitted after local changes through `model.api` are applied. Same as + * `.onLocalChange`, but this event buffered withing a microtask. */ + public readonly onLocalChanges = new MicrotaskBufferFanOut(this.onLocalChange); /** Emitted when the model changes. Combines `onReset`, `onPatch` and `onLocalChange`. */ public readonly onChange = new MergeFanOut([this.onReset, this.onPatch, this.onLocalChange]); /** Emitted when the model changes. Same as `.onChange`, but this event is emitted once per microtask. */ @@ -249,6 +252,7 @@ export class ModelApi implements SyncStore implements SyncStore(); + /** Emitted after transaction completes. */ + public readonly onTransaction = new FanOut(); + + public transaction(callback: () => void) { + this.onBeforeTransaction.emit(); + callback(); + this.onTransaction.emit(); + } + // ---------------------------------------------------------------- SyncStore public readonly subscribe = (callback: () => void) => this.onChanges.listen(() => callback()); From 15eaab8ef6ddfcfedffb0d76e0ce276be17710d1 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Dec 2023 00:51:07 +0100 Subject: [PATCH 13/16] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 2 +- src/json-crdt/file/PatchLog.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index f395a0f209..377d3b65c7 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -76,7 +76,7 @@ export class File implements Printable { this.log.push(patch); } - public sync(): (() => void) { + public sync(): () => void { const {model, log} = this; const api = model.api; const drain = () => { diff --git a/src/json-crdt/file/PatchLog.ts b/src/json-crdt/file/PatchLog.ts index e74929e6f5..a2040eb573 100644 --- a/src/json-crdt/file/PatchLog.ts +++ b/src/json-crdt/file/PatchLog.ts @@ -28,14 +28,13 @@ export class PatchLog implements Printable { public replayToEnd(): Model { const clone = this.start.clone(); - for (let node = first(this.patches.root); node; node = next(node)) - clone.applyPatch(node.v); + for (let node = first(this.patches.root); node; node = next(node)) clone.applyPatch(node.v); return clone; } public replayTo(ts: ITimestampStruct): Model { const clone = this.start.clone(); - for (let node = first(this.patches.root); node && (compare(ts, node.k) >= 0); node = next(node)) + for (let node = first(this.patches.root); node && compare(ts, node.k) >= 0; node = next(node)) clone.applyPatch(node.v); return clone; } From 247dab32da65ef7826d963d9b11144583f71d93e Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Dec 2023 01:26:24 +0100 Subject: [PATCH 14/16] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?utoflushing=20functionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/file/File.ts | 17 +++++-------- src/json-crdt/model/api/ModelApi.ts | 37 ++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/json-crdt/file/File.ts b/src/json-crdt/file/File.ts index 377d3b65c7..56e7157b9f 100644 --- a/src/json-crdt/file/File.ts +++ b/src/json-crdt/file/File.ts @@ -79,22 +79,17 @@ export class File implements Printable { public sync(): () => void { const {model, log} = this; const api = model.api; - const drain = () => { - if (!api.builder.patch.ops.length) return; - const patch = api.flush(); - this.log.push(patch); - }; + const autoflushUnsubscribe = api.autoFlush(); const onPatchUnsubscribe = api.onPatch.listen((patch) => { log.push(patch); }); - const onLocalChangesUnsubscribe = api.onLocalChanges.listen(drain); - const onBeforeTransactionUnsubscribe = api.onBeforeTransaction.listen(drain); - const onTransactionUnsubscribe = api.onTransaction.listen(drain); + const onFlushUnsubscribe = api.onFlush.listen((patch) => { + log.push(patch); + }); return () => { + autoflushUnsubscribe(); onPatchUnsubscribe(); - onLocalChangesUnsubscribe(); - onBeforeTransactionUnsubscribe(); - onTransactionUnsubscribe(); + onFlushUnsubscribe(); }; } diff --git a/src/json-crdt/model/api/ModelApi.ts b/src/json-crdt/model/api/ModelApi.ts index 37f87d6224..1c25b5597e 100644 --- a/src/json-crdt/model/api/ModelApi.ts +++ b/src/json-crdt/model/api/ModelApi.ts @@ -42,6 +42,10 @@ export class ModelApi implements SyncStore(this.onLocalChange); + /** Emitted before a transaction is started. */ + public readonly onBeforeTransaction = new FanOut(); + /** Emitted after transaction completes. */ + public readonly onTransaction = new FanOut(); /** Emitted when the model changes. Combines `onReset`, `onPatch` and `onLocalChange`. */ public readonly onChange = new MergeFanOut([this.onReset, this.onPatch, this.onLocalChange]); /** Emitted when the model changes. Same as `.onChange`, but this event is emitted once per microtask. */ @@ -248,6 +252,12 @@ export class ModelApi implements SyncStore void) { + this.onBeforeTransaction.emit(); + callback(); + this.onTransaction.emit(); + } + /** * Flushes the builder and returns a patch. * @@ -261,15 +271,26 @@ export class ModelApi implements SyncStore(); - /** Emitted after transaction completes. */ - public readonly onTransaction = new FanOut(); + public stopAutoFlush?: () => void = undefined; - public transaction(callback: () => void) { - this.onBeforeTransaction.emit(); - callback(); - this.onTransaction.emit(); + /** + * Begins to automatically flush buffered operations into patches, grouping + * operations by microtasks or by transactions. To capture the patch, listen + * to the `.onFlush` event. + * + * @returns Callback to stop auto flushing. + */ + public autoFlush(): (() => void) { + const drain = () => this.builder.patch.ops.length && this.flush(); + const onLocalChangesUnsubscribe = this.onLocalChanges.listen(drain); + const onBeforeTransactionUnsubscribe = this.onBeforeTransaction.listen(drain); + const onTransactionUnsubscribe = this.onTransaction.listen(drain); + return this.stopAutoFlush = () => { + this.stopAutoFlush = undefined; + onLocalChangesUnsubscribe(); + onBeforeTransactionUnsubscribe(); + onTransactionUnsubscribe(); + }; } // ---------------------------------------------------------------- SyncStore From 61b7bf62288453f534c809cb1c952ca30b69b226 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Dec 2023 01:27:18 +0100 Subject: [PATCH 15/16] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/model/api/ModelApi.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/json-crdt/model/api/ModelApi.ts b/src/json-crdt/model/api/ModelApi.ts index 1c25b5597e..f369c29965 100644 --- a/src/json-crdt/model/api/ModelApi.ts +++ b/src/json-crdt/model/api/ModelApi.ts @@ -280,17 +280,17 @@ export class ModelApi implements SyncStore void) { + public autoFlush(): () => void { const drain = () => this.builder.patch.ops.length && this.flush(); const onLocalChangesUnsubscribe = this.onLocalChanges.listen(drain); const onBeforeTransactionUnsubscribe = this.onBeforeTransaction.listen(drain); const onTransactionUnsubscribe = this.onTransaction.listen(drain); - return this.stopAutoFlush = () => { + return (this.stopAutoFlush = () => { this.stopAutoFlush = undefined; onLocalChangesUnsubscribe(); onBeforeTransactionUnsubscribe(); onTransactionUnsubscribe(); - }; + }); } // ---------------------------------------------------------------- SyncStore From 653dd809b30a78ac1d9b7dc97c37f3113bccf481 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 5 Dec 2023 01:30:27 +0100 Subject: [PATCH 16/16] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20fix=20?= =?UTF-8?q?JSDoc=20formatting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/model/api/ModelApi.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/json-crdt/model/api/ModelApi.ts b/src/json-crdt/model/api/ModelApi.ts index f369c29965..c84170098f 100644 --- a/src/json-crdt/model/api/ModelApi.ts +++ b/src/json-crdt/model/api/ModelApi.ts @@ -39,8 +39,10 @@ export class ModelApi implements SyncStore(); /** Emitted after local changes through `model.api` are applied. */ public readonly onLocalChange = new FanOut(); - /** Emitted after local changes through `model.api` are applied. Same as - * `.onLocalChange`, but this event buffered withing a microtask. */ + /** + * Emitted after local changes through `model.api` are applied. Same as + * `.onLocalChange`, but this event buffered withing a microtask. + */ public readonly onLocalChanges = new MicrotaskBufferFanOut(this.onLocalChange); /** Emitted before a transaction is started. */ public readonly onBeforeTransaction = new FanOut();