diff --git a/package.json b/package.json index b4c89b3eac..ae954ab3fe 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "arg": "^5.0.2", "hyperdyperid": "^1.2.0", "multibase": "^4.0.6", - "thingies": "^1.17.0" + "thingies": "^1.18.0" }, "devDependencies": { "@automerge/automerge": "2.1.7", diff --git a/src/json-pack/cbor/CborDecoderBase.ts b/src/json-pack/cbor/CborDecoderBase.ts index 3b27a6ea8d..f99bf7f0a5 100644 --- a/src/json-pack/cbor/CborDecoderBase.ts +++ b/src/json-pack/cbor/CborDecoderBase.ts @@ -13,7 +13,7 @@ export class CborDecoderBase { expect(val).toStrictEqual({a: 'a', b: 'b'}); }); - test('can encode CID using inlined custom class', () => { - class CID { - constructor(public readonly value: string) {} + class CID { + constructor(public readonly value: string) {} + } + class NotCID { + constructor(public readonly value: string) {} + } + + class IpfsCborEncoder extends CborEncoderDag { + public writeUnknown(val: unknown): void { + if (val instanceof CID) this.writeTag(42, val.value); + else throw new Error('Unknown value type'); } - const encoder = new (class extends CborEncoderDag { - public writeUnknown(val: unknown): void { - if (val instanceof CID) encoder.writeTag(42, val.value); - else throw new Error('Unknown value type'); - } - })(); + } + + class IpfsCborDecoder extends CborDecoderDag { + public readTagRaw(tag: number): CID | unknown { + if (tag === 42) return new CID(this.val() as any); + throw new Error('UNKNOWN_TAG'); + } + } + + test('can encode CID using inlined custom class', () => { + const encoder = new IpfsCborEncoder(); const encoded = encoder.encode({a: 'a', b: new JsonPackExtension(42, 'b')}); const val = decoder.read(encoded); expect(val).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); const encoded2 = encoder.encode({a: 'a', b: new CID('b')}); - const val2 = decoder.read(encoded2); + const val2 = decoder.decode(encoded2); expect(val).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); + expect(val2).toStrictEqual({a: 'a', b: new JsonPackExtension(42, 'b')}); + }); + + test('can encode CID inside a nested array', () => { + const encoder = new IpfsCborEncoder(); + const decoder = new IpfsCborDecoder(); + const cid = new CID('my-cid'); + const data = [1, [2, [3, cid, 4], 5], 6]; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toStrictEqual(data); }); test('can throw on unknown custom class', () => { - class CID { - constructor(public readonly value: string) {} - } - class NotCID { - constructor(public readonly value: string) {} - } - const encoder = new (class extends CborEncoderDag { - public writeUnknown(val: unknown): void { - if (val instanceof CID) encoder.writeTag(42, val.value); - else throw new Error('Unknown value type'); - } - })(); + const encoder = new IpfsCborEncoder(); const encoded1 = encoder.encode({a: 'a', b: new CID('b')}); expect(() => encoder.encode({a: 'a', b: new NotCID('b')})).toThrowError(new Error('Unknown value type')); }); diff --git a/src/json-pack/json/JsonDecoder.ts b/src/json-pack/json/JsonDecoder.ts index 79d0a8d7f1..f2c0b8ec9a 100644 --- a/src/json-pack/json/JsonDecoder.ts +++ b/src/json-pack/json/JsonDecoder.ts @@ -203,6 +203,11 @@ export class JsonDecoder implements BinaryJsonDecoder { return this.readAny(); } + public decode(uint8: Uint8Array): unknown { + this.reader.reset(uint8); + return this.readAny(); + } + public readAny(): PackValue { this.skipWhitespace(); const reader = this.reader; diff --git a/src/json-pack/types.ts b/src/json-pack/types.ts index 217c288bfd..b1e3dc9b2e 100644 --- a/src/json-pack/types.ts +++ b/src/json-pack/types.ts @@ -15,6 +15,7 @@ type PackArray = PackValue[] | readonly PackValue[]; type PackObject = {[key: string]: PackValue} | Readonly<{[key: string]: PackValue}>; export interface BinaryJsonEncoder { + encode(value: unknown): Uint8Array; writer: IWriter & IWriterGrowable; writeAny(value: unknown): void; writeNull(): void; @@ -52,6 +53,7 @@ export interface TlvBinaryJsonEncoder { } export interface BinaryJsonDecoder { + decode(uint8: Uint8Array): unknown; reader: IReader & IReaderResettable; read(uint8: Uint8Array): PackValue; } diff --git a/src/json-pack/ubjson/UbjsonDecoder.ts b/src/json-pack/ubjson/UbjsonDecoder.ts index 73c87ebf4f..927eb47120 100644 --- a/src/json-pack/ubjson/UbjsonDecoder.ts +++ b/src/json-pack/ubjson/UbjsonDecoder.ts @@ -11,6 +11,11 @@ export class UbjsonDecoder implements BinaryJsonDecoder { return this.readAny(); } + public decode(uint8: Uint8Array): unknown { + this.reader.reset(uint8); + return this.readAny(); + } + public readAny(): PackValue { const reader = this.reader; const octet = reader.u8(); diff --git a/src/util/buffers/b.ts b/src/util/buffers/b.ts new file mode 100644 index 0000000000..003f912314 --- /dev/null +++ b/src/util/buffers/b.ts @@ -0,0 +1 @@ +export const b = (...args: number[]) => new Uint8Array(args); diff --git a/src/util/buffers/cmpUint8Array2.ts b/src/util/buffers/cmpUint8Array2.ts new file mode 100644 index 0000000000..6bfcbcc38d --- /dev/null +++ b/src/util/buffers/cmpUint8Array2.ts @@ -0,0 +1,10 @@ +export const cmpUint8Array2 = (a: Uint8Array, b: Uint8Array): number => { + const len1 = a.length; + const len2 = b.length; + const len = Math.min(len1, len2); + for (let i = 0; i < len; i++) { + const diffChar = a[i] - b[i]; + if (diffChar !== 0) return diffChar; + } + return len1 - len2; +}; diff --git a/src/util/buffers/toBuf.ts b/src/util/buffers/toBuf.ts new file mode 100644 index 0000000000..576f8a517f --- /dev/null +++ b/src/util/buffers/toBuf.ts @@ -0,0 +1,8 @@ +import {encode} from './utf8/encode'; + +export const toBuf = (str: string): Uint8Array => { + const maxLength = str.length * 4; + const arr = new Uint8Array(maxLength); + const strBufferLength = encode(arr, str, 0, maxLength); + return arr.slice(0, strBufferLength); +}; diff --git a/src/web3/adl/hamt-crdt/Hamt.ts b/src/web3/adl/hamt-crdt/Hamt.ts new file mode 100644 index 0000000000..1097349c54 --- /dev/null +++ b/src/web3/adl/hamt-crdt/Hamt.ts @@ -0,0 +1,106 @@ +import {Defer} from 'thingies/es2020/Defer'; +import {concurrency} from 'thingies/es2020/concurrencyDecorator'; +import {HamtFrame} from './HamtFrame'; +import * as hlc from '../../hlc'; +import {Cid} from '../../multiformats'; +import {sha256} from '../../crypto'; +import {toBuf} from '../../../util/buffers/toBuf'; +import type {CidCasStruct} from '../../store/cas/CidCasStruct'; +import type * as types from './types'; + +export interface HamtDependencies { + cas: CidCasStruct; + hlcs: hlc.HlcFactory; +} + +export class Hamt implements types.HamtApi { + protected _root: HamtFrame; + protected _dirty: boolean = false; + protected _loading: Promise | null = null; + + public cid: Cid | null = null; + public prev: Cid | null = null; + public seq: number = 0; + public ops: types.HamtOp[] = []; + + constructor(protected readonly deps: HamtDependencies) { + this._root = new HamtFrame(deps.cas, null); + } + + public hasChanges(): boolean { + return this._dirty; + } + + // ------------------------------------------------------------------ HamtApi + + public async load(cid: Cid): Promise { + this.cid = cid; + const future = new Defer(); + this._loading = future.promise; + try { + const [prev, seq, ops, data] = (await this.deps.cas.get(cid)) as types.HamtRootFrameDto; + this.prev = prev; + this.seq = seq; + this._root.loadDto(data, null); + future.resolve(); + } catch (err) { + future.reject(err); + } finally { + this._loading = null; + } + return future.promise; + } + + @concurrency(1) + public async put(key: Uint8Array | string, val: unknown): Promise { + if (this._loading) await this._loading; + const hashedKey = await this._key(key); + const id = this.deps.hlcs.inc(); + const idDto = hlc.toDto(id); + const op: types.HamtOp = [hashedKey, val, idDto]; + const success = await this._root.put(op); + if (success) this.ops.push(op); + return success; + } + + public async get(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + const hashedKey = await this._key(key); + return await this._root.get(hashedKey); + } + + /** Convert any key to buffer and prefix with 4-byte hash. */ + protected async _key(key: Uint8Array | string): Promise { + const keyBuf = typeof key === 'string' ? toBuf(key) : key; + const hash = await sha256(keyBuf); + const buf = new Uint8Array(4 + keyBuf.length); + buf.set(hash.subarray(0, 4), 0); + buf.set(keyBuf, 4); + return buf; + } + + public async has(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + return (await this.get(key)) !== undefined; + } + + public async del(key: Uint8Array | string): Promise { + if (this._loading) await this._loading; + return await this.put(key, undefined); + } + + public async save(): Promise<[head: Cid, affected: Cid[]]> { + const [, affected] = await this._root.saveChildren(); + const prev = this.cid; + const seq = this.seq + 1; + const frameDto = this._root.toDto(); + const dto: types.HamtRootFrameDto = [prev, seq, this.ops, frameDto]; + const cid = await this.deps.cas.put(dto); + this.cid = cid; + this.prev = prev; + this.seq = seq; + this.ops = []; + affected.push(cid); + return [cid, affected]; + } +} diff --git a/src/web3/adl/hamt-crdt/HamtFactory.ts b/src/web3/adl/hamt-crdt/HamtFactory.ts new file mode 100644 index 0000000000..15397c35f6 --- /dev/null +++ b/src/web3/adl/hamt-crdt/HamtFactory.ts @@ -0,0 +1,12 @@ +import {Hamt, type HamtDependencies} from './Hamt'; + +export interface HamtFactoryDependencies extends HamtDependencies {} + +export class HamtFactory { + constructor(protected readonly deps: HamtFactoryDependencies) {} + + public make(): Hamt { + const hamt = new Hamt(this.deps); + return hamt; + } +} diff --git a/src/web3/adl/hamt-crdt/HamtFrame.ts b/src/web3/adl/hamt-crdt/HamtFrame.ts new file mode 100644 index 0000000000..d7db93f788 --- /dev/null +++ b/src/web3/adl/hamt-crdt/HamtFrame.ts @@ -0,0 +1,217 @@ +import {Defer} from 'thingies/es2020/Defer'; +import {cmpUint8Array2} from '../../../util/buffers/cmpUint8Array2'; +import {cmpDto} from '../../hlc'; +import {CidCasStruct} from '../../store/cas/CidCasStruct'; +import {Cid} from '../../multiformats'; +import {HamtConstraints} from './constants'; +import {mutex} from 'thingies/es2020/mutex'; +import type * as types from './types'; + +export class HamtFrame { + protected _entries: types.HamtFrameEntry[] = []; + protected _children: (HamtFrame | null)[] = [null]; + protected _loaded: boolean = false; + protected _loading: Defer | null = null; + /** Maybe instead of `_dirty`, just consider `id` === `null` to mean there are unsaved changes. */ + protected _dirty: boolean = false; + + constructor( + protected readonly cas: CidCasStruct, + public cid: Cid | null, + ) {} + + protected async ensureLoaded(): Promise { + if (this._loading) return this._loading.promise; + if (this._loaded) return; + if (!this.cid) return; + if (this.cid) await this.load(); + } + + /** + * Load the current node by CID from CAS. + * + * @param id CID of the node to load. + */ + @mutex + protected async load(): Promise { + const id = this.cid; + if (!id) throw new Error('ID_NOT_SET'); + this._loading = new Defer(); + const data = (await this.cas.get(id)) as types.HamtFrameDto; + this.loadDto(data, id); + this._loading.resolve(); + this._loading = null; + } + + /** + * Load the current node from known data. Provided data will be mutated + * internally, so it MUST not be used after this method is called. + * + * @param data Serialized data of the node to load. + * @param cid CID of the node to load, or null if CID is not known. + */ + public loadDto(data: types.HamtFrameDto, cid: Cid | null) { + this.cid = cid; + const [entries, children] = data; + this._entries = entries; + this._children = []; + const length = children.length; + for (let i = 0; i < length; i++) { + const childCid = children[i]; + const child = childCid ? new HamtFrame(this.cas, childCid) : null; + this._children.push(child); + } + this._loaded = true; + } + + public async getEntry(key: Uint8Array): Promise { + if (!this._loaded) await this.ensureLoaded(); + const entries = this._entries; + const length = entries.length; + for (let i = 0; i < length; i++) { + const entry = entries[i]; + const currentKey = entry[0]; + const comparison = cmpUint8Array2(currentKey, key); + if (comparison === 0) return entry; + const isKeySmallerThanCurrentKey = comparison > 0; + if (isKeySmallerThanCurrentKey) { + const child = this._children[i]; + if (!child) return undefined; + return await child.getEntry(key); + } + } + const lastChild = this._children[length]; + if (!lastChild) return undefined; + return await lastChild.getEntry(key); + } + + /** + * Recursively find a key value from current node or any of its children. + * + * @param key The key to fetch. + * @returns Returns the value if found, otherwise undefined. + */ + public async get(key: Uint8Array): Promise { + const entry = await this.getEntry(key); + if (!entry) return undefined; + return entry[1]; + } + + public async has(key: Uint8Array): Promise { + return (await this.get(key)) !== undefined; + } + + /** + * Insert or overwrite a key value pair in current node or any of its children. + * + * @param id HLC ID of the key. + * @param key Key to put. + * @param val Key value to put. + * @returns Returns true if the key was inserted. Insertion can fail if the + * ID of the insert operation is lower than the ID of the last write. + */ + public async put(op: types.HamtOp): Promise { + if (!this._loaded) await this.ensureLoaded(); + const [key, , id] = op; + const entries = this._entries; + const length = entries.length; + const insertInChild = length >= HamtConstraints.MaxEntriesPerFrame; + for (let i = 0; i < length; i++) { + const entry = entries[i]; + const currentKey = entry[0]; + const comparison = cmpUint8Array2(currentKey, key); + // Replace existing entry if keys are equal. + if (comparison === 0) { + const oldId = entry[2]; + if (cmpDto(oldId, id) >= 0) return false; + this._entries[i] = op; + this._markDirty(); + return true; + } + const isKeySmallerThanCurrentKey = comparison > 0; + if (isKeySmallerThanCurrentKey) { + if (insertInChild) { + // Insert at child node. + const wasInserted = await this._putAtChild(i, op); + if (wasInserted) this._markDirty(); + return wasInserted; + } else { + // Insert at current node, but shifting entries to the right. + this._entries.splice(i, 0, op); + this._children.splice(i, 0, null); + this._markDirty(); + return true; + } + } + } + // Insert into the last child. + if (insertInChild) { + const wasInserted = await this._putAtChild(length, op); + if (wasInserted) this._markDirty(); + return wasInserted; + } + // Append entry at the end of current block. + this._entries.push(op); + this._children.push(null); + this._markDirty(); + return true; + } + + protected _markDirty() { + this._dirty = true; + this.cid = null; + } + + private async _putAtChild(i: number, op: types.HamtOp): Promise { + let child = this._children[i]; + if (!child) child = this._children[i] = new HamtFrame(this.cas, null); + return await child.put(op); + } + + /** + * Save current node and all of its children. + * + * @returns Returns CID of current node, and a list of all affected CIDs, + * including the current CID. + */ + public async save(): Promise<[id: Cid, affected: Cid[]]> { + if (!this._loaded) await this.ensureLoaded(); + // TODO: Maybe throw if there are no changes. + if (this.cid && !this._dirty) return [this.cid, []]; + const [children, affected] = await this.saveChildren(); + const data: types.HamtFrameDto = [this._entries, children]; + const cid = await this.cas.put(data); + this.cid = cid; + affected.push(cid); + return [cid, affected]; + } + + /** + * Saves all "dirty" children and returns a list of all children. + * + * @returns Returns a list of stored CIDs and a all children of the current node, + * even the children which were not saved. + */ + public async saveChildren(): Promise<[children: (Cid | null)[], affected: Cid[]]> { + const ids: Cid[] = []; + const children: (Cid | null)[] = []; + const length = this._children.length; + for (let i = 0; i < length; i++) { + const child = this._children[i]; + if (!child) { + children.push(null); + continue; + } + const [childCid, affected] = await child.save(); + ids.push(...affected); + children.push(childCid); + } + return [children, ids]; + } + + public toDto(): types.HamtFrameDto { + const children = this._children.map((child) => (child ? child.cid : null)); + const dto: types.HamtFrameDto = [this._entries, children]; + return dto; + } +} diff --git a/src/web3/adl/hamt-crdt/README.md b/src/web3/adl/hamt-crdt/README.md new file mode 100644 index 0000000000..eb4d07b59b --- /dev/null +++ b/src/web3/adl/hamt-crdt/README.md @@ -0,0 +1,10 @@ +# HAMT CRDT + +HAMT CRDT is an infinitely scalable key-value store that is implemented using as +a HAMT data structure as a CRDT. Each HAMT node stores up to 16 *key-value-time* +3-tuple entries and up to 17 links to child nodes. + +It supports only a single `.put()` operation. To delete a key, one puts an +`undefined` value. Each operation is accompanied by an ID (timestamp), expressed +as hybrid logical clock (HLC) value. Each key is a single value LWW register, +where the HLC clock is used for determining the winner. diff --git a/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts new file mode 100644 index 0000000000..0fb09c947d --- /dev/null +++ b/src/web3/adl/hamt-crdt/__tests__/Hamt.spec.ts @@ -0,0 +1,265 @@ +import {b} from '../../../../util/buffers/b'; +import {HlcFactory} from '../../../hlc'; +import {CidCasMemory} from '../../../store/cas/CidCasMemory'; +import {CidCasStructCbor} from '../../../store/cas/CidCasStructCbor'; +import {HamtFactory} from '../HamtFactory'; +import {HamtRootFrameDto} from '../types'; + +const setup = () => { + const hlcs = new HlcFactory({}); + const cas0 = new CidCasMemory(); + const cas = new CidCasStructCbor(cas0); + const hamts = new HamtFactory({hlcs, cas}); + const hamt = hamts.make(); + return { + hlcs, + cas0, + cas, + hamts, + hamt, + }; +}; + +const toArr = (buf: Uint8Array): number[] => { + const arr: number[] = []; + for (let i = 0; i < buf.length; i++) arr.push(buf[i]); + return arr; +}; + +describe('HamtCrdt', () => { + test('new database has no changes', async () => { + const {hamt} = setup(); + const res = hamt.hasChanges(); + expect(res).toBe(false); + }); + + describe('.get()', () => { + test('returns undefined in empty database', async () => { + const {hamt} = setup(); + const res1 = await hamt.get(b(1, 2, 3)); + const res2 = await hamt.get('test'); + expect(res1).toBe(undefined); + expect(res2).toBe(undefined); + }); + + test('returns undefined in empty database', async () => { + const {hamt} = setup(); + const res1 = await hamt.get(b(1, 2, 3)); + const res2 = await hamt.get('test'); + expect(res1).toBe(undefined); + expect(res2).toBe(undefined); + }); + }); + + describe('.put()', () => { + test('can store a string key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put('test', b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get('test'); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can store a multiple keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.put('/@user1', b(1, 2, 3)); + const res2 = await hamt.put('/@user2', b(4, 5, 6)); + const res3 = await hamt.put('/@user3', b(7, 7, 7)); + expect(res1).toBe(true); + expect(res2).toBe(true); + expect(res3).toBe(true); + const res4 = await hamt.get('/@user1'); + const res5 = await hamt.get('/@user2'); + const res6 = await hamt.get('/@user3'); + expect(res4).toStrictEqual(b(1, 2, 3)); + expect(res5).toStrictEqual(b(4, 5, 6)); + expect(res6).toStrictEqual(b(7, 7, 7)); + }); + + test('can store into a binary key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(69), b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get(b(69)); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can store into an empty key', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(), b(1, 2, 3)); + expect(res1).toBe(true); + const res2 = await hamt.get(b()); + expect(res2).toStrictEqual(b(1, 2, 3)); + }); + + test('can overwrite a key', async () => { + const {hamt} = setup(); + await hamt.put('foo', b(1, 2, 3)); + await hamt.put('foo', b(4, 5, 6)); + const res2 = await hamt.get('foo'); + expect(res2).toStrictEqual(b(4, 5, 6)); + }); + + test('can add more than 16 keys', async () => { + const {hamt} = setup(); + for (let i = 0; i < 30; i++) { + await hamt.put('foo-' + i, b(i)); + } + for (let i = 0; i < 30; i++) { + const res = await hamt.get('foo-' + i); + expect(res).toStrictEqual(b(i)); + } + }); + + test('can store any serializable value', async () => { + const {hamt} = setup(); + const res1 = await hamt.put(b(), {foo: 123, bar: [true, false]}); + expect(res1).toBe(true); + const res2 = await hamt.get(b()); + expect(res2).toStrictEqual({foo: 123, bar: [true, false]}); + }); + }); + + describe('.has()', () => { + test('returns false for missing keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.has('a'); + const res2 = await hamt.has('b'); + expect(res1).toBe(false); + expect(res2).toBe(false); + }); + + test('returns true for existing keys', async () => { + const {hamt} = setup(); + await hamt.put('a', b()); + await hamt.put('b', b(1, 2, 3)); + const res1 = await hamt.has('a'); + const res2 = await hamt.has('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + }); + }); + + describe('.del()', () => { + test('can delete non-existing keys', async () => { + const {hamt} = setup(); + const res1 = await hamt.del('a'); + const res2 = await hamt.del('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + const res3 = await hamt.get('a'); + const res4 = await hamt.get('b'); + expect(res3).toBe(undefined); + expect(res4).toBe(undefined); + }); + + test('can delete existing key', async () => { + const {hamt} = setup(); + await hamt.put('a', b()); + await hamt.put('b', b(1, 2, 3)); + const res1 = await hamt.del('a'); + const res2 = await hamt.del('b'); + expect(res1).toBe(true); + expect(res2).toBe(true); + const res3 = await hamt.get('a'); + const res4 = await hamt.get('b'); + expect(res3).toBe(undefined); + expect(res4).toBe(undefined); + }); + }); + + describe('.save()', () => { + test('can persist empty HAMT', async () => { + const {hamt, hamts} = setup(); + const [cid] = await hamt.save(); + expect(cid).toBeDefined(); + const hamt2 = hamts.make(); + await hamt2.load(cid); + }); + + test('can save a single key', async () => { + const {hamt, cas0} = setup(); + const data = 111; + await hamt.put('a', b(data)); + const size = await (cas0 as CidCasMemory).size(); + expect(size).toBe(0); + const [cid] = await hamt.save(); + expect(await (cas0 as CidCasMemory).size()).toBe(size + 1); + const blob = await cas0.get(cid); + const found = toArr(blob).findIndex((octet) => octet === data); + expect(found > -1).toBe(true); + }); + + test('can load saved data', async () => { + const {hamt, hamts} = setup(); + await hamt.put('a', b(123)); + const [cid] = await hamt.save(); + const hamt2 = hamts.make(); + const res1 = await hamt2.get('a'); + expect(res1).toBe(undefined); + await hamt2.load(cid); + const res2 = await hamt2.get('a'); + expect(res2).toStrictEqual(b(123)); + }); + + test('can save and load more than 16 keys of data', async () => { + const {hamt, hamts} = setup(); + const keys = 1111; + for (let i = 0; i < keys; i++) { + await hamt.put('a:' + i, b(i, i + 1, i + 2)); + } + const [cid, all] = await hamt.save(); + const hamt2 = hamts.make(); + await hamt2.load(cid); + for (let i = 0; i < keys; i++) { + const res = await hamt2.get('a:' + i); + expect(res).toStrictEqual(b(i, i + 1, i + 2)); + } + }); + + test('can save and load more than 16 keys .save()"ed at periodic intervals', async () => { + const {hamt, hamts} = setup(); + const keysPerSave = 10; + const saves = 20; + for (let j = 0; j < saves; j++) { + for (let i = 0; i < keysPerSave; i++) { + const key = j * keysPerSave + i; + await hamt.put('abc:' + key, b(key, key + 1, key + 2)); + } + await hamt.save(); + } + const hamt2 = hamts.make(); + await hamt2.load(hamt.cid!); + for (let j = 0; j < saves; j++) { + for (let i = 0; i < keysPerSave; i++) { + const key = j * keysPerSave + i; + const res = await hamt2.get('abc:' + key); + expect(res).toStrictEqual(b(key, key + 1, key + 2)); + } + } + }); + }); + + describe('operations', () => { + test('stores operations as keys are edited', async () => { + const {hamt} = setup(); + expect(hamt.ops.length).toBe(0); + await hamt.put(b(0), 0); + expect(hamt.ops.length).toBe(1); + expect(hamt.ops).toStrictEqual([[expect.any(Uint8Array), 0, expect.any(Array)]]); + await hamt.put(b(1), 1); + expect(hamt.ops.length).toBe(2); + expect(hamt.ops).toStrictEqual([ + [expect.any(Uint8Array), 0, expect.any(Array)], + [expect.any(Uint8Array), 1, expect.any(Array)], + ]); + await hamt.del(b(0)); + expect(hamt.ops.length).toBe(3); + expect(hamt.ops).toStrictEqual([ + [expect.any(Uint8Array), 0, expect.any(Array)], + [expect.any(Uint8Array), 1, expect.any(Array)], + [hamt.ops[0][0], undefined, expect.any(Array)], + ]); + }); + }); +}); diff --git a/src/web3/adl/hamt-crdt/constants.ts b/src/web3/adl/hamt-crdt/constants.ts new file mode 100644 index 0000000000..c12b921b13 --- /dev/null +++ b/src/web3/adl/hamt-crdt/constants.ts @@ -0,0 +1,3 @@ +export const enum HamtConstraints { + MaxEntriesPerFrame = 16, +} diff --git a/src/web3/adl/hamt-crdt/types.ts b/src/web3/adl/hamt-crdt/types.ts new file mode 100644 index 0000000000..ee9e54d8d8 --- /dev/null +++ b/src/web3/adl/hamt-crdt/types.ts @@ -0,0 +1,64 @@ +import type {HlcDto} from '../../hlc'; +import type {Cid} from '../../multiformats'; + +export interface HamtApi { + load(id: Cid): Promise; + put(key: Uint8Array | string, val: unknown): Promise; + get(key: Uint8Array | string): Promise; + has(key: Uint8Array | string): Promise; + del(key: Uint8Array | string): Promise; + save(): Promise<[head: Cid, affected: Cid[]]>; +} + +/** Data of the root node of the HAMT. */ +export type HamtRootFrameDto = [ + /** + * CID of the previous state, previous root node. Zero, if there is no + * previous state. + */ + prev: Cid | null, + + /** + * Monotonically incrementing sequence number of the current state + * (increments with each new state). + */ + seq: number, + + /** + * An ordered list of operations which were performed on previous state to + * create the current state. Sorted, where the first operation is the oldest. + */ + ops: HamtOp[], + + /** + * Root level data of the HAMT. + */ + data: HamtFrameDto, +]; + +export type HamtFrameDto = [ + /** + * List of key value pairs stored in this node. + */ + entries: HamtFrameEntry[], + + /** + * Links to child nodes. This array must always be exactly one element larger + * than the `entries` array. Gaps are filled with nulls. + */ + children: (Cid | null)[], +]; + +export type HamtFrameEntry = [key: Uint8Array, val: unknown, id: HlcDto]; + +/** + * Key update operation. + */ +export type HamtOp = [ + /** Key that was updated. */ + key: Uint8Array, + /** New value of the key. */ + val: unknown, + /** ID of the operation as hybrid logical clock. */ + id: HlcDto, +]; diff --git a/src/web3/codec/Codecs.ts b/src/web3/codec/Codecs.ts new file mode 100644 index 0000000000..7229348474 --- /dev/null +++ b/src/web3/codec/Codecs.ts @@ -0,0 +1,20 @@ +import {MulticodecIpld} from '../multiformats'; +import type {IpldCodec} from './types'; + +export class Codecs { + protected readonly map = new Map(); + + public set(codec: MulticodecIpld, jsonCodec: IpldCodec): void { + this.map.set(codec, jsonCodec); + } + + public get(codec: MulticodecIpld): IpldCodec | undefined { + return this.map.get(codec); + } + + public getOrThrow(codec: MulticodecIpld): IpldCodec { + const jsonCodec = this.get(codec); + if (!jsonCodec) throw new Error(`Codec ${codec} (0x${codec.toString(16)}) not found`); + return jsonCodec; + } +} diff --git a/src/web3/codec/codecs/__tests__/cbor.spec.ts b/src/web3/codec/codecs/__tests__/cbor.spec.ts new file mode 100644 index 0000000000..bcfc278429 --- /dev/null +++ b/src/web3/codec/codecs/__tests__/cbor.spec.ts @@ -0,0 +1,22 @@ +import {Cid} from '../../../multiformats'; +import {cbor} from '../cbor'; + +test('can encode and decode CID', async () => { + const cid = await Cid.fromData(new Uint8Array([1, 2, 3, 4])); + const data = {foo: cid}; + const encoded = cbor.encoder.encode(data); + const decoded = cbor.decoder.decode(encoded); + expect(decoded).toStrictEqual(data); +}); + +test('can encode simplest fixture', async () => { + const data = [2]; + const encoded = cbor.encoder.encode(data); + const decoded = cbor.decoder.decode(encoded); + expect(decoded).toStrictEqual(data); + expect(encoded.length).toBe(2); + expect(encoded[0]).toBe(0x81); + expect(encoded[1]).toBe(0x02); + const cid = await Cid.fromDagCbor(encoded); + expect(cid.toText('base32')).toBe('bafyreihdb57fdysx5h35urvxz64ros7zvywshber7id6t6c6fek37jgyfe'); +}); diff --git a/src/web3/codec/codecs/cbor.ts b/src/web3/codec/codecs/cbor.ts new file mode 100644 index 0000000000..b5a6dbdcf4 --- /dev/null +++ b/src/web3/codec/codecs/cbor.ts @@ -0,0 +1,26 @@ +import {CborEncoderDag} from '../../../json-pack/cbor/CborEncoderDag'; +import {CborDecoderDag} from '../../../json-pack/cbor/CborDecoderDag'; +import {Cid} from '../../multiformats'; +import {writer} from './writer'; +import type {IpldCodec} from '../types'; + +const encoder = new (class extends CborEncoderDag { + public writeUnknown(val: unknown): void { + if (val instanceof Cid) this.writeTag(42, val.toBinary()); + else throw new Error('UNKNOWN_VALUE'); + } +})(writer); + +const decoder = new (class extends CborDecoderDag { + public readTagRaw(tag: number): Cid | unknown { + const value = this.val(); + if (tag === 42) return Cid.fromBinary(value as Uint8Array); + throw new Error('UNKNOWN_TAG'); + } +})(); + +export const cbor: IpldCodec = { + name: 'DAG-CBOR', + encoder, + decoder, +}; diff --git a/src/web3/codec/codecs/raw.ts b/src/web3/codec/codecs/raw.ts new file mode 100644 index 0000000000..18101aa0bc --- /dev/null +++ b/src/web3/codec/codecs/raw.ts @@ -0,0 +1,19 @@ +// import {bufferToUint8Array} from '../../../util/buffers/bufferToUint8Array'; +import type {IpldCodec} from '../types'; + +export const raw: IpldCodec = { + name: 'Raw', + encoder: { + encode: (value: unknown): Uint8Array => { + if (value instanceof Uint8Array) return value; + // if (typeof Buffer !== 'undefined') { + // if(Buffer.isBuffer(value)) return bufferToUint8Array(value as Buffer); + // return bufferToUint8Array(Buffer.from(String(value))); + // } + throw new Error('VALUE_NOT_SUPPORTED'); + }, + }, + decoder: { + decode: (data: Uint8Array): unknown => data, + }, +}; diff --git a/src/web3/codec/codecs/writer.ts b/src/web3/codec/codecs/writer.ts new file mode 100644 index 0000000000..fd58d8bb9f --- /dev/null +++ b/src/web3/codec/codecs/writer.ts @@ -0,0 +1,3 @@ +import {Writer} from '../../../util/buffers/Writer'; + +export const writer = new Writer(); diff --git a/src/web3/codec/index.ts b/src/web3/codec/index.ts new file mode 100644 index 0000000000..77f5672358 --- /dev/null +++ b/src/web3/codec/index.ts @@ -0,0 +1,13 @@ +import {MulticodecIpld} from '../multiformats'; +import {Codecs} from './Codecs'; +import {raw} from './codecs/raw'; +import {cbor} from './codecs/cbor'; + +export * from './types'; +export * from './Codecs'; + +export const codecs = new Codecs(); + +codecs.set(MulticodecIpld.Raw, raw); +codecs.set(MulticodecIpld.Cbor, cbor); +codecs.set(MulticodecIpld.DagCbor, cbor); diff --git a/src/web3/codec/types.ts b/src/web3/codec/types.ts new file mode 100644 index 0000000000..c02c6ae304 --- /dev/null +++ b/src/web3/codec/types.ts @@ -0,0 +1,7 @@ +import type {BinaryJsonDecoder, BinaryJsonEncoder} from '../../json-pack/types'; + +export interface IpldCodec { + name: string; + encoder: Pick; + decoder: Pick; +} diff --git a/src/web3/crypto/index.ts b/src/web3/crypto/index.ts index 51f41f6428..c7c41c22e3 100644 --- a/src/web3/crypto/index.ts +++ b/src/web3/crypto/index.ts @@ -1,8 +1,2 @@ -import {isNode} from '../constants'; - -/** - * Universal Node.js/browser Web Crypto API reference. - * - * @todo Maybe create an isomorphic package for this? - */ -export const crypto: Crypto = isNode ? require('node:crypto').webcrypto : window.crypto; +export * from './webcrypto'; +export * from './sha256'; diff --git a/src/web3/crypto/sha256.ts b/src/web3/crypto/sha256.ts new file mode 100644 index 0000000000..630abd2cb4 --- /dev/null +++ b/src/web3/crypto/sha256.ts @@ -0,0 +1,6 @@ +import {crypto} from './webcrypto'; + +export const sha256 = async (buf: Uint8Array): Promise => { + const ab = await crypto.subtle.digest('SHA-256', buf); + return new Uint8Array(ab); +}; diff --git a/src/web3/crypto/webcrypto.ts b/src/web3/crypto/webcrypto.ts new file mode 100644 index 0000000000..51f41f6428 --- /dev/null +++ b/src/web3/crypto/webcrypto.ts @@ -0,0 +1,8 @@ +import {isNode} from '../constants'; + +/** + * Universal Node.js/browser Web Crypto API reference. + * + * @todo Maybe create an isomorphic package for this? + */ +export const crypto: Crypto = isNode ? require('node:crypto').webcrypto : window.crypto; diff --git a/src/web3/multiformats/Cid.ts b/src/web3/multiformats/Cid.ts index 80c0241d22..7391a6bc1b 100644 --- a/src/web3/multiformats/Cid.ts +++ b/src/web3/multiformats/Cid.ts @@ -90,6 +90,7 @@ export class Cid { const contentType = this.ipldType; if (contentType >= 0b10000000) size += 1; if (contentType >= 0b10000000_0000000) size += 1; + if (contentType >= 0b10000000_0000000_0000000) throw new Error('UNSUPPORTED_IPLD_TYPE'); const hash = this.hash; const hashBuf = hash.buf; size += hashBuf.length; diff --git a/src/web3/store/cas/CidCasMemory.ts b/src/web3/store/cas/CidCasMemory.ts index 8d702f0634..dfec3d53ed 100644 --- a/src/web3/store/cas/CidCasMemory.ts +++ b/src/web3/store/cas/CidCasMemory.ts @@ -7,7 +7,7 @@ export class CidCasMemory implements CidCas { public async get(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); const value = this.store.get(key); if (!value) throw new Error(`No value for CID: ${key}`); return value; @@ -15,21 +15,25 @@ export class CidCasMemory implements CidCas { public async has(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); return this.store.has(key); } public async del(cid: Cid): Promise { const key = cid.toText(); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); this.store.delete(key); } public async put(value: Uint8Array, ipldType: MulticodecIpld = MulticodecIpld.Raw): Promise { const cid = await Cid.fromData(value, ipldType); - await new Promise((resolve) => setTimeout(resolve, 1)); + await new Promise((resolve) => setImmediate(resolve)); const key = cid.toText(); this.store.set(key, value); return cid; } + + public async size(): Promise { + return this.store.size; + } } diff --git a/src/web3/store/cas/CidCasStruct.ts b/src/web3/store/cas/CidCasStruct.ts index d0a2238a63..a8fdcbb918 100644 --- a/src/web3/store/cas/CidCasStruct.ts +++ b/src/web3/store/cas/CidCasStruct.ts @@ -1,17 +1,17 @@ import {Cid, MulticodecIpld} from '../../multiformats'; -import type {JsonValueCodec} from '../../../json-pack/codecs/types'; import type {CidCas} from './CidCas'; +import type {IpldCodec} from '../../codec'; export class CidCasStruct { constructor( protected readonly cas: CidCas, protected readonly ipldType: MulticodecIpld, - protected readonly codec: JsonValueCodec, + protected readonly codec: IpldCodec, ) {} public async get(cid: Cid): Promise { const blob = await this.cas.get(cid); - return this.codec.decoder.read(blob); + return this.codec.decoder.decode(blob); } public has(cid: Cid): Promise { @@ -23,9 +23,7 @@ export class CidCasStruct { } public async put(value: unknown): Promise { - const encoder = this.codec.encoder; - encoder.writeAny(value); - const blob = encoder.writer.flush(); + const blob = this.codec.encoder.encode(value); return this.cas.put(blob, this.ipldType); } } diff --git a/src/web3/store/cas/CidCasStructCbor.ts b/src/web3/store/cas/CidCasStructCbor.ts index ed1333e59f..08f7cd4f8e 100644 --- a/src/web3/store/cas/CidCasStructCbor.ts +++ b/src/web3/store/cas/CidCasStructCbor.ts @@ -1,13 +1,13 @@ import {MulticodecIpld} from '../../multiformats'; -import {CborJsonValueCodec} from '../../../json-pack/codecs/cbor'; -import {Writer} from '../../../util/buffers/Writer'; import {CidCasStruct} from './CidCasStruct'; +import {cbor} from '../../codec/codecs/cbor'; import type {CidCas} from './CidCas'; +import type {IpldCodec} from '../../codec'; export class CidCasStructCbor extends CidCasStruct { constructor( protected readonly cas: CidCas, - protected readonly codec: CborJsonValueCodec = new CborJsonValueCodec(new Writer(4096)), + protected readonly codec: IpldCodec = cbor, ) { super(cas, MulticodecIpld.Cbor, codec); } diff --git a/yarn.lock b/yarn.lock index f3ea9152e2..9643b61e24 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5318,10 +5318,10 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -thingies@^1.17.0: - version "1.17.0" - resolved "https://registry.yarnpkg.com/thingies/-/thingies-1.17.0.tgz#c172c423e5d3000a44666fc03723a29004ef0dcc" - integrity sha512-cpwd9eT9b7vcWDlJrIsfS+kBrcHEMST6DG08pOJoDkl/ToSwKYkW5x8RfBp5+k+iz1bT/dqEpoZOewg8I2lycw== +thingies@^1.18.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/thingies/-/thingies-1.18.0.tgz#827141872d96f3c3c2c0b432ab0dfdb581b4b4ac" + integrity sha512-WiB26BQP0MF47Bbvbq0P19KpyfrvdTK07L8xnltobpZ/aJPmu52CBGhYjLsnFgjyawmusJ0gVkTplnnoz2hBkQ== thunky@^1.0.2: version "1.1.0"