From a5e2fe02d702f12a91f5b3d7f70c39475272cab7 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 09:50:25 +0100 Subject: [PATCH 01/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20impleme?= =?UTF-8?q?nt=20stable=20JSON=20map=20encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/cbor/CborEncoderStable.ts | 9 ++------ src/json-pack/json/JsonEncoderStable.ts | 23 +++++++++++++++++++ .../json/__tests__/automated.spec.ts | 8 +++++-- src/json-pack/util/objKeyCmp.ts | 5 ++++ 4 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 src/json-pack/json/JsonEncoderStable.ts create mode 100644 src/json-pack/util/objKeyCmp.ts diff --git a/src/json-pack/cbor/CborEncoderStable.ts b/src/json-pack/cbor/CborEncoderStable.ts index 1f3a0ca514..45359574c4 100644 --- a/src/json-pack/cbor/CborEncoderStable.ts +++ b/src/json-pack/cbor/CborEncoderStable.ts @@ -1,12 +1,7 @@ import {CborEncoder} from './CborEncoder'; import {sort} from '../../util/sort/insertion2'; import {MAJOR_OVERLAY} from './constants'; - -const objectKeyComparator = (a: string, b: string): number => { - const len1 = a.length; - const len2 = b.length; - return len1 === len2 ? (a > b ? 1 : -1) : len1 - len2; -}; +import {objKeyCmp} from '../util/objKeyCmp'; const strHeaderLength = (strSize: number): 1 | 2 | 3 | 5 => { if (strSize <= 23) return 1; @@ -18,7 +13,7 @@ const strHeaderLength = (strSize: number): 1 | 2 | 3 | 5 => { export class CborEncoderStable extends CborEncoder { public writeObj(obj: Record): void { const keys = Object.keys(obj); - sort(keys, objectKeyComparator); + sort(keys, objKeyCmp); const length = keys.length; this.writeObjHdr(length); for (let i = 0; i < length; i++) { diff --git a/src/json-pack/json/JsonEncoderStable.ts b/src/json-pack/json/JsonEncoderStable.ts new file mode 100644 index 0000000000..203d07dcdf --- /dev/null +++ b/src/json-pack/json/JsonEncoderStable.ts @@ -0,0 +1,23 @@ +import {JsonEncoder} from './JsonEncoder'; +import {sort} from '../../util/sort/insertion2'; +import {objKeyCmp} from '../util/objKeyCmp'; + +export class JsonEncoderStable extends JsonEncoder { + public writeObj(obj: Record): void { + const writer = this.writer; + const keys = Object.keys(obj); + sort(keys, objKeyCmp); + const length = keys.length; + if (!length) return writer.u16(0x7b7d); // {} + writer.u8(0x7b); // { + for (let i = 0; i < length; i++) { + const key = keys[i]; + const value = obj[key]; + this.writeStr(key); + writer.u8(0x3a); // : + this.writeAny(value); + writer.u8(0x2c); // , + } + writer.uint8[writer.x - 1] = 0x7d; // } + } +} diff --git a/src/json-pack/json/__tests__/automated.spec.ts b/src/json-pack/json/__tests__/automated.spec.ts index 54892cef1c..534c942243 100644 --- a/src/json-pack/json/__tests__/automated.spec.ts +++ b/src/json-pack/json/__tests__/automated.spec.ts @@ -1,21 +1,25 @@ import {Writer} from '../../../util/buffers/Writer'; import {JsonValue} from '../../types'; import {JsonEncoder} from '../JsonEncoder'; +import {JsonEncoderStable} from '../JsonEncoderStable'; import {JsonDecoder} from '../JsonDecoder'; import {documents} from '../../../__tests__/json-documents'; import {binaryDocuments} from '../../../__tests__/binary-documents'; const writer = new Writer(8); const encoder = new JsonEncoder(writer); +const encoderStable = new JsonEncoderStable(writer); const decoder = new JsonDecoder(); const assertEncoder = (value: JsonValue) => { const encoded = encoder.encode(value); + const encoded2 = encoderStable.encode(value); // const json = Buffer.from(encoded).toString('utf-8'); // console.log('json', json); - decoder.reader.reset(encoded); - const decoded = decoder.readAny(); + const decoded = decoder.decode(encoded); + const decoded2 = decoder.decode(encoded2); expect(decoded).toEqual(value); + expect(decoded2).toEqual(value); }; describe('Sample JSON documents', () => { diff --git a/src/json-pack/util/objKeyCmp.ts b/src/json-pack/util/objKeyCmp.ts new file mode 100644 index 0000000000..18da990c7d --- /dev/null +++ b/src/json-pack/util/objKeyCmp.ts @@ -0,0 +1,5 @@ +export const objKeyCmp = (a: string, b: string): number => { + const len1 = a.length; + const len2 = b.length; + return len1 === len2 ? (a > b ? 1 : -1) : len1 - len2; +}; From 3020c1b80d9e9211ca12dc6c3609bd6cf71be4be Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 10:08:44 +0100 Subject: [PATCH 02/16] =?UTF-8?q?feat(util):=20=F0=9F=8E=B8=20improve=20ho?= =?UTF-8?q?w=20Base64=20padding=20is=20added?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/createToBase64.ts | 9 ++++++--- src/util/base64/toBase64Url.ts | 2 +- src/util/strings/flatstr.ts | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 src/util/strings/flatstr.ts diff --git a/src/util/base64/createToBase64.ts b/src/util/base64/createToBase64.ts index 6a8b8e62e1..994ab6a800 100644 --- a/src/util/base64/createToBase64.ts +++ b/src/util/base64/createToBase64.ts @@ -1,6 +1,7 @@ +import {flatstr} from '../strings/flatstr'; import {alphabet} from './constants'; -export const createToBase64 = (chars: string = alphabet, E: string = '=', EE: string = '==') => { +export const createToBase64 = (chars: string = alphabet, pad: string = '=') => { if (chars.length !== 64) throw new Error('chars must be 64 characters long'); const table = chars.split(''); @@ -8,12 +9,14 @@ export const createToBase64 = (chars: string = alphabet, E: string = '=', EE: st for (const c1 of table) { for (const c2 of table) { - const two = c1 + c2; - Number(two); + const two = flatstr(c1 + c2); table2.push(two); } } + const E: string = pad; + const EE: string = flatstr(pad + pad); + return (uint8: Uint8Array, length: number): string => { let out = ''; const extraLength = length % 3; diff --git a/src/util/base64/toBase64Url.ts b/src/util/base64/toBase64Url.ts index c638705583..4188ca42d0 100644 --- a/src/util/base64/toBase64Url.ts +++ b/src/util/base64/toBase64Url.ts @@ -1,3 +1,3 @@ import {createToBase64} from './createToBase64'; -export const toBase64Url = createToBase64('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', '', ''); +export const toBase64Url = createToBase64('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_', ''); diff --git a/src/util/strings/flatstr.ts b/src/util/strings/flatstr.ts new file mode 100644 index 0000000000..850dd5bcfa --- /dev/null +++ b/src/util/strings/flatstr.ts @@ -0,0 +1,5 @@ +export const flatstr = (s: string): string => { + s | 0; + Number(s); + return s; +}; From 433c33f0a410348c5b8d71f6c1f62fd856ef712c Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 10:17:56 +0100 Subject: [PATCH 03/16] =?UTF-8?q?perf(util):=20=E2=9A=A1=EF=B8=8F=20improv?= =?UTF-8?q?e=20padding=20handling=20in=20str=20Base64=20encoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/createToBase64.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/util/base64/createToBase64.ts b/src/util/base64/createToBase64.ts index 994ab6a800..83681524b4 100644 --- a/src/util/base64/createToBase64.ts +++ b/src/util/base64/createToBase64.ts @@ -29,17 +29,16 @@ export const createToBase64 = (chars: string = alphabet, pad: string = '=') => { const v2 = ((o2 & 0b1111) << 8) | o3; out += table2[v1] + table2[v2]; } - if (extraLength) { - if (extraLength === 1) { - const o1 = uint8[baseLength]; - out += table2[o1 << 4] + EE; - } else { - const o1 = uint8[baseLength]; - const o2 = uint8[baseLength + 1]; - const v1 = (o1 << 4) | (o2 >> 4); - const v2 = (o2 & 0b1111) << 2; - out += table2[v1] + table[v2] + E; - } + if (!extraLength) return out; + if (extraLength === 1) { + const o1 = uint8[baseLength]; + out += table2[o1 << 4] + EE; + } else { + const o1 = uint8[baseLength]; + const o2 = uint8[baseLength + 1]; + const v1 = (o1 << 4) | (o2 >> 4); + const v2 = (o2 & 0b1111) << 2; + out += table2[v1] + table[v2] + E; } return out; }; From 3501e3fefd7361b98268354698de26f20d6cd280 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 10:24:22 +0100 Subject: [PATCH 04/16] =?UTF-8?q?feat(util):=20=F0=9F=8E=B8=20support=20no?= =?UTF-8?q?=20padding=20in=20Base64=20encoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/__tests__/encode-bin.spec.ts | 7 ++++- src/util/base64/createToBase64Bin.ts | 33 ++++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/util/base64/__tests__/encode-bin.spec.ts b/src/util/base64/__tests__/encode-bin.spec.ts index 6a2b95f92a..c12f678963 100644 --- a/src/util/base64/__tests__/encode-bin.spec.ts +++ b/src/util/base64/__tests__/encode-bin.spec.ts @@ -1,8 +1,10 @@ import {toBase64} from '../toBase64'; import {createToBase64Bin} from '../createToBase64Bin'; import {bufferToUint8Array} from '../../buffers/bufferToUint8Array'; +import {copy} from '../../buffers/copy'; -const encode = createToBase64Bin(); +const encode = createToBase64Bin('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '='); +const encodeNoPadding = createToBase64Bin('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'); const generateBlob = (): Uint8Array => { const length = Math.floor(Math.random() * 100) + 1; @@ -19,6 +21,9 @@ test('works', () => { const result = bufferToUint8Array(Buffer.from(toBase64(blob))); const binWithBuffer = new Uint8Array(result.length + 3); encode(blob, 0, blob.length, new DataView(binWithBuffer.buffer), 3); + const dupe = copy(blob); + encodeNoPadding(blob, 0, blob.length, new DataView(binWithBuffer.buffer), 3); + expect(dupe).toEqual(blob); const encoded = binWithBuffer.subarray(3); // console.log(result); // console.log(binWithBuffer); diff --git a/src/util/base64/createToBase64Bin.ts b/src/util/base64/createToBase64Bin.ts index 865d09542b..d4ca9b35b1 100644 --- a/src/util/base64/createToBase64Bin.ts +++ b/src/util/base64/createToBase64Bin.ts @@ -1,6 +1,6 @@ import {alphabet} from './constants'; -export const createToBase64Bin = (chars: string = alphabet) => { +export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') => { if (chars.length !== 64) throw new Error('chars must be 64 characters long'); const table = chars.split('').map((c) => c.charCodeAt(0)); @@ -13,6 +13,10 @@ export const createToBase64Bin = (chars: string = alphabet) => { } } + const doAddPadding = pad.length === 1; + const E: number = doAddPadding ? pad.charCodeAt(0) : 0; + const EE: number = doAddPadding ? ((E << 8) | E) : 0 + return (uint8: Uint8Array, start: number, length: number, dest: DataView, offset: number): number => { const extraLength = length % 3; const baseLength = length - extraLength; @@ -25,18 +29,27 @@ export const createToBase64Bin = (chars: string = alphabet) => { dest.setInt32(offset, (table2[v1] << 16) + table2[v2]); offset += 4; } - if (extraLength) { - if (extraLength === 1) { - const o1 = uint8[baseLength]; - dest.setInt32(offset, (table2[o1 << 4] << 16) + 0x3d3d); + if (extraLength === 1) { + const o1 = uint8[baseLength]; + if (doAddPadding) { + dest.setInt32(offset, (table2[o1 << 4] << 16) + EE); return offset + 4; } else { - const o1 = uint8[baseLength]; - const o2 = uint8[baseLength + 1]; - const v1 = (o1 << 4) | (o2 >> 4); - const v2 = (o2 & 0b1111) << 2; - dest.setInt32(offset, (table2[v1] << 16) + (table[v2] << 8) + 0x3d); + dest.setInt16(offset, table2[o1 << 4]); + return offset + 2; + } + } else if (extraLength) { + const o1 = uint8[baseLength]; + const o2 = uint8[baseLength + 1]; + const v1 = (o1 << 4) | (o2 >> 4); + const v2 = (o2 & 0b1111) << 2; + if (doAddPadding) { + dest.setInt32(offset, (table2[v1] << 16) + (table[v2] << 8) + E); return offset + 4; + } else { + dest.setInt16(offset, table2[v1]); + dest.setInt8(offset, table[v2]); + return offset + 3; } } return offset; From 1e9e2adcd50e88af054cedd09623bf6f90c02db5 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 10:37:11 +0100 Subject: [PATCH 05/16] =?UTF-8?q?feat(util):=20=F0=9F=8E=B8=20implement=20?= =?UTF-8?q?binayr=20Base64=20encoder=20which=20uses=20Uint8Array?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/__tests__/encode-bin.spec.ts | 5 ++ src/util/base64/createToBase64BinUint8.ts | 56 ++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/util/base64/createToBase64BinUint8.ts diff --git a/src/util/base64/__tests__/encode-bin.spec.ts b/src/util/base64/__tests__/encode-bin.spec.ts index c12f678963..6ce22c6bdd 100644 --- a/src/util/base64/__tests__/encode-bin.spec.ts +++ b/src/util/base64/__tests__/encode-bin.spec.ts @@ -1,9 +1,11 @@ import {toBase64} from '../toBase64'; import {createToBase64Bin} from '../createToBase64Bin'; +import {createToBase64BinUint8} from '../createToBase64BinUint8'; import {bufferToUint8Array} from '../../buffers/bufferToUint8Array'; import {copy} from '../../buffers/copy'; const encode = createToBase64Bin('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '='); +const encodeUint8 = createToBase64BinUint8('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', '='); const encodeNoPadding = createToBase64Bin('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'); const generateBlob = (): Uint8Array => { @@ -24,6 +26,9 @@ test('works', () => { const dupe = copy(blob); encodeNoPadding(blob, 0, blob.length, new DataView(binWithBuffer.buffer), 3); expect(dupe).toEqual(blob); + const dupe2 = copy(blob); + encodeUint8(blob, 0, blob.length, binWithBuffer, 3); + expect(dupe2).toEqual(blob); const encoded = binWithBuffer.subarray(3); // console.log(result); // console.log(binWithBuffer); diff --git a/src/util/base64/createToBase64BinUint8.ts b/src/util/base64/createToBase64BinUint8.ts new file mode 100644 index 0000000000..d4616adbd3 --- /dev/null +++ b/src/util/base64/createToBase64BinUint8.ts @@ -0,0 +1,56 @@ +import {alphabet} from './constants'; + +export const createToBase64BinUint8 = (chars: string = alphabet, pad: string = '=') => { + if (chars.length !== 64) throw new Error('chars must be 64 characters long'); + + const table = chars.split('').map((c) => c.charCodeAt(0)); + const table2: number[] = []; + + for (const c1 of table) { + for (const c2 of table) { + const two = (c1 << 8) + c2; + table2.push(two); + } + } + + const E: number = pad.length === 1 ? pad.charCodeAt(0) : 0; + + return (uint8: Uint8Array, start: number, length: number, dest: Uint8Array, offset: number): number => { + const extraLength = length % 3; + const baseLength = length - extraLength; + for (; start < baseLength; start += 3) { + const o1 = uint8[start]; + const o2 = uint8[start + 1]; + const o3 = uint8[start + 2]; + const v1 = (o1 << 4) | (o2 >> 4); + const v2 = ((o2 & 0b1111) << 8) | o3; + let u16 = table2[v1]; + dest[offset++] = u16 >> 8; + dest[offset++] = u16; + u16 = table2[v2]; + dest[offset++] = u16 >> 8; + dest[offset++] = u16; + } + if (extraLength === 1) { + const o1 = uint8[baseLength]; + const u16 = table2[o1 << 4]; + dest[offset++] = u16 >> 8; + dest[offset++] = u16; + if (E) { + dest[offset++] = E; + dest[offset++] = E; + } + } else if (extraLength) { + const o1 = uint8[baseLength]; + const o2 = uint8[baseLength + 1]; + const v1 = (o1 << 4) | (o2 >> 4); + const v2 = (o2 & 0b1111) << 2; + const u16 = table2[v1]; + dest[offset++] = u16 >> 8; + dest[offset++] = u16; + dest[offset++] = table[v2]; + if (E) dest[offset++] = E; + } + return offset; + }; +}; From e3ba9b6680c5b97de8c3af9938341f3c1e0dc6c4 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 10:41:23 +0100 Subject: [PATCH 06/16] =?UTF-8?q?style(util):=20=F0=9F=92=84=20rename=20co?= =?UTF-8?q?nstant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/createToBase64BinUint8.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util/base64/createToBase64BinUint8.ts b/src/util/base64/createToBase64BinUint8.ts index d4616adbd3..efa77615b0 100644 --- a/src/util/base64/createToBase64BinUint8.ts +++ b/src/util/base64/createToBase64BinUint8.ts @@ -13,7 +13,7 @@ export const createToBase64BinUint8 = (chars: string = alphabet, pad: string = ' } } - const E: number = pad.length === 1 ? pad.charCodeAt(0) : 0; + const PAD: number = pad.length === 1 ? pad.charCodeAt(0) : 0; return (uint8: Uint8Array, start: number, length: number, dest: Uint8Array, offset: number): number => { const extraLength = length % 3; @@ -36,9 +36,9 @@ export const createToBase64BinUint8 = (chars: string = alphabet, pad: string = ' const u16 = table2[o1 << 4]; dest[offset++] = u16 >> 8; dest[offset++] = u16; - if (E) { - dest[offset++] = E; - dest[offset++] = E; + if (PAD) { + dest[offset++] = PAD; + dest[offset++] = PAD; } } else if (extraLength) { const o1 = uint8[baseLength]; @@ -49,7 +49,7 @@ export const createToBase64BinUint8 = (chars: string = alphabet, pad: string = ' dest[offset++] = u16 >> 8; dest[offset++] = u16; dest[offset++] = table[v2]; - if (E) dest[offset++] = E; + if (PAD) dest[offset++] = PAD; } return offset; }; From 9617448c78ad02ad7c8528096b0b4011eea096a6 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 11:11:55 +0100 Subject: [PATCH 07/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20support?= =?UTF-8?q?=20Bytes=20type=20encoding=20in=20DAG-JSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonEncoderDag.ts | 49 +++++++++++++++++++ .../json/__tests__/JsonEncoderDag.spec.ts | 22 +++++++++ 2 files changed, 71 insertions(+) create mode 100644 src/json-pack/json/JsonEncoderDag.ts create mode 100644 src/json-pack/json/__tests__/JsonEncoderDag.spec.ts diff --git a/src/json-pack/json/JsonEncoderDag.ts b/src/json-pack/json/JsonEncoderDag.ts new file mode 100644 index 0000000000..1cc190981a --- /dev/null +++ b/src/json-pack/json/JsonEncoderDag.ts @@ -0,0 +1,49 @@ +import {JsonEncoderStable} from './JsonEncoderStable'; +import {createToBase64BinUint8} from '../../util/base64/createToBase64BinUint8'; + +const objBaseLength = '{"/":{"bytes":""}}'.length; +const base64Encode = createToBase64BinUint8(undefined, ''); + +/** + * Base class for implementing DAG-JSON encoders. + * + * @see https://ipld.io/specs/codecs/dag-json/spec/ + */ +export class JsonEncoderDag extends JsonEncoderStable { + /** + * Encodes binary data as nested `["/", "bytes"]` object encoded in Base64 + * without padding. + * + * Example: + * + * ```json + * {"/":{"bytes":"aGVsbG8gd29ybGQ"}} + * ``` + * + * @param buf Binary data to write. + */ + public writeBin(buf: Uint8Array): void { + const writer = this.writer; + const length = buf.length; + writer.ensureCapacity(objBaseLength + (length << 1)); + const view = writer.view; + const uint8 = writer.uint8; + let x = writer.x; + view.setUint32(x, 0x7b222f22); // {"/" + x += 4; + view.setUint32(x, 0x3a7b2262); // :{"b + x += 4; + view.setUint32(x, 0x79746573); // ytes + x += 4; + view.setUint16(x, 0x223a); // ": + x += 2; + uint8[x] = 0x22; // " + x += 1; + x = base64Encode(buf, 0, length, uint8, x); + view.setUint16(x, 0x227d); // "} + x += 2; + uint8[x] = 0x7d; // } + x += 1; + writer.x = x; + } +} diff --git a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts new file mode 100644 index 0000000000..c6546512b3 --- /dev/null +++ b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts @@ -0,0 +1,22 @@ +import {Writer} from '../../../util/buffers/Writer'; +import {utf8} from '../../../util/buffers/strings'; +import {JsonEncoderDag} from '../JsonEncoderDag'; + +const writer = new Writer(16); +const encoder = new JsonEncoderDag(writer); + +test('can encode a simple buffer in object', () => { + const buf = utf8`hello world`; + const data = {foo: buf}; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('{"foo":{"/":{"bytes":"aGVsbG8gd29ybGQ"}}}'); +}); + +test('can encode a simple buffer in array', () => { + const buf = utf8`hello world`; + const data = [0, buf, 1]; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('[0,{"/":{"bytes":"aGVsbG8gd29ybGQ"}},1]'); +}); From 76668650c92e496ce251e28e5047d307b5acbb4c Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 11:16:35 +0100 Subject: [PATCH 08/16] =?UTF-8?q?fix(util):=20=F0=9F=90=9B=20correctly=20c?= =?UTF-8?q?ompute=20binary=20Base64=20last=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/createToBase64Bin.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/util/base64/createToBase64Bin.ts b/src/util/base64/createToBase64Bin.ts index d4ca9b35b1..31f371c7b5 100644 --- a/src/util/base64/createToBase64Bin.ts +++ b/src/util/base64/createToBase64Bin.ts @@ -33,10 +33,10 @@ export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') = const o1 = uint8[baseLength]; if (doAddPadding) { dest.setInt32(offset, (table2[o1 << 4] << 16) + EE); - return offset + 4; + offset + 4; } else { dest.setInt16(offset, table2[o1 << 4]); - return offset + 2; + offset + 2; } } else if (extraLength) { const o1 = uint8[baseLength]; @@ -45,11 +45,12 @@ export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') = const v2 = (o2 & 0b1111) << 2; if (doAddPadding) { dest.setInt32(offset, (table2[v1] << 16) + (table[v2] << 8) + E); - return offset + 4; + offset + 4; } else { dest.setInt16(offset, table2[v1]); + offset += 2; dest.setInt8(offset, table[v2]); - return offset + 3; + offset += 1; } } return offset; From e6c976c6d38d536223bce6aa57265bb52ceb902a Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 11:16:56 +0100 Subject: [PATCH 09/16] =?UTF-8?q?perf(json-pack):=20=E2=9A=A1=EF=B8=8F=20u?= =?UTF-8?q?se=20DataView=20for=20Base64=20chunk=20encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonEncoderDag.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/json-pack/json/JsonEncoderDag.ts b/src/json-pack/json/JsonEncoderDag.ts index 1cc190981a..1d95622421 100644 --- a/src/json-pack/json/JsonEncoderDag.ts +++ b/src/json-pack/json/JsonEncoderDag.ts @@ -1,8 +1,8 @@ import {JsonEncoderStable} from './JsonEncoderStable'; -import {createToBase64BinUint8} from '../../util/base64/createToBase64BinUint8'; +import {createToBase64Bin} from '../../util/base64/createToBase64Bin'; const objBaseLength = '{"/":{"bytes":""}}'.length; -const base64Encode = createToBase64BinUint8(undefined, ''); +const base64Encode = createToBase64Bin(undefined, ''); /** * Base class for implementing DAG-JSON encoders. @@ -39,7 +39,7 @@ export class JsonEncoderDag extends JsonEncoderStable { x += 2; uint8[x] = 0x22; // " x += 1; - x = base64Encode(buf, 0, length, uint8, x); + x = base64Encode(buf, 0, length, view, x); view.setUint16(x, 0x227d); // "} x += 2; uint8[x] = 0x7d; // } From 3c126e6c964a36e447863318a962ec8e3676b51b Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 13:13:29 +0100 Subject: [PATCH 10/16] =?UTF-8?q?feat(util):=20=F0=9F=8E=B8=20support=20bi?= =?UTF-8?q?nary=20Base64=20decoding=20without=20padding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/__tests__/decode-bin.spec.ts | 8 +++- src/util/base64/createFromBase64Bin.ts | 49 +++++++++++--------- src/util/base64/createToBase64Bin.ts | 6 +-- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/util/base64/__tests__/decode-bin.spec.ts b/src/util/base64/__tests__/decode-bin.spec.ts index 581d0bc4e9..ba0c5ffba9 100644 --- a/src/util/base64/__tests__/decode-bin.spec.ts +++ b/src/util/base64/__tests__/decode-bin.spec.ts @@ -16,10 +16,16 @@ test('works', () => { const dest = new Uint8Array(blob.length * 4); const length = toBase64Bin(blob, 0, blob.length, new DataView(dest.buffer), 0); const encoded = dest.subarray(0, length); - const decoded = fromBase64Bin(new DataView(encoded.buffer), 0, encoded.length); + const view = new DataView(encoded.buffer); + const decoded = fromBase64Bin(view, 0, encoded.length); + let padding = 0; + if (encoded.length > 0 && view.getUint8(encoded.length - 1) === 0x3d) padding++; + if (encoded.length > 1 && view.getUint8(encoded.length - 2) === 0x3d) padding++; + const decoded2 = fromBase64Bin(view, 0, encoded.length - padding); // console.log('blob', blob); // console.log('encoded', encoded); // console.log('decoded', decoded); expect(decoded).toEqual(blob); + expect(decoded2).toEqual(blob); } }); diff --git a/src/util/base64/createFromBase64Bin.ts b/src/util/base64/createFromBase64Bin.ts index 3b2ee1cc59..1707e676a3 100644 --- a/src/util/base64/createFromBase64Bin.ts +++ b/src/util/base64/createFromBase64Bin.ts @@ -1,6 +1,6 @@ import {alphabet} from './constants'; -export const createFromBase64Bin = (chars: string = alphabet, paddingOctet: number = 0x3d) => { +export const createFromBase64Bin = (chars: string = alphabet, pad: string = '=') => { if (chars.length !== 64) throw new Error('chars must be 64 characters long'); let max = 0; for (let i = 0; i < chars.length; i++) max = Math.max(max, chars.charCodeAt(i)); @@ -8,22 +8,26 @@ export const createFromBase64Bin = (chars: string = alphabet, paddingOctet: numb for (let i = 0; i <= max; i += 1) table[i] = -1; for (let i = 0; i < chars.length; i++) table[chars.charCodeAt(i)] = i; + const doExpectPadding = pad.length === 1; + const PAD = doExpectPadding ? pad.charCodeAt(0) : 0; + return (view: DataView, offset: number, length: number): Uint8Array => { if (!length) return new Uint8Array(0); - if (length % 4 !== 0) throw new Error('Base64 string length must be a multiple of 4'); - const end = offset + length; - const last = end - 1; - const lastOctet = view.getUint8(last); - const mainEnd = offset + (lastOctet !== paddingOctet ? length : length - 4); - let bufferLength = (length >> 2) * 3; let padding = 0; - if (last > 0 && view.getUint8(last - 1) === paddingOctet) { - padding = 2; - bufferLength -= 2; - } else if (lastOctet === paddingOctet) { - padding = 1; - bufferLength -= 1; + if (length % 4 !== 0) { + padding = 4 - (length % 4); + length += padding; + } else { + const end = offset + length; + const last = end - 1; + if (view.getUint8(last) === PAD) { + padding = 1; + if (length > 1 && view.getUint8(last - 1) === PAD) padding = 2; + } } + if (length % 4 !== 0) throw new Error('Base64 string length must be a multiple of 4'); + const mainEnd = offset + length - (padding ? 4 : 0); + let bufferLength = ((length >> 2) * 3) - padding; const buf = new Uint8Array(bufferLength); let j = 0; let i = offset; @@ -43,15 +47,8 @@ export const createFromBase64Bin = (chars: string = alphabet, paddingOctet: numb buf[j + 2] = (sextet2 << 6) | sextet3; j += 3; } - if (padding === 2) { - const word = view.getUint16(mainEnd); - const octet0 = word >> 8; - const octet1 = word & 0xff; - const sextet0 = table[octet0]; - const sextet1 = table[octet1]; - if (sextet0 < 0 || sextet1 < 0) throw new Error('INVALID_BASE64_SEQ'); - buf[j] = (sextet0 << 2) | (sextet1 >> 4); - } else if (padding === 1) { + if (!padding) return buf; + if (padding === 1) { const word = view.getUint16(mainEnd); const octet0 = word >> 8; const octet1 = word & 0xff; @@ -62,7 +59,15 @@ export const createFromBase64Bin = (chars: string = alphabet, paddingOctet: numb if (sextet0 < 0 || sextet1 < 0 || sextet2 < 0) throw new Error('INVALID_BASE64_SEQ'); buf[j] = (sextet0 << 2) | (sextet1 >> 4); buf[j + 1] = (sextet1 << 4) | (sextet2 >> 2); + return buf; } + const word = view.getUint16(mainEnd); + const octet0 = word >> 8; + const octet1 = word & 0xff; + const sextet0 = table[octet0]; + const sextet1 = table[octet1]; + if (sextet0 < 0 || sextet1 < 0) throw new Error('INVALID_BASE64_SEQ'); + buf[j] = (sextet0 << 2) | (sextet1 >> 4); return buf; }; }; diff --git a/src/util/base64/createToBase64Bin.ts b/src/util/base64/createToBase64Bin.ts index 31f371c7b5..fa64b8b264 100644 --- a/src/util/base64/createToBase64Bin.ts +++ b/src/util/base64/createToBase64Bin.ts @@ -33,10 +33,10 @@ export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') = const o1 = uint8[baseLength]; if (doAddPadding) { dest.setInt32(offset, (table2[o1 << 4] << 16) + EE); - offset + 4; + offset += 4; } else { dest.setInt16(offset, table2[o1 << 4]); - offset + 2; + offset += 2; } } else if (extraLength) { const o1 = uint8[baseLength]; @@ -45,7 +45,7 @@ export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') = const v2 = (o2 & 0b1111) << 2; if (doAddPadding) { dest.setInt32(offset, (table2[v1] << 16) + (table[v2] << 8) + E); - offset + 4; + offset += 4; } else { dest.setInt16(offset, table2[v1]); offset += 2; From 6fa62ccb5d3e904982e89d5c1cc9c12459f7fe2c Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 13:19:14 +0100 Subject: [PATCH 11/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20D?= =?UTF-8?q?AG-JSON=20decoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonDecoder.ts | 17 +---- src/json-pack/json/JsonDecoderDag.ts | 75 +++++++++++++++++++ .../json/__tests__/JsonDecoderDag.spec.ts | 30 ++++++++ src/json-pack/json/util.ts | 13 ++++ 4 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 src/json-pack/json/JsonDecoderDag.ts create mode 100644 src/json-pack/json/__tests__/JsonDecoderDag.spec.ts create mode 100644 src/json-pack/json/util.ts diff --git a/src/json-pack/json/JsonDecoder.ts b/src/json-pack/json/JsonDecoder.ts index f2c0b8ec9a..4642a17756 100644 --- a/src/json-pack/json/JsonDecoder.ts +++ b/src/json-pack/json/JsonDecoder.ts @@ -1,6 +1,7 @@ import {decodeUtf8} from '../../util/buffers/utf8/decodeUtf8'; import {Reader} from '../../util/buffers/Reader'; import {fromBase64Bin} from '../../util/base64/fromBase64Bin'; +import {findEndingQuote} from './util'; import type {BinaryJsonDecoder, PackValue} from '../types'; const REGEX_REPLACE_ESCAPED_CHARS = /\\(b|f|n|r|t|"|\/|\\)/g; @@ -104,20 +105,6 @@ const isUndefined = (u8: Uint8Array, x: number) => u8[x++] === 0x3d && // = u8[x++] === 0x22; // " -const findEndingQuote = (uint8: Uint8Array, x: number): number => { - const len = uint8.length; - let char = uint8[x]; - let prev = 0; - while (x < len) { - if (char === 34 && prev !== 92) break; - if (char === 92 && prev === 92) prev = 0; - else prev = char; - char = uint8[++x]; - } - if (x === len) throw new Error('Invalid JSON'); - return x; -}; - const fromCharCode = String.fromCharCode; const readShortUtf8StrAndUnescape = (reader: Reader): string => { @@ -670,7 +657,7 @@ export class JsonDecoder implements BinaryJsonDecoder { } } - public readObj(): Record { + public readObj(): PackValue | Record { const reader = this.reader; if (reader.u8() !== 0x7b) throw new Error('Invalid JSON'); const obj: Record = {}; diff --git a/src/json-pack/json/JsonDecoderDag.ts b/src/json-pack/json/JsonDecoderDag.ts new file mode 100644 index 0000000000..14241c48a2 --- /dev/null +++ b/src/json-pack/json/JsonDecoderDag.ts @@ -0,0 +1,75 @@ +import {JsonDecoder} from './JsonDecoder'; +import {findEndingQuote} from './util'; +import type {PackValue} from '../types'; +import {createFromBase64Bin} from '../../util/base64/createFromBase64Bin'; + +export const fromBase64Bin = createFromBase64Bin(undefined, ''); + +export class JsonDecoderDag extends JsonDecoder { + public readObj(): PackValue | Record | Uint8Array { + const bytes = this.tryReadBytes(); + if (bytes) return bytes; + return super.readObj(); + } + + protected tryReadBytes(): Uint8Array | undefined { + const reader = this.reader; + const x = reader.x; + if (reader.u8() !== 0x7b) { // { + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x22 || reader.u8() !== 0x2f || reader.u8() !== 0x22) { // "/" + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x3a) { // : + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x7b) { // { + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x22 || + reader.u8() !== 0x62 || + reader.u8() !== 0x79 || + reader.u8() !== 0x74 || + reader.u8() !== 0x65 || + reader.u8() !== 0x73 || + reader.u8() !== 0x22 + ) { // "bytes" + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x3a) { // : + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x22) { // " + reader.x = x; + return; + } + const bufStart = reader.x; + const bufEnd = findEndingQuote(reader.uint8, bufStart); + reader.x = 1 + bufEnd; + this.skipWhitespace(); + if (reader.u8() !== 0x7d) { // } + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x7d) { // } + reader.x = x; + return; + } + const bin = fromBase64Bin(reader.view, bufStart, bufEnd - bufStart); + return bin; + } +} diff --git a/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts b/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts new file mode 100644 index 0000000000..d228c78304 --- /dev/null +++ b/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts @@ -0,0 +1,30 @@ +import {Writer} from '../../../util/buffers/Writer'; +import {utf8} from '../../../util/buffers/strings'; +import {JsonEncoderDag} from '../JsonEncoderDag'; +import {JsonDecoderDag} from '../JsonDecoderDag'; + +const writer = new Writer(16); +const encoder = new JsonEncoderDag(writer); +const decoder = new JsonDecoderDag(); + +test('can decode a simple buffer in object', () => { + const buf = utf8`hello world`; + const data = {foo: buf}; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); +}); + +test('can decode buffers inside an array', () => { + const data = [0, utf8``, utf8`asdf`, 1]; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); +}); + +test('can decode buffer with whitespace surrounding literals', () => { + const json = ' { "foo" : { "/" : { "bytes" : "aGVsbG8gd29ybGQ" } } } '; + const encoded = Buffer.from(json); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual({foo: utf8`hello world`}); +}); diff --git a/src/json-pack/json/util.ts b/src/json-pack/json/util.ts new file mode 100644 index 0000000000..f708118e88 --- /dev/null +++ b/src/json-pack/json/util.ts @@ -0,0 +1,13 @@ +export const findEndingQuote = (uint8: Uint8Array, x: number): number => { + const len = uint8.length; + let char = uint8[x]; + let prev = 0; + while (x < len) { + if (char === 34 && prev !== 92) break; + if (char === 92 && prev === 92) prev = 0; + else prev = char; + char = uint8[++x]; + } + if (x === len) throw new Error('Invalid JSON'); + return x; +}; From 2ce6277a44ec1abdea88947126678fc06947c860 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 13:19:48 +0100 Subject: [PATCH 12/16] =?UTF-8?q?style:=20=F0=9F=92=84=20run=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonDecoderDag.ts | 30 +++++++++++++++++--------- src/json-pack/json/JsonEncoderDag.ts | 8 +++---- src/util/base64/createFromBase64Bin.ts | 2 +- src/util/base64/createToBase64Bin.ts | 2 +- src/util/strings/flatstr.ts | 2 +- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/json-pack/json/JsonDecoderDag.ts b/src/json-pack/json/JsonDecoderDag.ts index 14241c48a2..145721db5f 100644 --- a/src/json-pack/json/JsonDecoderDag.ts +++ b/src/json-pack/json/JsonDecoderDag.ts @@ -15,44 +15,52 @@ export class JsonDecoderDag extends JsonDecoder { protected tryReadBytes(): Uint8Array | undefined { const reader = this.reader; const x = reader.x; - if (reader.u8() !== 0x7b) { // { + if (reader.u8() !== 0x7b) { + // { reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x22 || reader.u8() !== 0x2f || reader.u8() !== 0x22) { // "/" + if (reader.u8() !== 0x22 || reader.u8() !== 0x2f || reader.u8() !== 0x22) { + // "/" reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x3a) { // : + if (reader.u8() !== 0x3a) { + // : reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x7b) { // { + if (reader.u8() !== 0x7b) { + // { reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x22 || + if ( + reader.u8() !== 0x22 || reader.u8() !== 0x62 || reader.u8() !== 0x79 || reader.u8() !== 0x74 || reader.u8() !== 0x65 || reader.u8() !== 0x73 || reader.u8() !== 0x22 - ) { // "bytes" + ) { + // "bytes" reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x3a) { // : + if (reader.u8() !== 0x3a) { + // : reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x22) { // " + if (reader.u8() !== 0x22) { + // " reader.x = x; return; } @@ -60,12 +68,14 @@ export class JsonDecoderDag extends JsonDecoder { const bufEnd = findEndingQuote(reader.uint8, bufStart); reader.x = 1 + bufEnd; this.skipWhitespace(); - if (reader.u8() !== 0x7d) { // } + if (reader.u8() !== 0x7d) { + // } reader.x = x; return; } this.skipWhitespace(); - if (reader.u8() !== 0x7d) { // } + if (reader.u8() !== 0x7d) { + // } reader.x = x; return; } diff --git a/src/json-pack/json/JsonEncoderDag.ts b/src/json-pack/json/JsonEncoderDag.ts index 1d95622421..c136c1f708 100644 --- a/src/json-pack/json/JsonEncoderDag.ts +++ b/src/json-pack/json/JsonEncoderDag.ts @@ -6,20 +6,20 @@ const base64Encode = createToBase64Bin(undefined, ''); /** * Base class for implementing DAG-JSON encoders. - * + * * @see https://ipld.io/specs/codecs/dag-json/spec/ */ export class JsonEncoderDag extends JsonEncoderStable { /** * Encodes binary data as nested `["/", "bytes"]` object encoded in Base64 * without padding. - * + * * Example: - * + * * ```json * {"/":{"bytes":"aGVsbG8gd29ybGQ"}} * ``` - * + * * @param buf Binary data to write. */ public writeBin(buf: Uint8Array): void { diff --git a/src/util/base64/createFromBase64Bin.ts b/src/util/base64/createFromBase64Bin.ts index 1707e676a3..9623a77b59 100644 --- a/src/util/base64/createFromBase64Bin.ts +++ b/src/util/base64/createFromBase64Bin.ts @@ -27,7 +27,7 @@ export const createFromBase64Bin = (chars: string = alphabet, pad: string = '=') } if (length % 4 !== 0) throw new Error('Base64 string length must be a multiple of 4'); const mainEnd = offset + length - (padding ? 4 : 0); - let bufferLength = ((length >> 2) * 3) - padding; + let bufferLength = (length >> 2) * 3 - padding; const buf = new Uint8Array(bufferLength); let j = 0; let i = offset; diff --git a/src/util/base64/createToBase64Bin.ts b/src/util/base64/createToBase64Bin.ts index fa64b8b264..95e3a92758 100644 --- a/src/util/base64/createToBase64Bin.ts +++ b/src/util/base64/createToBase64Bin.ts @@ -15,7 +15,7 @@ export const createToBase64Bin = (chars: string = alphabet, pad: string = '=') = const doAddPadding = pad.length === 1; const E: number = doAddPadding ? pad.charCodeAt(0) : 0; - const EE: number = doAddPadding ? ((E << 8) | E) : 0 + const EE: number = doAddPadding ? (E << 8) | E : 0; return (uint8: Uint8Array, start: number, length: number, dest: DataView, offset: number): number => { const extraLength = length % 3; diff --git a/src/util/strings/flatstr.ts b/src/util/strings/flatstr.ts index 850dd5bcfa..1479ac271a 100644 --- a/src/util/strings/flatstr.ts +++ b/src/util/strings/flatstr.ts @@ -1,5 +1,5 @@ export const flatstr = (s: string): string => { - s | 0; + (s) | 0; Number(s); return s; }; From d4f103ec9b924899f5e9a3b25760e710b2cc2cfb Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 13:20:12 +0100 Subject: [PATCH 13/16] =?UTF-8?q?style(util):=20=F0=9F=92=84=20prever=20co?= =?UTF-8?q?nst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/base64/createFromBase64Bin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/base64/createFromBase64Bin.ts b/src/util/base64/createFromBase64Bin.ts index 9623a77b59..ed9b018cb9 100644 --- a/src/util/base64/createFromBase64Bin.ts +++ b/src/util/base64/createFromBase64Bin.ts @@ -27,7 +27,7 @@ export const createFromBase64Bin = (chars: string = alphabet, pad: string = '=') } if (length % 4 !== 0) throw new Error('Base64 string length must be a multiple of 4'); const mainEnd = offset + length - (padding ? 4 : 0); - let bufferLength = (length >> 2) * 3 - padding; + const bufferLength = (length >> 2) * 3 - padding; const buf = new Uint8Array(bufferLength); let j = 0; let i = offset; From 651e22b158f5f5342a3d3481298ca7080148eb80 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 15:15:43 +0100 Subject: [PATCH 14/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20a?= =?UTF-8?q?bility=20to=20encode=20CIDs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonEncoder.ts | 15 +++++- src/json-pack/json/JsonEncoderDag.ts | 10 ++++ .../json/__tests__/JsonEncoderDag.spec.ts | 48 ++++++++++++++----- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/json-pack/json/JsonEncoder.ts b/src/json-pack/json/JsonEncoder.ts index f20daf3d03..a57566005a 100644 --- a/src/json-pack/json/JsonEncoder.ts +++ b/src/json-pack/json/JsonEncoder.ts @@ -12,6 +12,15 @@ export class JsonEncoder implements BinaryJsonEncoder, StreamingBinaryJsonEncode return writer.flush(); } + /** + * Called when the encoder encounters a value that it does not know how to encode. + * + * @param value Some JavaScript value. + */ + public writeUnknown(value: unknown): void { + this.writeNull(); + } + public writeAny(value: unknown): void { switch (typeof value) { case 'boolean': @@ -24,19 +33,21 @@ export class JsonEncoder implements BinaryJsonEncoder, StreamingBinaryJsonEncode if (value === null) return this.writeNull(); const constructor = value.constructor; switch (constructor) { + case Object: + return this.writeObj(value as Record); case Array: return this.writeArr(value as unknown[]); case Uint8Array: return this.writeBin(value as Uint8Array); default: - return this.writeObj(value as Record); + return this.writeUnknown(value); } } case 'undefined': { return this.writeUndef(); } default: - return this.writeNull(); + return this.writeUnknown(value); } } diff --git a/src/json-pack/json/JsonEncoderDag.ts b/src/json-pack/json/JsonEncoderDag.ts index c136c1f708..3c7dd4f51b 100644 --- a/src/json-pack/json/JsonEncoderDag.ts +++ b/src/json-pack/json/JsonEncoderDag.ts @@ -2,6 +2,7 @@ import {JsonEncoderStable} from './JsonEncoderStable'; import {createToBase64Bin} from '../../util/base64/createToBase64Bin'; const objBaseLength = '{"/":{"bytes":""}}'.length; +const cidBaseLength = '{"/":""}'.length; const base64Encode = createToBase64Bin(undefined, ''); /** @@ -46,4 +47,13 @@ export class JsonEncoderDag extends JsonEncoderStable { x += 1; writer.x = x; } + + public writeCid(cid: string): void { + const writer = this.writer; + writer.ensureCapacity(cidBaseLength + cid.length); + writer.u32(0x7b222f22); // {"/" + writer.u16(0x3a22); // :" + writer.ascii(cid); + writer.u16(0x227d); // "} + } } diff --git a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts index c6546512b3..8e08551278 100644 --- a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts +++ b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts @@ -5,18 +5,42 @@ import {JsonEncoderDag} from '../JsonEncoderDag'; const writer = new Writer(16); const encoder = new JsonEncoderDag(writer); -test('can encode a simple buffer in object', () => { - const buf = utf8`hello world`; - const data = {foo: buf}; - const encoded = encoder.encode(data); - const json = Buffer.from(encoded).toString(); - expect(json).toBe('{"foo":{"/":{"bytes":"aGVsbG8gd29ybGQ"}}}'); +describe('Bytes', () => { + test('can encode a simple buffer in object', () => { + const buf = utf8`hello world`; + const data = {foo: buf}; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('{"foo":{"/":{"bytes":"aGVsbG8gd29ybGQ"}}}'); + }); + + test('can encode a simple buffer in array', () => { + const buf = utf8`hello world`; + const data = [0, buf, 1]; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('[0,{"/":{"bytes":"aGVsbG8gd29ybGQ"}},1]'); + }); }); -test('can encode a simple buffer in array', () => { - const buf = utf8`hello world`; - const data = [0, buf, 1]; - const encoded = encoder.encode(data); - const json = Buffer.from(encoded).toString(); - expect(json).toBe('[0,{"/":{"bytes":"aGVsbG8gd29ybGQ"}},1]'); +describe('Cid', () => { + class CID { + constructor(public readonly value: string) {} + } + + class IpfsEncoder extends JsonEncoderDag { + public writeUnknown(value: unknown): void { + if (value instanceof CID) return this.writeCid(value.value); + else super.writeUnknown(value); + } + } + + const encoder = new IpfsEncoder(writer); + + test('can encode a simple buffer in array', () => { + const data = {id: new CID('QmXn5v3z')}; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('{"id":{"/":"QmXn5v3z"}}'); + }); }); From 41ba07d7d799629aaeeaa509de14c88cbc517c00 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 15:37:13 +0100 Subject: [PATCH 15/16] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20s?= =?UTF-8?q?upport=20for=20CID=20decoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/JsonDecoder.ts | 12 +-- src/json-pack/json/JsonDecoderDag.ts | 50 +++++++++++- .../json/__tests__/JsonDecoderDag.spec.ts | 76 ++++++++++++++----- .../json/__tests__/JsonEncoderDag.spec.ts | 9 ++- src/json-pack/types.ts | 2 +- 5 files changed, 123 insertions(+), 26 deletions(-) diff --git a/src/json-pack/json/JsonDecoder.ts b/src/json-pack/json/JsonDecoder.ts index 4642a17756..d595b5359e 100644 --- a/src/json-pack/json/JsonDecoder.ts +++ b/src/json-pack/json/JsonDecoder.ts @@ -185,7 +185,7 @@ const readShortUtf8StrAndUnescape = (reader: Reader): string => { export class JsonDecoder implements BinaryJsonDecoder { public reader = new Reader(); - public read(uint8: Uint8Array): PackValue { + public read(uint8: Uint8Array): unknown { this.reader.reset(uint8); return this.readAny(); } @@ -195,7 +195,7 @@ export class JsonDecoder implements BinaryJsonDecoder { return this.readAny(); } - public readAny(): PackValue { + public readAny(): unknown { this.skipWhitespace(); const reader = this.reader; const x = reader.x; @@ -640,10 +640,10 @@ export class JsonDecoder implements BinaryJsonDecoder { return bin; } - public readArr(): PackValue[] { + public readArr(): unknown[] { const reader = this.reader; if (reader.u8() !== 0x5b) throw new Error('Invalid JSON'); - const arr: PackValue[] = []; + const arr: unknown[] = []; const uint8 = reader.uint8; while (true) { this.skipWhitespace(); @@ -657,10 +657,10 @@ export class JsonDecoder implements BinaryJsonDecoder { } } - public readObj(): PackValue | Record { + public readObj(): PackValue | Record | unknown { const reader = this.reader; if (reader.u8() !== 0x7b) throw new Error('Invalid JSON'); - const obj: Record = {}; + const obj: Record = {}; const uint8 = reader.uint8; while (true) { this.skipWhitespace(); diff --git a/src/json-pack/json/JsonDecoderDag.ts b/src/json-pack/json/JsonDecoderDag.ts index 145721db5f..0735810f2e 100644 --- a/src/json-pack/json/JsonDecoderDag.ts +++ b/src/json-pack/json/JsonDecoderDag.ts @@ -6,9 +6,11 @@ import {createFromBase64Bin} from '../../util/base64/createFromBase64Bin'; export const fromBase64Bin = createFromBase64Bin(undefined, ''); export class JsonDecoderDag extends JsonDecoder { - public readObj(): PackValue | Record | Uint8Array { + public readObj(): PackValue | Record | Uint8Array | unknown { const bytes = this.tryReadBytes(); if (bytes) return bytes; + const cid = this.tryReadCid(); + if (cid) return cid; return super.readObj(); } @@ -82,4 +84,50 @@ export class JsonDecoderDag extends JsonDecoder { const bin = fromBase64Bin(reader.view, bufStart, bufEnd - bufStart); return bin; } + + protected tryReadCid(): undefined | unknown { + const reader = this.reader; + const x = reader.x; + if (reader.u8() !== 0x7b) { + // { + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x22 || reader.u8() !== 0x2f || reader.u8() !== 0x22) { + // "/" + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x3a) { + // : + reader.x = x; + return; + } + this.skipWhitespace(); + if (reader.u8() !== 0x22) { + // " + reader.x = x; + return; + } + const bufStart = reader.x; + const bufEnd = findEndingQuote(reader.uint8, bufStart); + reader.x = 1 + bufEnd; + this.skipWhitespace(); + if (reader.u8() !== 0x7d) { + // } + reader.x = x; + return; + } + const finalX = reader.x; + reader.x = bufStart; + const cid = reader.ascii(bufEnd - bufStart); + reader.x = finalX; + return this.readCid(cid); + } + + public readCid(cid: string): unknown { + return cid; + } } diff --git a/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts b/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts index d228c78304..a0a8f7adbc 100644 --- a/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts +++ b/src/json-pack/json/__tests__/JsonDecoderDag.spec.ts @@ -7,24 +7,66 @@ const writer = new Writer(16); const encoder = new JsonEncoderDag(writer); const decoder = new JsonDecoderDag(); -test('can decode a simple buffer in object', () => { - const buf = utf8`hello world`; - const data = {foo: buf}; - const encoded = encoder.encode(data); - const decoded = decoder.decode(encoded); - expect(decoded).toEqual(data); -}); +describe('Bytes', () => { + test('can decode a simple buffer in object', () => { + const buf = utf8`hello world`; + const data = {foo: buf}; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); + }); + + test('can decode buffers inside an array', () => { + const data = [0, utf8``, utf8`asdf`, 1]; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); + }); -test('can decode buffers inside an array', () => { - const data = [0, utf8``, utf8`asdf`, 1]; - const encoded = encoder.encode(data); - const decoded = decoder.decode(encoded); - expect(decoded).toEqual(data); + test('can decode buffer with whitespace surrounding literals', () => { + const json = ' { "foo" : { "/" : { "bytes" : "aGVsbG8gd29ybGQ" } } } '; + const encoded = Buffer.from(json); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual({foo: utf8`hello world`}); + }); }); -test('can decode buffer with whitespace surrounding literals', () => { - const json = ' { "foo" : { "/" : { "bytes" : "aGVsbG8gd29ybGQ" } } } '; - const encoded = Buffer.from(json); - const decoded = decoder.decode(encoded); - expect(decoded).toEqual({foo: utf8`hello world`}); +describe('Cid', () => { + class CID { + constructor(public readonly value: string) {} + } + + class IpfsEncoder extends JsonEncoderDag { + public writeUnknown(value: unknown): void { + if (value instanceof CID) return this.writeCid(value.value); + else super.writeUnknown(value); + } + } + + class IpfsDecoder extends JsonDecoderDag { + public readCid(cid: string): unknown { + return new CID(cid); + } + } + + const encoder = new IpfsEncoder(writer); + const decoder = new IpfsDecoder(); + + test('can decode a single CID', () => { + const data = new CID('Qm'); + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); + }); + + test('can decode a CID in object and array', () => { + const data = { + foo: 'bar', + baz: new CID('Qm'), + qux: [new CID('bu'), 'quux'], + }; + const encoded = encoder.encode(data); + const decoded = decoder.decode(encoded); + expect(decoded).toEqual(data); + }); }); diff --git a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts index 8e08551278..03e5bf3341 100644 --- a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts +++ b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts @@ -37,10 +37,17 @@ describe('Cid', () => { const encoder = new IpfsEncoder(writer); - test('can encode a simple buffer in array', () => { + test('can encode a CID as object key', () => { const data = {id: new CID('QmXn5v3z')}; const encoded = encoder.encode(data); const json = Buffer.from(encoded).toString(); expect(json).toBe('{"id":{"/":"QmXn5v3z"}}'); }); + + test('can encode a CID in array', () => { + const data = ['a', new CID('b'), 'c']; + const encoded = encoder.encode(data); + const json = Buffer.from(encoded).toString(); + expect(json).toBe('["a",{"/":"b"},"c"]'); + }); }); diff --git a/src/json-pack/types.ts b/src/json-pack/types.ts index b1e3dc9b2e..150ad9340b 100644 --- a/src/json-pack/types.ts +++ b/src/json-pack/types.ts @@ -55,5 +55,5 @@ export interface TlvBinaryJsonEncoder { export interface BinaryJsonDecoder { decode(uint8: Uint8Array): unknown; reader: IReader & IReaderResettable; - read(uint8: Uint8Array): PackValue; + read(uint8: Uint8Array): unknown; } From 1ff87659ec5c9191f953530b549083160bea57dd Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 12 Mar 2024 15:37:56 +0100 Subject: [PATCH 16/16] =?UTF-8?q?style(json-pack):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/json/__tests__/JsonEncoderDag.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts index 03e5bf3341..0368517866 100644 --- a/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts +++ b/src/json-pack/json/__tests__/JsonEncoderDag.spec.ts @@ -36,14 +36,14 @@ describe('Cid', () => { } const encoder = new IpfsEncoder(writer); - + test('can encode a CID as object key', () => { const data = {id: new CID('QmXn5v3z')}; const encoded = encoder.encode(data); const json = Buffer.from(encoded).toString(); expect(json).toBe('{"id":{"/":"QmXn5v3z"}}'); }); - + test('can encode a CID in array', () => { const data = ['a', new CID('b'), 'c']; const encoded = encoder.encode(data);