From bcffb1a29ba40022aefea4dfe1eec316c145ace5 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 01:19:44 +0100 Subject: [PATCH 01/44] =?UTF-8?q?perf(util):=20=E2=9A=A1=EF=B8=8F=20add=20?= =?UTF-8?q?HTTP/WebSocket=20/ping=20benchmark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/__bench__/ping.bench.ts | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/server/__bench__/ping.bench.ts diff --git a/src/server/__bench__/ping.bench.ts b/src/server/__bench__/ping.bench.ts new file mode 100644 index 0000000000..823b8ee79e --- /dev/null +++ b/src/server/__bench__/ping.bench.ts @@ -0,0 +1,54 @@ +// npx ts-node src/server/__bench__/ping.bench.ts + +/* tslint:disable no-console */ + +import {Suite} from 'benchmark'; +import {RpcPersistentClient, WebSocketChannel} from '../../reactive-rpc/common'; +import {Writer} from '../../util/buffers/Writer'; +import {BinaryRpcMessageCodec} from '../../reactive-rpc/common/codec/binary'; +import {CborJsonValueCodec} from '../../json-pack/codecs/cbor'; +import {RpcCodec} from '../../reactive-rpc/common/codec/RpcCodec'; +import {WebSocket} from 'ws'; + +const main = async () => { + const writer = new Writer(1024 * 4); + const msg = new BinaryRpcMessageCodec(); + const req = new CborJsonValueCodec(writer); + const codec = new RpcCodec(msg, req, req); + const client = new RpcPersistentClient({ + codec, + channel: { + newChannel: () => + new WebSocketChannel({ + newSocket: () => (new WebSocket('ws://localhost:9999/rpc', [codec.specifier()]) as any), + }), + }, + }); + client.start(); + + await client.call('util.ping', {}); // Warmup + + const suite = new Suite(); + suite + .add('fetch', async () => { + const res = await fetch('http://localhost:9999/ping', {keepalive: true}); + const pong = await res.text(); + if (pong !== '"pong"') throw new Error('Unexpected response'); + }) + .add('RpcPersistentClient', async () => { + const res = await client.call('util.ping', {}); + if (res !== 'pong') throw new Error('Unexpected response'); + }) + .on('cycle', (event: any) => { + console.log(String(event.target)); + }) + .on('complete', () => { + console.log('Fastest is ' + suite.filter('fastest').map('name')); + }) + .run({async: true}); +}; + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 41be9f3f446d36e332ff62007391ec6ce06c46e5 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 02:21:53 +0100 Subject: [PATCH 02/44] =?UTF-8?q?feat(util):=20=F0=9F=8E=B8=20implement=20?= =?UTF-8?q?base=20for=20streaming=201-byte=20reader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/buffers/StreamingOctetReader.ts | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/util/buffers/StreamingOctetReader.ts diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts new file mode 100644 index 0000000000..94398443a5 --- /dev/null +++ b/src/util/buffers/StreamingOctetReader.ts @@ -0,0 +1,40 @@ +export class StreamingOctetReader { + protected readonly chunks: Uint8Array[] = []; + + /** Total size of all chunks. */ + protected chunkSize: number = 0; + + protected x: number = 0; + + public size(): number { + return this.chunkSize - this.x; + } + + public push(uint8: Uint8Array): void { + this.chunks.push(uint8); + this.chunkSize += uint8.length; + } + + protected assertSize(size: number): void { + if (size > this.size()) throw new RangeError('OUT_OF_BOUNDS'); + } + + public u8(): number { + this.assertSize(1); + const chunk = this.chunks[0]!; + let x = this.x; + const octet = chunk[x++]; + if (x === chunk.length) { + this.chunks.shift(); + this.chunkSize -= chunk.length; + x = 0; + } + this.x = x; + return octet; + } + + public peak(): number { + this.assertSize(1); + return this.chunks[0]![this.x]; + } +} From ca1850a09a2c183036960c97a78e0a30bb6b72ca Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 02:44:37 +0100 Subject: [PATCH 03/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20star?= =?UTF-8?q?t=20Websocker=20frame=20decoder=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + src/reactive-rpc/server/ws/FrameHeader.ts | 8 ++ .../server/ws/WebsocketDecoder.ts | 45 ++++++++ .../server/ws/__tests__/decoder.spec.ts | 17 +++ src/util/buffers/StreamingOctetReader.ts | 29 ++++- yarn.lock | 106 +++++++++++++++++- 6 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 src/reactive-rpc/server/ws/FrameHeader.ts create mode 100644 src/reactive-rpc/server/ws/WebsocketDecoder.ts create mode 100644 src/reactive-rpc/server/ws/__tests__/decoder.spec.ts diff --git a/package.json b/package.json index c0a55128c9..9d63c3ab46 100644 --- a/package.json +++ b/package.json @@ -160,6 +160,7 @@ "webpack": "^5.84.1", "webpack-cli": "^5.1.1", "webpack-dev-server": "^4.15.0", + "websocket": "^1.0.34", "ws": "^8.14.2", "yjs": "13.6.9", "ywasm": "0.16.10" diff --git a/src/reactive-rpc/server/ws/FrameHeader.ts b/src/reactive-rpc/server/ws/FrameHeader.ts new file mode 100644 index 0000000000..cc8e8110a7 --- /dev/null +++ b/src/reactive-rpc/server/ws/FrameHeader.ts @@ -0,0 +1,8 @@ +export class FrameHeader { + constructor( + public readonly fin: 0 | 1, + public readonly opcode: number, + public readonly length: number, + public readonly mask: undefined | [number, number, number, number], + ) {} +} diff --git a/src/reactive-rpc/server/ws/WebsocketDecoder.ts b/src/reactive-rpc/server/ws/WebsocketDecoder.ts new file mode 100644 index 0000000000..274bbc68f3 --- /dev/null +++ b/src/reactive-rpc/server/ws/WebsocketDecoder.ts @@ -0,0 +1,45 @@ +import {StreamingOctetReader} from "../../../util/buffers/StreamingOctetReader"; +import {FrameHeader} from "./FrameHeader"; + +export class WebsocketDecoder { + public readonly reader = new StreamingOctetReader(); + + public push(uint8: Uint8Array): void { + this.reader.push(uint8); + } + + public readFrameHeader(): FrameHeader | undefined { + try { + const reader = this.reader; + if (reader.size() < 2) return undefined; + const b0 = reader.u8(); + const b1 = reader.u8(); + const fin = <0 | 1>(b0 >>> 7); + const opcode = b0 & 0b1111; + const mask = b1 >>> 7; + let length = b1 & 0b01111111; + if (length === 126) { + if (reader.size() < 2) return undefined; + length = reader.u8() << 8 | reader.u8(); + } else if (length === 127) { + if (reader.size() < 8) return undefined; + reader.skip(4); + length = reader.u32(); + } + let maskBytes: undefined | [number, number, number, number]; + if (mask) { + if (reader.size() < 4) return undefined; + maskBytes = [ + reader.u8(), + reader.u8(), + reader.u8(), + reader.u8(), + ]; + } + return new FrameHeader(fin, opcode, length, maskBytes); + } catch (err) { + if (err instanceof RangeError) return undefined; + throw err; + } + } +} diff --git a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts new file mode 100644 index 0000000000..3aedb50357 --- /dev/null +++ b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts @@ -0,0 +1,17 @@ +import {WebsocketDecoder} from "../WebsocketDecoder"; + +const {frame: WebSocketFrame} = require('websocket'); + +console.log(WebSocketFrame); + +test('...', () => { + const frame = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame.mask = true; + frame.binaryPayload = Buffer.from('hello'); + frame.opcode = 1; + const buf = frame.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const header = decoder.readFrameHeader(); + console.log(buf, header); +}); diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 94398443a5..684cf10583 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -10,9 +10,9 @@ export class StreamingOctetReader { return this.chunkSize - this.x; } - public push(uint8: Uint8Array): void { - this.chunks.push(uint8); - this.chunkSize += uint8.length; + public push(chunk: Uint8Array): void { + this.chunks.push(chunk); + this.chunkSize += chunk.length; } protected assertSize(size: number): void { @@ -33,6 +33,29 @@ export class StreamingOctetReader { return octet; } + public u32(): number { + const octet0 = this.u8(); + const octet1 = this.u8(); + const octet2 = this.u8(); + const octet3 = this.u8(); + return (octet0 * 0x1000000) + (octet1 << 16) + (octet2 << 8) | octet3; + } + + public skip(n: number): void { + this.assertSize(n); + const chunk = this.chunks[0]!; + let x = this.x + n; + const length = chunk.length; + if (x < length) { + this.x = x; + return; + } + this.x = 0; + this.chunks.shift(); + this.chunkSize -= length; + this.skip(x - length); + } + public peak(): number { this.assertSize(1); return this.chunks[0]![this.x]; diff --git a/yarn.lock b/yarn.lock index c1538be51b..e1cde4b6c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1682,6 +1682,13 @@ buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" +bufferutil@^4.0.1: + version "4.0.8" + resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.8.tgz#1de6a71092d65d7766c4d8a522b261a6e787e8ea" + integrity sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw== + dependencies: + node-gyp-build "^4.3.0" + builtin-modules@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" @@ -2084,6 +2091,14 @@ csstype@^3.0.2, csstype@^3.0.6: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== +d@1, d@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a" + integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA== + dependencies: + es5-ext "^0.10.50" + type "^1.0.1" + date-fns@^2.30.0: version "2.30.0" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.30.0.tgz#f367e644839ff57894ec6ac480de40cae4b0f4d0" @@ -2091,7 +2106,7 @@ date-fns@^2.30.0: dependencies: "@babel/runtime" "^7.21.0" -debug@2.6.9: +debug@2.6.9, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -2338,6 +2353,32 @@ es-module-lexer@^1.2.1: resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.4.1.tgz#41ea21b43908fe6a287ffcbe4300f790555331f5" integrity sha512-cXLGjP0c4T3flZJKQSuziYoq7MlT+rnvfZjfp7h+I7K9BNX54kP9nyWvdbwjQ4u1iWbOL4u96fgeZLToQlZC7w== +es5-ext@^0.10.35, es5-ext@^0.10.50: + version "0.10.62" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.62.tgz#5e6adc19a6da524bf3d1e02bbc8960e5eb49a9a5" + integrity sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA== + dependencies: + es6-iterator "^2.0.3" + es6-symbol "^3.1.3" + next-tick "^1.1.0" + +es6-iterator@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" + integrity sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g== + dependencies: + d "1" + es5-ext "^0.10.35" + es6-symbol "^3.1.1" + +es6-symbol@^3.1.1, es6-symbol@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.3.tgz#bad5d3c1bcdac28269f4cb331e431c78ac705d18" + integrity sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA== + dependencies: + d "^1.0.1" + ext "^1.1.2" + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -2496,6 +2537,13 @@ express@^4.17.3: utils-merge "1.0.1" vary "~1.1.2" +ext@^1.1.2: + version "1.7.0" + resolved "https://registry.yarnpkg.com/ext/-/ext-1.7.0.tgz#0ea4383c0103d60e70be99e9a7f11027a33c4f5f" + integrity sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw== + dependencies: + type "^2.7.2" + fast-decode-uri-component@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz#46f8b6c22b30ff7a81357d4f59abfae938202543" @@ -3184,6 +3232,11 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +is-typedarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== + is-wsl@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" @@ -4114,6 +4167,11 @@ neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== +next-tick@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" + integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== + no-case@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d" @@ -4147,6 +4205,11 @@ node-gyp-build-optional-packages@5.0.7: resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz#5d2632bbde0ab2f6e22f1bbac2199b07244ae0b3" integrity sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w== +node-gyp-build@^4.3.0: + version "4.7.1" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.7.1.tgz#cd7d2eb48e594874053150a9418ac85af83ca8f7" + integrity sha512-wTSrZ+8lsRRa3I3H8Xr65dLWSgCvY2l4AOnaeKdPA9TB/WYMPaTcrzf3rXvFoVvjKNVnu0CcWSx54qq9GKRUYg== + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5424,6 +5487,23 @@ type-is@~1.6.18: media-typer "0.3.0" mime-types "~2.1.24" +type@^1.0.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0" + integrity sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg== + +type@^2.7.2: + version "2.7.2" + resolved "https://registry.yarnpkg.com/type/-/type-2.7.2.tgz#2376a15a3a28b1efa0f5350dcf72d24df6ef98d0" + integrity sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw== + +typedarray-to-buffer@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" + integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== + dependencies: + is-typedarray "^1.0.0" + typescript@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" @@ -5483,6 +5563,13 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +utf-8-validate@^5.0.2: + version "5.0.10" + resolved "https://registry.yarnpkg.com/utf-8-validate/-/utf-8-validate-5.0.10.tgz#d7d10ea39318171ca982718b6b96a8d2442571a2" + integrity sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ== + dependencies: + node-gyp-build "^4.3.0" + util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -5673,6 +5760,18 @@ websocket-extensions@>=0.1.1: resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== +websocket@^1.0.34: + version "1.0.34" + resolved "https://registry.yarnpkg.com/websocket/-/websocket-1.0.34.tgz#2bdc2602c08bf2c82253b730655c0ef7dcab3111" + integrity sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ== + dependencies: + bufferutil "^4.0.1" + debug "^2.2.0" + es5-ext "^0.10.50" + typedarray-to-buffer "^3.1.5" + utf-8-validate "^5.0.2" + yaeti "^0.0.6" + which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" @@ -5726,6 +5825,11 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yaeti@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/yaeti/-/yaeti-0.0.6.tgz#f26f484d72684cf42bedfb76970aa1608fbf9577" + integrity sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug== + yallist@4.0.0, yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" From 450f85f5ffa804680c8856d8ef1a2ceea2fb0a13 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 13:57:19 +0100 Subject: [PATCH 04/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20basic=20frame=20data=20reading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/WebsocketDecoder.ts | 30 +++++ .../server/ws/__tests__/decoder.spec.ts | 124 ++++++++++++++++-- src/reactive-rpc/server/ws/constants.ts | 7 + 3 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 src/reactive-rpc/server/ws/constants.ts diff --git a/src/reactive-rpc/server/ws/WebsocketDecoder.ts b/src/reactive-rpc/server/ws/WebsocketDecoder.ts index 274bbc68f3..e7479eeb9d 100644 --- a/src/reactive-rpc/server/ws/WebsocketDecoder.ts +++ b/src/reactive-rpc/server/ws/WebsocketDecoder.ts @@ -42,4 +42,34 @@ export class WebsocketDecoder { throw err; } } + + /** + * Read application data of a frame and copy it to the destination buffer. + * Receives the frame header and the number of bytes that still need to be + * copied, returns back the number of bytes that still need to be copied in + * subsequent calls. + * + * @param frame Frame header. + * @param remaining How many bytes are remaining to be copied. + * @param dst The destination buffer to write to. + * @param pos Position in the destination buffer to start writing to. + * @returns The number of bytes that still need to be copied in the next call. + */ + public readFrameData(frame: FrameHeader, remaining: number, dst: Uint8Array, pos: number): number { + const reader = this.reader; + const mask = frame.mask; + const readSize = Math.min(reader.size(), remaining); + if (!mask) { + for (let i = 0; i < readSize; i++) dst[pos++] = reader.u8(); + } else { + const alreadyRead = frame.length - remaining; + for (let i = 0; i < readSize; i++) { + const octet = reader.u8(); + const j = (i + alreadyRead) % 4; + const unmasked = octet ^ mask[j]; + dst[pos++] = unmasked; + } + } + return remaining - readSize; + } } diff --git a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts index 3aedb50357..c060d96b05 100644 --- a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts @@ -2,16 +2,122 @@ import {WebsocketDecoder} from "../WebsocketDecoder"; const {frame: WebSocketFrame} = require('websocket'); -console.log(WebSocketFrame); +test('can read final text packet with mask', () => { + // const frame = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + // frame.mask = true; + // frame.binaryPayload = Buffer.from('hello'); + // frame.opcode = 1; + // const buf = frame.toBuffer(); + // const buf = Buffer.from(new Uint8Array([129, 8, 118, 101, 114, 57, 48, 48, 48])); + const buf = Buffer.from(new Uint8Array([ + 129, 136, // Header + 136, 35, 93, 205, // Mask + 231, 85, 56, 191, 177, 19, 109, 253, // Payload + ])); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(8); + expect(frame.mask).toEqual([136, 35, 93, 205]); + expect(dst.toString()).toBe('over9000'); +}); + +test('can read final text packet without mask', () => { + const buf = Buffer.from(new Uint8Array([129, 8, 111, 118, 101, 114, 57, 48, 48, 48])); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(8); + expect(frame.mask).toEqual(undefined); + expect(dst.toString()).toBe('over9000'); +}); + +test('can read final masked text frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 1; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); +}); + +test('can read non-final masked text frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 1; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); +}); + +test('can read non-final masked binary frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 2; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(2); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); +}); -test('...', () => { - const frame = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - frame.mask = true; - frame.binaryPayload = Buffer.from('hello'); - frame.opcode = 1; - const buf = frame.toBuffer(); +test('can read non-final non-masked binary frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = false; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 2; + const buf = frame0.toBuffer(); const decoder = new WebsocketDecoder(); decoder.push(buf); - const header = decoder.readFrameHeader(); - console.log(buf, header); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(2); + expect(frame.length).toBe(11); + expect(frame.mask).toBe(undefined); + expect(dst.toString()).toBe('hello world'); }); diff --git a/src/reactive-rpc/server/ws/constants.ts b/src/reactive-rpc/server/ws/constants.ts new file mode 100644 index 0000000000..82852bacb9 --- /dev/null +++ b/src/reactive-rpc/server/ws/constants.ts @@ -0,0 +1,7 @@ +export const enum WebsocketFrameOpcode { + TEXT = 1, + BINARY = 2, + CLOSE = 8, + PING = 9, + PONG = 10, +} \ No newline at end of file From 02108f522f29af09e9a43fd7742247618df8bfdf Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 23:00:22 +0100 Subject: [PATCH 05/44] =?UTF-8?q?perf(reactive-rpc):=20=E2=9A=A1=EF=B8=8F?= =?UTF-8?q?=20implement=20more=20efficient=20payload=20copying?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/WebsocketDecoder.ts | 12 +--- .../server/ws/__tests__/decoder.spec.ts | 6 -- src/util/buffers/StreamingOctetReader.ts | 57 ++++++++++++++++++- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/reactive-rpc/server/ws/WebsocketDecoder.ts b/src/reactive-rpc/server/ws/WebsocketDecoder.ts index e7479eeb9d..effc46b292 100644 --- a/src/reactive-rpc/server/ws/WebsocketDecoder.ts +++ b/src/reactive-rpc/server/ws/WebsocketDecoder.ts @@ -59,16 +59,10 @@ export class WebsocketDecoder { const reader = this.reader; const mask = frame.mask; const readSize = Math.min(reader.size(), remaining); - if (!mask) { - for (let i = 0; i < readSize; i++) dst[pos++] = reader.u8(); - } else { + if (!mask) reader.copy(readSize, dst, pos); + else { const alreadyRead = frame.length - remaining; - for (let i = 0; i < readSize; i++) { - const octet = reader.u8(); - const j = (i + alreadyRead) % 4; - const unmasked = octet ^ mask[j]; - dst[pos++] = unmasked; - } + reader.copyXor(readSize, dst, pos, mask, alreadyRead); } return remaining - readSize; } diff --git a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts index c060d96b05..1d81a6b97b 100644 --- a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts @@ -3,12 +3,6 @@ import {WebsocketDecoder} from "../WebsocketDecoder"; const {frame: WebSocketFrame} = require('websocket'); test('can read final text packet with mask', () => { - // const frame = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - // frame.mask = true; - // frame.binaryPayload = Buffer.from('hello'); - // frame.opcode = 1; - // const buf = frame.toBuffer(); - // const buf = Buffer.from(new Uint8Array([129, 8, 118, 101, 114, 57, 48, 48, 48])); const buf = Buffer.from(new Uint8Array([ 129, 136, // Header 136, 35, 93, 205, // Mask diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 684cf10583..19d467c8c0 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -41,8 +41,56 @@ export class StreamingOctetReader { return (octet0 * 0x1000000) + (octet1 << 16) + (octet2 << 8) | octet3; } - public skip(n: number): void { - this.assertSize(n); + public copy(size: number, dst: Uint8Array, pos: number): void { + if (!size) return; + this.assertSize(size); + const chunk0 = this.chunks[0]!; + const size0 = Math.min(chunk0.length - this.x, size); + dst.set(chunk0.subarray(this.x, this.x + size0), pos); + size -= size0; + if (size <= 0) { + this.skipUnsafe(size0); + return; + } + let chunkIndex = 1; + while (size > 0) { + const chunk1 = this.chunks[chunkIndex]!; + const size1 = Math.min(chunk1.length, size); + dst.set(chunk1.subarray(0, size1), pos + size0); + size -= size1; + chunkIndex++; + } + this.skipUnsafe(size); + } + + public copyXor(size: number, dst: Uint8Array, pos: number, mask: [number, number, number, number], maskIndex: number): void { + if (!size) return; + this.assertSize(size); + const chunk0 = this.chunks[0]!; + const size0 = Math.min(chunk0.length - this.x, size); + for (let i = 0; i < size0; i++) + dst[pos + i] = chunk0[this.x + i] ^ mask[maskIndex++ % 4]; + size -= size0; + pos += size0; + if (size <= 0) { + this.skipUnsafe(size0); + return; + } + let chunkIndex = 1; + while (size > 0) { + const chunk1 = this.chunks[chunkIndex]!; + const size1 = Math.min(chunk1.length, size); + for (let i = 0; i < size1; i++) + dst[pos + size0 + i] = chunk1[i] ^ mask[maskIndex++ % 4]; + size -= size1; + pos += size1; + chunkIndex++; + } + this.skipUnsafe(size); + } + + public skipUnsafe(n: number): void { + if (!n) return; const chunk = this.chunks[0]!; let x = this.x + n; const length = chunk.length; @@ -56,6 +104,11 @@ export class StreamingOctetReader { this.skip(x - length); } + public skip(n: number): void { + this.assertSize(n); + this.skipUnsafe(n); + } + public peak(): number { this.assertSize(1); return this.chunks[0]![this.x]; From 9e05aebc94c8d4839b1c08fe79d745f8b66aed01 Mon Sep 17 00:00:00 2001 From: streamich Date: Sun, 24 Dec 2023 23:50:26 +0100 Subject: [PATCH 06/44] =?UTF-8?q?test(reactive-rpc):=20=F0=9F=92=8D=20add?= =?UTF-8?q?=20continuation=20frame=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/__tests__/decoder.spec.ts | 244 ++++++++++-------- 1 file changed, 137 insertions(+), 107 deletions(-) diff --git a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts index 1d81a6b97b..c81678dd45 100644 --- a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts @@ -2,116 +2,146 @@ import {WebsocketDecoder} from "../WebsocketDecoder"; const {frame: WebSocketFrame} = require('websocket'); -test('can read final text packet with mask', () => { - const buf = Buffer.from(new Uint8Array([ - 129, 136, // Header - 136, 35, 93, 205, // Mask - 231, 85, 56, 191, 177, 19, 109, 253, // Payload - ])); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(1); - expect(frame.opcode).toBe(1); - expect(frame.length).toBe(8); - expect(frame.mask).toEqual([136, 35, 93, 205]); - expect(dst.toString()).toBe('over9000'); -}); +describe('data frames', () => { + test('can read final text packet with mask', () => { + const buf = Buffer.from(new Uint8Array([ + 129, 136, // Header + 136, 35, 93, 205, // Mask + 231, 85, 56, 191, 177, 19, 109, 253, // Payload + ])); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(8); + expect(frame.mask).toEqual([136, 35, 93, 205]); + expect(dst.toString()).toBe('over9000'); + }); -test('can read final text packet without mask', () => { - const buf = Buffer.from(new Uint8Array([129, 8, 111, 118, 101, 114, 57, 48, 48, 48])); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(1); - expect(frame.opcode).toBe(1); - expect(frame.length).toBe(8); - expect(frame.mask).toEqual(undefined); - expect(dst.toString()).toBe('over9000'); -}); + test('can read final text packet without mask', () => { + const buf = Buffer.from(new Uint8Array([129, 8, 111, 118, 101, 114, 57, 48, 48, 48])); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(8); + expect(frame.mask).toEqual(undefined); + expect(dst.toString()).toBe('over9000'); + }); -test('can read final masked text frame', () => { - const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - frame0.fin = true; - frame0.mask = true; - frame0.binaryPayload = Buffer.from('hello world'); - frame0.opcode = 1; - const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(1); - expect(frame.opcode).toBe(1); - expect(frame.length).toBe(11); - expect(frame.mask).toBeInstanceOf(Array); - expect(dst.toString()).toBe('hello world'); -}); + test('can read final masked text frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 1; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); + }); -test('can read non-final masked text frame', () => { - const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - frame0.fin = false; - frame0.mask = true; - frame0.binaryPayload = Buffer.from('hello world'); - frame0.opcode = 1; - const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(0); - expect(frame.opcode).toBe(1); - expect(frame.length).toBe(11); - expect(frame.mask).toBeInstanceOf(Array); - expect(dst.toString()).toBe('hello world'); -}); + test('can read non-final masked text frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 1; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(1); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); + }); -test('can read non-final masked binary frame', () => { - const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - frame0.fin = false; - frame0.mask = true; - frame0.binaryPayload = Buffer.from('hello world'); - frame0.opcode = 2; - const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(0); - expect(frame.opcode).toBe(2); - expect(frame.length).toBe(11); - expect(frame.mask).toBeInstanceOf(Array); - expect(dst.toString()).toBe('hello world'); -}); + test('can read non-final masked binary frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 2; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(2); + expect(frame.length).toBe(11); + expect(frame.mask).toBeInstanceOf(Array); + expect(dst.toString()).toBe('hello world'); + }); + + test('can read non-final non-masked binary frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = false; + frame0.binaryPayload = Buffer.from('hello world'); + frame0.opcode = 2; + const buf = frame0.toBuffer(); + const decoder = new WebsocketDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + const dst = Buffer.alloc(frame.length); + let remaining = frame.length; + remaining = decoder.readFrameData(frame, remaining, dst, 0); + expect(frame.fin).toBe(0); + expect(frame.opcode).toBe(2); + expect(frame.length).toBe(11); + expect(frame.mask).toBe(undefined); + expect(dst.toString()).toBe('hello world'); + }); -test('can read non-final non-masked binary frame', () => { - const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); - frame0.fin = false; - frame0.mask = false; - frame0.binaryPayload = Buffer.from('hello world'); - frame0.opcode = 2; - const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); - decoder.push(buf); - const frame = decoder.readFrameHeader()!; - const dst = Buffer.alloc(frame.length); - let remaining = frame.length; - remaining = decoder.readFrameData(frame, remaining, dst, 0); - expect(frame.fin).toBe(0); - expect(frame.opcode).toBe(2); - expect(frame.length).toBe(11); - expect(frame.mask).toBe(undefined); - expect(dst.toString()).toBe('hello world'); + test('can decode a frame with a continuation frame', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = false; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('hello '); + frame0.opcode = 2; + const frame1 = new WebSocketFrame(Buffer.alloc(4), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame1.fin = true; + frame1.mask = true; + frame1.binaryPayload = Buffer.from('world'); + frame1.opcode = 0; + const buf0 = frame0.toBuffer(); + const buf1 = frame1.toBuffer(); + const dst = Buffer.alloc(11); + const decoder = new WebsocketDecoder(); + decoder.push(buf0); + const header0 = decoder.readFrameHeader()!; + let remaining0 = header0.length; + remaining0 = decoder.readFrameData(header0, remaining0, dst, 0); + expect(header0.fin).toBe(0); + decoder.push(buf1); + const header1 = decoder.readFrameHeader()!; + let remaining1 = header1.length; + remaining1 = decoder.readFrameData(header1, remaining1, dst, 6); + expect(header1.fin).toBe(1); + expect(dst.toString()).toBe('hello world'); + }); }); From 1b7db26d59451d7ed23fe52ebdf6f796ad097bbc Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 00:08:59 +0100 Subject: [PATCH 07/44] =?UTF-8?q?refactor(reactive-rpc):=20=F0=9F=92=A1=20?= =?UTF-8?q?improve=20ws=20folder=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/ws/FrameHeader.ts | 8 ---- .../WsFrameDecoder.ts} | 12 ++--- .../ws/{ => codec}/__tests__/decoder.spec.ts | 16 +++---- .../server/ws/{ => codec}/constants.ts | 2 +- src/reactive-rpc/server/ws/codec/errors.ts | 1 + src/reactive-rpc/server/ws/codec/frames.ts | 44 +++++++++++++++++++ src/reactive-rpc/server/ws/codec/index.ts | 4 ++ src/reactive-rpc/server/ws/server.ts | 0 8 files changed, 64 insertions(+), 23 deletions(-) delete mode 100644 src/reactive-rpc/server/ws/FrameHeader.ts rename src/reactive-rpc/server/ws/{WebsocketDecoder.ts => codec/WsFrameDecoder.ts} (83%) rename src/reactive-rpc/server/ws/{ => codec}/__tests__/decoder.spec.ts (93%) rename src/reactive-rpc/server/ws/{ => codec}/constants.ts (61%) create mode 100644 src/reactive-rpc/server/ws/codec/errors.ts create mode 100644 src/reactive-rpc/server/ws/codec/frames.ts create mode 100644 src/reactive-rpc/server/ws/codec/index.ts create mode 100644 src/reactive-rpc/server/ws/server.ts diff --git a/src/reactive-rpc/server/ws/FrameHeader.ts b/src/reactive-rpc/server/ws/FrameHeader.ts deleted file mode 100644 index cc8e8110a7..0000000000 --- a/src/reactive-rpc/server/ws/FrameHeader.ts +++ /dev/null @@ -1,8 +0,0 @@ -export class FrameHeader { - constructor( - public readonly fin: 0 | 1, - public readonly opcode: number, - public readonly length: number, - public readonly mask: undefined | [number, number, number, number], - ) {} -} diff --git a/src/reactive-rpc/server/ws/WebsocketDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts similarity index 83% rename from src/reactive-rpc/server/ws/WebsocketDecoder.ts rename to src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index effc46b292..c904c2c14f 100644 --- a/src/reactive-rpc/server/ws/WebsocketDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -1,14 +1,14 @@ -import {StreamingOctetReader} from "../../../util/buffers/StreamingOctetReader"; -import {FrameHeader} from "./FrameHeader"; +import {StreamingOctetReader} from "../../../../util/buffers/StreamingOctetReader"; +import {WsFrameHeader} from "./frames"; -export class WebsocketDecoder { +export class WsFrameDecoder { public readonly reader = new StreamingOctetReader(); public push(uint8: Uint8Array): void { this.reader.push(uint8); } - public readFrameHeader(): FrameHeader | undefined { + public readFrameHeader(): WsFrameHeader | undefined { try { const reader = this.reader; if (reader.size() < 2) return undefined; @@ -36,7 +36,7 @@ export class WebsocketDecoder { reader.u8(), ]; } - return new FrameHeader(fin, opcode, length, maskBytes); + return new WsFrameHeader(fin, opcode, length, maskBytes); } catch (err) { if (err instanceof RangeError) return undefined; throw err; @@ -55,7 +55,7 @@ export class WebsocketDecoder { * @param pos Position in the destination buffer to start writing to. * @returns The number of bytes that still need to be copied in the next call. */ - public readFrameData(frame: FrameHeader, remaining: number, dst: Uint8Array, pos: number): number { + public readFrameData(frame: WsFrameHeader, remaining: number, dst: Uint8Array, pos: number): number { const reader = this.reader; const mask = frame.mask; const readSize = Math.min(reader.size(), remaining); diff --git a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts similarity index 93% rename from src/reactive-rpc/server/ws/__tests__/decoder.spec.ts rename to src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index c81678dd45..5e37e01cba 100644 --- a/src/reactive-rpc/server/ws/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,4 +1,4 @@ -import {WebsocketDecoder} from "../WebsocketDecoder"; +import {WsFrameDecoder} from "../WsFrameDecoder"; const {frame: WebSocketFrame} = require('websocket'); @@ -9,7 +9,7 @@ describe('data frames', () => { 136, 35, 93, 205, // Mask 231, 85, 56, 191, 177, 19, 109, 253, // Payload ])); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -24,7 +24,7 @@ describe('data frames', () => { test('can read final text packet without mask', () => { const buf = Buffer.from(new Uint8Array([129, 8, 111, 118, 101, 114, 57, 48, 48, 48])); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -44,7 +44,7 @@ describe('data frames', () => { frame0.binaryPayload = Buffer.from('hello world'); frame0.opcode = 1; const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -64,7 +64,7 @@ describe('data frames', () => { frame0.binaryPayload = Buffer.from('hello world'); frame0.opcode = 1; const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -84,7 +84,7 @@ describe('data frames', () => { frame0.binaryPayload = Buffer.from('hello world'); frame0.opcode = 2; const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -104,7 +104,7 @@ describe('data frames', () => { frame0.binaryPayload = Buffer.from('hello world'); frame0.opcode = 2; const buf = frame0.toBuffer(); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); @@ -131,7 +131,7 @@ describe('data frames', () => { const buf0 = frame0.toBuffer(); const buf1 = frame1.toBuffer(); const dst = Buffer.alloc(11); - const decoder = new WebsocketDecoder(); + const decoder = new WsFrameDecoder(); decoder.push(buf0); const header0 = decoder.readFrameHeader()!; let remaining0 = header0.length; diff --git a/src/reactive-rpc/server/ws/constants.ts b/src/reactive-rpc/server/ws/codec/constants.ts similarity index 61% rename from src/reactive-rpc/server/ws/constants.ts rename to src/reactive-rpc/server/ws/codec/constants.ts index 82852bacb9..d937fb9ded 100644 --- a/src/reactive-rpc/server/ws/constants.ts +++ b/src/reactive-rpc/server/ws/codec/constants.ts @@ -1,4 +1,4 @@ -export const enum WebsocketFrameOpcode { +export const enum WsFrameOpcode { TEXT = 1, BINARY = 2, CLOSE = 8, diff --git a/src/reactive-rpc/server/ws/codec/errors.ts b/src/reactive-rpc/server/ws/codec/errors.ts new file mode 100644 index 0000000000..69c6d34e93 --- /dev/null +++ b/src/reactive-rpc/server/ws/codec/errors.ts @@ -0,0 +1 @@ +export class WsFrameDecodingError extends Error {} diff --git a/src/reactive-rpc/server/ws/codec/frames.ts b/src/reactive-rpc/server/ws/codec/frames.ts new file mode 100644 index 0000000000..ed560cdc0f --- /dev/null +++ b/src/reactive-rpc/server/ws/codec/frames.ts @@ -0,0 +1,44 @@ +export class WsFrameHeader { + constructor( + public readonly fin: 0 | 1, + public readonly opcode: number, + public readonly length: number, + public readonly mask: undefined | [number, number, number, number], + ) {} +} + +export class WsPingFrame extends WsFrameHeader { + constructor( + fin: 0 | 1, + opcode: number, + length: number, + mask: undefined | [number, number, number, number], + public readonly data: Uint8Array, + ) { + super(fin, opcode, length, mask); + } +} + +export class WsPongFrame extends WsFrameHeader { + constructor( + fin: 0 | 1, + opcode: number, + length: number, + mask: undefined | [number, number, number, number], + public readonly data: Uint8Array, + ) { + super(fin, opcode, length, mask); + } +} + +export class WsCloseFrame extends WsFrameHeader { + constructor( + fin: 0 | 1, + opcode: number, + length: number, + mask: undefined | [number, number, number, number], + public readonly data: Uint8Array, + ) { + super(fin, opcode, length, mask); + } +} \ No newline at end of file diff --git a/src/reactive-rpc/server/ws/codec/index.ts b/src/reactive-rpc/server/ws/codec/index.ts new file mode 100644 index 0000000000..2593a1cd37 --- /dev/null +++ b/src/reactive-rpc/server/ws/codec/index.ts @@ -0,0 +1,4 @@ +export * from './constants'; +export * from './errors'; +export * from './frames'; +export * from './WsFrameDecoder'; \ No newline at end of file diff --git a/src/reactive-rpc/server/ws/server.ts b/src/reactive-rpc/server/ws/server.ts new file mode 100644 index 0000000000..e69de29bb2 From 64d926dc703a0d3944ab443a59c60a4c57345538 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 00:09:24 +0100 Subject: [PATCH 08/44] =?UTF-8?q?style(reactive-rpc):=20=F0=9F=92=84=20run?= =?UTF-8?q?=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 13 +++------- .../server/ws/codec/__tests__/decoder.spec.ts | 25 ++++++++++++++----- src/reactive-rpc/server/ws/codec/constants.ts | 2 +- src/reactive-rpc/server/ws/codec/frames.ts | 2 +- src/reactive-rpc/server/ws/codec/index.ts | 2 +- src/server/__bench__/ping.bench.ts | 2 +- src/util/buffers/StreamingOctetReader.ts | 16 +++++++----- 7 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index c904c2c14f..925ba591ac 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -1,5 +1,5 @@ -import {StreamingOctetReader} from "../../../../util/buffers/StreamingOctetReader"; -import {WsFrameHeader} from "./frames"; +import {StreamingOctetReader} from '../../../../util/buffers/StreamingOctetReader'; +import {WsFrameHeader} from './frames'; export class WsFrameDecoder { public readonly reader = new StreamingOctetReader(); @@ -20,7 +20,7 @@ export class WsFrameDecoder { let length = b1 & 0b01111111; if (length === 126) { if (reader.size() < 2) return undefined; - length = reader.u8() << 8 | reader.u8(); + length = (reader.u8() << 8) | reader.u8(); } else if (length === 127) { if (reader.size() < 8) return undefined; reader.skip(4); @@ -29,12 +29,7 @@ export class WsFrameDecoder { let maskBytes: undefined | [number, number, number, number]; if (mask) { if (reader.size() < 4) return undefined; - maskBytes = [ - reader.u8(), - reader.u8(), - reader.u8(), - reader.u8(), - ]; + maskBytes = [reader.u8(), reader.u8(), reader.u8(), reader.u8()]; } return new WsFrameHeader(fin, opcode, length, maskBytes); } catch (err) { diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index 5e37e01cba..b898232111 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,14 +1,27 @@ -import {WsFrameDecoder} from "../WsFrameDecoder"; +import {WsFrameDecoder} from '../WsFrameDecoder'; const {frame: WebSocketFrame} = require('websocket'); describe('data frames', () => { test('can read final text packet with mask', () => { - const buf = Buffer.from(new Uint8Array([ - 129, 136, // Header - 136, 35, 93, 205, // Mask - 231, 85, 56, 191, 177, 19, 109, 253, // Payload - ])); + const buf = Buffer.from( + new Uint8Array([ + 129, + 136, // Header + 136, + 35, + 93, + 205, // Mask + 231, + 85, + 56, + 191, + 177, + 19, + 109, + 253, // Payload + ]), + ); const decoder = new WsFrameDecoder(); decoder.push(buf); const frame = decoder.readFrameHeader()!; diff --git a/src/reactive-rpc/server/ws/codec/constants.ts b/src/reactive-rpc/server/ws/codec/constants.ts index d937fb9ded..37f72a4fd9 100644 --- a/src/reactive-rpc/server/ws/codec/constants.ts +++ b/src/reactive-rpc/server/ws/codec/constants.ts @@ -4,4 +4,4 @@ export const enum WsFrameOpcode { CLOSE = 8, PING = 9, PONG = 10, -} \ No newline at end of file +} diff --git a/src/reactive-rpc/server/ws/codec/frames.ts b/src/reactive-rpc/server/ws/codec/frames.ts index ed560cdc0f..46267e45a8 100644 --- a/src/reactive-rpc/server/ws/codec/frames.ts +++ b/src/reactive-rpc/server/ws/codec/frames.ts @@ -41,4 +41,4 @@ export class WsCloseFrame extends WsFrameHeader { ) { super(fin, opcode, length, mask); } -} \ No newline at end of file +} diff --git a/src/reactive-rpc/server/ws/codec/index.ts b/src/reactive-rpc/server/ws/codec/index.ts index 2593a1cd37..ddd48ff815 100644 --- a/src/reactive-rpc/server/ws/codec/index.ts +++ b/src/reactive-rpc/server/ws/codec/index.ts @@ -1,4 +1,4 @@ export * from './constants'; export * from './errors'; export * from './frames'; -export * from './WsFrameDecoder'; \ No newline at end of file +export * from './WsFrameDecoder'; diff --git a/src/server/__bench__/ping.bench.ts b/src/server/__bench__/ping.bench.ts index 823b8ee79e..48e2b54464 100644 --- a/src/server/__bench__/ping.bench.ts +++ b/src/server/__bench__/ping.bench.ts @@ -20,7 +20,7 @@ const main = async () => { channel: { newChannel: () => new WebSocketChannel({ - newSocket: () => (new WebSocket('ws://localhost:9999/rpc', [codec.specifier()]) as any), + newSocket: () => new WebSocket('ws://localhost:9999/rpc', [codec.specifier()]) as any, }), }, }); diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 19d467c8c0..a1e19a19b3 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -38,7 +38,7 @@ export class StreamingOctetReader { const octet1 = this.u8(); const octet2 = this.u8(); const octet3 = this.u8(); - return (octet0 * 0x1000000) + (octet1 << 16) + (octet2 << 8) | octet3; + return (octet0 * 0x1000000 + (octet1 << 16) + (octet2 << 8)) | octet3; } public copy(size: number, dst: Uint8Array, pos: number): void { @@ -63,13 +63,18 @@ export class StreamingOctetReader { this.skipUnsafe(size); } - public copyXor(size: number, dst: Uint8Array, pos: number, mask: [number, number, number, number], maskIndex: number): void { + public copyXor( + size: number, + dst: Uint8Array, + pos: number, + mask: [number, number, number, number], + maskIndex: number, + ): void { if (!size) return; this.assertSize(size); const chunk0 = this.chunks[0]!; const size0 = Math.min(chunk0.length - this.x, size); - for (let i = 0; i < size0; i++) - dst[pos + i] = chunk0[this.x + i] ^ mask[maskIndex++ % 4]; + for (let i = 0; i < size0; i++) dst[pos + i] = chunk0[this.x + i] ^ mask[maskIndex++ % 4]; size -= size0; pos += size0; if (size <= 0) { @@ -80,8 +85,7 @@ export class StreamingOctetReader { while (size > 0) { const chunk1 = this.chunks[chunkIndex]!; const size1 = Math.min(chunk1.length, size); - for (let i = 0; i < size1; i++) - dst[pos + size0 + i] = chunk1[i] ^ mask[maskIndex++ % 4]; + for (let i = 0; i < size1; i++) dst[pos + size0 + i] = chunk1[i] ^ mask[maskIndex++ % 4]; size -= size1; pos += size1; chunkIndex++; From 9df304f52be3b26241bf707f35d020539bfd9dec Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 00:40:25 +0100 Subject: [PATCH 09/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20decode=20close=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 33 +++++++++- .../server/ws/codec/__tests__/decoder.spec.ts | 60 +++++++++++++++++++ src/reactive-rpc/server/ws/codec/constants.ts | 4 ++ src/reactive-rpc/server/ws/codec/errors.ts | 6 +- src/reactive-rpc/server/ws/codec/frames.ts | 3 +- src/util/buffers/StreamingOctetReader.ts | 42 +++++++++++++ 6 files changed, 145 insertions(+), 3 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index 925ba591ac..fcfadedce3 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -1,5 +1,7 @@ import {StreamingOctetReader} from '../../../../util/buffers/StreamingOctetReader'; -import {WsFrameHeader} from './frames'; +import {WsFrameOpcode} from './constants'; +import {WsFrameDecodingError} from './errors'; +import {WsCloseFrame, WsFrameHeader} from './frames'; export class WsFrameDecoder { public readonly reader = new StreamingOctetReader(); @@ -31,6 +33,16 @@ export class WsFrameDecoder { if (reader.size() < 4) return undefined; maskBytes = [reader.u8(), reader.u8(), reader.u8(), reader.u8()]; } + if (opcode >= WsFrameOpcode.MIN_CONTROL_OPCODE) { + switch (opcode) { + case WsFrameOpcode.CLOSE: { + return new WsCloseFrame(fin, opcode, length, maskBytes, 0, ''); + } + default: { + throw new WsFrameDecodingError(); + } + } + } return new WsFrameHeader(fin, opcode, length, maskBytes); } catch (err) { if (err instanceof RangeError) return undefined; @@ -61,4 +73,23 @@ export class WsFrameDecoder { } return remaining - readSize; } + + public readCloseFrameData(frame: WsCloseFrame): void { + let length = frame.length; + if (length > 125) throw new WsFrameDecodingError(); + let code: number = 0; + let reason: string = ''; + if (length > 0) { + const reader = this.reader; + if (length < 2) throw new WsFrameDecodingError(); + const mask = frame.mask; + const octet1 = reader.u8() ^ (mask ? mask[0] : 0); + const octet2 = reader.u8() ^ (mask ? mask[1] : 0); + code = (octet1 << 8) | octet2; + length -= 2; + if (length) reason = reader.utf8(length, mask ?? [0, 0, 0, 0], 2); + } + frame.code = code; + frame.reason = reason; + } } diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index b898232111..2739a32e23 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,4 +1,6 @@ import {WsFrameDecoder} from '../WsFrameDecoder'; +import {WsFrameOpcode} from '../constants'; +import {WsCloseFrame} from '../frames'; const {frame: WebSocketFrame} = require('websocket'); @@ -158,3 +160,61 @@ describe('data frames', () => { expect(dst.toString()).toBe('hello world'); }); }); + +describe('control frames', () => { + test('can read CLOSE frame with masked UTF-8 payload', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = true; + frame0.binaryPayload = Buffer.from('something 🤷‍♂️ happened'); + frame0.closeStatus = 1000; + frame0.opcode = WsFrameOpcode.CLOSE; + const buf = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(frame0.binaryPayload.length + 2); + expect(frame.mask).toBeInstanceOf(Array); + expect((frame as WsCloseFrame).code).toBe(0); + expect((frame as WsCloseFrame).reason).toBe(''); + decoder.readCloseFrameData(frame as WsCloseFrame); + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(frame0.binaryPayload.length + 2); + expect(frame.mask).toBeInstanceOf(Array); + expect((frame as WsCloseFrame).code).toBe(1000); + expect((frame as WsCloseFrame).reason).toBe('something 🤷‍♂️ happened'); + }); + + test('can read CLOSE frame with un-masked UTF-8 payload', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = false; + frame0.binaryPayload = Buffer.from('something 🤷‍♂️ happened'); + frame0.closeStatus = 1000; + frame0.opcode = WsFrameOpcode.CLOSE; + const buf = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(frame0.binaryPayload.length + 2); + expect(frame.mask).toBe(undefined); + expect((frame as WsCloseFrame).code).toBe(0); + expect((frame as WsCloseFrame).reason).toBe(''); + decoder.readCloseFrameData(frame as WsCloseFrame); + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(frame0.binaryPayload.length + 2); + expect(frame.mask).toBe(undefined); + expect((frame as WsCloseFrame).code).toBe(1000); + expect((frame as WsCloseFrame).reason).toBe('something 🤷‍♂️ happened'); + }); +}); diff --git a/src/reactive-rpc/server/ws/codec/constants.ts b/src/reactive-rpc/server/ws/codec/constants.ts index 37f72a4fd9..24601e833e 100644 --- a/src/reactive-rpc/server/ws/codec/constants.ts +++ b/src/reactive-rpc/server/ws/codec/constants.ts @@ -1,6 +1,10 @@ export const enum WsFrameOpcode { + // Data frames TEXT = 1, BINARY = 2, + + // Control frames + MIN_CONTROL_OPCODE = 8, CLOSE = 8, PING = 9, PONG = 10, diff --git a/src/reactive-rpc/server/ws/codec/errors.ts b/src/reactive-rpc/server/ws/codec/errors.ts index 69c6d34e93..8311fa4597 100644 --- a/src/reactive-rpc/server/ws/codec/errors.ts +++ b/src/reactive-rpc/server/ws/codec/errors.ts @@ -1 +1,5 @@ -export class WsFrameDecodingError extends Error {} +export class WsFrameDecodingError extends Error { + constructor() { + super('WS_FRAME_DECODING'); + } +} diff --git a/src/reactive-rpc/server/ws/codec/frames.ts b/src/reactive-rpc/server/ws/codec/frames.ts index 46267e45a8..e8c8f4e84f 100644 --- a/src/reactive-rpc/server/ws/codec/frames.ts +++ b/src/reactive-rpc/server/ws/codec/frames.ts @@ -37,7 +37,8 @@ export class WsCloseFrame extends WsFrameHeader { opcode: number, length: number, mask: undefined | [number, number, number, number], - public readonly data: Uint8Array, + public code: number, + public reason: string, ) { super(fin, opcode, length, mask); } diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index a1e19a19b3..eb375a84be 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -1,3 +1,5 @@ +const fromCharCode = String.fromCharCode; + export class StreamingOctetReader { protected readonly chunks: Uint8Array[] = []; @@ -117,4 +119,44 @@ export class StreamingOctetReader { this.assertSize(1); return this.chunks[0]![this.x]; } + + public utf8(length: number, mask: [number, number, number, number], maskIndex: number): string { + this.assertSize(length); + let i = 0; + const points: number[] = []; + while (i < length) { + let code = this.u8() ^ mask[maskIndex++ % 4]; + i++; + if ((code & 0x80) !== 0) { + const octet2 = (this.u8() ^ mask[maskIndex++ % 4]) & 0x3f; + i++; + if ((code & 0xe0) === 0xc0) { + code = ((code & 0x1f) << 6) | octet2; + } else { + const octet3 = (this.u8() ^ mask[maskIndex++ % 4]) & 0x3f; + i++ + if ((code & 0xf0) === 0xe0) { + code = ((code & 0x1f) << 12) | (octet2 << 6) | octet3; + } else { + if ((code & 0xf8) === 0xf0) { + const octet4 = (this.u8() ^ mask[maskIndex++ % 4]) & 0x3f; + i++; + let unit = ((code & 0x07) << 0x12) | (octet2 << 0x0c) | (octet3 << 0x06) | octet4; + if (unit > 0xffff) { + unit -= 0x10000; + const unit0 = ((unit >>> 10) & 0x3ff) | 0xd800; + code = 0xdc00 | (unit & 0x3ff); + points.push(unit0); + } else { + code = unit; + } + } + } + } + } + points.push(code); + } + return fromCharCode.apply(String, points); + }; + } From b36919b78994bc339621329f8ba15e76bfb062db Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 00:41:11 +0100 Subject: [PATCH 10/44] =?UTF-8?q?style(reactive-rpc):=20=F0=9F=92=84=20run?= =?UTF-8?q?=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/buffers/StreamingOctetReader.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index eb375a84be..5ee05d78c3 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -134,7 +134,7 @@ export class StreamingOctetReader { code = ((code & 0x1f) << 6) | octet2; } else { const octet3 = (this.u8() ^ mask[maskIndex++ % 4]) & 0x3f; - i++ + i++; if ((code & 0xf0) === 0xe0) { code = ((code & 0x1f) << 12) | (octet2 << 6) | octet3; } else { @@ -157,6 +157,5 @@ export class StreamingOctetReader { points.push(code); } return fromCharCode.apply(String, points); - }; - + } } From 738c90492b5aa4eb2ca043300516714a5e559092 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 01:07:03 +0100 Subject: [PATCH 11/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20parse=20PING=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 21 ++++++---- .../server/ws/codec/__tests__/decoder.spec.ts | 38 ++++++++++++++++++- src/util/buffers/StreamingOctetReader.ts | 14 +++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index fcfadedce3..37205a7de5 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -1,7 +1,7 @@ import {StreamingOctetReader} from '../../../../util/buffers/StreamingOctetReader'; import {WsFrameOpcode} from './constants'; import {WsFrameDecodingError} from './errors'; -import {WsCloseFrame, WsFrameHeader} from './frames'; +import {WsCloseFrame, WsFrameHeader, WsPingFrame} from './frames'; export class WsFrameDecoder { public readonly reader = new StreamingOctetReader(); @@ -18,7 +18,7 @@ export class WsFrameDecoder { const b1 = reader.u8(); const fin = <0 | 1>(b0 >>> 7); const opcode = b0 & 0b1111; - const mask = b1 >>> 7; + const maskBit = b1 >>> 7; let length = b1 & 0b01111111; if (length === 126) { if (reader.size() < 2) return undefined; @@ -28,22 +28,27 @@ export class WsFrameDecoder { reader.skip(4); length = reader.u32(); } - let maskBytes: undefined | [number, number, number, number]; - if (mask) { + let mask: undefined | [number, number, number, number]; + if (maskBit) { if (reader.size() < 4) return undefined; - maskBytes = [reader.u8(), reader.u8(), reader.u8(), reader.u8()]; + mask = [reader.u8(), reader.u8(), reader.u8(), reader.u8()]; } if (opcode >= WsFrameOpcode.MIN_CONTROL_OPCODE) { switch (opcode) { case WsFrameOpcode.CLOSE: { - return new WsCloseFrame(fin, opcode, length, maskBytes, 0, ''); + return new WsCloseFrame(fin, opcode, length, mask, 0, ''); + } + case WsFrameOpcode.PING: { + if (length > 125) throw new WsFrameDecodingError(); + const data = mask ? reader.bufXor(length, mask, 0) : reader.buf(length); + return new WsPingFrame(fin, opcode, length, mask, data); } default: { throw new WsFrameDecodingError(); } } } - return new WsFrameHeader(fin, opcode, length, maskBytes); + return new WsFrameHeader(fin, opcode, length, mask); } catch (err) { if (err instanceof RangeError) return undefined; throw err; @@ -80,8 +85,8 @@ export class WsFrameDecoder { let code: number = 0; let reason: string = ''; if (length > 0) { - const reader = this.reader; if (length < 2) throw new WsFrameDecodingError(); + const reader = this.reader; const mask = frame.mask; const octet1 = reader.u8() ^ (mask ? mask[0] : 0); const octet2 = reader.u8() ^ (mask ? mask[1] : 0); diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index 2739a32e23..73d729c8d8 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,6 +1,6 @@ import {WsFrameDecoder} from '../WsFrameDecoder'; import {WsFrameOpcode} from '../constants'; -import {WsCloseFrame} from '../frames'; +import {WsCloseFrame, WsPingFrame} from '../frames'; const {frame: WebSocketFrame} = require('websocket'); @@ -217,4 +217,40 @@ describe('control frames', () => { expect((frame as WsCloseFrame).code).toBe(1000); expect((frame as WsCloseFrame).reason).toBe('something 🤷‍♂️ happened'); }); + + test('can read PING frame with masked bytes', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = true; + frame0.binaryPayload = new Uint8Array([1, 2, 3]); + frame0.opcode = WsFrameOpcode.PING; + const buf0 = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf0); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPingFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PING); + expect(frame.length).toBe(3); + expect(frame.mask).toBeInstanceOf(Array); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3])); + }); + + test('can read PING frame with un-masked bytes', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = false; + frame0.binaryPayload = Buffer.from(new Uint8Array([1, 2, 3])); + frame0.opcode = WsFrameOpcode.PING; + const buf0 = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf0); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPingFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PING); + expect(frame.length).toBe(3); + expect(frame.mask).toBe(undefined); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3])); + }); }); diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 5ee05d78c3..97ab898ad3 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -95,6 +95,20 @@ export class StreamingOctetReader { this.skipUnsafe(size); } + public buf(size: number): Uint8Array { + this.assertSize(size); + const buf = new Uint8Array(size); + this.copy(size, buf, 0); + return buf; + } + + public bufXor(size: number, mask: [number, number, number, number], maskIndex: number): Uint8Array { + this.assertSize(size); + const buf = new Uint8Array(size); + this.copyXor(size, buf, 0, mask, maskIndex); + return buf; + } + public skipUnsafe(n: number): void { if (!n) return; const chunk = this.chunks[0]!; From 4eaf4f2d5a5dfa6af95768c2864a7ce18a750edf Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 01:16:52 +0100 Subject: [PATCH 12/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20decode=20PONG=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 7 +++- .../server/ws/codec/__tests__/decoder.spec.ts | 38 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index 37205a7de5..e7cd61d880 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -1,7 +1,7 @@ import {StreamingOctetReader} from '../../../../util/buffers/StreamingOctetReader'; import {WsFrameOpcode} from './constants'; import {WsFrameDecodingError} from './errors'; -import {WsCloseFrame, WsFrameHeader, WsPingFrame} from './frames'; +import {WsCloseFrame, WsFrameHeader, WsPingFrame, WsPongFrame} from './frames'; export class WsFrameDecoder { public readonly reader = new StreamingOctetReader(); @@ -43,6 +43,11 @@ export class WsFrameDecoder { const data = mask ? reader.bufXor(length, mask, 0) : reader.buf(length); return new WsPingFrame(fin, opcode, length, mask, data); } + case WsFrameOpcode.PONG: { + if (length > 125) throw new WsFrameDecodingError(); + const data = mask ? reader.bufXor(length, mask, 0) : reader.buf(length); + return new WsPongFrame(fin, opcode, length, mask, data); + } default: { throw new WsFrameDecodingError(); } diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index 73d729c8d8..9bd33ad92f 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,6 +1,6 @@ import {WsFrameDecoder} from '../WsFrameDecoder'; import {WsFrameOpcode} from '../constants'; -import {WsCloseFrame, WsPingFrame} from '../frames'; +import {WsCloseFrame, WsPingFrame, WsPongFrame} from '../frames'; const {frame: WebSocketFrame} = require('websocket'); @@ -253,4 +253,40 @@ describe('control frames', () => { expect(frame.mask).toBe(undefined); expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3])); }); + + test('can read PONG frame with masked bytes', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = true; + frame0.binaryPayload = new Uint8Array([1, 2, 3]); + frame0.opcode = WsFrameOpcode.PONG; + const buf0 = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf0); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPongFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PONG); + expect(frame.length).toBe(3); + expect(frame.mask).toBeInstanceOf(Array); + expect((frame as WsPongFrame).data).toEqual(new Uint8Array([1, 2, 3])); + }); + + test('can read PONG frame with un-masked bytes', () => { + const frame0 = new WebSocketFrame(Buffer.alloc(256), Buffer.alloc(128), {maxReceivedFrameSize: 1000000}); + frame0.fin = true; + frame0.mask = false; + frame0.binaryPayload = Buffer.from(new Uint8Array([1, 2, 3])); + frame0.opcode = WsFrameOpcode.PONG; + const buf0 = frame0.toBuffer(); + const decoder = new WsFrameDecoder(); + decoder.push(buf0); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPongFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PONG); + expect(frame.length).toBe(3); + expect(frame.mask).toBe(undefined); + expect((frame as WsPongFrame).data).toEqual(new Uint8Array([1, 2, 3])); + }); }); From 0d5e7d3522b6439009ecaa57b66584cd9d023e68 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 01:46:01 +0100 Subject: [PATCH 13/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20encode=20PIGN=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameEncoder.ts | 37 +++++++++++++++++++ .../server/ws/codec/__tests__/encoder.spec.ts | 34 +++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts create mode 100644 src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts new file mode 100644 index 0000000000..efc8dfe255 --- /dev/null +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -0,0 +1,37 @@ +import {Writer} from "../../../../util/buffers/Writer"; +import {WsFrameOpcode} from "./constants"; +import type {IWriter, IWriterGrowable} from "../../../../util/buffers"; + +export class WsFrameEncoder { + constructor(public readonly writer: W = new Writer() as any) {} + + public encodePing(data: Uint8Array | null): Uint8Array { + this.writePing(data); + return this.writer.flush(); + } + + public writePing(data: Uint8Array | null): void { + let length = 0; + if (data && (length = data.length)) { + this.writeHeader(1, WsFrameOpcode.PING, length, 0); + this.writer.buf(data, length); + } else { + this.writeHeader(1, WsFrameOpcode.PING, 0, 0); + } + } + + public writeHeader(fin: 0 | 1, opcode: WsFrameOpcode, length: number, mask: number): void { + const octet1 = (fin << 7) | opcode; + const octet2 = (mask ? 0b10000000 : 0) | (length < 126 ? length : length < 0x10000 ? 126 : 127); + const writer = this.writer; + writer.u16((octet1 << 8) | octet2); + if (length >= 126) { + if (length < 0x10000) writer.u16(length); + else { + writer.u32(0); + writer.u32(length); + } + } + if (mask) writer.u32(0); + } +} diff --git a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts new file mode 100644 index 0000000000..e3da25b598 --- /dev/null +++ b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts @@ -0,0 +1,34 @@ +import {WsFrameDecoder} from '../WsFrameDecoder'; +import {WsFrameEncoder} from '../WsFrameEncoder'; +import {WsFrameOpcode} from '../constants'; +import {WsPingFrame} from '../frames'; + +describe('control frames', () => { + test('can encode an empty PING frame', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodePing(null); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPingFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PING); + expect(frame.length).toBe(0); + expect(frame.mask).toBeUndefined(); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array(0)); + }); + + test('can encode a PING frame with data', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodePing(new Uint8Array([1, 2, 3, 4])); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPingFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PING); + expect(frame.length).toBe(4); + expect(frame.mask).toBeUndefined(); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3, 4])); + }); +}); From 6d20f8e9f2995fb056641568029885a80e8ebe2b Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 01:47:51 +0100 Subject: [PATCH 14/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20encode=20PONG=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameEncoder.ts | 15 ++++++++++ .../server/ws/codec/__tests__/encoder.spec.ts | 30 ++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index efc8dfe255..d4425bf155 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -10,6 +10,11 @@ export class WsFrameEncoder { test('can encode an empty PING frame', () => { @@ -31,4 +31,32 @@ describe('control frames', () => { expect(frame.mask).toBeUndefined(); expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3, 4])); }); + + test('can encode an empty PONG frame', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodePong(null); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPongFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PONG); + expect(frame.length).toBe(0); + expect(frame.mask).toBeUndefined(); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array(0)); + }); + + test('can encode a PONG frame with data', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodePong(new Uint8Array([1, 2, 3, 4])); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsPongFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.PONG); + expect(frame.length).toBe(4); + expect(frame.mask).toBeUndefined(); + expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3, 4])); + }); }); From 9f3ad5c76183a538c0f117ca28db7c9dd51f688c Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 02:01:06 +0100 Subject: [PATCH 15/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20encode=20CLOSE=20Websocket=20frame?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameEncoder.ts | 37 +++++++++++++++++-- .../server/ws/codec/__tests__/encoder.spec.ts | 31 +++++++++++++++- src/reactive-rpc/server/ws/codec/errors.ts | 6 +++ src/util/buffers/StreamingOctetReader.ts | 2 +- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index d4425bf155..f79064cec1 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -1,6 +1,7 @@ -import {Writer} from "../../../../util/buffers/Writer"; -import {WsFrameOpcode} from "./constants"; -import type {IWriter, IWriterGrowable} from "../../../../util/buffers"; +import {Writer} from '../../../../util/buffers/Writer'; +import {WsFrameOpcode} from './constants'; +import type {IWriter, IWriterGrowable} from '../../../../util/buffers'; +import {WsFrameEncodingError} from './errors'; export class WsFrameEncoder { constructor(public readonly writer: W = new Writer() as any) {} @@ -15,6 +16,11 @@ export class WsFrameEncoder 126 - 2) throw new WsFrameEncodingError(); + writer.uint8[lengthX] = (writer.uint8[lengthX] & 0b10000000) | (utf8Length + 2); + } + } + } else { + this.writeHeader(1, WsFrameOpcode.CLOSE, 0, 0); + } + } + public writeHeader(fin: 0 | 1, opcode: WsFrameOpcode, length: number, mask: number): void { const octet1 = (fin << 7) | opcode; const octet2 = (mask ? 0b10000000 : 0) | (length < 126 ? length : length < 0x10000 ? 126 : 127); diff --git a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts index dcf4a69c13..070c8cd60a 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts @@ -1,7 +1,7 @@ import {WsFrameDecoder} from '../WsFrameDecoder'; import {WsFrameEncoder} from '../WsFrameEncoder'; import {WsFrameOpcode} from '../constants'; -import {WsPingFrame, WsPongFrame} from '../frames'; +import {WsCloseFrame, WsPingFrame, WsPongFrame} from '../frames'; describe('control frames', () => { test('can encode an empty PING frame', () => { @@ -59,4 +59,33 @@ describe('control frames', () => { expect(frame.mask).toBeUndefined(); expect((frame as WsPingFrame).data).toEqual(new Uint8Array([1, 2, 3, 4])); }); + + test('can encode an empty CLOSE frame', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodeClose(''); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(0); + expect(frame.mask).toBeUndefined(); + }); + + test('can encode a CLOSE frame with code and reason', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodeClose('gg wp', 123); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + decoder.readCloseFrameData(frame as WsCloseFrame); + expect(frame).toBeInstanceOf(WsCloseFrame); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.CLOSE); + expect(frame.length).toBe(2 + 5); + expect(frame.mask).toBeUndefined(); + expect((frame as WsCloseFrame).code).toBe(123); + expect((frame as WsCloseFrame).reason).toBe('gg wp'); + }); }); diff --git a/src/reactive-rpc/server/ws/codec/errors.ts b/src/reactive-rpc/server/ws/codec/errors.ts index 8311fa4597..dec8534b17 100644 --- a/src/reactive-rpc/server/ws/codec/errors.ts +++ b/src/reactive-rpc/server/ws/codec/errors.ts @@ -3,3 +3,9 @@ export class WsFrameDecodingError extends Error { super('WS_FRAME_DECODING'); } } + +export class WsFrameEncodingError extends Error { + constructor() { + super('WS_FRAME_ENCODING'); + } +} diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 97ab898ad3..73e1bbaccd 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -112,7 +112,7 @@ export class StreamingOctetReader { public skipUnsafe(n: number): void { if (!n) return; const chunk = this.chunks[0]!; - let x = this.x + n; + const x = this.x + n; const length = chunk.length; if (x < length) { this.x = x; From 4f67df82f0816d933227b87a11968878ab000262 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 13:51:14 +0100 Subject: [PATCH 16/44] =?UTF-8?q?perf(reactive-rpc):=20=E2=9A=A1=EF=B8=8F?= =?UTF-8?q?=20improve=20header=20encoding=20speed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameEncoder.ts | 59 ++++++++++++++----- .../server/ws/codec/__tests__/encoder.spec.ts | 5 ++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index f79064cec1..a1018c3893 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -24,20 +24,20 @@ export class WsFrameEncoder= 126) { - if (length < 0x10000) writer.u16(length); - else { - writer.u32(0); - writer.u32(length); - } + if (length < 126) { + const octet2 = maskBit | length; + writer.u16((octet1 << 8) | octet2); + return; + } else if (length < 0x10000) { + const octet2 = maskBit | 126; + writer.u32((((octet1 << 8) | octet2) * 0x10000) + length); + return; + } else { + const octet2 = maskBit | 126; + writer.u16((octet1 << 8) | octet2); + writer.u32(0); + writer.u32(length); } - if (mask) writer.u32(0); + if (mask) writer.u32(mask); + } + + public writeDataMsgHdrFast(length: number): void { + const writer = this.writer; + if (length < 126) { + const octet1 = 0b10000000 + WsFrameOpcode.BINARY; + const octet2 = length; + writer.u16((octet1 << 8) | octet2); + return; + } + if (length < 0x10000) { + const octet1 = 0b10000000 + WsFrameOpcode.BINARY; + const octet2 = 126; + writer.u32((((octet1 << 8) | octet2) * 0x10000) + length); + return; + } + const octet1 = 0b10000000 + WsFrameOpcode.BINARY; + const octet2 = 127; + writer.u16((octet1 << 8) | octet2); + writer.u32(0); + writer.u32(length); } } diff --git a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts index 070c8cd60a..220e8ee61f 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts @@ -89,3 +89,8 @@ describe('control frames', () => { expect((frame as WsCloseFrame).reason).toBe('gg wp'); }); }); + +test.todo('fuzz message sizes'); +test.todo('test with masking'); +test.todo('test with fragmented messages'); +test.todo('test with fragmented messages and control frames in between'); From 078e59b6136aae86565db6acff61e2e9c8b166c2 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 14:14:26 +0100 Subject: [PATCH 17/44] =?UTF-8?q?fix(reactive-rpc):=20=F0=9F=90=9B=20corre?= =?UTF-8?q?ct=20message=20size=20encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameEncoder.ts | 26 ++++--- .../server/ws/codec/__tests__/encoder.spec.ts | 71 ++++++++++++++++++- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index a1018c3893..c5e94496ad 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -21,6 +21,16 @@ export class WsFrameEncoder { test('can encode an empty PING frame', () => { @@ -90,7 +90,74 @@ describe('control frames', () => { }); }); -test.todo('fuzz message sizes'); + +describe('data frames', () => { + test('can encode an empty BINARY data frame', () => { + const encoder = new WsFrameEncoder(); + const encoded = encoder.encodeHdr(1, WsFrameOpcode.BINARY, 0, 0); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsFrameHeader); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame.length).toBe(0); + expect(frame.mask).toBeUndefined(); + }); + + test('can encode a BINARY data frame with data', () => { + const encoder = new WsFrameEncoder(); + encoder.writeHdr(1, WsFrameOpcode.BINARY, 5, 0); + encoder.writer.buf(new Uint8Array([1, 2, 3, 4, 5]), 5); + const encoded = encoder.writer.flush(); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsFrameHeader); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame.length).toBe(5); + expect(frame.mask).toBeUndefined(); + const data = decoder.reader.buf(5); + expect(data).toEqual(new Uint8Array([1, 2, 3, 4, 5])); + }); + + test('can encode a fast BINARY data frame with data', () => { + const encoder = new WsFrameEncoder(); + const data = new Uint8Array(333); + encoder.writeDataMsgHdrFast(data.length); + encoder.writer.buf(data, data.length); + const encoded = encoder.writer.flush(); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsFrameHeader); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame.length).toBe(data.length); + expect(frame.mask).toBeUndefined(); + const data2 = decoder.reader.buf(frame.length); + expect(data2).toEqual(data); + }); + + describe('can encode different message sizes', () => { + const sizes = [0, 1, 2, 125, 126, 127, 128, 129, 255, 1234, 65535, 65536, 65537, 7777777, 2 ** 31 - 1]; + const encoder = new WsFrameEncoder(); + const decoder = new WsFrameDecoder(); + for (const size of sizes) { + test(`size ${size}`, () => { + const encoded = encoder.encodeHdr(1, WsFrameOpcode.BINARY, size, 0); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsFrameHeader); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame.length).toBe(size); + }); + } + }); +}); + test.todo('test with masking'); test.todo('test with fragmented messages'); test.todo('test with fragmented messages and control frames in between'); From c4c4549ff30b7704eb8dbfc2f483627bd1a39ab5 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 14:30:08 +0100 Subject: [PATCH 18/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20encode=20masked=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 6 ++++++ .../server/ws/codec/WsFrameEncoder.ts | 16 +++++++++++++-- .../server/ws/codec/__tests__/encoder.spec.ts | 20 ++++++++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index e7cd61d880..e7813fde05 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -84,6 +84,12 @@ export class WsFrameDecoder { return remaining - readSize; } + /** + * Reads application data of the CLOSE frame and sets the code and reason + * properties of the frame. + * + * @param frame Close frame. + */ public readCloseFrameData(frame: WsCloseFrame): void { let length = frame.length; if (length > 125) throw new WsFrameDecodingError(); diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index c5e94496ad..c9724356b4 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -3,6 +3,9 @@ import {WsFrameOpcode} from './constants'; import type {IWriter, IWriterGrowable} from '../../../../util/buffers'; import {WsFrameEncodingError} from './errors'; +const maskBuf = new Uint8Array(4); +const maskBufView = new DataView(maskBuf.buffer, maskBuf.byteOffset, maskBuf.byteLength); + export class WsFrameEncoder { constructor(public readonly writer: W = new Writer() as any) {} @@ -83,11 +86,9 @@ export class WsFrameEncoder { }); } }); + + test('can encode a masked frame', () => { + const encoder = new WsFrameEncoder(); + const data = new Uint8Array([1, 2, 3, 4, 5]); + const mask = 123456789; + encoder.writeHdr(1, WsFrameOpcode.BINARY, data.length, mask); + encoder.writeBufXor(data, mask); + const encoded = encoder.writer.flush(); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame = decoder.readFrameHeader()!; + expect(frame).toBeInstanceOf(WsFrameHeader); + expect(frame.fin).toBe(1); + expect(frame.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame.length).toBe(data.length); + expect(frame.mask).toEqual([7, 91, 205, 21]); + const data2 = decoder.reader.bufXor(frame.length, frame.mask!, 0); + expect(data2).toEqual(data); + }); }); -test.todo('test with masking'); test.todo('test with fragmented messages'); test.todo('test with fragmented messages and control frames in between'); From 590d20976a55a848e2dfc850a476c8dc675e0af8 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 14:38:36 +0100 Subject: [PATCH 19/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20ability=20to=20encode=20fragmented=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/__tests__/encoder.spec.ts | 34 +++++++++++++++++-- src/reactive-rpc/server/ws/codec/constants.ts | 3 ++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts index 2942807a7d..7edc0999a4 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/encoder.spec.ts @@ -175,7 +175,35 @@ describe('data frames', () => { const data2 = decoder.reader.bufXor(frame.length, frame.mask!, 0); expect(data2).toEqual(data); }); -}); -test.todo('test with fragmented messages'); -test.todo('test with fragmented messages and control frames in between'); + test('can encode and decode a fragmented message', () => { + const encoder = new WsFrameEncoder(); + const data1 = new Uint8Array([1, 2, 3]); + const data2 = new Uint8Array([4, 5]); + const mask1 = 333444555; + const mask2 = 123123123; + encoder.writeHdr(0, WsFrameOpcode.BINARY, data1.length, mask1); + encoder.writeBufXor(data1, mask1); + encoder.writeHdr(1, WsFrameOpcode.CONTINUE, data2.length, mask2); + encoder.writeBufXor(data2, mask2); + const encoded = encoder.writer.flush(); + const decoder = new WsFrameDecoder(); + decoder.push(encoded); + const frame0 = decoder.readFrameHeader()!; + expect(frame0).toBeInstanceOf(WsFrameHeader); + expect(frame0.fin).toBe(0); + expect(frame0.opcode).toBe(WsFrameOpcode.BINARY); + expect(frame0.length).toBe(data1.length); + expect(frame0.mask).toEqual([19, 223, 245, 203]); + const data3 = decoder.reader.bufXor(frame0.length, frame0.mask!, 0); + expect(data3).toEqual(data1); + const frame1 = decoder.readFrameHeader()!; + expect(frame1).toBeInstanceOf(WsFrameHeader); + expect(frame1.fin).toBe(1); + expect(frame1.opcode).toBe(WsFrameOpcode.CONTINUE); + expect(frame1.length).toBe(data2.length); + expect(frame1.mask).toEqual([7, 86, 181, 179]); + const data4 = decoder.reader.bufXor(frame1.length, frame1.mask!, 0); + expect(data4).toEqual(data2); + }); +}); diff --git a/src/reactive-rpc/server/ws/codec/constants.ts b/src/reactive-rpc/server/ws/codec/constants.ts index 24601e833e..ccd1296261 100644 --- a/src/reactive-rpc/server/ws/codec/constants.ts +++ b/src/reactive-rpc/server/ws/codec/constants.ts @@ -1,4 +1,7 @@ export const enum WsFrameOpcode { + // Continuation fragment of a data frame + CONTINUE = 0, + // Data frames TEXT = 1, BINARY = 2, From 5a61e50328ee50c7f60658a5fa58319a99e418aa Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 25 Dec 2023 15:41:46 +0100 Subject: [PATCH 20/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20star?= =?UTF-8?q?t=20WebSocketConnection=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 6 + src/reactive-rpc/server/ws/server.ts | 0 src/reactive-rpc/server/ws/server/WsServer.ts | 17 +++ .../server/ws/server/WsServerConnection.ts | 103 ++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 src/reactive-rpc/__demos__/ws.ts delete mode 100644 src/reactive-rpc/server/ws/server.ts create mode 100644 src/reactive-rpc/server/ws/server/WsServer.ts create mode 100644 src/reactive-rpc/server/ws/server/WsServerConnection.ts diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts new file mode 100644 index 0000000000..fad04a6e55 --- /dev/null +++ b/src/reactive-rpc/__demos__/ws.ts @@ -0,0 +1,6 @@ +// npx ts-node src/reactive-rpc/__demos__/ws.ts + +import {WsServer} from '../server/ws/server/WsServer'; + +const server = new WsServer(); +server.start(); diff --git a/src/reactive-rpc/server/ws/server.ts b/src/reactive-rpc/server/ws/server.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/reactive-rpc/server/ws/server/WsServer.ts b/src/reactive-rpc/server/ws/server/WsServer.ts new file mode 100644 index 0000000000..0d5edf1e3a --- /dev/null +++ b/src/reactive-rpc/server/ws/server/WsServer.ts @@ -0,0 +1,17 @@ +import * as http from 'http'; + +export class WsServer { + public start(): void { + const server = http.createServer(); + server.on('request', (req, res) => { + console.log('REQUEST'); + }); + server.on('upgrade', (req, socket, head) => { + console.log('UPGRADE'); + }); + server.on('clientError', (err, socket) => { + socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); + }); + server.listen(8000); + } +} diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts new file mode 100644 index 0000000000..06e3e196c4 --- /dev/null +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -0,0 +1,103 @@ +import * as net from 'net'; +import {WsCloseFrame, WsFrameDecoder, WsFrameHeader, WsFrameOpcode, WsPingFrame, WsPongFrame} from '../codec'; +import {utf8Size} from '../../../../util/strings/utf8'; +import {FanOut} from 'thingies/es2020/fanout'; +import type {WsFrameEncoder} from '../codec/WsFrameEncoder'; + +export class WsServerConnection { + protected readonly decoder: WsFrameDecoder; + + public closed: WsCloseFrame | null = null; + + /** + * If this is not null, then the connection is receiving a stream: a sequence + * of fragment frames. + */ + protected stream: FanOut | null = null; + + public readonly defaultOnPing = (data: Uint8Array | null): void => { + this.sendPong(data); + }; + + public readonly defaultOnClose = (frame: WsCloseFrame): void => { + this.closed = frame; + this.socket.end(); + }; + + public onmessage: (data: Uint8Array, isUtf8: boolean) => void = () => {}; + public onping: (data: Uint8Array | null) => void = this.defaultOnPing; + public onpong: (data: Uint8Array | null) => void = () => {}; + public onclose: (frame: WsCloseFrame) => void = this.defaultOnClose; + + constructor( + protected readonly encoder: WsFrameEncoder, + public readonly socket: net.Socket, + ) { + const decoder = this.decoder = new WsFrameDecoder(); + socket.on('data', (data) => { + decoder.push(data); + while (true) { + const frame = decoder.readFrameHeader(); + if (!frame) break; + else if (frame instanceof WsPingFrame) { + this.onping(frame.data); + } else if (frame instanceof WsPongFrame) { + this.onpong(frame.data); + } else if (frame instanceof WsCloseFrame) { + this.onclose(frame); + } else if (frame instanceof WsFrameHeader) { + if (this.stream) { + if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); + return; + } + console.log('Data frame received'); + // switch (frame.opcode) { + // case WsFrameOpcode.BINARY: { + // const payload = decoder.readFrameData(frame, ) + // break; + // } + // case WsFrameOpcode.TEXT: { + // break; + // } + // default: { + // throw new Error('WRONG_OPCODE'); + // } + // } + } + } + }); + } + + public sendPing(data: Uint8Array | null): void { + const frame = this.encoder.encodePing(data); + this.socket.write(frame); + } + + public sendPong(data: Uint8Array | null): void { + const frame = this.encoder.encodePong(data); + this.socket.write(frame); + } + + public sendBinMsg(data: Uint8Array): void { + const encoder = this.encoder; + const socket = this.socket; + const header = encoder.encodeDataMsgHdrFast(data.length); + // TODO: benchmark if corking helps + // TODO: maybe cork and uncork on macro task boundary + socket.cork(); + socket.write(header); + socket.write(data); + socket.uncork(); + } + + public sendTxtMsg(txt: string): void { + const encoder = this.encoder; + const writer = encoder.writer; + const size = utf8Size(txt); + encoder.writeHdr(1, WsFrameOpcode.TEXT, size, 0); + writer.ensureCapacity(size); + writer.utf8(txt); + const buf = writer.flush(); + this.socket.write(buf); + } +} From 1e92ddb15f9ab1bc8c2cac8c2a3cad7aa51a399c Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 26 Dec 2023 02:17:37 +0100 Subject: [PATCH 21/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20basic=20socket=20upgrade=20mechanism?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 5 +-- src/reactive-rpc/server/http/HttpServer.ts | 43 +++++++++++++++++++ .../server/ws/server/WsServerConnection.ts | 5 +++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 src/reactive-rpc/server/http/HttpServer.ts diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index fad04a6e55..307b6fbff1 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -1,6 +1,5 @@ // npx ts-node src/reactive-rpc/__demos__/ws.ts -import {WsServer} from '../server/ws/server/WsServer'; +import {HttpServer} from '../server/http/HttpServer'; -const server = new WsServer(); -server.start(); +const server = HttpServer.start(); diff --git a/src/reactive-rpc/server/http/HttpServer.ts b/src/reactive-rpc/server/http/HttpServer.ts new file mode 100644 index 0000000000..9565386eb5 --- /dev/null +++ b/src/reactive-rpc/server/http/HttpServer.ts @@ -0,0 +1,43 @@ +import * as http from 'http'; +import * as net from 'net'; +import * as crypto from 'crypto'; +import {WsServerConnection} from '../ws/server/WsServerConnection'; +import {WsFrameEncoder} from '../ws/codec/WsFrameEncoder'; + +export interface HttpServerOpts { + server: http.Server; +} + +export class HttpServer { + public static start(opts: http.ServerOptions = {}, port = 8000): HttpServer { + const server = http.createServer(opts); + server.listen(port); + return new HttpServer({server}); + } + + constructor(protected readonly opts: HttpServerOpts) { + const server = opts.server; + server.on('request', (req, res) => { + console.log('REQUEST'); + }); + server.on('upgrade', (req, socket, head) => { + const accept = req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; + const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); + socket.write('HTTP/1.1 101 Switching Protocols\r\n' + + 'Upgrade: websocket\r\n' + + 'Connection: Upgrade\r\n' + + 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + + '\r\n' + ); + const encoder = new WsFrameEncoder(); + const connection = new WsServerConnection(encoder, socket as net.Socket); + console.log('head', head); + socket.on('data', (data) => { + console.log('DATA', data); + }); + }); + server.on('clientError', (err, socket) => { + socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); + }); + } +} diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 06e3e196c4..ff38843c68 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -50,7 +50,12 @@ export class WsServerConnection { if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); return; } + const length = frame.length; + // if (length) { + // decoder. + // } console.log('Data frame received'); + console.log(frame); // switch (frame.opcode) { // case WsFrameOpcode.BINARY: { // const payload = decoder.readFrameData(frame, ) From 2b4b34fca6ac4a720e8ecea0d4b1e7a62d7907fd Mon Sep 17 00:00:00 2001 From: streamich Date: Wed, 27 Dec 2023 17:51:34 +0100 Subject: [PATCH 22/44] =?UTF-8?q?refactor(reactive-rpc):=20=F0=9F=92=A1=20?= =?UTF-8?q?rename=20HTTP=20server=20classes=20to=20HTTP1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 4 +-- .../HttpServer.ts => http1/Http1Server.ts} | 25 +++++++++++++------ src/reactive-rpc/server/ws/server/WsServer.ts | 17 ------------- 3 files changed, 19 insertions(+), 27 deletions(-) rename src/reactive-rpc/server/{http/HttpServer.ts => http1/Http1Server.ts} (66%) delete mode 100644 src/reactive-rpc/server/ws/server/WsServer.ts diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index 307b6fbff1..bf78854743 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -1,5 +1,5 @@ // npx ts-node src/reactive-rpc/__demos__/ws.ts -import {HttpServer} from '../server/http/HttpServer'; +import {Http1Server} from '../server/http1/HttpServer'; -const server = HttpServer.start(); +const server = Http1Server.start(); diff --git a/src/reactive-rpc/server/http/HttpServer.ts b/src/reactive-rpc/server/http1/Http1Server.ts similarity index 66% rename from src/reactive-rpc/server/http/HttpServer.ts rename to src/reactive-rpc/server/http1/Http1Server.ts index 9565386eb5..712693f780 100644 --- a/src/reactive-rpc/server/http/HttpServer.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -3,20 +3,30 @@ import * as net from 'net'; import * as crypto from 'crypto'; import {WsServerConnection} from '../ws/server/WsServerConnection'; import {WsFrameEncoder} from '../ws/codec/WsFrameEncoder'; +import {Writer} from '../../../util/buffers/Writer'; -export interface HttpServerOpts { +export interface Http1ServerOpts { server: http.Server; } -export class HttpServer { - public static start(opts: http.ServerOptions = {}, port = 8000): HttpServer { +export class Http1Server { + public static start(opts: http.ServerOptions = {}, port = 8000): Http1Server { const server = http.createServer(opts); server.listen(port); - return new HttpServer({server}); + return new Http1Server({server}); } - constructor(protected readonly opts: HttpServerOpts) { - const server = opts.server; + public readonly server: http.Server; + protected readonly wsEncoder: WsFrameEncoder; + + constructor(protected readonly opts: Http1ServerOpts) { + this.server = opts.server; + const writer = new Writer(); + this.wsEncoder = new WsFrameEncoder(writer); + } + + public start(): void { + const server = this.server; server.on('request', (req, res) => { console.log('REQUEST'); }); @@ -29,8 +39,7 @@ export class HttpServer { 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + '\r\n' ); - const encoder = new WsFrameEncoder(); - const connection = new WsServerConnection(encoder, socket as net.Socket); + const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket); console.log('head', head); socket.on('data', (data) => { console.log('DATA', data); diff --git a/src/reactive-rpc/server/ws/server/WsServer.ts b/src/reactive-rpc/server/ws/server/WsServer.ts deleted file mode 100644 index 0d5edf1e3a..0000000000 --- a/src/reactive-rpc/server/ws/server/WsServer.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as http from 'http'; - -export class WsServer { - public start(): void { - const server = http.createServer(); - server.on('request', (req, res) => { - console.log('REQUEST'); - }); - server.on('upgrade', (req, socket, head) => { - console.log('UPGRADE'); - }); - server.on('clientError', (err, socket) => { - socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); - }); - server.listen(8000); - } -} From e249d90cef5308113d961c5963e90a062c9da0ae Mon Sep 17 00:00:00 2001 From: streamich Date: Wed, 27 Dec 2023 23:57:13 +0100 Subject: [PATCH 23/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20setu?= =?UTF-8?q?p=20HTTP1=20server=20socket=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 11 +++- src/reactive-rpc/server/http1/Http1Server.ts | 63 +++++++++++++------ .../server/ws/server/WsServerConnection.ts | 13 ++++ src/util/router/codegen.ts | 2 +- src/util/router/router.ts | 12 ++-- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index bf78854743..a1a39040b2 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -1,5 +1,14 @@ // npx ts-node src/reactive-rpc/__demos__/ws.ts -import {Http1Server} from '../server/http1/HttpServer'; +import {Http1Server} from '../server/http1/Http1Server'; const server = Http1Server.start(); + +server.ws({ + path: '/ws', + onConnect: (connection) => { + console.log('CONNECTED'); + }, +}); + +server.start(); diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 712693f780..883bf89fd8 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -1,9 +1,10 @@ import * as http from 'http'; import * as net from 'net'; -import * as crypto from 'crypto'; import {WsServerConnection} from '../ws/server/WsServerConnection'; import {WsFrameEncoder} from '../ws/codec/WsFrameEncoder'; import {Writer} from '../../../util/buffers/Writer'; +import {RouteMatcher} from '../../../util/router/codegen'; +import {Router} from '../../../util/router'; export interface Http1ServerOpts { server: http.Server; @@ -11,13 +12,16 @@ export interface Http1ServerOpts { export class Http1Server { public static start(opts: http.ServerOptions = {}, port = 8000): Http1Server { - const server = http.createServer(opts); - server.listen(port); - return new Http1Server({server}); + const rawServer = http.createServer(opts); + rawServer.listen(port); + const server = new Http1Server({server: rawServer}); + return server; } public readonly server: http.Server; protected readonly wsEncoder: WsFrameEncoder; + protected wsRouter = new Router(); + protected wsMatcher: RouteMatcher = () => undefined; constructor(protected readonly opts: Http1ServerOpts) { this.server = opts.server; @@ -27,26 +31,45 @@ export class Http1Server { public start(): void { const server = this.server; + this.wsMatcher = this.wsRouter.compile(); server.on('request', (req, res) => { - console.log('REQUEST'); - }); - server.on('upgrade', (req, socket, head) => { - const accept = req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; - const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); - socket.write('HTTP/1.1 101 Switching Protocols\r\n' + - 'Upgrade: websocket\r\n' + - 'Connection: Upgrade\r\n' + - 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + - '\r\n' - ); - const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket); - console.log('head', head); - socket.on('data', (data) => { - console.log('DATA', data); - }); + console.log('REQUEST', req.method, req.url); }); + server.on('upgrade', this.onWsUpgrade); server.on('clientError', (err, socket) => { socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); }); } + + private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket, head: Buffer) => { + const route = req.url || ''; + console.log('route', route); + const match = this.wsMatcher(route); + console.log('match', match); + if (!match) { + socket.end(); + return; + } + const def = match.data; + const headers = req.headers; + const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket); + if (def.onUpgrade) def.onUpgrade(req, connection); + else { + const secWebSocketKey = headers['sec-websocket-key'] ?? ''; + const secWebSocketProtocol = headers['sec-websocket-protocol'] ?? ''; + const secWebSocketExtensions = headers['sec-websocket-extensions'] ?? ''; + connection.upgrade(secWebSocketKey, secWebSocketProtocol, secWebSocketExtensions); + } + }; + + public ws(def: WsEndpointDefinition): void { + this.wsRouter.add(def.path, def); + } } + +export interface WsEndpointDefinition { + path: string; + maxPayload?: number; + onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; + onConnect(connection: WsServerConnection): void; +} \ No newline at end of file diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index ff38843c68..ed033bb3f7 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -1,4 +1,5 @@ import * as net from 'net'; +import * as crypto from 'crypto'; import {WsCloseFrame, WsFrameDecoder, WsFrameHeader, WsFrameOpcode, WsPingFrame, WsPongFrame} from '../codec'; import {utf8Size} from '../../../../util/strings/utf8'; import {FanOut} from 'thingies/es2020/fanout'; @@ -35,6 +36,7 @@ export class WsServerConnection { ) { const decoder = this.decoder = new WsFrameDecoder(); socket.on('data', (data) => { + console.log('DATA', data); decoder.push(data); while (true) { const frame = decoder.readFrameHeader(); @@ -73,6 +75,17 @@ export class WsServerConnection { }); } + public upgrade(secWebSocketKey: string, secWebSocketProtocol: string, secWebSocketExtensions: string): void { + const accept = secWebSocketKey + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; + const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); + this.socket.write('HTTP/1.1 101 Switching Protocols\r\n' + + 'Upgrade: websocket\r\n' + + 'Connection: Upgrade\r\n' + + 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + + '\r\n' + ); + } + public sendPing(data: Uint8Array | null): void { const frame = this.encoder.encodePing(data); this.socket.write(frame); diff --git a/src/util/router/codegen.ts b/src/util/router/codegen.ts index 2af58ce26e..a2329e08af 100644 --- a/src/util/router/codegen.ts +++ b/src/util/router/codegen.ts @@ -2,7 +2,7 @@ import {Codegen} from '../codegen'; import {JsExpression} from '../codegen/util/JsExpression'; import type {Match} from './router'; -export type RouteMatcher = (route: string) => undefined | Match; +export type RouteMatcher = (route: string) => undefined | Match; export class RouterCodegenCtx { public readonly codegen: Codegen; diff --git a/src/util/router/router.ts b/src/util/router/router.ts index d3bf5968c2..2176a905d0 100644 --- a/src/util/router/router.ts +++ b/src/util/router/router.ts @@ -10,12 +10,12 @@ export interface RouterOptions { defaultUntil?: string; } -export class Router implements Printable { +export class Router implements Printable { public readonly destinations: Destination[] = []; constructor(public readonly options: RouterOptions = {}) {} - public add(route: string | string[], data: unknown) { + public add(route: string | string[], data: Data) { const destination = Destination.from(route, data, this.options.defaultUntil); this.destinations.push(destination); } @@ -34,12 +34,12 @@ export class Router implements Printable { return tree; } - public compile(): RouteMatcher { + public compile(): RouteMatcher { const ctx = new RouterCodegenCtx(); const node = new RouterCodegenOpts(new JsExpression(() => 'str'), '0'); const tree = this.tree(); tree.codegen(ctx, node); - return ctx.codegen.compile(); + return ctx.codegen.compile() as RouteMatcher; } public toString(tab: string = '') { @@ -130,6 +130,6 @@ export class Route implements Printable { } } -export class Match { - constructor(public readonly data: unknown, public params: string[] | null) {} +export class Match { + constructor(public readonly data: Data, public params: string[] | null) {} } From f7bb2ec16d8a4ec2519104400be0341d8f68cb6b Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 00:26:27 +0100 Subject: [PATCH 24/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20deco?= =?UTF-8?q?de=20ws=20messages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 3 ++ src/reactive-rpc/server/http1/Http1Server.ts | 1 + .../server/ws/server/WsServerConnection.ts | 40 ++++++++++--------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index a1a39040b2..2c11bd5293 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -8,6 +8,9 @@ server.ws({ path: '/ws', onConnect: (connection) => { console.log('CONNECTED'); + connection.onmessage = (data, isUtf8) => { + console.log('MESSAGE', data, isUtf8); + }; }, }); diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 883bf89fd8..34cc3a8446 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -60,6 +60,7 @@ export class Http1Server { const secWebSocketExtensions = headers['sec-websocket-extensions'] ?? ''; connection.upgrade(secWebSocketKey, secWebSocketProtocol, secWebSocketExtensions); } + def.onConnect(connection); }; public ws(def: WsEndpointDefinition): void { diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index ed033bb3f7..f07ed264c8 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -4,6 +4,7 @@ import {WsCloseFrame, WsFrameDecoder, WsFrameHeader, WsFrameOpcode, WsPingFrame, import {utf8Size} from '../../../../util/strings/utf8'; import {FanOut} from 'thingies/es2020/fanout'; import type {WsFrameEncoder} from '../codec/WsFrameEncoder'; +import {Reader} from '../../../../util/buffers/Reader'; export class WsServerConnection { protected readonly decoder: WsFrameDecoder; @@ -35,9 +36,19 @@ export class WsServerConnection { public readonly socket: net.Socket, ) { const decoder = this.decoder = new WsFrameDecoder(); + let currentFrame: WsFrameHeader | null = null; socket.on('data', (data) => { - console.log('DATA', data); decoder.push(data); + if (currentFrame) { + const length = currentFrame.length; + if (length <= decoder.reader.size()) { + const buf = new Uint8Array(length); + decoder.readFrameData(currentFrame, length, buf, 0); + const isText = currentFrame.opcode === WsFrameOpcode.TEXT; + currentFrame = null; + this.onmessage(buf, isText); + } + } while (true) { const frame = decoder.readFrameHeader(); if (!frame) break; @@ -50,26 +61,17 @@ export class WsServerConnection { } else if (frame instanceof WsFrameHeader) { if (this.stream) { if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); - return; + throw new Error('streaming not implemented'); } const length = frame.length; - // if (length) { - // decoder. - // } - console.log('Data frame received'); - console.log(frame); - // switch (frame.opcode) { - // case WsFrameOpcode.BINARY: { - // const payload = decoder.readFrameData(frame, ) - // break; - // } - // case WsFrameOpcode.TEXT: { - // break; - // } - // default: { - // throw new Error('WRONG_OPCODE'); - // } - // } + if (length <= decoder.reader.size()) { + const buf = new Uint8Array(length); + decoder.readFrameData(frame, length, buf, 0); + const isText = frame.opcode === WsFrameOpcode.TEXT; + this.onmessage(buf, isText); + } else { + currentFrame = frame; + } } } }); From 024e15d3a8046b66f77c630d0285b4147829f815 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 00:32:05 +0100 Subject: [PATCH 25/44] =?UTF-8?q?refactor(reactive-rpc):=20=F0=9F=92=A1=20?= =?UTF-8?q?improve=20ws?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts | 11 +++++++++++ .../server/ws/server/WsServerConnection.ts | 9 +++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index e7813fde05..4d215da3a4 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -84,6 +84,17 @@ export class WsFrameDecoder { return remaining - readSize; } + public copyFrameData(frame: WsFrameHeader, dst: Uint8Array, pos: number): void { + const reader = this.reader; + const mask = frame.mask; + const readSize = reader.size(); + if (!mask) reader.copy(readSize, dst, pos); + else { + const alreadyRead = frame.length - readSize; + reader.copyXor(readSize, dst, pos, mask, alreadyRead); + } + } + /** * Reads application data of the CLOSE frame and sets the code and reason * properties of the frame. diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index f07ed264c8..6fb50c5a2f 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -37,13 +37,13 @@ export class WsServerConnection { ) { const decoder = this.decoder = new WsFrameDecoder(); let currentFrame: WsFrameHeader | null = null; - socket.on('data', (data) => { + const onData = (data: Uint8Array): void => { decoder.push(data); if (currentFrame) { const length = currentFrame.length; if (length <= decoder.reader.size()) { const buf = new Uint8Array(length); - decoder.readFrameData(currentFrame, length, buf, 0); + decoder.copyFrameData(currentFrame, buf, 0); const isText = currentFrame.opcode === WsFrameOpcode.TEXT; currentFrame = null; this.onmessage(buf, isText); @@ -66,7 +66,7 @@ export class WsServerConnection { const length = frame.length; if (length <= decoder.reader.size()) { const buf = new Uint8Array(length); - decoder.readFrameData(frame, length, buf, 0); + decoder.copyFrameData(frame, buf, 0); const isText = frame.opcode === WsFrameOpcode.TEXT; this.onmessage(buf, isText); } else { @@ -74,7 +74,8 @@ export class WsServerConnection { } } } - }); + }; + socket.on('data', onData); } public upgrade(secWebSocketKey: string, secWebSocketProtocol: string, secWebSocketExtensions: string): void { From 2e1bcf0acc7adf3b3580d56a13c02133cc2c8532 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 10:57:47 +0100 Subject: [PATCH 26/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20buffering=20to=20ws=20writes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/server/WsServerConnection.ts | 87 ++++++++++++------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 6fb50c5a2f..153ad684f2 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -4,12 +4,9 @@ import {WsCloseFrame, WsFrameDecoder, WsFrameHeader, WsFrameOpcode, WsPingFrame, import {utf8Size} from '../../../../util/strings/utf8'; import {FanOut} from 'thingies/es2020/fanout'; import type {WsFrameEncoder} from '../codec/WsFrameEncoder'; -import {Reader} from '../../../../util/buffers/Reader'; export class WsServerConnection { - protected readonly decoder: WsFrameDecoder; - - public closed: WsCloseFrame | null = null; + public closed: boolean = false; /** * If this is not null, then the connection is receiving a stream: a sequence @@ -21,23 +18,18 @@ export class WsServerConnection { this.sendPong(data); }; - public readonly defaultOnClose = (frame: WsCloseFrame): void => { - this.closed = frame; - this.socket.end(); - }; - public onmessage: (data: Uint8Array, isUtf8: boolean) => void = () => {}; public onping: (data: Uint8Array | null) => void = this.defaultOnPing; public onpong: (data: Uint8Array | null) => void = () => {}; - public onclose: (frame: WsCloseFrame) => void = this.defaultOnClose; + public onclose: (frame?: WsCloseFrame) => void = () => {}; constructor( protected readonly encoder: WsFrameEncoder, public readonly socket: net.Socket, ) { - const decoder = this.decoder = new WsFrameDecoder(); + const decoder = new WsFrameDecoder(); let currentFrame: WsFrameHeader | null = null; - const onData = (data: Uint8Array): void => { + const handleData = (data: Uint8Array): void => { decoder.push(data); if (currentFrame) { const length = currentFrame.length; @@ -52,13 +44,10 @@ export class WsServerConnection { while (true) { const frame = decoder.readFrameHeader(); if (!frame) break; - else if (frame instanceof WsPingFrame) { - this.onping(frame.data); - } else if (frame instanceof WsPongFrame) { - this.onpong(frame.data); - } else if (frame instanceof WsCloseFrame) { - this.onclose(frame); - } else if (frame instanceof WsFrameHeader) { + else if (frame instanceof WsPingFrame) this.onping(frame.data); + else if (frame instanceof WsPongFrame) this.onpong(frame.data); + else if (frame instanceof WsCloseFrame) this.onClose(frame); + else if (frame instanceof WsFrameHeader) { if (this.stream) { if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); throw new Error('streaming not implemented'); @@ -75,9 +64,28 @@ export class WsServerConnection { } } }; - socket.on('data', onData); + const handleClose = (hadError: boolean): void => { + if (this.closed) return; + this.onClose(); + }; + socket.on('data', handleData); + socket.on('close', handleClose); + } + + private onClose(frame?: WsCloseFrame): void { + this.closed = true; + if (this.__writeTimer) { + clearImmediate(this.__writeTimer); + this.__writeTimer = null; + } + const socket = this.socket; + socket.removeAllListeners(); + if (!socket.destroyed) socket.destroy(); + this.onclose(frame); } + // ----------------------------------------------------------- Handle upgrade + public upgrade(secWebSocketKey: string, secWebSocketProtocol: string, secWebSocketExtensions: string): void { const accept = secWebSocketKey + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); @@ -89,26 +97,45 @@ export class WsServerConnection { ); } + // ---------------------------------------------------------- Write to socket + + private __buffer: Uint8Array[] = []; + private __writeTimer: NodeJS.Immediate | null = null; + + public write(buf: Uint8Array): void { + if (this.closed) return; + this.__buffer.push(buf); + if (this.__writeTimer) return; + this.__writeTimer = setImmediate(() => { + this.__writeTimer = null; + const buffer = this.__buffer; + this.__buffer = []; + if (!buffer.length) return; + const socket = this.socket; + // TODO: benchmark if corking helps + socket.cork(); + for (let i = 0, len = buffer.length; i < len; i++) socket.write(buffer[i]); + socket.uncork(); + }); + } + + // ------------------------------------------------- Write WebSocket messages + public sendPing(data: Uint8Array | null): void { const frame = this.encoder.encodePing(data); - this.socket.write(frame); + this.write(frame); } public sendPong(data: Uint8Array | null): void { const frame = this.encoder.encodePong(data); - this.socket.write(frame); + this.write(frame); } public sendBinMsg(data: Uint8Array): void { const encoder = this.encoder; - const socket = this.socket; const header = encoder.encodeDataMsgHdrFast(data.length); - // TODO: benchmark if corking helps - // TODO: maybe cork and uncork on macro task boundary - socket.cork(); - socket.write(header); - socket.write(data); - socket.uncork(); + this.write(header); + this.write(data); } public sendTxtMsg(txt: string): void { @@ -119,6 +146,6 @@ export class WsServerConnection { writer.ensureCapacity(size); writer.utf8(txt); const buf = writer.flush(); - this.socket.write(buf); + this.write(buf); } } From 60ce7860e064d5b3308b668a46d4bc2a8854091a Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 11:02:52 +0100 Subject: [PATCH 27/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20hide?= =?UTF-8?q?=20close=20frame=20implementation=20details?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/server/WsServerConnection.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 153ad684f2..0d1aecfae0 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -21,7 +21,7 @@ export class WsServerConnection { public onmessage: (data: Uint8Array, isUtf8: boolean) => void = () => {}; public onping: (data: Uint8Array | null) => void = this.defaultOnPing; public onpong: (data: Uint8Array | null) => void = () => {}; - public onclose: (frame?: WsCloseFrame) => void = () => {}; + public onclose: (code: number, reason: string) => void = () => {}; constructor( protected readonly encoder: WsFrameEncoder, @@ -46,7 +46,7 @@ export class WsServerConnection { if (!frame) break; else if (frame instanceof WsPingFrame) this.onping(frame.data); else if (frame instanceof WsPongFrame) this.onpong(frame.data); - else if (frame instanceof WsCloseFrame) this.onClose(frame); + else if (frame instanceof WsCloseFrame) this.onClose(frame.code, frame.reason); else if (frame instanceof WsFrameHeader) { if (this.stream) { if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); @@ -66,13 +66,13 @@ export class WsServerConnection { }; const handleClose = (hadError: boolean): void => { if (this.closed) return; - this.onClose(); + this.onClose(hadError ? 1001 : 1002, 'END'); }; socket.on('data', handleData); socket.on('close', handleClose); } - private onClose(frame?: WsCloseFrame): void { + private onClose(code: number, reason: string): void { this.closed = true; if (this.__writeTimer) { clearImmediate(this.__writeTimer); @@ -81,7 +81,7 @@ export class WsServerConnection { const socket = this.socket; socket.removeAllListeners(); if (!socket.destroyed) socket.destroy(); - this.onclose(frame); + this.onclose(code, reason); } // ----------------------------------------------------------- Handle upgrade From daaba2a925517ddff56f371101a02310c9502e3b Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 11:06:29 +0100 Subject: [PATCH 28/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20hand?= =?UTF-8?q?le=20parsing=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/server/WsServerConnection.ts | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 0d1aecfae0..a3f68176ca 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -30,38 +30,42 @@ export class WsServerConnection { const decoder = new WsFrameDecoder(); let currentFrame: WsFrameHeader | null = null; const handleData = (data: Uint8Array): void => { - decoder.push(data); - if (currentFrame) { - const length = currentFrame.length; - if (length <= decoder.reader.size()) { - const buf = new Uint8Array(length); - decoder.copyFrameData(currentFrame, buf, 0); - const isText = currentFrame.opcode === WsFrameOpcode.TEXT; - currentFrame = null; - this.onmessage(buf, isText); - } - } - while (true) { - const frame = decoder.readFrameHeader(); - if (!frame) break; - else if (frame instanceof WsPingFrame) this.onping(frame.data); - else if (frame instanceof WsPongFrame) this.onpong(frame.data); - else if (frame instanceof WsCloseFrame) this.onClose(frame.code, frame.reason); - else if (frame instanceof WsFrameHeader) { - if (this.stream) { - if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); - throw new Error('streaming not implemented'); - } - const length = frame.length; + try { + decoder.push(data); + if (currentFrame) { + const length = currentFrame.length; if (length <= decoder.reader.size()) { const buf = new Uint8Array(length); - decoder.copyFrameData(frame, buf, 0); - const isText = frame.opcode === WsFrameOpcode.TEXT; + decoder.copyFrameData(currentFrame, buf, 0); + const isText = currentFrame.opcode === WsFrameOpcode.TEXT; + currentFrame = null; this.onmessage(buf, isText); - } else { - currentFrame = frame; } } + while (true) { + const frame = decoder.readFrameHeader(); + if (!frame) break; + else if (frame instanceof WsPingFrame) this.onping(frame.data); + else if (frame instanceof WsPongFrame) this.onpong(frame.data); + else if (frame instanceof WsCloseFrame) this.onClose(frame.code, frame.reason); + else if (frame instanceof WsFrameHeader) { + if (this.stream) { + if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); + throw new Error('streaming not implemented'); + } + const length = frame.length; + if (length <= decoder.reader.size()) { + const buf = new Uint8Array(length); + decoder.copyFrameData(frame, buf, 0); + const isText = frame.opcode === WsFrameOpcode.TEXT; + this.onmessage(buf, isText); + } else { + currentFrame = frame; + } + } + } + } catch (error) { + this.onClose(1002, 'DATA'); } }; const handleClose = (hadError: boolean): void => { From 74a5bfca148d308237dcb6c4832cb01a5fb78f0c Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 13:26:41 +0100 Subject: [PATCH 29/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20impr?= =?UTF-8?q?ove=20ws=20definition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 8 +++----- src/reactive-rpc/server/ws/server/WsServerConnection.ts | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 34cc3a8446..a59a5ec7d4 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -43,16 +43,14 @@ export class Http1Server { private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket, head: Buffer) => { const route = req.url || ''; - console.log('route', route); const match = this.wsMatcher(route); - console.log('match', match); if (!match) { socket.end(); return; } const def = match.data; const headers = req.headers; - const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket); + const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket, head); if (def.onUpgrade) def.onUpgrade(req, connection); else { const secWebSocketKey = headers['sec-websocket-key'] ?? ''; @@ -60,7 +58,7 @@ export class Http1Server { const secWebSocketExtensions = headers['sec-websocket-extensions'] ?? ''; connection.upgrade(secWebSocketKey, secWebSocketProtocol, secWebSocketExtensions); } - def.onConnect(connection); + def.onConnect(connection, req); }; public ws(def: WsEndpointDefinition): void { @@ -72,5 +70,5 @@ export interface WsEndpointDefinition { path: string; maxPayload?: number; onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; - onConnect(connection: WsServerConnection): void; + onConnect(connection: WsServerConnection, req: http.IncomingMessage): void; } \ No newline at end of file diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index a3f68176ca..1be5fcfeeb 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -26,8 +26,10 @@ export class WsServerConnection { constructor( protected readonly encoder: WsFrameEncoder, public readonly socket: net.Socket, + head: Buffer, ) { const decoder = new WsFrameDecoder(); + if (head.length) decoder.push(head); let currentFrame: WsFrameHeader | null = null; const handleData = (data: Uint8Array): void => { try { From f9762d3e463eb5a74f8cb36b6daf36b560a2bc50 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 14:32:46 +0100 Subject: [PATCH 30/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20impr?= =?UTF-8?q?ove=20HTTP=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 95 +++++++++++++++++--- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index a59a5ec7d4..b4ca26d835 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -6,6 +6,39 @@ import {Writer} from '../../../util/buffers/Writer'; import {RouteMatcher} from '../../../util/router/codegen'; import {Router} from '../../../util/router'; +export type Http1Handler = (params: null | string[], req: http.IncomingMessage, res: http.ServerResponse) => void; +export type Http1NotFoundHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; +export type Http1InternalErrorHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; + +export class Http1EndpointMatch { + constructor(public readonly handler: Http1Handler) {} +} + +export interface Http1EndpointDefinition { + /** + * The HTTP method to match. If not specified, then the handler will be + * invoked for any method. + */ + method?: string | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'OPTIONS' | 'TRACE' | 'CONNECT'; + + /** + * The path to match. Should start with a slash. + */ + path: string; + + /** + * The handler function. + */ + handler: Http1Handler; +} + +export interface WsEndpointDefinition { + path: string; + maxPayload?: number; + onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; + onConnect(connection: WsServerConnection, req: http.IncomingMessage): void; +} + export interface Http1ServerOpts { server: http.Server; } @@ -19,9 +52,6 @@ export class Http1Server { } public readonly server: http.Server; - protected readonly wsEncoder: WsFrameEncoder; - protected wsRouter = new Router(); - protected wsMatcher: RouteMatcher = () => undefined; constructor(protected readonly opts: Http1ServerOpts) { this.server = opts.server; @@ -31,16 +61,62 @@ export class Http1Server { public start(): void { const server = this.server; + this.httpMatcher = this.httpRouter.compile(); this.wsMatcher = this.wsRouter.compile(); - server.on('request', (req, res) => { - console.log('REQUEST', req.method, req.url); - }); + server.on('request', this.onRequest); server.on('upgrade', this.onWsUpgrade); server.on('clientError', (err, socket) => { socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); }); } + // ------------------------------------------------------------- HTTP routing + + public onnotfound: Http1NotFoundHandler = (res) => { + res.writeHead(404, 'Not Found'); + res.end(); + }; + + public oninternalerror: Http1InternalErrorHandler = (res) => { + res.writeHead(500, 'Internal Server Error'); + res.end(); + }; + + protected readonly httpRouter = new Router(); + protected httpMatcher: RouteMatcher = () => undefined; + + public route(def: Http1EndpointDefinition): void { + let path = def.path; + if (path[0] !== '/') path = '/' + path; + const method = def.method ? def.method.toUpperCase() : 'GET'; + const route = method + path; + Number(route); + const match = new Http1EndpointMatch(def.handler); + this.httpRouter.add(route, match); + } + + private readonly onRequest = (req: http.IncomingMessage, res: http.ServerResponse) => { + try { + const route = (req.method || '') + (req.url || ''); + const match = this.httpMatcher(route); + if (!match) { + this.onnotfound(res, req); + return; + } + const params = match.params; + const handler = match.data.handler; + handler(params, req, res); + } catch (error) { + this.oninternalerror(res, req); + } + }; + + // --------------------------------------------------------------- WebSockets + + protected readonly wsEncoder: WsFrameEncoder; + protected readonly wsRouter = new Router(); + protected wsMatcher: RouteMatcher = () => undefined; + private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket, head: Buffer) => { const route = req.url || ''; const match = this.wsMatcher(route); @@ -65,10 +141,3 @@ export class Http1Server { this.wsRouter.add(def.path, def); } } - -export interface WsEndpointDefinition { - path: string; - maxPayload?: number; - onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; - onConnect(connection: WsServerConnection, req: http.IncomingMessage): void; -} \ No newline at end of file From b1c8467ac699369d343aa2996dd7502c788b19b7 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 14:48:26 +0100 Subject: [PATCH 31/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20make?= =?UTF-8?q?=20HTTP1=20server=20printable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 10 ++++++++++ src/reactive-rpc/server/http1/Http1Server.ts | 13 ++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index 2c11bd5293..6365d44741 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -14,4 +14,14 @@ server.ws({ }, }); +server.route({ + path: '/hello', + handler: (params, req, res) => { + res.statusCode = 200; + res.end('Hello World\n'); + }, +}); + server.start(); + +console.log(server + ''); diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index b4ca26d835..b61b7290a2 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -5,6 +5,8 @@ import {WsFrameEncoder} from '../ws/codec/WsFrameEncoder'; import {Writer} from '../../../util/buffers/Writer'; import {RouteMatcher} from '../../../util/router/codegen'; import {Router} from '../../../util/router'; +import {Printable} from '../../../util/print/types'; +import {printTree} from '../../../util/print/printTree'; export type Http1Handler = (params: null | string[], req: http.IncomingMessage, res: http.ServerResponse) => void; export type Http1NotFoundHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; @@ -43,7 +45,7 @@ export interface Http1ServerOpts { server: http.Server; } -export class Http1Server { +export class Http1Server implements Printable { public static start(opts: http.ServerOptions = {}, port = 8000): Http1Server { const rawServer = http.createServer(opts); rawServer.listen(port); @@ -140,4 +142,13 @@ export class Http1Server { public ws(def: WsEndpointDefinition): void { this.wsRouter.add(def.path, def); } + + // ---------------------------------------------------------------- Printable + + public toString(tab: string = ''): string { + return `${this.constructor.name}` + printTree(tab, [ + (tab) => `HTTP/1.1 ${this.httpRouter.toString(tab)}`, + (tab) => `WebSocket ${this.wsRouter.toString(tab)}`, + ]); + } } From c23973a937224b0367efe50d0d78a9f6c07f7ca7 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 15:01:00 +0100 Subject: [PATCH 32/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20do?= =?UTF-8?q?=20not=20send=20date=20header=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index b61b7290a2..c331fd48bc 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -99,6 +99,7 @@ export class Http1Server implements Printable { private readonly onRequest = (req: http.IncomingMessage, res: http.ServerResponse) => { try { + res.sendDate = false; const route = (req.method || '') + (req.url || ''); const match = this.httpMatcher(route); if (!match) { From 4e84b006e2fb60a4d488138abb27a555acac2635 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 16:54:51 +0100 Subject: [PATCH 33/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20add?= =?UTF-8?q?=20concept=20of=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 4 +- src/reactive-rpc/server/http1/Http1Server.ts | 108 +++++++++++++++++-- src/reactive-rpc/server/http1/context.ts | 91 ++++++++++++++++ src/reactive-rpc/server/http1/errors.ts | 5 + src/reactive-rpc/server/http1/util.ts | 34 ++++++ 5 files changed, 230 insertions(+), 12 deletions(-) create mode 100644 src/reactive-rpc/server/http1/context.ts create mode 100644 src/reactive-rpc/server/http1/errors.ts create mode 100644 src/reactive-rpc/server/http1/util.ts diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index 6365d44741..9547ee09d0 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -16,12 +16,14 @@ server.ws({ server.route({ path: '/hello', - handler: (params, req, res) => { + handler: ({res}) => { res.statusCode = 200; res.end('Hello World\n'); }, }); +server.enableHttpPing(); + server.start(); console.log(server + ''); diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index c331fd48bc..f651d62a3b 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -7,10 +7,17 @@ import {RouteMatcher} from '../../../util/router/codegen'; import {Router} from '../../../util/router'; import {Printable} from '../../../util/print/types'; import {printTree} from '../../../util/print/printTree'; - -export type Http1Handler = (params: null | string[], req: http.IncomingMessage, res: http.ServerResponse) => void; +import {PayloadTooLarge} from './errors'; +import {findTokenInText} from './util'; +import {Http1ConnectionContext} from './context'; +import {RpcCodecs} from '../../common/codec/RpcCodecs'; +import {Codecs} from '../../../json-pack/codecs/Codecs'; +import {RpcMessageCodecs} from '../../common/codec/RpcMessageCodecs'; +import {NullObject} from '../../../util/NullObject'; + +export type Http1Handler = (ctx: Http1ConnectionContext) => void; export type Http1NotFoundHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; -export type Http1InternalErrorHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; +export type Http1InternalErrorHandler = (error: unknown, res: http.ServerResponse, req: http.IncomingMessage) => void; export class Http1EndpointMatch { constructor(public readonly handler: Http1Handler) {} @@ -43,6 +50,8 @@ export interface WsEndpointDefinition { export interface Http1ServerOpts { server: http.Server; + codecs?: RpcCodecs; + writer?: Writer; } export class Http1Server implements Printable { @@ -53,11 +62,13 @@ export class Http1Server implements Printable { return server; } + public readonly codecs: RpcCodecs; public readonly server: http.Server; constructor(protected readonly opts: Http1ServerOpts) { this.server = opts.server; - const writer = new Writer(); + const writer = opts.writer ?? new Writer(); + this.codecs = opts.codecs ?? new RpcCodecs(opts.codecs ?? new Codecs(writer), new RpcMessageCodecs()); this.wsEncoder = new WsFrameEncoder(writer); } @@ -79,8 +90,15 @@ export class Http1Server implements Printable { res.end(); }; - public oninternalerror: Http1InternalErrorHandler = (res) => { - res.writeHead(500, 'Internal Server Error'); + public oninternalerror: Http1InternalErrorHandler = (error: unknown, res) => { + if (error instanceof PayloadTooLarge) { + res.statusCode = 413; + res.statusMessage = 'Payload Too Large'; + res.end(); + return; + } + res.statusCode = 500; + res.statusMessage = 'Internal Server Error'; res.end(); }; @@ -100,17 +118,28 @@ export class Http1Server implements Printable { private readonly onRequest = (req: http.IncomingMessage, res: http.ServerResponse) => { try { res.sendDate = false; - const route = (req.method || '') + (req.url || ''); + const url = req.url ?? ''; + const queryStartIndex = url.indexOf('?'); + let path = url; + let query = ''; + if (queryStartIndex >= 0) { + path = url.slice(0, queryStartIndex); + query = url.slice(queryStartIndex + 1); + } + const route = (req.method || '') + path; const match = this.httpMatcher(route); if (!match) { this.onnotfound(res, req); return; } - const params = match.params; + const codecs = this.codecs; + const ip = this.findIp(req); + const token = this.findToken(req); + const ctx = new Http1ConnectionContext(res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.jsonRpc2); const handler = match.data.handler; - handler(params, req, res); + handler(ctx); } catch (error) { - this.oninternalerror(res, req); + this.oninternalerror(error, res, req); } }; @@ -144,11 +173,68 @@ export class Http1Server implements Printable { this.wsRouter.add(def.path, def); } + // ------------------------------------------------------- Context management + + public findIp(req: http.IncomingMessage): string { + const headers = req.headers; + const ip = ( + headers['x-forwarded-for'] || + headers['x-real-ip'] || + req.socket.remoteAddress || + '' + ); + return ip instanceof Array ? ip[0] : ip; + } + + /** + * Looks for an authentication token in the following places: + * + * 1. The `Authorization` header. + * 2. The URI query parameters. + * 3. The `Cookie` header. + * 4. The `Sec-Websocket-Protocol` header. + * + * @param req HTTP request + * @returns Authentication token, if any. + */ + public findToken(req: http.IncomingMessage): string { + let token: string = ''; + let text: string = ''; + const headers = req.headers; + let header: string | string[] | undefined; + header = headers['authorization']; + text = typeof header === 'string' ? header : header?.[0] ?? ''; + if (text) token = findTokenInText(text); + if (token) return token; + text = req.url || ''; + if (text) token = findTokenInText(text); + if (token) return token; + header = headers['cookie']; + text = typeof header === 'string' ? header : header?.[0] ?? ''; + if (text) token = findTokenInText(text); + if (token) return token; + header = headers['sec-websocket-protocol']; + text = typeof header === 'string' ? header : header?.[0] ?? ''; + if (text) token = findTokenInText(text); + return token; + } + + // ------------------------------------------------------- High-level routing + + public enableHttpPing(path: string = '/ping') { + this.route({ + path, + handler: (ctx) => { + ctx.res.end('pong'); + }, + }); + } + // ---------------------------------------------------------------- Printable public toString(tab: string = ''): string { return `${this.constructor.name}` + printTree(tab, [ - (tab) => `HTTP/1.1 ${this.httpRouter.toString(tab)}`, + (tab) => `HTTP ${this.httpRouter.toString(tab)}`, (tab) => `WebSocket ${this.wsRouter.toString(tab)}`, ]); } diff --git a/src/reactive-rpc/server/http1/context.ts b/src/reactive-rpc/server/http1/context.ts new file mode 100644 index 0000000000..ea6747b451 --- /dev/null +++ b/src/reactive-rpc/server/http1/context.ts @@ -0,0 +1,91 @@ +import type * as http from 'http'; +import type {JsonValueCodec} from '../../../json-pack/codecs/types'; +import type {RpcMessageCodec} from '../../common/codec/types'; +import type {RpcCodecs} from '../../common/codec/RpcCodecs'; + +const REGEX_CODECS_SPECIFIER = /rpc\.(\w{0,32})\.(\w{0,32})\.(\w{0,32})(?:\-(\w{0,32}))?/; + +export interface ConnectionContext> { + path: string; + query: string; + ip: string; + token: string; + params: string[] | null; + meta: Meta; + reqCodec: JsonValueCodec; + resCodec: JsonValueCodec; + msgCodec: RpcMessageCodec; +} + +export class Http1ConnectionContext> implements ConnectionContext { + constructor( + public readonly res: http.ServerResponse, + public path: string, + public query: string, + public readonly ip: string, + public token: string, + public readonly params: string[] | null, + public readonly meta: Meta, + public reqCodec: JsonValueCodec, + public resCodec: JsonValueCodec, + public msgCodec: RpcMessageCodec, + ) {} + + /** + * @param specifier A string which may contain a codec specifier. For example: + * - `rpc.rx.compact.cbor` for Rx-RPC with compact messages and CBOR values. + * - `rpc.json2.verbose.json` for JSON-RPC 2.0 with verbose messages encoded as JSON. + */ + public setCodecs(specifier: string, codecs: RpcCodecs): void { + const match = REGEX_CODECS_SPECIFIER.exec(specifier); + if (!match) return; + const [, protocol, messageFormat, request, response] = match; + switch (protocol) { + case 'rx': { + switch (messageFormat) { + case 'compact': { + this.msgCodec = codecs.messages.compact; + break; + } + case 'binary': { + this.msgCodec = codecs.messages.binary; + break; + } + } + break; + } + case 'json2': { + this.msgCodec = codecs.messages.jsonRpc2; + break; + } + } + switch (request) { + case 'cbor': { + this.resCodec = this.reqCodec = codecs.value.cbor; + break; + } + case 'json': { + this.resCodec = this.reqCodec = codecs.value.json; + break; + } + case 'msgpack': { + this.resCodec = this.reqCodec = codecs.value.msgpack; + break; + } + } + switch (response) { + case 'cbor': { + this.resCodec = codecs.value.cbor; + break; + } + case 'json': { + this.resCodec = codecs.value.json; + break; + } + case 'msgpack': { + this.resCodec = codecs.value.msgpack; + break; + } + } + } +} diff --git a/src/reactive-rpc/server/http1/errors.ts b/src/reactive-rpc/server/http1/errors.ts new file mode 100644 index 0000000000..01fff751a0 --- /dev/null +++ b/src/reactive-rpc/server/http1/errors.ts @@ -0,0 +1,5 @@ +export class PayloadTooLarge extends Error { + constructor() { + super('TOO_LARGE'); + } +} diff --git a/src/reactive-rpc/server/http1/util.ts b/src/reactive-rpc/server/http1/util.ts new file mode 100644 index 0000000000..2d07b12480 --- /dev/null +++ b/src/reactive-rpc/server/http1/util.ts @@ -0,0 +1,34 @@ +import {PayloadTooLarge} from './errors'; +import type * as http from 'http'; + +export const getBody = (request: http.IncomingMessage, max: number): Promise => { + return new Promise((resolve, reject) => { + let size: number = 0; + const chunks: Buffer[] = []; + request.on('error', (error) => { + request.removeAllListeners(); + reject(error); + }); + request.on('data', (chunk) => { + size += chunk.length; + if (size > max) { + request.removeAllListeners(); + reject(new PayloadTooLarge()); + return; + } + chunks.push(chunk); + }); + request.on('end', () => { + // request.removeAllListeners(); + resolve(chunks); + }); + }); +}; + +const REGEX_AUTH_TOKEN_SPECIFIER = /tkn\.([a-zA-Z0-9\-_]+)(?:[^a-zA-Z0-9\-_]|$)/; + +export const findTokenInText = (text: string): string => { + const match = REGEX_AUTH_TOKEN_SPECIFIER.exec(text); + if (!match) return ''; + return match[1] || ''; +}; From 00680d7011f07d23a456e9ae19e733a7991c2642 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 17:48:21 +0100 Subject: [PATCH 34/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20impl?= =?UTF-8?q?ement=20server=20builder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 31 +---- .../common/rpc/caller/error/RpcError.ts | 2 +- src/reactive-rpc/server/http1/Http1Server.ts | 2 +- src/reactive-rpc/server/http1/RpcServer.ts | 123 ++++++++++++++++++ src/reactive-rpc/server/http1/context.ts | 9 ++ src/reactive-rpc/server/http1/types.ts | 4 + 6 files changed, 144 insertions(+), 27 deletions(-) create mode 100644 src/reactive-rpc/server/http1/RpcServer.ts create mode 100644 src/reactive-rpc/server/http1/types.ts diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index 9547ee09d0..f46644dfec 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -1,29 +1,10 @@ // npx ts-node src/reactive-rpc/__demos__/ws.ts -import {Http1Server} from '../server/http1/Http1Server'; +import {createCaller} from '../common/rpc/__tests__/sample-api'; +import {RpcServer} from '../server/http1/RpcServer'; -const server = Http1Server.start(); - -server.ws({ - path: '/ws', - onConnect: (connection) => { - console.log('CONNECTED'); - connection.onmessage = (data, isUtf8) => { - console.log('MESSAGE', data, isUtf8); - }; - }, -}); - -server.route({ - path: '/hello', - handler: ({res}) => { - res.statusCode = 200; - res.end('Hello World\n'); - }, +RpcServer.startWithDefaults({ + port: 3000, + caller: createCaller(), + logger: console, }); - -server.enableHttpPing(); - -server.start(); - -console.log(server + ''); diff --git a/src/reactive-rpc/common/rpc/caller/error/RpcError.ts b/src/reactive-rpc/common/rpc/caller/error/RpcError.ts index bb6f138944..b272370634 100644 --- a/src/reactive-rpc/common/rpc/caller/error/RpcError.ts +++ b/src/reactive-rpc/common/rpc/caller/error/RpcError.ts @@ -34,7 +34,7 @@ export enum RpcErrorCodes { export type RpcErrorValue = RpcValue; export class RpcError extends Error implements IRpcError { - public static from(error: unknown) { + public static from(error: unknown): RpcError { if (error instanceof RpcError) return error; return RpcError.internal(error); } diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index f651d62a3b..1ae35435af 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -135,7 +135,7 @@ export class Http1Server implements Printable { const codecs = this.codecs; const ip = this.findIp(req); const token = this.findToken(req); - const ctx = new Http1ConnectionContext(res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.jsonRpc2); + const ctx = new Http1ConnectionContext(req, res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.jsonRpc2); const handler = match.data.handler; handler(ctx); } catch (error) { diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts new file mode 100644 index 0000000000..4f9431b9dd --- /dev/null +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -0,0 +1,123 @@ +import * as http from 'http'; +import {Printable} from '../../../util/print/types'; +import {printTree} from '../../../util/print/printTree'; +import {Http1Server} from './Http1Server'; +import {RpcError} from '../../common/rpc/caller'; +import {IncomingBatchMessage, RpcMessageBatchProcessor} from '../../common'; +import {ConnectionContext} from './context'; +import type {RpcCaller} from '../../common/rpc/caller/RpcCaller'; +import type {ServerLogger} from './types'; + +const DEFAULT_MAX_PAYLOAD = 4 * 1024 * 1024; + +export interface RpcServerOpts { + http1: Http1Server; + caller: RpcCaller; + logger?: ServerLogger; +} + +export interface RpcServerStartOpts extends Omit { + port?: number; + server?: http.Server; +} + +export class RpcServer implements Printable { + public static readonly create = (opts: RpcServerOpts) => { + const server = new RpcServer(opts); + opts.http1.enableHttpPing(); + return server; + }; + + public static readonly startWithDefaults = (opts: RpcServerStartOpts) => { + const port = opts.port ?? 8080; + const logger = opts.logger ?? console; + const server = http.createServer(); + const http1Server = new Http1Server({ + server, + }); + const rpcServer = RpcServer.create({ + caller: opts.caller, + http1: http1Server, + logger, + }); + rpcServer.enableDefaults(); + http1Server.start(); + server.listen(port, () => { + let host = server.address() || 'localhost'; + if (typeof host === 'object') host = (host as any).address; + logger.log({msg: 'SERVER_STARTED', host, port}); + }); + }; + + public readonly http1: Http1Server; + protected readonly batchProcessor: RpcMessageBatchProcessor; + + constructor (protected readonly opts: RpcServerOpts) { + const http1 = this.http1 = opts.http1; + const onInternalError = http1.oninternalerror; + http1.oninternalerror = (error, res, req) => { + if (error instanceof RpcError) { + res.statusCode = 400; + const data = JSON.stringify(error.toJson()); + res.end(data); + return; + } + onInternalError(error, res, req); + }; + this.batchProcessor = new RpcMessageBatchProcessor({caller: opts.caller}); + } + + public enableHttpPing(): void { + this.http1.enableHttpPing(); + } + + public enableHttpRpc(path: string = '/rpc'): void { + const batchProcessor = this.batchProcessor; + const logger = this.opts.logger ?? console; + this.http1.route({ + method: 'POST', + path, + handler: async (ctx) => { + const res = ctx.res; + const body = await ctx.body(DEFAULT_MAX_PAYLOAD); + if (!res.socket) return; + try { + const messageCodec = ctx.msgCodec; + const incomingMessages = messageCodec.decodeBatch(ctx.reqCodec, body); + try { + const outgoingMessages = await batchProcessor.onBatch(incomingMessages as IncomingBatchMessage[], ctx); + if (!res.socket) return; + const resCodec = ctx.resCodec; + messageCodec.encodeBatch(resCodec, outgoingMessages); + const buf = resCodec.encoder.writer.flush(); + if (!res.socket) return; + res.end(buf); + } catch (error) { + logger.error('HTTP_RPC_PROCESSING', error, {messages: incomingMessages}); + throw RpcError.from(error); + } + } catch (error) { + if (typeof error === 'object' && error) + if ((error as any).message === 'Invalid JSON') throw RpcError.badRequest(); + throw RpcError.from(error); + } + }, + }); + } + + public enableDefaults(): void { + // this.enableCors(); + this.enableHttpPing(); + this.enableHttpRpc(); + // this.enableWsRpc(); + // this.startRouting(); + } + + // ---------------------------------------------------------------- Printable + + public toString(tab: string = ''): string { + return `${this.constructor.name}` + printTree(tab, [ + (tab) => `HTTP/1.1 ${this.http1.toString(tab)}`, + ]); + } +} diff --git a/src/reactive-rpc/server/http1/context.ts b/src/reactive-rpc/server/http1/context.ts index ea6747b451..739934e089 100644 --- a/src/reactive-rpc/server/http1/context.ts +++ b/src/reactive-rpc/server/http1/context.ts @@ -1,3 +1,5 @@ +import {getBody} from './util'; +import {listToUint8} from '../../../util/buffers/concat'; import type * as http from 'http'; import type {JsonValueCodec} from '../../../json-pack/codecs/types'; import type {RpcMessageCodec} from '../../common/codec/types'; @@ -19,6 +21,7 @@ export interface ConnectionContext> { export class Http1ConnectionContext> implements ConnectionContext { constructor( + public readonly req: http.IncomingMessage, public readonly res: http.ServerResponse, public path: string, public query: string, @@ -88,4 +91,10 @@ export class Http1ConnectionContext> implements C } } } + + public async body(maxPayload: number): Promise { + const list = await getBody(this.req, maxPayload); + const bodyUint8 = listToUint8(list); + return bodyUint8; + } } diff --git a/src/reactive-rpc/server/http1/types.ts b/src/reactive-rpc/server/http1/types.ts new file mode 100644 index 0000000000..dcb64f5180 --- /dev/null +++ b/src/reactive-rpc/server/http1/types.ts @@ -0,0 +1,4 @@ +export interface ServerLogger { + log(msg: unknown): void; + error(kind: string, error?: Error | unknown | null, meta?: unknown): void; +} From 4d5041558f774a74f5fe05ab00fd3a5a88c02207 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 18:11:37 +0100 Subject: [PATCH 35/44] =?UTF-8?q?fix(reactive-rpc):=20=F0=9F=90=9B=20handl?= =?UTF-8?q?e=20route=20handler=20async=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 1ae35435af..924a81411b 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -15,7 +15,7 @@ import {Codecs} from '../../../json-pack/codecs/Codecs'; import {RpcMessageCodecs} from '../../common/codec/RpcMessageCodecs'; import {NullObject} from '../../../util/NullObject'; -export type Http1Handler = (ctx: Http1ConnectionContext) => void; +export type Http1Handler = (ctx: Http1ConnectionContext) => void | Promise; export type Http1NotFoundHandler = (res: http.ServerResponse, req: http.IncomingMessage) => void; export type Http1InternalErrorHandler = (error: unknown, res: http.ServerResponse, req: http.IncomingMessage) => void; @@ -115,7 +115,7 @@ export class Http1Server implements Printable { this.httpRouter.add(route, match); } - private readonly onRequest = (req: http.IncomingMessage, res: http.ServerResponse) => { + private readonly onRequest = async (req: http.IncomingMessage, res: http.ServerResponse) => { try { res.sendDate = false; const url = req.url ?? ''; @@ -135,9 +135,9 @@ export class Http1Server implements Printable { const codecs = this.codecs; const ip = this.findIp(req); const token = this.findToken(req); - const ctx = new Http1ConnectionContext(req, res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.jsonRpc2); + const ctx = new Http1ConnectionContext(req, res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.compact); const handler = match.data.handler; - handler(ctx); + await handler(ctx); } catch (error) { this.oninternalerror(error, res, req); } From d25c6b210fb795b59beaecd497daf7826f31ba16 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 18:18:20 +0100 Subject: [PATCH 36/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20extr?= =?UTF-8?q?act=20codec=20types=20from=20Content-Type=20header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 924a81411b..1516398fd0 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -136,6 +136,9 @@ export class Http1Server implements Printable { const ip = this.findIp(req); const token = this.findToken(req); const ctx = new Http1ConnectionContext(req, res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.compact); + const headers = req.headers; + const contentType = headers['content-type']; + if (typeof contentType === 'string') ctx.setCodecs(contentType, codecs); const handler = match.data.handler; await handler(ctx); } catch (error) { From 72ac248751867ed09f6c6249c3841d789517e780 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 18:18:57 +0100 Subject: [PATCH 37/44] =?UTF-8?q?style(reactive-rpc):=20=F0=9F=92=84=20run?= =?UTF-8?q?=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 32 ++++++++++++------- src/reactive-rpc/server/http1/RpcServer.ts | 8 ++--- .../server/ws/codec/WsFrameEncoder.ts | 4 +-- .../server/ws/codec/__tests__/encoder.spec.ts | 3 +- .../server/ws/server/WsServerConnection.ts | 19 ++++++----- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 1516398fd0..2c823322d2 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -135,7 +135,19 @@ export class Http1Server implements Printable { const codecs = this.codecs; const ip = this.findIp(req); const token = this.findToken(req); - const ctx = new Http1ConnectionContext(req, res, path, query, ip, token, match.params, new NullObject(), codecs.value.json, codecs.value.json, codecs.messages.compact); + const ctx = new Http1ConnectionContext( + req, + res, + path, + query, + ip, + token, + match.params, + new NullObject(), + codecs.value.json, + codecs.value.json, + codecs.messages.compact, + ); const headers = req.headers; const contentType = headers['content-type']; if (typeof contentType === 'string') ctx.setCodecs(contentType, codecs); @@ -180,12 +192,7 @@ export class Http1Server implements Printable { public findIp(req: http.IncomingMessage): string { const headers = req.headers; - const ip = ( - headers['x-forwarded-for'] || - headers['x-real-ip'] || - req.socket.remoteAddress || - '' - ); + const ip = headers['x-forwarded-for'] || headers['x-real-ip'] || req.socket.remoteAddress || ''; return ip instanceof Array ? ip[0] : ip; } @@ -236,9 +243,12 @@ export class Http1Server implements Printable { // ---------------------------------------------------------------- Printable public toString(tab: string = ''): string { - return `${this.constructor.name}` + printTree(tab, [ - (tab) => `HTTP ${this.httpRouter.toString(tab)}`, - (tab) => `WebSocket ${this.wsRouter.toString(tab)}`, - ]); + return ( + `${this.constructor.name}` + + printTree(tab, [ + (tab) => `HTTP ${this.httpRouter.toString(tab)}`, + (tab) => `WebSocket ${this.wsRouter.toString(tab)}`, + ]) + ); } } diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts index 4f9431b9dd..8963bbb1db 100644 --- a/src/reactive-rpc/server/http1/RpcServer.ts +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -52,8 +52,8 @@ export class RpcServer implements Printable { public readonly http1: Http1Server; protected readonly batchProcessor: RpcMessageBatchProcessor; - constructor (protected readonly opts: RpcServerOpts) { - const http1 = this.http1 = opts.http1; + constructor(protected readonly opts: RpcServerOpts) { + const http1 = (this.http1 = opts.http1); const onInternalError = http1.oninternalerror; http1.oninternalerror = (error, res, req) => { if (error instanceof RpcError) { @@ -116,8 +116,6 @@ export class RpcServer implements Printable { // ---------------------------------------------------------------- Printable public toString(tab: string = ''): string { - return `${this.constructor.name}` + printTree(tab, [ - (tab) => `HTTP/1.1 ${this.http1.toString(tab)}`, - ]); + return `${this.constructor.name}` + printTree(tab, [(tab) => `HTTP/1.1 ${this.http1.toString(tab)}`]); } } diff --git a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts index c9724356b4..b706d1acf0 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameEncoder.ts @@ -85,10 +85,10 @@ export class WsFrameEncoder { }); }); - describe('data frames', () => { test('can encode an empty BINARY data frame', () => { const encoder = new WsFrameEncoder(); @@ -139,7 +138,7 @@ describe('data frames', () => { const data2 = decoder.reader.buf(frame.length); expect(data2).toEqual(data); }); - + describe('can encode different message sizes', () => { const sizes = [0, 1, 2, 125, 126, 127, 128, 129, 255, 1234, 65535, 65536, 65537, 7777777, 2 ** 31 - 1]; const encoder = new WsFrameEncoder(); diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 1be5fcfeeb..9b75e59a7b 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -23,11 +23,7 @@ export class WsServerConnection { public onpong: (data: Uint8Array | null) => void = () => {}; public onclose: (code: number, reason: string) => void = () => {}; - constructor( - protected readonly encoder: WsFrameEncoder, - public readonly socket: net.Socket, - head: Buffer, - ) { + constructor(protected readonly encoder: WsFrameEncoder, public readonly socket: net.Socket, head: Buffer) { const decoder = new WsFrameDecoder(); if (head.length) decoder.push(head); let currentFrame: WsFrameHeader | null = null; @@ -95,11 +91,14 @@ export class WsServerConnection { public upgrade(secWebSocketKey: string, secWebSocketProtocol: string, secWebSocketExtensions: string): void { const accept = secWebSocketKey + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); - this.socket.write('HTTP/1.1 101 Switching Protocols\r\n' + - 'Upgrade: websocket\r\n' + - 'Connection: Upgrade\r\n' + - 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + - '\r\n' + this.socket.write( + 'HTTP/1.1 101 Switching Protocols\r\n' + + 'Upgrade: websocket\r\n' + + 'Connection: Upgrade\r\n' + + 'Sec-WebSocket-Accept: ' + + acceptSha1 + + '\r\n' + + '\r\n', ); } From eed1662eb2139a338285d3a3762515b21bfb552f Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 21:14:25 +0100 Subject: [PATCH 38/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20inte?= =?UTF-8?q?grate=20WebSocket=20into=20RPC=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 60 +++++++++++---- src/reactive-rpc/server/http1/RpcServer.ts | 59 +++++++++++++- src/reactive-rpc/server/http1/context.ts | 77 ++++--------------- src/reactive-rpc/server/http1/util.ts | 63 +++++++++++++++ .../server/ws/server/WsServerConnection.ts | 20 ++++- 5 files changed, 199 insertions(+), 80 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 2c823322d2..3f3ae610bd 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -8,8 +8,8 @@ import {Router} from '../../../util/router'; import {Printable} from '../../../util/print/types'; import {printTree} from '../../../util/print/printTree'; import {PayloadTooLarge} from './errors'; -import {findTokenInText} from './util'; -import {Http1ConnectionContext} from './context'; +import {findTokenInText, setCodecs} from './util'; +import {Http1ConnectionContext, WsConnectionContext} from './context'; import {RpcCodecs} from '../../common/codec/RpcCodecs'; import {Codecs} from '../../../json-pack/codecs/Codecs'; import {RpcMessageCodecs} from '../../common/codec/RpcMessageCodecs'; @@ -43,9 +43,10 @@ export interface Http1EndpointDefinition { export interface WsEndpointDefinition { path: string; - maxPayload?: number; + maxIncomingMessage?: number; + maxOutgoingBackpressure?: number; onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; - onConnect(connection: WsServerConnection, req: http.IncomingMessage): void; + onConnect(ctx: WsConnectionContext, req: http.IncomingMessage): void; } export interface Http1ServerOpts { @@ -150,7 +151,7 @@ export class Http1Server implements Printable { ); const headers = req.headers; const contentType = headers['content-type']; - if (typeof contentType === 'string') ctx.setCodecs(contentType, codecs); + if (typeof contentType === 'string') setCodecs(ctx, contentType, codecs); const handler = match.data.handler; await handler(ctx); } catch (error) { @@ -165,7 +166,15 @@ export class Http1Server implements Printable { protected wsMatcher: RouteMatcher = () => undefined; private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket, head: Buffer) => { - const route = req.url || ''; + const url = req.url ?? ''; + const queryStartIndex = url.indexOf('?'); + let path = url; + let query = ''; + if (queryStartIndex >= 0) { + path = url.slice(0, queryStartIndex); + query = url.slice(queryStartIndex + 1); + } + const route = (req.method || '') + path; const match = this.wsMatcher(route); if (!match) { socket.end(); @@ -174,6 +183,8 @@ export class Http1Server implements Printable { const def = match.data; const headers = req.headers; const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket, head); + connection.maxIncomingMessage = def.maxIncomingMessage ?? 2 * 1024 * 1024; + connection.maxBackpressure = def.maxOutgoingBackpressure ?? 2 * 1024 * 1024; if (def.onUpgrade) def.onUpgrade(req, connection); else { const secWebSocketKey = headers['sec-websocket-key'] ?? ''; @@ -181,7 +192,28 @@ export class Http1Server implements Printable { const secWebSocketExtensions = headers['sec-websocket-extensions'] ?? ''; connection.upgrade(secWebSocketKey, secWebSocketProtocol, secWebSocketExtensions); } - def.onConnect(connection, req); + const codecs = this.codecs; + const ip = this.findIp(req); + const token = this.findToken(req); + const ctx = new WsConnectionContext( + connection, + path, + query, + ip, + token, + match.params, + new NullObject(), + codecs.value.json, + codecs.value.json, + codecs.messages.compact, + ); + const contentType = headers['content-type']; + if (typeof contentType === 'string') setCodecs(ctx, contentType, codecs); + else { + const secWebSocketProtocol = headers['sec-websocket-protocol'] ?? ''; + if (typeof secWebSocketProtocol === 'string') setCodecs(ctx, secWebSocketProtocol, codecs); + } + def.onConnect(ctx, req); }; public ws(def: WsEndpointDefinition): void { @@ -209,23 +241,19 @@ export class Http1Server implements Printable { */ public findToken(req: http.IncomingMessage): string { let token: string = ''; - let text: string = ''; const headers = req.headers; let header: string | string[] | undefined; header = headers['authorization']; - text = typeof header === 'string' ? header : header?.[0] ?? ''; - if (text) token = findTokenInText(text); + if (typeof header === 'string') token = findTokenInText(header); if (token) return token; - text = req.url || ''; - if (text) token = findTokenInText(text); + const url = req.url; + if (typeof url === 'string') token = findTokenInText(url); if (token) return token; header = headers['cookie']; - text = typeof header === 'string' ? header : header?.[0] ?? ''; - if (text) token = findTokenInText(text); + if (typeof header === 'string') token = findTokenInText(header); if (token) return token; header = headers['sec-websocket-protocol']; - text = typeof header === 'string' ? header : header?.[0] ?? ''; - if (text) token = findTokenInText(text); + if (typeof header === 'string') token = findTokenInText(header); return token; } diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts index 8963bbb1db..c5f09eba4e 100644 --- a/src/reactive-rpc/server/http1/RpcServer.ts +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -3,8 +3,8 @@ import {Printable} from '../../../util/print/types'; import {printTree} from '../../../util/print/printTree'; import {Http1Server} from './Http1Server'; import {RpcError} from '../../common/rpc/caller'; -import {IncomingBatchMessage, RpcMessageBatchProcessor} from '../../common'; -import {ConnectionContext} from './context'; +import {IncomingBatchMessage, ReactiveRpcClientMessage, ReactiveRpcMessage, RpcMessageBatchProcessor, RpcMessageStreamProcessor} from '../../common'; +import {ConnectionContext, WsConnectionContext} from './context'; import type {RpcCaller} from '../../common/rpc/caller/RpcCaller'; import type {ServerLogger} from './types'; @@ -105,6 +105,61 @@ export class RpcServer implements Printable { }); } + public enableWsRpc(path: string = '/rpc'): void { + const opts = this.opts; + const logger = opts.logger ?? console; + const caller = opts.caller; + this.http1.ws({ + path, + maxIncomingMessage: 2 * 1024 * 1024, + maxOutgoingBackpressure: 2 * 1024 * 1024, + onConnect: (ctx: WsConnectionContext, req: http.IncomingMessage) => { + const connection = ctx.connection; + const reqCodec = ctx.reqCodec; + const resCodec = ctx.resCodec; + const msgCodec = ctx.msgCodec; + const encoder = resCodec.encoder; + const rpc = new RpcMessageStreamProcessor({ + caller, + send: (messages: ReactiveRpcMessage[]) => { + try { + const writer = encoder.writer; + writer.reset(); + msgCodec.encodeBatch(resCodec, messages); + const encoded = writer.flush(); + connection.sendBinMsg(encoded); + } catch (error) { + logger.error('WS_SEND', error, {messages}); + connection.close(); + } + }, + bufferSize: 1, + bufferTime: 0, + }); + connection.onmessage = (uint8: Uint8Array, isUtf8: boolean) => { + let messages: ReactiveRpcClientMessage[]; + try { + messages = msgCodec.decodeBatch(reqCodec, uint8) as ReactiveRpcClientMessage[]; + } catch (error) { + logger.error('RX_RPC_DECODING', error, {codec: reqCodec.id, buf: Buffer.from(uint8).toString('base64')}); + connection.close(); + return; + } + try { + rpc.onMessages(messages, ctx); + } catch (error) { + logger.error('RX_RPC_PROCESSING', error, messages!); + connection.close(); + return; + } + }; + connection.onclose = (code: number, reason: string) => { + rpc.stop(); + }; + }, + }); + } + public enableDefaults(): void { // this.enableCors(); this.enableHttpPing(); diff --git a/src/reactive-rpc/server/http1/context.ts b/src/reactive-rpc/server/http1/context.ts index 739934e089..ad1acf19e5 100644 --- a/src/reactive-rpc/server/http1/context.ts +++ b/src/reactive-rpc/server/http1/context.ts @@ -3,9 +3,7 @@ import {listToUint8} from '../../../util/buffers/concat'; import type * as http from 'http'; import type {JsonValueCodec} from '../../../json-pack/codecs/types'; import type {RpcMessageCodec} from '../../common/codec/types'; -import type {RpcCodecs} from '../../common/codec/RpcCodecs'; - -const REGEX_CODECS_SPECIFIER = /rpc\.(\w{0,32})\.(\w{0,32})\.(\w{0,32})(?:\-(\w{0,32}))?/; +import type {WsServerConnection} from '../ws/server/WsServerConnection'; export interface ConnectionContext> { path: string; @@ -34,67 +32,24 @@ export class Http1ConnectionContext> implements C public msgCodec: RpcMessageCodec, ) {} - /** - * @param specifier A string which may contain a codec specifier. For example: - * - `rpc.rx.compact.cbor` for Rx-RPC with compact messages and CBOR values. - * - `rpc.json2.verbose.json` for JSON-RPC 2.0 with verbose messages encoded as JSON. - */ - public setCodecs(specifier: string, codecs: RpcCodecs): void { - const match = REGEX_CODECS_SPECIFIER.exec(specifier); - if (!match) return; - const [, protocol, messageFormat, request, response] = match; - switch (protocol) { - case 'rx': { - switch (messageFormat) { - case 'compact': { - this.msgCodec = codecs.messages.compact; - break; - } - case 'binary': { - this.msgCodec = codecs.messages.binary; - break; - } - } - break; - } - case 'json2': { - this.msgCodec = codecs.messages.jsonRpc2; - break; - } - } - switch (request) { - case 'cbor': { - this.resCodec = this.reqCodec = codecs.value.cbor; - break; - } - case 'json': { - this.resCodec = this.reqCodec = codecs.value.json; - break; - } - case 'msgpack': { - this.resCodec = this.reqCodec = codecs.value.msgpack; - break; - } - } - switch (response) { - case 'cbor': { - this.resCodec = codecs.value.cbor; - break; - } - case 'json': { - this.resCodec = codecs.value.json; - break; - } - case 'msgpack': { - this.resCodec = codecs.value.msgpack; - break; - } - } - } - public async body(maxPayload: number): Promise { const list = await getBody(this.req, maxPayload); const bodyUint8 = listToUint8(list); return bodyUint8; } } + +export class WsConnectionContext> implements ConnectionContext { + constructor( + public readonly connection: WsServerConnection, + public path: string, + public query: string, + public readonly ip: string, + public token: string, + public readonly params: string[] | null, + public readonly meta: Meta, + public reqCodec: JsonValueCodec, + public resCodec: JsonValueCodec, + public msgCodec: RpcMessageCodec, + ) {} +} diff --git a/src/reactive-rpc/server/http1/util.ts b/src/reactive-rpc/server/http1/util.ts index 2d07b12480..1d2f878278 100644 --- a/src/reactive-rpc/server/http1/util.ts +++ b/src/reactive-rpc/server/http1/util.ts @@ -1,5 +1,7 @@ import {PayloadTooLarge} from './errors'; +import type {ConnectionContext} from './context'; import type * as http from 'http'; +import type {RpcCodecs} from '../../common/codec/RpcCodecs'; export const getBody = (request: http.IncomingMessage, max: number): Promise => { return new Promise((resolve, reject) => { @@ -32,3 +34,64 @@ export const findTokenInText = (text: string): string => { if (!match) return ''; return match[1] || ''; }; + + +const REGEX_CODECS_SPECIFIER = /rpc\.(\w{0,32})\.(\w{0,32})\.(\w{0,32})(?:\-(\w{0,32}))?/; + +/** + * @param specifier A string which may contain a codec specifier. For example: + * - `rpc.rx.compact.cbor` for Rx-RPC with compact messages and CBOR values. + * - `rpc.json2.verbose.json` for JSON-RPC 2.0 with verbose messages encoded as JSON. + */ +export const setCodecs = (ctx: ConnectionContext, specifier: string, codecs: RpcCodecs): void => { + const match = REGEX_CODECS_SPECIFIER.exec(specifier); + if (!match) return; + const [, protocol, messageFormat, request, response] = match; + switch (protocol) { + case 'rx': { + switch (messageFormat) { + case 'compact': { + ctx.msgCodec = codecs.messages.compact; + break; + } + case 'binary': { + ctx.msgCodec = codecs.messages.binary; + break; + } + } + break; + } + case 'json2': { + ctx.msgCodec = codecs.messages.jsonRpc2; + break; + } + } + switch (request) { + case 'cbor': { + ctx.resCodec = ctx.reqCodec = codecs.value.cbor; + break; + } + case 'json': { + ctx.resCodec = ctx.reqCodec = codecs.value.json; + break; + } + case 'msgpack': { + ctx.resCodec = ctx.reqCodec = codecs.value.msgpack; + break; + } + } + switch (response) { + case 'cbor': { + ctx.resCodec = codecs.value.cbor; + break; + } + case 'json': { + ctx.resCodec = codecs.value.json; + break; + } + case 'msgpack': { + ctx.resCodec = codecs.value.msgpack; + break; + } + } +}; diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 9b75e59a7b..348926a387 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -7,6 +7,8 @@ import type {WsFrameEncoder} from '../codec/WsFrameEncoder'; export class WsServerConnection { public closed: boolean = false; + public maxIncomingMessage: number = 2 * 1024 * 1024; + public maxBackpressure: number = 2 * 1024 * 1024; /** * If this is not null, then the connection is receiving a stream: a sequence @@ -48,10 +50,17 @@ export class WsServerConnection { else if (frame instanceof WsCloseFrame) this.onClose(frame.code, frame.reason); else if (frame instanceof WsFrameHeader) { if (this.stream) { - if (frame.opcode !== WsFrameOpcode.CONTINUE) throw new Error('WRONG_OPCODE'); + if (frame.opcode !== WsFrameOpcode.CONTINUE) { + this.onClose(1002, 'DATA'); + return; + } throw new Error('streaming not implemented'); } const length = frame.length; + if (length > this.maxIncomingMessage) { + this.onClose(1009, 'TOO_LARGE'); + return; + } if (length <= decoder.reader.size()) { const buf = new Uint8Array(length); decoder.copyFrameData(frame, buf, 0); @@ -74,6 +83,14 @@ export class WsServerConnection { socket.on('close', handleClose); } + public close(): void { + const code = 1000; + const reason = 'CLOSE'; + const frame = this.encoder.encodeClose(reason, code); + this.socket.write(frame); + this.onClose(code, reason); + } + private onClose(code: number, reason: string): void { this.closed = true; if (this.__writeTimer) { @@ -117,6 +134,7 @@ export class WsServerConnection { this.__buffer = []; if (!buffer.length) return; const socket = this.socket; + if (socket.writableLength > this.maxBackpressure) this.onClose(1009, 'BACKPRESSURE'); // TODO: benchmark if corking helps socket.cork(); for (let i = 0, len = buffer.length; i < len; i++) socket.write(buffer[i]); From d7555e3696ca5c6515fde564fb9ad916129296d3 Mon Sep 17 00:00:00 2001 From: streamich Date: Thu, 28 Dec 2023 21:24:49 +0100 Subject: [PATCH 39/44] =?UTF-8?q?feat(reactive-rpc):=20=F0=9F=8E=B8=20enab?= =?UTF-8?q?le=20CORS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 4 ++- src/reactive-rpc/server/http1/RpcServer.ts | 30 +++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index f46644dfec..97af075e41 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -3,8 +3,10 @@ import {createCaller} from '../common/rpc/__tests__/sample-api'; import {RpcServer} from '../server/http1/RpcServer'; -RpcServer.startWithDefaults({ +const server = RpcServer.startWithDefaults({ port: 3000, caller: createCaller(), logger: console, }); + +console.log(server + ''); diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts index c5f09eba4e..d0c2ed8f4e 100644 --- a/src/reactive-rpc/server/http1/RpcServer.ts +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -28,14 +28,14 @@ export class RpcServer implements Printable { return server; }; - public static readonly startWithDefaults = (opts: RpcServerStartOpts) => { + public static readonly startWithDefaults = (opts: RpcServerStartOpts): RpcServer => { const port = opts.port ?? 8080; const logger = opts.logger ?? console; const server = http.createServer(); const http1Server = new Http1Server({ server, }); - const rpcServer = RpcServer.create({ + const rpcServer = new RpcServer({ caller: opts.caller, http1: http1Server, logger, @@ -47,6 +47,7 @@ export class RpcServer implements Printable { if (typeof host === 'object') host = (host as any).address; logger.log({msg: 'SERVER_STARTED', host, port}); }); + return rpcServer; }; public readonly http1: Http1Server; @@ -71,6 +72,24 @@ export class RpcServer implements Printable { this.http1.enableHttpPing(); } + public enableCors(): void { + this.http1.route({ + method: 'OPTIONS', + path: '/{::\n}', + handler: (ctx) => { + const res = ctx.res; + res.writeHead(200, 'OK', { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Credentials': 'true', + // 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + // 'Access-Control-Allow-Headers': 'Content-Type', + // 'Access-Control-Max-Age': '86400', + }); + res.end(); + }, + }); + } + public enableHttpRpc(path: string = '/rpc'): void { const batchProcessor = this.batchProcessor; const logger = this.opts.logger ?? console; @@ -161,16 +180,15 @@ export class RpcServer implements Printable { } public enableDefaults(): void { - // this.enableCors(); + this.enableCors(); this.enableHttpPing(); this.enableHttpRpc(); - // this.enableWsRpc(); - // this.startRouting(); + this.enableWsRpc(); } // ---------------------------------------------------------------- Printable public toString(tab: string = ''): string { - return `${this.constructor.name}` + printTree(tab, [(tab) => `HTTP/1.1 ${this.http1.toString(tab)}`]); + return `${this.constructor.name}` + printTree(tab, [(tab) => this.http1.toString(tab)]); } } From 653869ead4275dbaafe773481a7afe7efdff5b25 Mon Sep 17 00:00:00 2001 From: streamich Date: Fri, 29 Dec 2023 00:54:47 +0100 Subject: [PATCH 40/44] =?UTF-8?q?fix(reactive-rpc):=20=F0=9F=90=9B=20route?= =?UTF-8?q?=20WebSocket=20traffic=20correctly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 7 +++---- src/reactive-rpc/server/http1/RpcServer.ts | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 3f3ae610bd..851fb233ce 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -46,7 +46,7 @@ export interface WsEndpointDefinition { maxIncomingMessage?: number; maxOutgoingBackpressure?: number; onUpgrade?(req: http.IncomingMessage, connection: WsServerConnection): void; - onConnect(ctx: WsConnectionContext, req: http.IncomingMessage): void; + handler(ctx: WsConnectionContext, req: http.IncomingMessage): void; } export interface Http1ServerOpts { @@ -174,8 +174,7 @@ export class Http1Server implements Printable { path = url.slice(0, queryStartIndex); query = url.slice(queryStartIndex + 1); } - const route = (req.method || '') + path; - const match = this.wsMatcher(route); + const match = this.wsMatcher(path); if (!match) { socket.end(); return; @@ -213,7 +212,7 @@ export class Http1Server implements Printable { const secWebSocketProtocol = headers['sec-websocket-protocol'] ?? ''; if (typeof secWebSocketProtocol === 'string') setCodecs(ctx, secWebSocketProtocol, codecs); } - def.onConnect(ctx, req); + def.handler(ctx, req); }; public ws(def: WsEndpointDefinition): void { diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts index d0c2ed8f4e..4dc61b841b 100644 --- a/src/reactive-rpc/server/http1/RpcServer.ts +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -132,7 +132,7 @@ export class RpcServer implements Printable { path, maxIncomingMessage: 2 * 1024 * 1024, maxOutgoingBackpressure: 2 * 1024 * 1024, - onConnect: (ctx: WsConnectionContext, req: http.IncomingMessage) => { + handler: (ctx: WsConnectionContext, req: http.IncomingMessage) => { const connection = ctx.connection; const reqCodec = ctx.reqCodec; const resCodec = ctx.resCodec; From 83e1629af3c5984d81e242ed2860f48a60ede7d6 Mon Sep 17 00:00:00 2001 From: streamich Date: Fri, 29 Dec 2023 14:11:23 +0100 Subject: [PATCH 41/44] =?UTF-8?q?fix(reactive-rpc):=20=F0=9F=90=9B=20corre?= =?UTF-8?q?ctly=20copy=20frame=20payloads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/ws/codec/WsFrameDecoder.ts | 7 +- .../server/ws/codec/__tests__/decoder.spec.ts | 70 ++++++++++++++++++- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts index 4d215da3a4..e9bedbf767 100644 --- a/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts +++ b/src/reactive-rpc/server/ws/codec/WsFrameDecoder.ts @@ -87,12 +87,9 @@ export class WsFrameDecoder { public copyFrameData(frame: WsFrameHeader, dst: Uint8Array, pos: number): void { const reader = this.reader; const mask = frame.mask; - const readSize = reader.size(); + const readSize = frame.length; if (!mask) reader.copy(readSize, dst, pos); - else { - const alreadyRead = frame.length - readSize; - reader.copyXor(readSize, dst, pos, mask, alreadyRead); - } + else reader.copyXor(readSize, dst, pos, mask, 0); } /** diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index 9bd33ad92f..fcb9692f8b 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -1,6 +1,6 @@ import {WsFrameDecoder} from '../WsFrameDecoder'; import {WsFrameOpcode} from '../constants'; -import {WsCloseFrame, WsPingFrame, WsPongFrame} from '../frames'; +import {WsCloseFrame, WsFrameHeader, WsPingFrame, WsPongFrame} from '../frames'; const {frame: WebSocketFrame} = require('websocket'); @@ -37,6 +37,60 @@ describe('data frames', () => { expect(dst.toString()).toBe('over9000'); }); + test('can decode multiple chunks', () => { + const decoder = new WsFrameDecoder(); + const chunks: string[] = [ + 'gpbMadbAlzLn7P1F9LW4ALruvAC4p+5Frb2RNA==', + 'gv4IkyOW2h54zesyEbr4a1f/tjBT/7R5AbqhY366gS8PpfY8VuKzcg3ms3BEtPZlXsv2RRK67jIB4653T7iqd03x+DJY64cyeKf2Kw+0r2pK+vRuSvi9PA/tp0MPzesyFbr4a1f/tjBT/7R5AbqhY366gS8PofY8VuKzcg3ms3BEtPZlXsv2RRK64jIB4653T7iqd03x+DJY64cyeKf2Jw+0r2pK+vRuSvi9PA/tp0MPzesyEqb2PFbis3IN5rNwRLT2ZV7L9kUSuusvD7Svakr69G5K+L08D+2nQw/N6zISpPY8VuKzcg3ms3BEtPZlXsv2RRK66y0PtK9qSvr0bkr4vTwP7adDD83rMhKi9jxW4rNyDeazcES09mVey/ZFErrrKw+0r2pK+vRuSvi9PA/tp0MPzesyEqD2PFbis3IN5rNwRLT2ZV7L9kUSuuspD7Svakr69G5K+L08D+2nQw/N6zISrvY8VuKzcg3ms3BEtPZlXsv2RRK66ycPtK9qSvr0bkr4vTwP7adDD83rMhGm9jxW4rNyDeazcES09mVey/ZFErroLw+0r2pK+vRuSvi9PA/tp0MPzesyEaT2PFbis3IN5rNwRLT2ZV7L9kUSuugtD7Svakr69G5K+L08D+2nQw/N6zIRovY8VuKzcg3ms3BEtPZlXsv2RRK66CsPtK9qSvr0bkr4vTwP7adDD83rMhGg9jxW4rNyDeazcES09mVey/ZFErroKQ+0r2pK+vRuSvi9PA/tp0MPzesyEa72PFbis3IN5rNwRLT2ZV7L9kUSuugnD7Svakr69G5K+L08D+2nQw/N6zIQpvY8VuKzcg3ms3BEtPZlXsv2RRK66S8PtK9qSvr0bkr4vTwP7adDD83rMhCk9jxW4rNyDeazcES09mVey/ZFErrpLQ+0r2pK+vRuSvi9PA/tp0MPzesyEKL2PFbis3IN5rNwRLT2ZV7L9kUSuukrD7Svakr69G5K+L08D+2nQw/N6zIQoPY8VuKzcg3ms3BEtPZlXsv2RRK66SkPtK9qSvr0bkr4vTwP7adDD83rMhCu9jxW4rNyDeazcES09mVey/ZFErrpJw+0r2pK+vRuSvi9PA/tp0MPzesyF6b2PFbis3IN5rNwRLT2ZV7L9kUSuu4vD7Svakr69G5K+L08D+2nQw/N6zIXpPY8VuKzcg3ms3BEtPZlXsv2RRK67i0PtK9qSvr0bkr4vTwP7adDD83rMhei9jxW4rNyDeazcES09mVey/ZFErruKw+0r2pK+vRuSvi9PA/tp0MPzesyF6D2PFbis3IN5rNwRLT2ZV7L9kUSuu4pD7Svakr69G5K+L08D+2nQw/N6zIXrvY8VuKzcg3ms3BEtPZlXsv2RRK67icPtK9qSvr0bkr4vTwP7adDD83rMham9jxW4rNyDeazcES09mVey/ZFErrvLw+0r2pK+vRuSvi9PA/tp0MPzesyFqT2PFbis3IN5rNwRLT2ZV7L9kUSuu8tD7Svakr69G5K+L08D+2nQw/N6zIWovY8VuKzcg3ms3BEtPZlXsv2RRK67ysPtK9qSvr0bkr4vTwP7adDD83rMhag9jxW4rNyDeazcES09mVey/ZFErrvKQ+0r2pK+vRuSvi9PA/tp0MPzesyFq72PFbis3IN5rNwRLT2ZV7L9kUSuu8nD7Svakr69G5K+L08D+2nQw/N6zIVpvY8VuKzcg3ms3BEtPZlXsv2RRK67C8PtK9qSvr0bkr4vTwP7adDD83rMhWk9jxW4rNyDeazcES09mVey/ZFErrsLQ+0r2pK+vRuSvi9PA/tp0MPzesyFaL2PFbis3IN5rNwRLT2ZV7L9kUSuuwrD7Svakr69G5K+L08D+2nQw/N6zIVoPY8VuKzcg3ms3BEtPZlXsv2RRK67CkPtK9qSvr0bkr4vTwP7adDD83rMhWu9jxW4rNyDeazcES09mVey/ZFErrsJw+0r2pK+vRuSvi9PA/tp0MPzesyFKb2PFbis3IN5rNwRLT2ZV7L9kUSuu0vD7Svakr69G5K+L08D+2nQw/N6zIUpPY8VuKzcg3ms3BEtPZlXsv2RRK67S0PtK9qSvr0bkr4vTwP7adDD83rMhSi9jxW4rNyDeazcES09mVey/ZFErrtKw+0r2pK+vRuSvi9PA/tp0MPzesyFKD2PFbis3IN5rNwRLT2ZV7L9kUSuu0pD7Svakr69G5K+L08D+2nQw/N6zIUrvY8VuKzcg3ms3BEtPZlXsv2RRK67ScPtK9qSvr0bkr4vTwP7adDD83rMhum9jxW4rNyDeazcES09mVey/ZFErriLw+0r2pK+vRuSvi9PA/tp0MPzesyG6T2PFbis3IN5rNwRLT2ZV7L9kUSuuItD7Svakr69G5K+L08D+2nQw/N6zIbovY8VuKzcg3ms3BEtPZlXsv2RRK64isPtK9qSvr0bkr4vTwP7adDD83rMhug9jxW4rNyDeazcES09mVey/ZFErriKQ+0r2pK+vRuSvi9PA/tp0MPzesyG672PFbis3IN5rNwRLT2ZV7L9kUSuuInD7Svakr69G5K+L08D+2nQw/N6zIapvY8VuKzcg3ms3BEtPZlXsv2RRK64y8PtK9qSvr0bkr4vTwP7adDD83rMhqk9jxW4rNyDeazcES09mVey/ZFErrjLQ+0r2pK+vRuSvi9PA/tp0MPzesyGqL2PFbis3IN5rNwRLT2ZV7L9kUSuuMrD7Svakr69G5K+L08D+2nQw/N6zIaoPY8VuKzcg3ms3BEtPZlXsv2RRK64ykPtK9qSvr0bkr4vTwP7adDD83rMhqu9jxW4rNyDeazcES09mVey/ZFErrjJw+0r2pK+vRuSvi9PA/tp0MPzesyEqbqMgHjrndPuKp3TfH4MljrhzJ4p/YvE6f2PFbis3IN5rNwRLT2ZV7Lhw==', + 'gv4I/eI8WRu5Z2g30wxrN8BJLXKOEilyjFt7N5lBBDe5DXUq0g91OZdIMHfMTDB1hR51YJ9hdUDTEGgr1hB7bpZVNTWSVTd8wBAiZr8QAirODWkuzh4sb4tQd2uLUj45zkckRs5naDfTDG83wEktco4SKXKMW3s3mUEEN7kNdSrSC3U5l0gwd8xMMHWFHnVgn2F1QNMQaCvaEHtullU1NZJVN3zAECJmvxACKs4NaSLOHixvi1B3a4tSPjnORyRGzmdoN9MNaTfASS1yjhIpcoxbezeZQQQ3uQ11KtMNdTmXSDB3zEwwdYUedWCfYXVA0xBoKtAQe26WVTU1klU3fMAQIma/EAIqzg1oKM4eLG+LUHdri1I+Oc5HJEbOZ2g30w1tN8BJLXKOEilyjFt7N5lBBDe5DXUq0wl1OZdIMHfMTDB1hR51YJ9hdUDTEGgq1BB7bpZVNTWSVTd8wBAiZr8QAirODWgszh4sb4tQd2uLUj45zkckRs5naDfTDWE3wEktco4SKXKMW3s3mUEEN7kNdSrTBXU5l0gwd8xMMHWFHnVgn2F1QNMQaCnSEHtullU1NZJVN3zAECJmvxACKs4NayrOHixvi1B3a4tSPjnORyRGzmdoN9MOazfASS1yjhIpcoxbezeZQQQ3uQ11KtAPdTmXSDB3zEwwdYUedWCfYXVA0xBoKdYQe26WVTU1klU3fMAQIma/EAIqzg1rLs4eLG+LUHdri1I+Oc5HJEbOZ2g30w5vN8BJLXKOEilyjFt7N5lBBDe5DXUq0At1OZdIMHfMTDB1hR51YJ9hdUDTEGgp2hB7bpZVNTWSVTd8wBAiZr8QAirODWsizh4sb4tQd2uLUj45zkckRs5naDfTD2k3wEktco4SKXKMW3s3mUEEN7kNdSrRDXU5l0gwd8xMMHWFHnVgn2F1QNMQaCjQEHtullU1NZJVN3zAECJmvxACKs4NaijOHixvi1B3a4tSPjnORyRGzmdoN9MPbTfASS1yjhIpcoxbezeZQQQ3uQ11KtEJdTmXSDB3zEwwdYUedWCfYXVA0xBoKNQQe26WVTU1klU3fMAQIma/EAIqzg1qLM4eLG+LUHdri1I+Oc5HJEbOZ2g30w9hN8BJLXKOEilyjFt7N5lBBDe5DXUq0QV1OZdIMHfMTDB1hR51YJ9hdUDTEGgv0hB7bpZVNTWSVTd8wBAiZr8QAirODW0qzh4sb4tQd2uLUj45zkckRs5naDfTCGs3wEktco4SKXKMW3s3mUEEN7kNdSrWD3U5l0gwd8xMMHWFHnVgn2F1QNMQaC/WEHtullU1NZJVN3zAECJmvxACKs4NbS7OHixvi1B3a4tSPjnORyRGzmdoN9MIbzfASS1yjhIpcoxbezeZQQQ3uQ11KtYLdTmXSDB3zEwwdYUedWCfYXVA0xBoL9oQe26WVTU1klU3fMAQIma/EAIqzg1tIs4eLG+LUHdri1I+Oc5HJEbOZ2g30wlpN8BJLXKOEilyjFt7N5lBBDe5DXUq1w11OZdIMHfMTDB1hR51YJ9hdUDTEGgu0BB7bpZVNTWSVTd8wBAiZr8QAirODWwozh4sb4tQd2uLUj45zkckRs5naDfTCW03wEktco4SKXKMW3s3mUEEN7kNdSrXCXU5l0gwd8xMMHWFHnVgn2F1QNMQaC7UEHtullU1NZJVN3zAECJmvxACKs4NbCzOHixvi1B3a4tSPjnORyRGzmdoN9MJYTfASS1yjhIpcoxbezeZQQQ3uQ11KtcFdTmXSDB3zEwwdYUedWCfYXVA0xBoLdIQe26WVTU1klU3fMAQIma/EAIqzg1vKs4eLG+LUHdri1I+Oc5HJEbOZ2g30wprN8BJLXKOEilyjFt7N5lBBDe5DXUq1A91OZdIMHfMTDB1hR51YJ9hdUDTEGgt1hB7bpZVNTWSVTd8wBAiZr8QAirODW8uzh4sb4tQd2uLUj45zkckRs5naDfTCm83wEktco4SKXKMW3s3mUEEN7kNdSrUC3U5l0gwd8xMMHWFHnVgn2F1QNMQaC3aEHtullU1NZJVN3zAECJmvxACKs4NbyLOHixvi1B3a4tSPjnORyRGzmdoN9MLaTfASS1yjhIpcoxbezeZQQQ3uQ11KtUNdTmXSDB3zEwwdYUedWCfYXVA0xBoLNAQe26WVTU1klU3fMAQIma/EAIqzg1uKM4eLG+LUHdri1I+Oc5HJEbOZ2g30wttN8BJLXKOEilyjFt7N5lBBDe5DXUq1Ql1OZdIMHfMTDB1hR51YJ9hdUDTEGgs1BB7bpZVNTWSVTd8wBAiZr8QAirODW4szh4sb4tQd2uLUj45zkckRs5naDfTC2E3wEktco4SKXKMW3s3mUEEN7kNdSrVBXU5l0gwd8xMMHWFHnVgn2F1QNMQaCPSEHtullU1NZJVN3zAECJmvxACKs4NYSrOHixvi1B3a4tSPjnORyRGzmdoN9MEazfASS1yjhIpcoxbezeZQQQ3uQ11KtoPdTmXSDB3zEwwdYUedWCfYXVA0xBoI9YQe26WVTU1klU3fMAQIma/EAIqzg1hLs4eLG+LUHdri1I+Oc5HJEbOZ2g30wRvN8BJLXKOEilyjFt7N5lBBDe5DXUq2gt1OZdIMHfMTDB1hR51YJ9hdUDTEGgj2hB7bpZVNTWSVTd8wBAiZr8QAirODWEizh4sb4tQd2uLUj45zkckRs5naDfTBWk3wEktco4SKXKMW3s3mUEEN7kNdSrbDXU5l0gwd8xMMHWFHnVgn2F1QNMQaCLQEHtullU1NZJVN3zAECJmvxACKs4NYCjOHixvi1B3a4tSPjnORyRGzmdoN9MFbTfASS1yjhIpcoxbezeZQQQ3uQ11KtsJdTmXSDB3zEwwdYUedWCfYXVA0xBoItQQe26WVTU1klU3fMAQIma/EAIqzg1gLM4eLG+LUHdri1I+Oc5HJEbOZ2g30wVhN8BJLXKOEilyjFt7N5lBBDe5DXUq2wV1OZdIMHfMTDB1hR51YJ9hdUDTEGsr0hB7bpZVNTWSVTd8wBAiZr8QAirODmkqzh4sb4tQd2uLUj45zkckRr+C/gj9scVY1uqeafqD9Wr6k7Asv93rKL/fonr6yrgF+ur0dOSB9nT0xLExup+1MbjW53StzJh0jYDpauaF6Xqjxaw0+MGsNrGT6SOr7OkD5533aOOd5y2i2Kl2ptirP/SdviWLnZ5p+oP1bvqTsCy/3esov9+ievrKuAX66vR05IHydPTEsTG6n7UxuNbndK3MmHSNgOlq5onpeqPFrDT4waw2sZPpI6vs6QPnnfdo753nLaLYqXam2Ks/9J2+JYudnmn6g/Ro+pOwLL/d6yi/36J6+sq4Bfrq9HTkgPR09MSxMbqftTG41ud0rcyYdI2A6Wrng+l6o8WsNPjBrDaxk+kjq+zpA+ed92nlnectotipdqbYqz/0nb4li52eafqD9Gz6k7Asv93rKL/fonr6yrgF+ur0dOSA8HT0xLExup+1MbjW53StzJh0jYDpaueH6Xqjxaw0+MGsNrGT6SOr7OkD5533aeGd5y2i2Kl2ptirP/SdviWLnZ5p+oP0YPqTsCy/3esov9+ievrKuAX66vR05ID8dPTEsTG6n7UxuNbndK3MmHSNgOlq5IHpeqPFrDT4waw2sZPpI6vs6QPnnfdq553nLaLYqXam2Ks/9J2+JYudnmn6g/dq+pOwLL/d6yi/36J6+sq4Bfrq9HTkg/Z09MSxMbqftTG41ud0rcyYdI2A6Wrkhel6o8WsNPjBrDaxk+kjq+zpA+ed92rjnectotipdqbYqz/0nb4li52eafqD9276k7Asv93rKL/fonr6yrgF+ur0dOSD8nT0xLExup+1MbjW53StzJh0jYDpauSJ6Xqjxaw0+MGsNrGT6SOr7OkD5533au+d5y2i2Kl2ptirP/SdviWLnZ5p+oP2aPqTsCy/3esov9+ievrKuAX66vR05IL0dPTEsTG6n7UxuNbndK3MmHSNgOlq5YPpeqPFrDT4waw2sZPpI6vs6QPnnfdr5Z3nLaLYqXam2Ks/9J2+JYudnmn6g/Zs+pOwLL/d6yi/36J6+sq4Bfrq9HTkgvB09MSxMbqftTG41ud0rcyYdI2A6Wrlh+l6o8WsNPjBrDaxk+kjq+zpA+ed92vhnectotipdqbYqz/0nb4li52eafqD9mD6k7Asv93rKL/fonr6yrgF+ur0dOSC/HT0xLExup+1MbjW53StzJh0jYDpauKB6Xqjxaw0+MGsNrGT6SOr7OkD5533bOed5y2i2Kl2ptirP/SdviWLnZ5p+oPxavqTsCy/3esov9+ievrKuAX66vR05IX2dPTEsTG6n7UxuNbndK3MmHSNgOlq4oXpeqPFrDT4waw2sZPpI6vs6QPnnfds453nLaLYqXam2Ks/9J2+JYudnmn6g/Fu+pOwLL/d6yi/36J6+sq4Bfrq9HTkhfJ09MSxMbqftTG41ud0rcyYdI2A6Wriiel6o8WsNPjBrDaxk+kjq+zpA+ed92zvnectotipdqbYqz/0nb4li52eafqD8Gj6k7Asv93rKL/fonr6yrgF+ur0dOSE9HT0xLExup+1MbjW53StzJh0jYDpauOD6Xqjxaw0+MGsNrGT6SOr7OkD5533beWd5y2i2Kl2ptirP/SdviWLnZ5p+oPwbPqTsCy/3esov9+ievrKuAX66vR05ITwdPTEsTG6n7UxuNbndK3MmHSNgOlq44fpeqPFrDT4waw2sZPpI6vs6QPnnfdt4Z3nLaLYqXam2Ks/9J2+JYudnmn6g/Bg+pOwLL/d6yi/36J6+sq4Bfrq9HTkhPx09MSxMbqftTG41ud0rcyYdI2A6Wrggel6o8WsNPjBrDaxk+kjq+zpA+ed927nnectotipdqbYqz/0nb4li52eafqD82r6k7Asv93rKL/fonr6yrgF+ur0dOSH9nT0xLExup+1MbjW53StzJh0jYDpauCF6Xqjxaw0+MGsNrGT6SOr7OkD5533buOd5y2i2Kl2ptirP/SdviWLnZ5p+oPzbvqTsCy/3esov9+ievrKuAX66vR05IfydPTEsTG6n7UxuNbndK3MmHSNgOlq4InpeqPFrDT4waw2sZPpI6vs6QPnnfdu753nLaLYqXam2Ks/9J2+JYudnmn6g/Jo+pOwLL/d6yi/36J6+sq4Bfrq9HTkhvR09MSxMbqftTG41ud0rcyYdI2A6Wrhg+l6o8WsNPjBrDaxk+kjq+zpA+ed92/lnectotipdqbYqz/0nb4li52eafqD8mz6k7Asv93rKL/fonr6yrgF+ur0dOSG8HT0xLExup+1MbjW53StzJh0jYDpauGH6Xqjxaw0+MGsNrGT6SOr7OkD5533b+Gd5y2i2Kl2ptirP/SdviWLnZ5p+oPyYPqTsCy/3esov9+ievrKuAX66vR05Ib8dPTEsTG6n7UxuNbndK3MmHSNgOlq7oHpeqPFrDT4waw2sZPpI6vs6QPnnfdg553nLaLYqXam2Ks/9J2+JYudnmn6g/1q+pOwLL/d6yi/36J6+sq4Bfrq9HTkifZ09MSxMbqftTG41ud0rcyYdI2A6Wruhel6o8WsNPjBrDaxk+kjq+zpA+ed92DjnectotipdqbYqz/0nb4li52eafqD/W76k7Asv93rKL/fonr6yrgF+ur0dOSJ8nT0xLExup+1MbjW53StzJh0jYDpau6J6Xqjxaw0+MGsNrGT6SOr7OkD5533YO+d5y2i2Kl2ptirP/SdviWLnZ5p+oP8aPqTsCy/3esov9+ievrKuAX66vR05Ij0dPTEsTG6n7UxuNbndK3MmHSNgOlq74PpeqPFrDT4waw2sZPpI6vs6QPnnfdh5Z3nLaLYqXam2Ks/9J2+JYudnmn6g/xs+pOwLL/d6yi/36J6+sq4Bfrq9HTkiPB09MSxMbqftTG41ud0rcyYdI2A6Wrvh+l6o8WsNPjBrDaxk+kjq+zpA+ed92HhnectotipdqbYqz/0nb4li52eafqD/GD6k7Asv93rKL/fonr6yrgF+ur0dOSI/HT0xLExup+1MbjW53StzJh0jYDpa+aB6Xqjxaw0+MGsNrGT6SOr7OkD5532aOed5y2i2Kl2ptirP/SdviWL7A==', + 'gv4HgfpnciGhPEMNy1VCE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VCGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VDGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VAGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VBGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VGGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VHGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VEGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFENZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFE9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFEtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFFdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFFNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFF9ZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFFtZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFGdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VFGNZFB1WTC1xRkwkVA9YcD3zWPEMNy1VKEdZFB1WTC1xRkwkVA9YcD3zWPEMNy1VKENZFB1WTC1xRkwkVA9YcD3yn', + ]; + for (const chunk of chunks) { + const buf = Buffer.from(chunk, 'base64'); + decoder.push(buf); + } + const frames: WsFrameHeader[] = []; + const payloads: Uint8Array[] = []; + let currentFrame: WsFrameHeader | undefined; + while (true) { + if (currentFrame) { + const length = currentFrame.length; + if (length <= decoder.reader.size()) { + const buf = new Uint8Array(length); + decoder.copyFrameData(currentFrame, buf, 0); + payloads.push(buf); + currentFrame = undefined; + } else break; + } + const frame = decoder.readFrameHeader(); + if (!frame) break; + else if (frame instanceof WsFrameHeader) { + frames.push(frame); + if (frame.length) currentFrame = frame; + } + } + expect(frames.length).toBe(5); + expect(frames[0].fin).toBe(1); + expect(frames[1].fin).toBe(1); + expect(frames[2].fin).toBe(1); + expect(frames[3].fin).toBe(1); + expect(frames[4].fin).toBe(1); + expect(frames[0].opcode).toBe(2); + expect(frames[1].opcode).toBe(2); + expect(frames[2].opcode).toBe(2); + expect(frames[3].opcode).toBe(2); + expect(frames[4].opcode).toBe(2); + expect(frames[0].length).toBe(22); + expect(frames[1].length).toBe(2195); + expect(frames[2].length).toBe(2301); + expect(frames[3].length).toBe(2301); + expect(frames[4].length).toBe(1921); + expect(Buffer.from(payloads[0]).toString()).toBe('[[1,1,"util.ping",{}]]'); + expect(Buffer.from(payloads[1]).toString()).toBe('[[1,2,"util.ping",{}],[1,3,"util.ping",{}],[1,4,"util.ping",{}],[1,5,"util.ping",{}],[1,6,"util.ping",{}],[1,7,"util.ping",{}],[1,8,"util.ping",{}],[1,9,"util.ping",{}],[1,10,"util.ping",{}],[1,11,"util.ping",{}],[1,12,"util.ping",{}],[1,13,"util.ping",{}],[1,14,"util.ping",{}],[1,15,"util.ping",{}],[1,16,"util.ping",{}],[1,17,"util.ping",{}],[1,18,"util.ping",{}],[1,19,"util.ping",{}],[1,20,"util.ping",{}],[1,21,"util.ping",{}],[1,22,"util.ping",{}],[1,23,"util.ping",{}],[1,24,"util.ping",{}],[1,25,"util.ping",{}],[1,26,"util.ping",{}],[1,27,"util.ping",{}],[1,28,"util.ping",{}],[1,29,"util.ping",{}],[1,30,"util.ping",{}],[1,31,"util.ping",{}],[1,32,"util.ping",{}],[1,33,"util.ping",{}],[1,34,"util.ping",{}],[1,35,"util.ping",{}],[1,36,"util.ping",{}],[1,37,"util.ping",{}],[1,38,"util.ping",{}],[1,39,"util.ping",{}],[1,40,"util.ping",{}],[1,41,"util.ping",{}],[1,42,"util.ping",{}],[1,43,"util.ping",{}],[1,44,"util.ping",{}],[1,45,"util.ping",{}],[1,46,"util.ping",{}],[1,47,"util.ping",{}],[1,48,"util.ping",{}],[1,49,"util.ping",{}],[1,50,"util.ping",{}],[1,51,"util.ping",{}],[1,52,"util.ping",{}],[1,53,"util.ping",{}],[1,54,"util.ping",{}],[1,55,"util.ping",{}],[1,56,"util.ping",{}],[1,57,"util.ping",{}],[1,58,"util.ping",{}],[1,59,"util.ping",{}],[1,60,"util.ping",{}],[1,61,"util.ping",{}],[1,62,"util.ping",{}],[1,63,"util.ping",{}],[1,64,"util.ping",{}],[1,65,"util.ping",{}],[1,66,"util.ping",{}],[1,67,"util.ping",{}],[1,68,"util.ping",{}],[1,69,"util.ping",{}],[1,70,"util.ping",{}],[1,71,"util.ping",{}],[1,72,"util.ping",{}],[1,73,"util.ping",{}],[1,74,"util.ping",{}],[1,75,"util.ping",{}],[1,76,"util.ping",{}],[1,77,"util.ping",{}],[1,78,"util.ping",{}],[1,79,"util.ping",{}],[1,80,"util.ping",{}],[1,81,"util.ping",{}],[1,82,"util.ping",{}],[1,83,"util.ping",{}],[1,84,"util.ping",{}],[1,85,"util.ping",{}],[1,86,"util.ping",{}],[1,87,"util.ping",{}],[1,88,"util.ping",{}],[1,89,"util.ping",{}],[1,90,"util.ping",{}],[1,91,"util.ping",{}],[1,92,"util.ping",{}],[1,93,"util.ping",{}],[1,94,"util.ping",{}],[1,95,"util.ping",{}],[1,96,"util.ping",{}],[1,97,"util.ping",{}],[1,98,"util.ping",{}],[1,99,"util.ping",{}],[1,100,"util.ping",{}],[1,101,"util.ping",{}]]'); + expect(Buffer.from(payloads[2]).toString()).toBe('[[1,102,"util.ping",{}],[1,103,"util.ping",{}],[1,104,"util.ping",{}],[1,105,"util.ping",{}],[1,106,"util.ping",{}],[1,107,"util.ping",{}],[1,108,"util.ping",{}],[1,109,"util.ping",{}],[1,110,"util.ping",{}],[1,111,"util.ping",{}],[1,112,"util.ping",{}],[1,113,"util.ping",{}],[1,114,"util.ping",{}],[1,115,"util.ping",{}],[1,116,"util.ping",{}],[1,117,"util.ping",{}],[1,118,"util.ping",{}],[1,119,"util.ping",{}],[1,120,"util.ping",{}],[1,121,"util.ping",{}],[1,122,"util.ping",{}],[1,123,"util.ping",{}],[1,124,"util.ping",{}],[1,125,"util.ping",{}],[1,126,"util.ping",{}],[1,127,"util.ping",{}],[1,128,"util.ping",{}],[1,129,"util.ping",{}],[1,130,"util.ping",{}],[1,131,"util.ping",{}],[1,132,"util.ping",{}],[1,133,"util.ping",{}],[1,134,"util.ping",{}],[1,135,"util.ping",{}],[1,136,"util.ping",{}],[1,137,"util.ping",{}],[1,138,"util.ping",{}],[1,139,"util.ping",{}],[1,140,"util.ping",{}],[1,141,"util.ping",{}],[1,142,"util.ping",{}],[1,143,"util.ping",{}],[1,144,"util.ping",{}],[1,145,"util.ping",{}],[1,146,"util.ping",{}],[1,147,"util.ping",{}],[1,148,"util.ping",{}],[1,149,"util.ping",{}],[1,150,"util.ping",{}],[1,151,"util.ping",{}],[1,152,"util.ping",{}],[1,153,"util.ping",{}],[1,154,"util.ping",{}],[1,155,"util.ping",{}],[1,156,"util.ping",{}],[1,157,"util.ping",{}],[1,158,"util.ping",{}],[1,159,"util.ping",{}],[1,160,"util.ping",{}],[1,161,"util.ping",{}],[1,162,"util.ping",{}],[1,163,"util.ping",{}],[1,164,"util.ping",{}],[1,165,"util.ping",{}],[1,166,"util.ping",{}],[1,167,"util.ping",{}],[1,168,"util.ping",{}],[1,169,"util.ping",{}],[1,170,"util.ping",{}],[1,171,"util.ping",{}],[1,172,"util.ping",{}],[1,173,"util.ping",{}],[1,174,"util.ping",{}],[1,175,"util.ping",{}],[1,176,"util.ping",{}],[1,177,"util.ping",{}],[1,178,"util.ping",{}],[1,179,"util.ping",{}],[1,180,"util.ping",{}],[1,181,"util.ping",{}],[1,182,"util.ping",{}],[1,183,"util.ping",{}],[1,184,"util.ping",{}],[1,185,"util.ping",{}],[1,186,"util.ping",{}],[1,187,"util.ping",{}],[1,188,"util.ping",{}],[1,189,"util.ping",{}],[1,190,"util.ping",{}],[1,191,"util.ping",{}],[1,192,"util.ping",{}],[1,193,"util.ping",{}],[1,194,"util.ping",{}],[1,195,"util.ping",{}],[1,196,"util.ping",{}],[1,197,"util.ping",{}],[1,198,"util.ping",{}],[1,199,"util.ping",{}],[1,200,"util.ping",{}],[1,201,"util.ping",{}]]'); + expect(Buffer.from(payloads[3]).toString()).toBe('[[1,202,"util.ping",{}],[1,203,"util.ping",{}],[1,204,"util.ping",{}],[1,205,"util.ping",{}],[1,206,"util.ping",{}],[1,207,"util.ping",{}],[1,208,"util.ping",{}],[1,209,"util.ping",{}],[1,210,"util.ping",{}],[1,211,"util.ping",{}],[1,212,"util.ping",{}],[1,213,"util.ping",{}],[1,214,"util.ping",{}],[1,215,"util.ping",{}],[1,216,"util.ping",{}],[1,217,"util.ping",{}],[1,218,"util.ping",{}],[1,219,"util.ping",{}],[1,220,"util.ping",{}],[1,221,"util.ping",{}],[1,222,"util.ping",{}],[1,223,"util.ping",{}],[1,224,"util.ping",{}],[1,225,"util.ping",{}],[1,226,"util.ping",{}],[1,227,"util.ping",{}],[1,228,"util.ping",{}],[1,229,"util.ping",{}],[1,230,"util.ping",{}],[1,231,"util.ping",{}],[1,232,"util.ping",{}],[1,233,"util.ping",{}],[1,234,"util.ping",{}],[1,235,"util.ping",{}],[1,236,"util.ping",{}],[1,237,"util.ping",{}],[1,238,"util.ping",{}],[1,239,"util.ping",{}],[1,240,"util.ping",{}],[1,241,"util.ping",{}],[1,242,"util.ping",{}],[1,243,"util.ping",{}],[1,244,"util.ping",{}],[1,245,"util.ping",{}],[1,246,"util.ping",{}],[1,247,"util.ping",{}],[1,248,"util.ping",{}],[1,249,"util.ping",{}],[1,250,"util.ping",{}],[1,251,"util.ping",{}],[1,252,"util.ping",{}],[1,253,"util.ping",{}],[1,254,"util.ping",{}],[1,255,"util.ping",{}],[1,256,"util.ping",{}],[1,257,"util.ping",{}],[1,258,"util.ping",{}],[1,259,"util.ping",{}],[1,260,"util.ping",{}],[1,261,"util.ping",{}],[1,262,"util.ping",{}],[1,263,"util.ping",{}],[1,264,"util.ping",{}],[1,265,"util.ping",{}],[1,266,"util.ping",{}],[1,267,"util.ping",{}],[1,268,"util.ping",{}],[1,269,"util.ping",{}],[1,270,"util.ping",{}],[1,271,"util.ping",{}],[1,272,"util.ping",{}],[1,273,"util.ping",{}],[1,274,"util.ping",{}],[1,275,"util.ping",{}],[1,276,"util.ping",{}],[1,277,"util.ping",{}],[1,278,"util.ping",{}],[1,279,"util.ping",{}],[1,280,"util.ping",{}],[1,281,"util.ping",{}],[1,282,"util.ping",{}],[1,283,"util.ping",{}],[1,284,"util.ping",{}],[1,285,"util.ping",{}],[1,286,"util.ping",{}],[1,287,"util.ping",{}],[1,288,"util.ping",{}],[1,289,"util.ping",{}],[1,290,"util.ping",{}],[1,291,"util.ping",{}],[1,292,"util.ping",{}],[1,293,"util.ping",{}],[1,294,"util.ping",{}],[1,295,"util.ping",{}],[1,296,"util.ping",{}],[1,297,"util.ping",{}],[1,298,"util.ping",{}],[1,299,"util.ping",{}],[1,300,"util.ping",{}],[1,301,"util.ping",{}]]'); + }); + test('can read final text packet without mask', () => { const buf = Buffer.from(new Uint8Array([129, 8, 111, 118, 101, 114, 57, 48, 48, 48])); const decoder = new WsFrameDecoder(); @@ -80,7 +134,14 @@ describe('data frames', () => { frame0.opcode = 1; const buf = frame0.toBuffer(); const decoder = new WsFrameDecoder(); - decoder.push(buf); + const slice1 = buf.slice(0, 2); + const slice2 = buf.slice(2, 6); + const slice3 = buf.slice(6, 10); + const slice4 = buf.slice(10); + decoder.push(slice1); + decoder.push(slice2); + decoder.push(slice3); + decoder.push(slice4); const frame = decoder.readFrameHeader()!; const dst = Buffer.alloc(frame.length); let remaining = frame.length; @@ -279,8 +340,11 @@ describe('control frames', () => { frame0.binaryPayload = Buffer.from(new Uint8Array([1, 2, 3])); frame0.opcode = WsFrameOpcode.PONG; const buf0 = frame0.toBuffer(); + const slice0 = buf0.slice(0, 2); + const slice1 = buf0.slice(2); const decoder = new WsFrameDecoder(); - decoder.push(buf0); + decoder.push(slice0); + decoder.push(slice1); const frame = decoder.readFrameHeader()!; expect(frame).toBeInstanceOf(WsPongFrame); expect(frame.fin).toBe(1); From 0617436bc565fd88ab91c7b4398fc2a8fc6bcbc2 Mon Sep 17 00:00:00 2001 From: streamich Date: Fri, 29 Dec 2023 14:12:48 +0100 Subject: [PATCH 42/44] =?UTF-8?q?fix(reactive-rpc):=20=F0=9F=90=9B=20impro?= =?UTF-8?q?ve=20ws=20frame=20reading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/server/http1/Http1Server.ts | 6 ++--- .../server/ws/server/WsServerConnection.ts | 10 +++---- src/util/buffers/StreamingOctetReader.ts | 26 +++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 851fb233ce..1fce6a8cac 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -165,7 +165,7 @@ export class Http1Server implements Printable { protected readonly wsRouter = new Router(); protected wsMatcher: RouteMatcher = () => undefined; - private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket, head: Buffer) => { + private readonly onWsUpgrade = (req: http.IncomingMessage, socket: net.Socket) => { const url = req.url ?? ''; const queryStartIndex = url.indexOf('?'); let path = url; @@ -181,7 +181,7 @@ export class Http1Server implements Printable { } const def = match.data; const headers = req.headers; - const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket, head); + const connection = new WsServerConnection(this.wsEncoder, socket as net.Socket); connection.maxIncomingMessage = def.maxIncomingMessage ?? 2 * 1024 * 1024; connection.maxBackpressure = def.maxOutgoingBackpressure ?? 2 * 1024 * 1024; if (def.onUpgrade) def.onUpgrade(req, connection); @@ -262,7 +262,7 @@ export class Http1Server implements Printable { this.route({ path, handler: (ctx) => { - ctx.res.end('pong'); + ctx.res.end('"pong"'); }, }); } diff --git a/src/reactive-rpc/server/ws/server/WsServerConnection.ts b/src/reactive-rpc/server/ws/server/WsServerConnection.ts index 348926a387..11fbf6bd4b 100644 --- a/src/reactive-rpc/server/ws/server/WsServerConnection.ts +++ b/src/reactive-rpc/server/ws/server/WsServerConnection.ts @@ -25,9 +25,8 @@ export class WsServerConnection { public onpong: (data: Uint8Array | null) => void = () => {}; public onclose: (code: number, reason: string) => void = () => {}; - constructor(protected readonly encoder: WsFrameEncoder, public readonly socket: net.Socket, head: Buffer) { + constructor(protected readonly encoder: WsFrameEncoder, public readonly socket: net.Socket) { const decoder = new WsFrameDecoder(); - if (head.length) decoder.push(head); let currentFrame: WsFrameHeader | null = null; const handleData = (data: Uint8Array): void => { try { @@ -108,13 +107,14 @@ export class WsServerConnection { public upgrade(secWebSocketKey: string, secWebSocketProtocol: string, secWebSocketExtensions: string): void { const accept = secWebSocketKey + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; const acceptSha1 = crypto.createHash('sha1').update(accept).digest('base64'); + // prettier-ignore this.socket.write( 'HTTP/1.1 101 Switching Protocols\r\n' + 'Upgrade: websocket\r\n' + 'Connection: Upgrade\r\n' + - 'Sec-WebSocket-Accept: ' + - acceptSha1 + - '\r\n' + + 'Sec-WebSocket-Accept: ' + acceptSha1 + '\r\n' + + (secWebSocketProtocol ? 'Sec-WebSocket-Protocol: ' + secWebSocketProtocol + '\r\n' : '') + + // 'Sec-WebSocket-Extensions: ""\r\n' + '\r\n', ); } diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 73e1bbaccd..2fc06718e2 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -75,22 +75,21 @@ export class StreamingOctetReader { if (!size) return; this.assertSize(size); const chunk0 = this.chunks[0]!; - const size0 = Math.min(chunk0.length - this.x, size); - for (let i = 0; i < size0; i++) dst[pos + i] = chunk0[this.x + i] ^ mask[maskIndex++ % 4]; + let x = this.x; + const size0 = Math.min(chunk0.length - x, size); + const end = x + size0; + for (; x < end;) dst[pos++] = chunk0[x++] ^ mask[maskIndex++ % 4]; size -= size0; - pos += size0; if (size <= 0) { this.skipUnsafe(size0); return; } let chunkIndex = 1; while (size > 0) { - const chunk1 = this.chunks[chunkIndex]!; + const chunk1 = this.chunks[chunkIndex++]!; const size1 = Math.min(chunk1.length, size); - for (let i = 0; i < size1; i++) dst[pos + size0 + i] = chunk1[i] ^ mask[maskIndex++ % 4]; + for (let x = 0; x < size1;) dst[pos++] = chunk1[x++] ^ mask[maskIndex++ % 4]; size -= size1; - pos += size1; - chunkIndex++; } this.skipUnsafe(size); } @@ -112,16 +111,17 @@ export class StreamingOctetReader { public skipUnsafe(n: number): void { if (!n) return; const chunk = this.chunks[0]!; - const x = this.x + n; - const length = chunk.length; - if (x < length) { - this.x = x; + const chunkLength = chunk.length; + const remaining = chunkLength - this.x; + if (remaining > n) { + this.x = this.x + n; return; } this.x = 0; this.chunks.shift(); - this.chunkSize -= length; - this.skip(x - length); + this.chunkSize -= chunkLength; + n -= remaining; + this.skipUnsafe(n); } public skip(n: number): void { From 83cb97447a03778c394d607d59c7f250ef173020 Mon Sep 17 00:00:00 2001 From: streamich Date: Fri, 29 Dec 2023 14:20:51 +0100 Subject: [PATCH 43/44] =?UTF-8?q?perf(reactive-rpc):=20=E2=9A=A1=EF=B8=8F?= =?UTF-8?q?=20benchmark=20new=20ws=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/__bench__/ping.bench.ts | 10 +++++++--- src/server/index.ts | 7 +++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/server/__bench__/ping.bench.ts b/src/server/__bench__/ping.bench.ts index 48e2b54464..552bccd315 100644 --- a/src/server/__bench__/ping.bench.ts +++ b/src/server/__bench__/ping.bench.ts @@ -6,21 +6,25 @@ import {Suite} from 'benchmark'; import {RpcPersistentClient, WebSocketChannel} from '../../reactive-rpc/common'; import {Writer} from '../../util/buffers/Writer'; import {BinaryRpcMessageCodec} from '../../reactive-rpc/common/codec/binary'; +import {CompactRpcMessageCodec} from '../../reactive-rpc/common/codec/compact'; import {CborJsonValueCodec} from '../../json-pack/codecs/cbor'; +import {JsonJsonValueCodec} from '../../json-pack/codecs/json'; import {RpcCodec} from '../../reactive-rpc/common/codec/RpcCodec'; import {WebSocket} from 'ws'; const main = async () => { const writer = new Writer(1024 * 4); - const msg = new BinaryRpcMessageCodec(); - const req = new CborJsonValueCodec(writer); + // const msg = new BinaryRpcMessageCodec(); + // const req = new CborJsonValueCodec(writer); + const msg = new CompactRpcMessageCodec(); + const req = new JsonJsonValueCodec(writer); const codec = new RpcCodec(msg, req, req); const client = new RpcPersistentClient({ codec, channel: { newChannel: () => new WebSocketChannel({ - newSocket: () => new WebSocket('ws://localhost:9999/rpc', [codec.specifier()]) as any, + newSocket: () => new WebSocket('ws://localhost:9999/rpc', [codec.specifier()], {perMessageDeflate: false}) as any, }), }, }); diff --git a/src/server/index.ts b/src/server/index.ts index 5d9bff49fe..0aef51667e 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -5,9 +5,16 @@ import {RpcApp} from '../reactive-rpc/server/uws/RpcApp'; import {createCaller} from './routes'; import {Services} from './services/Services'; import type {MyCtx} from './services/types'; +import {RpcServer} from '../reactive-rpc/server/http1/RpcServer'; const app = new RpcApp({ uws: App({}), caller: createCaller(new Services()).caller, }); app.startWithDefaults(); + +// const server = RpcServer.startWithDefaults({ +// port: 9999, +// caller: createCaller(new Services()).caller, +// logger: console, +// }); From c3ac50ca51a244410de8c45f2c6e098952ed8073 Mon Sep 17 00:00:00 2001 From: streamich Date: Fri, 29 Dec 2023 14:22:19 +0100 Subject: [PATCH 44/44] =?UTF-8?q?style(reactive-rpc):=20=F0=9F=92=84=20run?= =?UTF-8?q?=20prettier=20and=20linter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/reactive-rpc/__demos__/ws.ts | 1 + src/reactive-rpc/server/http1/Http1Server.ts | 4 ++-- src/reactive-rpc/server/http1/RpcServer.ts | 12 +++++++++--- src/reactive-rpc/server/http1/util.ts | 1 - .../server/ws/codec/__tests__/decoder.spec.ts | 12 +++++++++--- src/server/__bench__/ping.bench.ts | 3 ++- src/util/buffers/StreamingOctetReader.ts | 4 ++-- 7 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/reactive-rpc/__demos__/ws.ts b/src/reactive-rpc/__demos__/ws.ts index 97af075e41..4c0a205a8d 100644 --- a/src/reactive-rpc/__demos__/ws.ts +++ b/src/reactive-rpc/__demos__/ws.ts @@ -9,4 +9,5 @@ const server = RpcServer.startWithDefaults({ logger: console, }); +// tslint:disable-next-line no-console console.log(server + ''); diff --git a/src/reactive-rpc/server/http1/Http1Server.ts b/src/reactive-rpc/server/http1/Http1Server.ts index 1fce6a8cac..04fca023d4 100644 --- a/src/reactive-rpc/server/http1/Http1Server.ts +++ b/src/reactive-rpc/server/http1/Http1Server.ts @@ -242,13 +242,13 @@ export class Http1Server implements Printable { let token: string = ''; const headers = req.headers; let header: string | string[] | undefined; - header = headers['authorization']; + header = headers.authorization; if (typeof header === 'string') token = findTokenInText(header); if (token) return token; const url = req.url; if (typeof url === 'string') token = findTokenInText(url); if (token) return token; - header = headers['cookie']; + header = headers.cookie; if (typeof header === 'string') token = findTokenInText(header); if (token) return token; header = headers['sec-websocket-protocol']; diff --git a/src/reactive-rpc/server/http1/RpcServer.ts b/src/reactive-rpc/server/http1/RpcServer.ts index 4dc61b841b..890d30bfe4 100644 --- a/src/reactive-rpc/server/http1/RpcServer.ts +++ b/src/reactive-rpc/server/http1/RpcServer.ts @@ -3,7 +3,13 @@ import {Printable} from '../../../util/print/types'; import {printTree} from '../../../util/print/printTree'; import {Http1Server} from './Http1Server'; import {RpcError} from '../../common/rpc/caller'; -import {IncomingBatchMessage, ReactiveRpcClientMessage, ReactiveRpcMessage, RpcMessageBatchProcessor, RpcMessageStreamProcessor} from '../../common'; +import { + IncomingBatchMessage, + ReactiveRpcClientMessage, + ReactiveRpcMessage, + RpcMessageBatchProcessor, + RpcMessageStreamProcessor, +} from '../../common'; import {ConnectionContext, WsConnectionContext} from './context'; import type {RpcCaller} from '../../common/rpc/caller/RpcCaller'; import type {ServerLogger} from './types'; @@ -158,7 +164,7 @@ export class RpcServer implements Printable { connection.onmessage = (uint8: Uint8Array, isUtf8: boolean) => { let messages: ReactiveRpcClientMessage[]; try { - messages = msgCodec.decodeBatch(reqCodec, uint8) as ReactiveRpcClientMessage[]; + messages = msgCodec.decodeBatch(reqCodec, uint8) as ReactiveRpcClientMessage[]; } catch (error) { logger.error('RX_RPC_DECODING', error, {codec: reqCodec.id, buf: Buffer.from(uint8).toString('base64')}); connection.close(); @@ -172,7 +178,7 @@ export class RpcServer implements Printable { return; } }; - connection.onclose = (code: number, reason: string) => { + connection.onclose = (code: number, reason: string) => { rpc.stop(); }; }, diff --git a/src/reactive-rpc/server/http1/util.ts b/src/reactive-rpc/server/http1/util.ts index 1d2f878278..7c45acea16 100644 --- a/src/reactive-rpc/server/http1/util.ts +++ b/src/reactive-rpc/server/http1/util.ts @@ -35,7 +35,6 @@ export const findTokenInText = (text: string): string => { return match[1] || ''; }; - const REGEX_CODECS_SPECIFIER = /rpc\.(\w{0,32})\.(\w{0,32})\.(\w{0,32})(?:\-(\w{0,32}))?/; /** diff --git a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts index fcb9692f8b..d96ab60739 100644 --- a/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts +++ b/src/reactive-rpc/server/ws/codec/__tests__/decoder.spec.ts @@ -86,9 +86,15 @@ describe('data frames', () => { expect(frames[3].length).toBe(2301); expect(frames[4].length).toBe(1921); expect(Buffer.from(payloads[0]).toString()).toBe('[[1,1,"util.ping",{}]]'); - expect(Buffer.from(payloads[1]).toString()).toBe('[[1,2,"util.ping",{}],[1,3,"util.ping",{}],[1,4,"util.ping",{}],[1,5,"util.ping",{}],[1,6,"util.ping",{}],[1,7,"util.ping",{}],[1,8,"util.ping",{}],[1,9,"util.ping",{}],[1,10,"util.ping",{}],[1,11,"util.ping",{}],[1,12,"util.ping",{}],[1,13,"util.ping",{}],[1,14,"util.ping",{}],[1,15,"util.ping",{}],[1,16,"util.ping",{}],[1,17,"util.ping",{}],[1,18,"util.ping",{}],[1,19,"util.ping",{}],[1,20,"util.ping",{}],[1,21,"util.ping",{}],[1,22,"util.ping",{}],[1,23,"util.ping",{}],[1,24,"util.ping",{}],[1,25,"util.ping",{}],[1,26,"util.ping",{}],[1,27,"util.ping",{}],[1,28,"util.ping",{}],[1,29,"util.ping",{}],[1,30,"util.ping",{}],[1,31,"util.ping",{}],[1,32,"util.ping",{}],[1,33,"util.ping",{}],[1,34,"util.ping",{}],[1,35,"util.ping",{}],[1,36,"util.ping",{}],[1,37,"util.ping",{}],[1,38,"util.ping",{}],[1,39,"util.ping",{}],[1,40,"util.ping",{}],[1,41,"util.ping",{}],[1,42,"util.ping",{}],[1,43,"util.ping",{}],[1,44,"util.ping",{}],[1,45,"util.ping",{}],[1,46,"util.ping",{}],[1,47,"util.ping",{}],[1,48,"util.ping",{}],[1,49,"util.ping",{}],[1,50,"util.ping",{}],[1,51,"util.ping",{}],[1,52,"util.ping",{}],[1,53,"util.ping",{}],[1,54,"util.ping",{}],[1,55,"util.ping",{}],[1,56,"util.ping",{}],[1,57,"util.ping",{}],[1,58,"util.ping",{}],[1,59,"util.ping",{}],[1,60,"util.ping",{}],[1,61,"util.ping",{}],[1,62,"util.ping",{}],[1,63,"util.ping",{}],[1,64,"util.ping",{}],[1,65,"util.ping",{}],[1,66,"util.ping",{}],[1,67,"util.ping",{}],[1,68,"util.ping",{}],[1,69,"util.ping",{}],[1,70,"util.ping",{}],[1,71,"util.ping",{}],[1,72,"util.ping",{}],[1,73,"util.ping",{}],[1,74,"util.ping",{}],[1,75,"util.ping",{}],[1,76,"util.ping",{}],[1,77,"util.ping",{}],[1,78,"util.ping",{}],[1,79,"util.ping",{}],[1,80,"util.ping",{}],[1,81,"util.ping",{}],[1,82,"util.ping",{}],[1,83,"util.ping",{}],[1,84,"util.ping",{}],[1,85,"util.ping",{}],[1,86,"util.ping",{}],[1,87,"util.ping",{}],[1,88,"util.ping",{}],[1,89,"util.ping",{}],[1,90,"util.ping",{}],[1,91,"util.ping",{}],[1,92,"util.ping",{}],[1,93,"util.ping",{}],[1,94,"util.ping",{}],[1,95,"util.ping",{}],[1,96,"util.ping",{}],[1,97,"util.ping",{}],[1,98,"util.ping",{}],[1,99,"util.ping",{}],[1,100,"util.ping",{}],[1,101,"util.ping",{}]]'); - expect(Buffer.from(payloads[2]).toString()).toBe('[[1,102,"util.ping",{}],[1,103,"util.ping",{}],[1,104,"util.ping",{}],[1,105,"util.ping",{}],[1,106,"util.ping",{}],[1,107,"util.ping",{}],[1,108,"util.ping",{}],[1,109,"util.ping",{}],[1,110,"util.ping",{}],[1,111,"util.ping",{}],[1,112,"util.ping",{}],[1,113,"util.ping",{}],[1,114,"util.ping",{}],[1,115,"util.ping",{}],[1,116,"util.ping",{}],[1,117,"util.ping",{}],[1,118,"util.ping",{}],[1,119,"util.ping",{}],[1,120,"util.ping",{}],[1,121,"util.ping",{}],[1,122,"util.ping",{}],[1,123,"util.ping",{}],[1,124,"util.ping",{}],[1,125,"util.ping",{}],[1,126,"util.ping",{}],[1,127,"util.ping",{}],[1,128,"util.ping",{}],[1,129,"util.ping",{}],[1,130,"util.ping",{}],[1,131,"util.ping",{}],[1,132,"util.ping",{}],[1,133,"util.ping",{}],[1,134,"util.ping",{}],[1,135,"util.ping",{}],[1,136,"util.ping",{}],[1,137,"util.ping",{}],[1,138,"util.ping",{}],[1,139,"util.ping",{}],[1,140,"util.ping",{}],[1,141,"util.ping",{}],[1,142,"util.ping",{}],[1,143,"util.ping",{}],[1,144,"util.ping",{}],[1,145,"util.ping",{}],[1,146,"util.ping",{}],[1,147,"util.ping",{}],[1,148,"util.ping",{}],[1,149,"util.ping",{}],[1,150,"util.ping",{}],[1,151,"util.ping",{}],[1,152,"util.ping",{}],[1,153,"util.ping",{}],[1,154,"util.ping",{}],[1,155,"util.ping",{}],[1,156,"util.ping",{}],[1,157,"util.ping",{}],[1,158,"util.ping",{}],[1,159,"util.ping",{}],[1,160,"util.ping",{}],[1,161,"util.ping",{}],[1,162,"util.ping",{}],[1,163,"util.ping",{}],[1,164,"util.ping",{}],[1,165,"util.ping",{}],[1,166,"util.ping",{}],[1,167,"util.ping",{}],[1,168,"util.ping",{}],[1,169,"util.ping",{}],[1,170,"util.ping",{}],[1,171,"util.ping",{}],[1,172,"util.ping",{}],[1,173,"util.ping",{}],[1,174,"util.ping",{}],[1,175,"util.ping",{}],[1,176,"util.ping",{}],[1,177,"util.ping",{}],[1,178,"util.ping",{}],[1,179,"util.ping",{}],[1,180,"util.ping",{}],[1,181,"util.ping",{}],[1,182,"util.ping",{}],[1,183,"util.ping",{}],[1,184,"util.ping",{}],[1,185,"util.ping",{}],[1,186,"util.ping",{}],[1,187,"util.ping",{}],[1,188,"util.ping",{}],[1,189,"util.ping",{}],[1,190,"util.ping",{}],[1,191,"util.ping",{}],[1,192,"util.ping",{}],[1,193,"util.ping",{}],[1,194,"util.ping",{}],[1,195,"util.ping",{}],[1,196,"util.ping",{}],[1,197,"util.ping",{}],[1,198,"util.ping",{}],[1,199,"util.ping",{}],[1,200,"util.ping",{}],[1,201,"util.ping",{}]]'); - expect(Buffer.from(payloads[3]).toString()).toBe('[[1,202,"util.ping",{}],[1,203,"util.ping",{}],[1,204,"util.ping",{}],[1,205,"util.ping",{}],[1,206,"util.ping",{}],[1,207,"util.ping",{}],[1,208,"util.ping",{}],[1,209,"util.ping",{}],[1,210,"util.ping",{}],[1,211,"util.ping",{}],[1,212,"util.ping",{}],[1,213,"util.ping",{}],[1,214,"util.ping",{}],[1,215,"util.ping",{}],[1,216,"util.ping",{}],[1,217,"util.ping",{}],[1,218,"util.ping",{}],[1,219,"util.ping",{}],[1,220,"util.ping",{}],[1,221,"util.ping",{}],[1,222,"util.ping",{}],[1,223,"util.ping",{}],[1,224,"util.ping",{}],[1,225,"util.ping",{}],[1,226,"util.ping",{}],[1,227,"util.ping",{}],[1,228,"util.ping",{}],[1,229,"util.ping",{}],[1,230,"util.ping",{}],[1,231,"util.ping",{}],[1,232,"util.ping",{}],[1,233,"util.ping",{}],[1,234,"util.ping",{}],[1,235,"util.ping",{}],[1,236,"util.ping",{}],[1,237,"util.ping",{}],[1,238,"util.ping",{}],[1,239,"util.ping",{}],[1,240,"util.ping",{}],[1,241,"util.ping",{}],[1,242,"util.ping",{}],[1,243,"util.ping",{}],[1,244,"util.ping",{}],[1,245,"util.ping",{}],[1,246,"util.ping",{}],[1,247,"util.ping",{}],[1,248,"util.ping",{}],[1,249,"util.ping",{}],[1,250,"util.ping",{}],[1,251,"util.ping",{}],[1,252,"util.ping",{}],[1,253,"util.ping",{}],[1,254,"util.ping",{}],[1,255,"util.ping",{}],[1,256,"util.ping",{}],[1,257,"util.ping",{}],[1,258,"util.ping",{}],[1,259,"util.ping",{}],[1,260,"util.ping",{}],[1,261,"util.ping",{}],[1,262,"util.ping",{}],[1,263,"util.ping",{}],[1,264,"util.ping",{}],[1,265,"util.ping",{}],[1,266,"util.ping",{}],[1,267,"util.ping",{}],[1,268,"util.ping",{}],[1,269,"util.ping",{}],[1,270,"util.ping",{}],[1,271,"util.ping",{}],[1,272,"util.ping",{}],[1,273,"util.ping",{}],[1,274,"util.ping",{}],[1,275,"util.ping",{}],[1,276,"util.ping",{}],[1,277,"util.ping",{}],[1,278,"util.ping",{}],[1,279,"util.ping",{}],[1,280,"util.ping",{}],[1,281,"util.ping",{}],[1,282,"util.ping",{}],[1,283,"util.ping",{}],[1,284,"util.ping",{}],[1,285,"util.ping",{}],[1,286,"util.ping",{}],[1,287,"util.ping",{}],[1,288,"util.ping",{}],[1,289,"util.ping",{}],[1,290,"util.ping",{}],[1,291,"util.ping",{}],[1,292,"util.ping",{}],[1,293,"util.ping",{}],[1,294,"util.ping",{}],[1,295,"util.ping",{}],[1,296,"util.ping",{}],[1,297,"util.ping",{}],[1,298,"util.ping",{}],[1,299,"util.ping",{}],[1,300,"util.ping",{}],[1,301,"util.ping",{}]]'); + expect(Buffer.from(payloads[1]).toString()).toBe( + '[[1,2,"util.ping",{}],[1,3,"util.ping",{}],[1,4,"util.ping",{}],[1,5,"util.ping",{}],[1,6,"util.ping",{}],[1,7,"util.ping",{}],[1,8,"util.ping",{}],[1,9,"util.ping",{}],[1,10,"util.ping",{}],[1,11,"util.ping",{}],[1,12,"util.ping",{}],[1,13,"util.ping",{}],[1,14,"util.ping",{}],[1,15,"util.ping",{}],[1,16,"util.ping",{}],[1,17,"util.ping",{}],[1,18,"util.ping",{}],[1,19,"util.ping",{}],[1,20,"util.ping",{}],[1,21,"util.ping",{}],[1,22,"util.ping",{}],[1,23,"util.ping",{}],[1,24,"util.ping",{}],[1,25,"util.ping",{}],[1,26,"util.ping",{}],[1,27,"util.ping",{}],[1,28,"util.ping",{}],[1,29,"util.ping",{}],[1,30,"util.ping",{}],[1,31,"util.ping",{}],[1,32,"util.ping",{}],[1,33,"util.ping",{}],[1,34,"util.ping",{}],[1,35,"util.ping",{}],[1,36,"util.ping",{}],[1,37,"util.ping",{}],[1,38,"util.ping",{}],[1,39,"util.ping",{}],[1,40,"util.ping",{}],[1,41,"util.ping",{}],[1,42,"util.ping",{}],[1,43,"util.ping",{}],[1,44,"util.ping",{}],[1,45,"util.ping",{}],[1,46,"util.ping",{}],[1,47,"util.ping",{}],[1,48,"util.ping",{}],[1,49,"util.ping",{}],[1,50,"util.ping",{}],[1,51,"util.ping",{}],[1,52,"util.ping",{}],[1,53,"util.ping",{}],[1,54,"util.ping",{}],[1,55,"util.ping",{}],[1,56,"util.ping",{}],[1,57,"util.ping",{}],[1,58,"util.ping",{}],[1,59,"util.ping",{}],[1,60,"util.ping",{}],[1,61,"util.ping",{}],[1,62,"util.ping",{}],[1,63,"util.ping",{}],[1,64,"util.ping",{}],[1,65,"util.ping",{}],[1,66,"util.ping",{}],[1,67,"util.ping",{}],[1,68,"util.ping",{}],[1,69,"util.ping",{}],[1,70,"util.ping",{}],[1,71,"util.ping",{}],[1,72,"util.ping",{}],[1,73,"util.ping",{}],[1,74,"util.ping",{}],[1,75,"util.ping",{}],[1,76,"util.ping",{}],[1,77,"util.ping",{}],[1,78,"util.ping",{}],[1,79,"util.ping",{}],[1,80,"util.ping",{}],[1,81,"util.ping",{}],[1,82,"util.ping",{}],[1,83,"util.ping",{}],[1,84,"util.ping",{}],[1,85,"util.ping",{}],[1,86,"util.ping",{}],[1,87,"util.ping",{}],[1,88,"util.ping",{}],[1,89,"util.ping",{}],[1,90,"util.ping",{}],[1,91,"util.ping",{}],[1,92,"util.ping",{}],[1,93,"util.ping",{}],[1,94,"util.ping",{}],[1,95,"util.ping",{}],[1,96,"util.ping",{}],[1,97,"util.ping",{}],[1,98,"util.ping",{}],[1,99,"util.ping",{}],[1,100,"util.ping",{}],[1,101,"util.ping",{}]]', + ); + expect(Buffer.from(payloads[2]).toString()).toBe( + '[[1,102,"util.ping",{}],[1,103,"util.ping",{}],[1,104,"util.ping",{}],[1,105,"util.ping",{}],[1,106,"util.ping",{}],[1,107,"util.ping",{}],[1,108,"util.ping",{}],[1,109,"util.ping",{}],[1,110,"util.ping",{}],[1,111,"util.ping",{}],[1,112,"util.ping",{}],[1,113,"util.ping",{}],[1,114,"util.ping",{}],[1,115,"util.ping",{}],[1,116,"util.ping",{}],[1,117,"util.ping",{}],[1,118,"util.ping",{}],[1,119,"util.ping",{}],[1,120,"util.ping",{}],[1,121,"util.ping",{}],[1,122,"util.ping",{}],[1,123,"util.ping",{}],[1,124,"util.ping",{}],[1,125,"util.ping",{}],[1,126,"util.ping",{}],[1,127,"util.ping",{}],[1,128,"util.ping",{}],[1,129,"util.ping",{}],[1,130,"util.ping",{}],[1,131,"util.ping",{}],[1,132,"util.ping",{}],[1,133,"util.ping",{}],[1,134,"util.ping",{}],[1,135,"util.ping",{}],[1,136,"util.ping",{}],[1,137,"util.ping",{}],[1,138,"util.ping",{}],[1,139,"util.ping",{}],[1,140,"util.ping",{}],[1,141,"util.ping",{}],[1,142,"util.ping",{}],[1,143,"util.ping",{}],[1,144,"util.ping",{}],[1,145,"util.ping",{}],[1,146,"util.ping",{}],[1,147,"util.ping",{}],[1,148,"util.ping",{}],[1,149,"util.ping",{}],[1,150,"util.ping",{}],[1,151,"util.ping",{}],[1,152,"util.ping",{}],[1,153,"util.ping",{}],[1,154,"util.ping",{}],[1,155,"util.ping",{}],[1,156,"util.ping",{}],[1,157,"util.ping",{}],[1,158,"util.ping",{}],[1,159,"util.ping",{}],[1,160,"util.ping",{}],[1,161,"util.ping",{}],[1,162,"util.ping",{}],[1,163,"util.ping",{}],[1,164,"util.ping",{}],[1,165,"util.ping",{}],[1,166,"util.ping",{}],[1,167,"util.ping",{}],[1,168,"util.ping",{}],[1,169,"util.ping",{}],[1,170,"util.ping",{}],[1,171,"util.ping",{}],[1,172,"util.ping",{}],[1,173,"util.ping",{}],[1,174,"util.ping",{}],[1,175,"util.ping",{}],[1,176,"util.ping",{}],[1,177,"util.ping",{}],[1,178,"util.ping",{}],[1,179,"util.ping",{}],[1,180,"util.ping",{}],[1,181,"util.ping",{}],[1,182,"util.ping",{}],[1,183,"util.ping",{}],[1,184,"util.ping",{}],[1,185,"util.ping",{}],[1,186,"util.ping",{}],[1,187,"util.ping",{}],[1,188,"util.ping",{}],[1,189,"util.ping",{}],[1,190,"util.ping",{}],[1,191,"util.ping",{}],[1,192,"util.ping",{}],[1,193,"util.ping",{}],[1,194,"util.ping",{}],[1,195,"util.ping",{}],[1,196,"util.ping",{}],[1,197,"util.ping",{}],[1,198,"util.ping",{}],[1,199,"util.ping",{}],[1,200,"util.ping",{}],[1,201,"util.ping",{}]]', + ); + expect(Buffer.from(payloads[3]).toString()).toBe( + '[[1,202,"util.ping",{}],[1,203,"util.ping",{}],[1,204,"util.ping",{}],[1,205,"util.ping",{}],[1,206,"util.ping",{}],[1,207,"util.ping",{}],[1,208,"util.ping",{}],[1,209,"util.ping",{}],[1,210,"util.ping",{}],[1,211,"util.ping",{}],[1,212,"util.ping",{}],[1,213,"util.ping",{}],[1,214,"util.ping",{}],[1,215,"util.ping",{}],[1,216,"util.ping",{}],[1,217,"util.ping",{}],[1,218,"util.ping",{}],[1,219,"util.ping",{}],[1,220,"util.ping",{}],[1,221,"util.ping",{}],[1,222,"util.ping",{}],[1,223,"util.ping",{}],[1,224,"util.ping",{}],[1,225,"util.ping",{}],[1,226,"util.ping",{}],[1,227,"util.ping",{}],[1,228,"util.ping",{}],[1,229,"util.ping",{}],[1,230,"util.ping",{}],[1,231,"util.ping",{}],[1,232,"util.ping",{}],[1,233,"util.ping",{}],[1,234,"util.ping",{}],[1,235,"util.ping",{}],[1,236,"util.ping",{}],[1,237,"util.ping",{}],[1,238,"util.ping",{}],[1,239,"util.ping",{}],[1,240,"util.ping",{}],[1,241,"util.ping",{}],[1,242,"util.ping",{}],[1,243,"util.ping",{}],[1,244,"util.ping",{}],[1,245,"util.ping",{}],[1,246,"util.ping",{}],[1,247,"util.ping",{}],[1,248,"util.ping",{}],[1,249,"util.ping",{}],[1,250,"util.ping",{}],[1,251,"util.ping",{}],[1,252,"util.ping",{}],[1,253,"util.ping",{}],[1,254,"util.ping",{}],[1,255,"util.ping",{}],[1,256,"util.ping",{}],[1,257,"util.ping",{}],[1,258,"util.ping",{}],[1,259,"util.ping",{}],[1,260,"util.ping",{}],[1,261,"util.ping",{}],[1,262,"util.ping",{}],[1,263,"util.ping",{}],[1,264,"util.ping",{}],[1,265,"util.ping",{}],[1,266,"util.ping",{}],[1,267,"util.ping",{}],[1,268,"util.ping",{}],[1,269,"util.ping",{}],[1,270,"util.ping",{}],[1,271,"util.ping",{}],[1,272,"util.ping",{}],[1,273,"util.ping",{}],[1,274,"util.ping",{}],[1,275,"util.ping",{}],[1,276,"util.ping",{}],[1,277,"util.ping",{}],[1,278,"util.ping",{}],[1,279,"util.ping",{}],[1,280,"util.ping",{}],[1,281,"util.ping",{}],[1,282,"util.ping",{}],[1,283,"util.ping",{}],[1,284,"util.ping",{}],[1,285,"util.ping",{}],[1,286,"util.ping",{}],[1,287,"util.ping",{}],[1,288,"util.ping",{}],[1,289,"util.ping",{}],[1,290,"util.ping",{}],[1,291,"util.ping",{}],[1,292,"util.ping",{}],[1,293,"util.ping",{}],[1,294,"util.ping",{}],[1,295,"util.ping",{}],[1,296,"util.ping",{}],[1,297,"util.ping",{}],[1,298,"util.ping",{}],[1,299,"util.ping",{}],[1,300,"util.ping",{}],[1,301,"util.ping",{}]]', + ); }); test('can read final text packet without mask', () => { diff --git a/src/server/__bench__/ping.bench.ts b/src/server/__bench__/ping.bench.ts index 552bccd315..6a8fce31f8 100644 --- a/src/server/__bench__/ping.bench.ts +++ b/src/server/__bench__/ping.bench.ts @@ -24,7 +24,8 @@ const main = async () => { channel: { newChannel: () => new WebSocketChannel({ - newSocket: () => new WebSocket('ws://localhost:9999/rpc', [codec.specifier()], {perMessageDeflate: false}) as any, + newSocket: () => + new WebSocket('ws://localhost:9999/rpc', [codec.specifier()], {perMessageDeflate: false}) as any, }), }, }); diff --git a/src/util/buffers/StreamingOctetReader.ts b/src/util/buffers/StreamingOctetReader.ts index 2fc06718e2..b254a75647 100644 --- a/src/util/buffers/StreamingOctetReader.ts +++ b/src/util/buffers/StreamingOctetReader.ts @@ -78,7 +78,7 @@ export class StreamingOctetReader { let x = this.x; const size0 = Math.min(chunk0.length - x, size); const end = x + size0; - for (; x < end;) dst[pos++] = chunk0[x++] ^ mask[maskIndex++ % 4]; + for (; x < end; ) dst[pos++] = chunk0[x++] ^ mask[maskIndex++ % 4]; size -= size0; if (size <= 0) { this.skipUnsafe(size0); @@ -88,7 +88,7 @@ export class StreamingOctetReader { while (size > 0) { const chunk1 = this.chunks[chunkIndex++]!; const size1 = Math.min(chunk1.length, size); - for (let x = 0; x < size1;) dst[pos++] = chunk1[x++] ^ mask[maskIndex++ % 4]; + for (let x = 0; x < size1; ) dst[pos++] = chunk1[x++] ^ mask[maskIndex++ % 4]; size -= size1; } this.skipUnsafe(size);