diff --git a/README.md b/README.md index b22e629..d695b19 100644 --- a/README.md +++ b/README.md @@ -2,73 +2,90 @@ tinybuf icon showing binary peeking out from behind a square. -Compressed, static-typed binary buffers in HTML5 / Node.js +Fast, tiny binary serialization for Node.js and HTML5 – based on [js-binary](https://www.npmjs.com/package/js-binary) -- πŸš€ Designed for real-time HTML5 games (via [geckos.io](https://github.com/geckosio/geckos.io), [peer.js](https://github.com/peers/peerjs) or [socket.io](https://github.com/socketio/socket.io)) -- πŸ—œοΈ Lossless and lossy compression, up to ~50% smaller than [FlatBuffers](https://github.com/google/flatbuffers) or [Protocol Buffers](https://protobuf.dev/) -- ✨ Out-of-the-box boolean packing, 16-bit floats, 8-bit scalars, and more -- 🚦 Compile-time safety & runtime validation +- πŸš€ Suitable for real-time HTML5 games (e.g. [geckos.io](https://github.com/geckosio/geckos.io), [socket.io](https://github.com/socketio/socket.io), [peer.js](https://github.com/peers/peerjs)) +- ✨ Inferred typings, built-in validation and transforms +- πŸ—œοΈ Fast, highly-compressed encoding: ~50% smaller than [FlatBuffers/protobuf](#-comparison-table) +- 🚦 Headerless encodings, safe for property mangling (e.g. [terser](https://terser.org/)) -> **tinybuf** is safe for use with property mangling & code minification like [terser](https://terser.org/) +## Basic Usage -## Why? - -**tinybuf** is small, fast and extensible. Unlike _FlatBuffers_ and _Protocol Buffers_ - which focus on cross-platform languages, limited encoding choices, and generated code - **tinybuf** is focused soley on fast, native serialization to compressed formats. See [comparison table](#-comparison-table). - -## Sample Usage -*Easily send and receive custom binary formats.* - -**Define formats:** +**Define encoding formats:** ```ts -import { encoder, Type } from 'tinybuf'; +import { defineFormat, optional, Type } from 'tinybuf'; + -const GameWorldState = encoder({ - time: Type.UInt, - players: [{ /* ... */ }] +const GameWorldData = defineFormat({ + world: { + seqNo: Type.UInt, + time: Type.Float16, + }, + players: [{ + id: Type.UInt, + inputs: Type.Bools, // [up, left, down, right] + position: optional({ + x: Type.Float32, + y: Type.Float32, + }), + }] }); ``` -**Sending:** +> [!NOTE] +> **Objects** are flat-encoded, so additional nesting incurs a 0 byte overhead. **Arrays** incur a negligible (1 byte) +> overhead. + +**Serialize:** ```ts -// Encode: -const bytes = GameWorldState.encode(myWorld); +// encode +const bytes = GameWorldData.encode(myGameWorld) + +bytes.byteLength +// 16 ``` -**Receiving:** +**Deserialize:** ```ts -// Decode: -const myWorldData = GameWorldState.decode(bytes); +// decode +const data = GameWorldData.decode(bytes) + +// { +// world: { seqNo: number, time: number }, +// players: Array<{ id: number, inputs: boolean[], position?: { x: number, y: number } }> +// } ``` -**Receiving (many):** +**Deserialize multiple formats:** ```ts -import { decoder } from 'tinybuf'; +import { bufferParser } from 'tinybuf' -// Create a decoder: -const myDecoder = decoder() +// subscribe to formats +const parser = bufferParser() .on(GameWorldState, (data) => myWorld.update(data)) - .on(ChatMessage, (data) => myHud.onChatMessage(data)); + .on(ChatMessage, (data) => myHud.onChatMessage(data)) -// Handle incoming: -myDecoder.processBuffer(bytes); +// process incoming data +parser.processBuffer(bytes) ``` -## Getting Started -*Everything you need to quickly encode and decode strongly-typed message formats.* +## Get Started -The only requirement for **tinybuf** is that encoding formats are known by clients, servers and/or peers. You should define encoding formats in some shared module. +**tinybuf** achieves its tiny size by serializing to a schemaless encoding format; This means both the client and server +(or peers) must share common encoding definitions. You might typically put these into some common, shared module. -Then all you need is: +Use te following -1. **[encoder](#define-formats)** (+[types](#types)): _Define flexible, static-typed encoding formats_ -2. **[decoder](#use-decoder)**: _Parse incoming binary in registered formats_ -3. **[Compression/serialization](#%EF%B8%8F-compression-and-serialization)**: _Various tips & techniques for making data small_ +1. **[defineFormat](#define-formats)**: _Define flexible, static-typed encoding formats_ +2. **[bufferParser](#use-decoder)**: _Parse incoming binary in registered formats_ +3. **[Compression/serialization](#%EF%B8%8F-compression-and-serialization)**: _Various tips & techniques for making data small_ -> For more information on additional pre/post-processing rules, check out [Validation and Transforms](#-validation--transforms). +> [!TIP] +> For additional validation and post-processing, see [Validation and Transforms](#-validation--transforms) ## Installation @@ -87,23 +104,31 @@ yarn add tinybuf Create an encoding format like so: ```ts -import { encoder, Type, Optional } from 'tinybuf'; - -// Define your format: -const GameWorldData = encoder({ - time: Type.UInt, - players: [{ - id: Type.UInt, - isJumping: Type.Boolean, - position: Optional({ - x: Type.Float, - y: Type.Float - }) - }] -}); +import { encoder, Type, Optional } from 'tinybuf' + +// define reusable snippets with `as const` +const Vec2 = { + x: Type.Float32, + y: Type.Float32, +} as const + +const Player = { + id: Type.UInt, + inputs: Type.Bools, + position: Vec2, + velocity: Vec2, +} as const + +const GameWorldData = defineFormat({ + world: { + seqNo: Type.UInt, + time: Type.Float16 + }, + players: [Player], +}) ``` -Then call `encode()` to turn it into binary (as `ArrayBuffer`). +Use `encode(data)` to serialize to binary (as `ArrayBuffer`). ```ts // Encode: @@ -116,10 +141,14 @@ const bytes = GameWorldData.encode({ position: { x: 110.57345, y: -93.5366 + }, + velocity: { + x: 11, + y: 22.12 } } ] -}); +}) bytes.byteLength // 14 @@ -129,7 +158,7 @@ And you can also decode it directly from the encoding type. ``` // Decode: -const data = GameWorldData.decode(bytes); +const data = GameWorldData.decode(bytes) ``` ### Inferred types @@ -155,7 +184,7 @@ For example, the return type of `GameWorldData.decode(...)` from the above examp You can also use the `Decoded` helper to add inferred types to any custom method/handler: ```ts -import { Decoded } from 'tinybuf'; +import { Decoded } from 'tinybuf' function updateGameWorld(data: Decoded) { // e.g. Access `data.players[0].position?.x` @@ -166,7 +195,7 @@ function updateGameWorld(data: Decoded) { *Serialize data as a number of lossless (and lossy!) data types* | **Type** | **Inferred JavaScript Type** | **Bytes** | **About** | -| :-----------------: | :-----------------: | :---------------------------------------: | ------------------------------------------------------------------------------------------------------------------- | +| :----------------- | :-----------------: | :---------------------------------------: | ------------------------------------------------------------------------------------------------------------------- | | `Type.Int` | `number` | 1-8\* | Integer between `-Number.MAX_SAFE_INTEGER` and `Number.MAX_SAFE_INTEGER`. | | `Type.Int8` | `number` | 1 | Integer between -127 to 128. | | `Type.Int16` | `number` | 2 | Integer between -32,767 to 32,767. | @@ -180,17 +209,17 @@ function updateGameWorld(data: Decoded) { | `Type.Float64` / `Type.Double` | `number` | 8 | Default JavaScript `number` type. A 64-bit "double" precision floating point number. | | `Type.Float32` / `Type.Float` | `number` | 4 | A 32-bit "single" precision floating point number. | | `Type.Float16` / `Type.Half` | `number` | 2 | A 16-bit "half" precision floating point number.
**Important Note:** Low decimal precision. Max. large values Β±65,500. | +| `Type.Bool` | `boolean` | 1 | A single boolean. | +| `Type.Bools` | `boolean[]` | 1ΒΆ | Variable-length array of boolean values packed into 1ΒΆ byte. | +| `Type.Bools8` | `boolean[]` | 1 | Array of 1 - 8 booleans. | +| `Type.Bools16` | `boolean[]` | 2 | Array of 1 - 16 booleans. | +| `Type.Bools32` | `boolean[]` | 4 | Array of 1 - 32 booleans. | +| `Type.Buffer` | `ArrayBuffer` | 1† + n | JavaScript `ArrayBuffer` data. | | `Type.String` | `string` | 1† + n | A UTF-8 string. | -| `Type.Boolean` | `boolean` | 1 | A single boolean. | -| `Type.BooleanTuple` | `boolean[]` | 1ΒΆ | Variable-length array/tuple of boolean values packed into 1ΒΆ byte. | -| `Type.Bitmask8` | `boolean[]` | 1 | 8 booleans. | -| `Type.Bitmask16` | `boolean[]` | 2 | 16 booleans. | -| `Type.Bitmask32` | `boolean[]` | 4 | 32 booleans. | | `Type.JSON` | `any` | 1† + n | Arbitrary [JSON](http://json.org/) data, encoded as a UTF-8 string. | -| `Type.Binary` | `ArrayBuffer` | 1† + n | JavaScript `ArrayBuffer` data. | | `Type.RegExp` | `RegExp` | 1† + n + 1 | JavaScript `RegExp` object. | | `Type.Date` | `Date` | 8 | JavaScript `Date` object. | -| `Optional(T)` | `T \| undefined` | 1 | Any optional field. Use the `Optional(...)` helper. Array elements cannot be optional. | +| `optional(T)` | `T \| undefined` | 1 | Any optional field. Use the `Optional(...)` helper. Array elements cannot be optional. | | `[T]` | `Array` | 1† + n | Use array syntax. Any array. | | `{}` | `object` | _none_ | Use object syntax. No overhead to using object types. Buffers are ordered, flattened structures. | @@ -215,48 +244,29 @@ In JavaScript, all numbers are stored as 64-bit (8-byte) floating-point numbers Most of the meaningful gains will come out of compressing floats, including those in 2D or 3D vectors and quaternions. You can compress all visual-only quantities without issue - i.e. if you are using [Snapshot Compression Netcode](https://gafferongames.com/post/snapshot_compression/), or updating elements of a [HUD](https://en.wikipedia.org/wiki/Head-up_display). -### Quantizing Physics +#### Quantizing Physics -If you are running a deterministic physics simulation (i.e. [State Synchronization / Rollback Netcode](https://gafferongames.com/post/state_synchronization/)), you may need to apply the same quantization to your physics simulation to avoid desynchronization issues or rollback "pops". +If you are running a deterministic physics simulation (e.g. [State Synchronization / Rollback Netcode](https://gafferongames.com/post/state_synchronization/)), +you may need to _quantize_ your floating-point numbers before comparing them. -Or as Glenn Fiedler suggests, apply the deserialized state on every phyiscs `update()` as if it had come over the network: +As [Glenn Fiedler](https://gafferongames.com) suggests, you could simply apply the deserialized state on every phyiscs `update()` as if it had come over the network: ```ts -update() { - // Do physics updates... +updateLoop() { + // do physics here ... - // Quantize: - const serialized = GameWorldFormat.encode(this.getState()); - const deserialized = GameWorldFormat.decode(serialized); - this.setState(deserialized); + // quantize + const encoded = GameWorldFormat.encode(world) + world.update(GameWorldFormat.decode(encoded)) } ``` -Or for simple cases, you can apply the rounding function to the physics simulation: - -```ts -update() { - // Do physics updates... - - // Quantize: - quantize(); -} - -quantize() { - for (const entity of this.worldEntities) { - // Round everything to the nearest 32-bit representation: - entity.position.set( Math.fround(player.position.x), Math.fround(player.position.y) ); - entity.velocity.set( Math.fround(player.velocity.x), Math.fround(player.velocity.y) ); - } -} -``` - -For reference here are the is a list of the various quantization (rounding) functions for each number type: +For more manual approaches, here are the is a list of the various quantization (rounding) functions for each number type: | **Type** | **Bytes** | **Quantization function** | **Use Cases** | | --- | :-: | --- | --- | | `Type.Float64` | **8** | _n/a_ | Physics values. | -| `Type.Float32` | **4** | `Math.fround(x)` | Visual values, physics values. | +| `Type.Float32` | **4** | `Math.fround(x)` (built-in) | Visual values, physics values. | | `Type.Float16` | **2** | `fround16(x)` | Limited visual values, limited physics values - i.e. safe for numbers in the range Β±65,504, with the smallest precision Β±0.00011839976. | | `Type.Scalar` | **1** | `scalarRound(x)` | Player inputs - e.g. _analog player input (joystick)_. Values from -1.00 to 1.00. | | `Type.UScalar` | **1** | `uScalarRound(x)` | Visual values - e.g. _a health bar_. Values from 0.00 to 1.00. | @@ -273,84 +283,108 @@ What we could do instead is set custom [transforms](#transforms) that utilize mu ```ts // Example transform functions that boosts precision by x20,000 by putting // values into the range Β±~62,832, prior to serializing as a 16-bit float. -const toSpecialRange = x => (x * 20_000) - 62_832; -const fromSpecialRange = x => (x + 62_832) / 20_000; +const toSpecialRange = x => (x * 20_000) - 62_832 +const fromSpecialRange = x => (x + 62_832) / 20_000 -const MyState = encoder({ +const MyState = defineFormat({ myRotation: Type.Float16 }) - .setTransforms({ myRotation: [ toSpecialRange, fromSpecialRange ]}); + .setTransforms({ myRotation: [ toSpecialRange, fromSpecialRange ]}) ``` ## ✨ Parsing formats -By default, each encoder encodes a 2-byte identifier based on the shape of the data. +By default, each encoding includes a 2-byte identifier based on the shape of the data which is used to decode the packet. + +These identifiers are shape-based, so they will collide for identical-shaped encodings. -You can explicitly set `Id` in the `encoder(Id, definition)` to any 2-byte string or unsigned integer (or disable entirely by passing `null`). +You can explicitly set the header to any 2-byte string or u16 integer in the `defineFormat({ header: 'Ab' }, definition)`. -### Use Decoder +### bufferParser() -Handle multiple binary formats at once using a `decoder`: +Handle multiple binary formats at once using a `bufferParser`: ```ts -import { decoder } from 'tinybuf'; +import { bufferParser } from 'tinybuf' -const myDecoder = decoder() +const myDecoder = bufferParser() .on(MyFormatA, data => onMessageA(data)) - .on(MyFormatB, data => onMessageB(data)); + .on(MyFormatB, data => onMessageB(data)) // Trigger handler (or throw UnhandledBinaryDecodeError) -myDecoder.processBuffer(binary); +myDecoder.processBuffer(binary) ``` -> Note: Cannot be used with formats where `Id` was disabled. +> Note: Cannot be used with headerless formats. ### Manual handling -You can manually read message identifers from incoming buffers with the static function `BinaryCoder.peekIntId(...)` (or `BinaryCoder.peekStrId(...)`): +You can check headers on raw buffers using `peekHeader(): number` and `peekHeaderStr(): string`: ```ts -import { BinaryCoder } from 'tinybuf'; +import { peekHeader } from 'tinybuf' -if (BinaryCoder.peekStrId(incomingBinary) === MyMessageFormat.Id) { +if (peekHeader(incomingBinary) === MyMessageFormat.header) { // Do something special. } ``` -### πŸ’₯ Id Collisions +### πŸ’₯ Header Collisions -By default `Id` is based on a hash code of the encoding format. So the following two messages would have identical Ids: +The default `header` is based on the shape of the encoding format, so the following two formats would have identical headers: ```ts -const Person = encoder({ - firstName: Type.String, - lastName: Type.String -}); +const User = defineFormat({ + name: Type.String, + age: Type.UInt +}) -const FavoriteColor = encoder({ - fullName: Type.String, - color: Type.String -}); +const Color = defineFormat({ + name: Type.String, + hex: Type.UInt +}) -NameCoder.Id === ColorCoder.Id - // true +User.header === Color.header +// true ``` -If two identical formats with different handlers is a requirement, you can explicitly set unique identifiers. +You can explicitly set unique headers, as an integer 0 -> 65,535, or a 2-byte string (e.g. `'AB'`). ```ts -const Person = encoder(1, { - firstName: Type.String, - lastName: Type.String -}); +const User = defineFormat(123, { + name: Type.String, + age: Type.UInt +}) -const FavoriteColor = encoder(2, { - fullName: Type.String, - color: Type.String -}); +const Color = defineFormat('Co', { + name: Type.String, + hex: Type.UInt +}) + +User.header === Color.header +// false ``` -> Identifiers can either be a 2-byte string (e.g. `'AB'`), an unsigned integer (0 -> 65,535). +e.g. using a `const enum`: + +```ts +const enum Formats { + User, + Color, +} + +const User = defineFormat(Formats.User, { + name: Type.String, + age: Type.UInt +}) + +const Color = defineFormat(Formats.Color, { + name: Type.String, + hex: Type.UInt +}) +``` + +> ## ✨ Validation / Transforms @@ -360,7 +394,7 @@ The great thing about binary encoders is that data is implicitly type-validated, validation rules using `setValidation()`: ```ts -const UserMessage = encoder({ +const UserMessage = defineFormat({ uuid: Type.String, name: Optional(Type.String), // ... @@ -368,10 +402,10 @@ const UserMessage = encoder({ .setValidation({ uuid: (x) => { if (!isValidUUIDv4(x)) { - throw new Error('Invalid UUIDv4: ' + x); + throw new Error('Invalid UUIDv4: ' + x) } } -}); +}) ``` ### Transforms @@ -381,11 +415,11 @@ You can also apply additional encode/decode transforms. Here is an example where we're stripping out all whitespace: ```ts -const PositionMessage = encoder({ name: Type.String }) - .setTransforms({ name: a => a.replace(/\s+/g, '') }); +const PositionMessage = defineFormat({ name: Type.String }) + .setTransforms({ name: a => a.replace(/\s+/g, '') }) let binary = PositionMessage.encode({ name: 'Hello There' }) -let data = PositionMessage.decode(binary); +let data = PositionMessage.decode(binary) data.name // "HelloThere" @@ -398,16 +432,16 @@ The transform function is only applied on **encode()**, but you can provide two Here is an example which cuts the number of bytes required from `10` to `5`: ```ts -const PercentMessage = encoder(null, { value: Type.String }) +const PercentMessage = defineFormat(null, { value: Type.String }) .setTransforms({ value: [ (before) => before.replace(/\$|USD/g, '').trim(), (after) => '$' + after + ' USD' ] - }); + }) let binary = PercentMessage.encode({ value: ' $45.53 USD' }) -let data = PercentMessage.decode(binary); +let data = PercentMessage.decode(binary) binary.byteLength // 5 @@ -431,7 +465,7 @@ Here are some use cases stacked uup. | **Reference data size†** | 34 bytes | 68 bytes | 72 bytes | 175 bytes (minified) | | **Fast & efficient** | 🟒 | 🟒 | 🟒 | πŸ”΄ | | **16-bit floats** | 🟒 | πŸ”΄ | πŸ”΄ | πŸ”΄ | -| **Boolean-packing** | 🟒 | πŸ”΄ | πŸ”΄ | πŸ”΄ | +| **Packed booleans** | 🟒 | πŸ”΄ | πŸ”΄ | πŸ”΄ | | **Arbitrary JSON** | 🟒 | πŸ”΄ | πŸ”΄ | 🟒 | | **Property mangling** | 🟒 | πŸ”΄ | πŸ”΄ | πŸ”΄ | | **Suitable for real-time data** | 🟒 | 🟒 | πŸ”΄ | πŸ”΄ | @@ -484,7 +518,7 @@ Here are some use cases stacked uup. **tinybuf** ```ts -const ExampleMessage = encoder({ +const ExampleMessage = defineFormat({ players: [ { id: Type.UInt, @@ -501,56 +535,56 @@ const ExampleMessage = encoder({ health: Type.UScalar }, ], -}); +}) ``` **FlatBuffers** ```fbs // ExampleMessage.fbs -namespace ExampleNamespace; +namespace ExampleNamespace table Vec3 { - x: float; - y: float; - z: float; + x: float + y: float + z: float } table Player { - id: uint; - position: Vec3; - velocity: Vec3; - health: float; + id: uint + position: Vec3 + velocity: Vec3 + health: float } table ExampleMessage { - players: [Player]; + players: [Player] } -root_type ExampleMessage; +root_type ExampleMessage ``` **Protocol Buffers (Proto3)** ```proto -syntax = "proto3"; +syntax = "proto3" -package example; +package example message Vec3 { - float x = 1; - float y = 2; - float z = 3; + float x = 1 + float y = 2 + float z = 3 } message Player { - uint32 id = 1; - Vec3 position = 2; - Vec3 velocity = 3; - float health = 4; + uint32 id = 1 + Vec3 position = 2 + Vec3 velocity = 3 + float health = 4 } message ExampleMessage { - repeated Player players = 1; + repeated Player players = 1 } ``` @@ -562,4 +596,4 @@ See [docs/ENCODING.md](docs/ENCODING.md) for an overview on how most formats are ## Credits -Developed from a hard-fork of Guilherme Souza's [js-binary](https://github.com/sitegui/js-binary). +Hard-forked from Guilherme Souza's [js-binary](https://github.com/sitegui/js-binary). diff --git a/package.json b/package.json index a241770..731a748 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { "name": "tinybuf", - "version": "1.6.6", + "version": "1.6.7", "author": "Reece Como ", "authors": [ "Reece Como ", "Sitegui " ], "license": "MIT", - "description": "Fast, lightweight binary encoders for JavaScript", + "description": "Fast, tiny binary serialization for Node.js and HTML5", "main": "./dist/index.js", "repository": { "type": "git", diff --git a/src/core/BinaryCoder.ts b/src/core/BinaryCoder.ts index f1d6cd2..8aee210 100644 --- a/src/core/BinaryCoder.ts +++ b/src/core/BinaryCoder.ts @@ -1,10 +1,6 @@ import * as coders from './lib/coders'; -import { Field } from './Field'; -import { - djb2HashUInt16, - hashCodeTo2CharStr, - strToHashCode -} from './lib/hashCode'; +import { djb2HashUInt16, strToHashCode } from './lib/hashCode'; +import { peekHeader, peekHeaderStr } from './lib/peek'; import { MutableArrayBuffer } from './MutableArrayBuffer'; import { ReadState } from './ReadState'; import { @@ -16,16 +12,18 @@ import { InferredTransformConfig, InferredValidationConfig, ValidationFn, - Transforms + Transforms, + FieldDefinition } from './Type'; +export type BinaryCoderHeader = string | number; + /** - * Infer the decoded type of a BinaryCoder. - * - * @example - * let onData = (data: Infer) => {...}; + * Decoded type of a binary encoding. + * @example let onData = (data: Decoded) => {...}; */ export type Decoded = FromBinaryCoder extends BinaryCoder ? InferredDecodedType : never; + /** @deprecated use Decoded */ export type Infer = Decoded; @@ -33,35 +31,39 @@ export type Infer = Decoded; * BinaryCoder is a utility class for encoding and decoding binary data based * on a provided encoding format. * - * @see {Id} + * @see {header} * @see {encode(data)} * @see {decode(binary)} */ -export class BinaryCoder { +export class BinaryCoder { + /** + * A unique identifier encoded as the first 2 bytes (or `undefined` if headerless). + * + * @see {peekHeader(...)} + * @see {peekHeaderStr(...)} + * @see {BinaryCoder.hashCode} + */ + public readonly header?: HeaderType; + protected readonly type: Type; protected readonly fields: Field[]; protected _hash?: number; protected _format?: string; - protected _id?: IdType; protected _transforms?: Transforms | undefined; protected _validationFn?: ValidationFn | undefined; - /** - * @param encoderDefinition A defined encoding format. - * @param Id Defaults to hash code. Set `null` to disable. Must be a 16-bit unsigned integer. - */ public constructor( encoderDefinition: EncoderType, - Id?: IdType | null + header?: HeaderType | null ) { if ( - (typeof Id === 'number' && (Math.floor(Id) !== Id || Id < 0 || Id > 65_535)) - || (typeof Id === 'string' && new TextEncoder().encode(Id).byteLength !== 2) - || (Id !== undefined && Id !== null && !['string', 'number'].includes(typeof Id)) + (typeof header === 'number' && (Math.floor(header) !== header || header < 0 || header > 65_535)) + || (typeof header === 'string' && new TextEncoder().encode(header).byteLength !== 2) + || (header !== undefined && header !== null && !['string', 'number'].includes(typeof header)) ) { - throw new TypeError(`Id must be an unsigned 16-bit integer, a 2-byte string, or \`null\`. Received: ${Id}`); + throw new TypeError(`header must be an unsigned 16-bit integer, a 2-byte string, or \`null\`. Received: ${header}`); } else if (encoderDefinition instanceof OptionalType) { throw new TypeError("Invalid type given. Root object must not be an Optional."); @@ -79,60 +81,51 @@ export class BinaryCoder>(arrayBuffer: ArrayBuffer | ArrayBufferView): DecodedType { return this.read(new ReadState( arrayBuffer instanceof ArrayBuffer ? arrayBuffer : arrayBuffer.buffer, - this.Id === undefined ? 0 : 2 + this.header === undefined ? 0 : 2 )); } @@ -279,14 +272,14 @@ export class BinaryCoder { switch (type) { case Type.Binary: return coders.arrayBufferCoder; - case Type.Bitmask16: return coders.bitmask16Coder; - case Type.Bitmask32: return coders.bitmask32Coder; - case Type.Bitmask8: return coders.bitmask8Coder; - case Type.Boolean: return coders.booleanCoder; - case Type.BooleanTuple: return coders.booleanArrayCoder; + case Type.Bool: return coders.booleanCoder; + case Type.Bools: return coders.booleanArrayCoder; + case Type.Bools8: return coders.bitmask8Coder; + case Type.Bools16: return coders.bitmask16Coder; + case Type.Bools32: return coders.bitmask32Coder; case Type.Date: return coders.dateCoder; case Type.Float16: return coders.float16Coder; case Type.Float32: return coders.float32Coder; @@ -454,4 +447,51 @@ export class BinaryCoder; + public readonly isOptional: boolean; + public readonly isArray: boolean; + + protected _format?: string; + + public constructor(name: string, rawType: FieldDefinition) { + this.isOptional = rawType instanceof OptionalType; + + let type = rawType instanceof OptionalType ? rawType.type : rawType; + + this.name = name; + + if (Array.isArray(type)) { + if (type.length !== 1) { + throw new TypeError('Invalid array definition, it must have exactly one element'); + } + + type = type[0]; + this.isArray = true; + } + else { + this.isArray = false; + } + + this.coder = new BinaryCoder(type); + } + + /** + * @returns A string identifying the encoding format. + * @example "{str,uint16,bool}[]?" + */ + public get format(): string { + if (this._format === undefined) { + this._format = `${(this.coder as any).format}${this.isArray ? '[]' : ''}${this.isOptional ? '?' : ''}`; + } + + return this._format; + } +} + + +export default BinaryCoder; diff --git a/src/core/BinaryFormatHandler.ts b/src/core/BinaryFormatHandler.ts index 5619578..3843a88 100644 --- a/src/core/BinaryFormatHandler.ts +++ b/src/core/BinaryFormatHandler.ts @@ -1,12 +1,13 @@ import BinaryCoder from "./BinaryCoder"; import { EncoderDefinition, InferredDecodedType } from "./Type"; -import { hashCodeTo2CharStr, strToHashCode } from "./lib/hashCode"; +import { hashCodeToStr, strToHashCode } from "./lib/hashCode"; +import { peekHeader } from "./lib/peek"; -type BinaryCoderId = number; +type BCHeader = number; type BinaryCoderOnDataHandler = (data: InferredDecodedType) => any; export class UnhandledBinaryDecodeError extends Error {} -export class BinaryCoderIdCollisionError extends Error {} +export class FormatHeaderCollisionError extends Error {} /** * A utility that facilitates the management and handling of multiple binary formats. @@ -14,7 +15,7 @@ export class BinaryCoderIdCollisionError extends Error {} * It provides a central handler for encoding, decoding and routing. */ export class BinaryFormatHandler { - private coders = new Map, BinaryCoderOnDataHandler]>(); + private coders = new Map, BinaryCoderOnDataHandler]>(); /** All available coders. */ public get available(): Set> { @@ -28,17 +29,17 @@ export class BinaryFormatHandler { coder: BinaryCoder, onDataHandler: (data: DecodedType) => any ): this { - if (coder.Id === undefined) { - throw new TypeError('Cannot register a BinaryCoder with Id disabled.'); + if (coder.header === undefined) { + throw new TypeError('Cannot register a headerless encoding format.'); } - const intId = typeof coder.Id === 'string' ? strToHashCode(coder.Id) : coder.Id; + const intHeader = typeof coder.header === 'string' ? strToHashCode(coder.header) : coder.header; - if (this.coders.has(intId)) { - throw new BinaryCoderIdCollisionError(`Coder was already registered with matching Id: ${coder.Id}`); + if (this.coders.has(intHeader)) { + throw new FormatHeaderCollisionError(`Format with identical header was already registered: ${coder.header}`); } - this.coders.set(intId, [coder, onDataHandler]); + this.coders.set(intHeader, [coder, onDataHandler]); return this; } @@ -52,15 +53,13 @@ export class BinaryFormatHandler { * @throws {RangeError} If buffer has < 2 bytes. */ public processBuffer(buffer: ArrayBuffer | ArrayBufferView): void { - const id: number = BinaryCoder.peekIntId(buffer); - const tuple = this.coders.get(id); + const header: number = peekHeader(buffer); - if (!tuple) { - const strId = hashCodeTo2CharStr(id); - throw new UnhandledBinaryDecodeError(`Failed to process buffer with Id ${id} ('${strId}').`); + if (!this.coders.has(header)) { + throw new UnhandledBinaryDecodeError(`Failed to process buffer. Header: ${header} ('${hashCodeToStr(header)}').`); } - const [coder, onDataHandler] = tuple; + const [coder, onDataHandler] = this.coders.get(header); const data = coder.decode(buffer); onDataHandler(data); diff --git a/src/core/Field.ts b/src/core/Field.ts deleted file mode 100644 index 813ba82..0000000 --- a/src/core/Field.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { BinaryCoder } from './BinaryCoder'; -import { OptionalType, FieldDefinition } from './Type'; - -/** - * Parse and represent an object field. See example in Type.js - */ -export class Field { - public readonly name: string; - public readonly coder: BinaryCoder; - public readonly isOptional: boolean; - public readonly isArray: boolean; - - protected _format?: string; - - public constructor(name: string, rawType: FieldDefinition) { - this.isOptional = rawType instanceof OptionalType; - - let type = rawType instanceof OptionalType ? rawType.type : rawType; - - this.name = name; - - if (Array.isArray(type)) { - if (type.length !== 1) { - throw new TypeError('Invalid array definition, it must have exactly one element'); - } - - type = type[0]; - this.isArray = true; - } - else { - this.isArray = false; - } - - this.coder = new BinaryCoder(type); - } - - /** - * @returns A string identifying the encoding format. - * @example "{str,uint16,bool}[]?" - */ - public get format(): string { - if (this._format === undefined) { - this._format = `${(this.coder as any).format}${this.isArray ? '[]' : ''}${this.isOptional ? '?' : ''}`; - } - - return this._format; - } -} - -export default Field; diff --git a/src/core/Type.ts b/src/core/Type.ts index c6f10f0..2733518 100644 --- a/src/core/Type.ts +++ b/src/core/Type.ts @@ -19,11 +19,11 @@ export type ValueTypes = { [Type.UScalar]: number; [Type.Scalar]: number; // Boolean - [Type.Boolean]: boolean; - [Type.BooleanTuple]: boolean[]; - [Type.Bitmask8]: boolean[]; - [Type.Bitmask16]: boolean[]; - [Type.Bitmask32]: boolean[]; + [Type.Bool]: boolean; + [Type.Bools]: boolean[]; + [Type.Bools8]: boolean[]; + [Type.Bools16]: boolean[]; + [Type.Bools32]: boolean[]; // Other [Type.String]: string; [Type.Date]: Date; @@ -47,7 +47,7 @@ export class OptionalType { /** * Wrap any definition as optional. */ -export function Optional(t: T): OptionalType { +export function optional(t: T): OptionalType { return new OptionalType(t); } @@ -128,14 +128,13 @@ export type InferredValidationConfig = { */ export const enum Type { /** - * A single boolean, encoded as 1 byte. + * A boolean, encoded as 1 byte. * - * To pack multiple booleans into a single byte, see: + * Use `Bools8` to pack multiple booleans into 1 byte. * - * @see {Type.BooleanTuple} - * @see {Type.Bitmask8} + * @see {Type.Bools8} */ - Boolean = 'bool', + Bool = 'bool', /** A string. */ String = 'str', @@ -223,29 +222,20 @@ export const enum Type { */ RegExp = 'regex', - /** - * Any JSON-serializable data. - */ + /** Any JSON-serializable data. Encoded as a UTF-8 string. */ JSON = 'json', - /** - * A tuple/array of booleans. - * - * Automatically packs into the minimal amount of bytes (with a 2-bit header): - * - For arrays with 0 -> 6 values uses 1 bytes. - * - For arrays with 7 -> 12 values uses 2 bytes. - * - And so forth... - */ - BooleanTuple = 'booltuple', + /** Array of 1 - 8 booleans, encoded to 1 byte. */ + Bools8 = 'bitmask8', - /** An array containing up to 8 booleans, encoded as a single UInt8. */ - Bitmask8 = 'bitmask8', + /** Array of 1 - 16 booleans, encoded to 2 bytes. */ + Bools16 = 'bitmask16', - /** An array containing up to 16 booleans, encoded as a single UInt16. */ - Bitmask16 = 'bitmask16', + /** Array of 1 - 32 booleans, encoded to 4 bytes. */ + Bools32 = 'bitmask32', - /** An array containing up to 32 booleans, encoded as a single UInt32. */ - Bitmask32 = 'bitmask32', + /** Array of booleans (arbitrarily long), encoded with a 2-bit header. */ + Bools = 'booltuple', // ----- Data structures: ----- @@ -263,11 +253,29 @@ export const enum Type { /** Alias for `Type.Float32` @see {Float32} */ Single = 'float32', + /** Alias for `Type.Float32` @see {Float32} */ + Float = 'float32', + /** Alias for `Type.Float64` @see {Float64} */ Double = 'float64', - /** Alias for `Type.Float32` @see {Float32} */ - Float = 'float32', + /** Alias for `Type.Bool` @see {Bool} */ + Boolean = 'bool', + + /** Alias for 'Type.UInt' @see {UInt} */ + Enum = 'uint', + + /** @deprecated Alias for `Type.BoolArray` @see {BoolArray} */ + BooleanTuple = 'booltuple', + + /** @deprecated Alias for `Type.Bools8` @see {Bools8} */ + Bitmask8 = 'bitmask8', + + /** @deprecated Alias for `Type.Bools16` @see {Bools16} */ + Bitmask16 = 'bitmask16', + + /** @deprecated Alias for `Type.Bools32` @see {Bools32} */ + Bitmask32 = 'bitmask32', } export const ValidValueTypes: readonly string[] = [ @@ -288,11 +296,11 @@ export const ValidValueTypes: readonly string[] = [ Type.UScalar, Type.Scalar, // Boolean - Type.Boolean, - Type.BooleanTuple, - Type.Bitmask8, - Type.Bitmask16, - Type.Bitmask32, + Type.Bool, + Type.Bools, + Type.Bools8, + Type.Bools16, + Type.Bools32, // Other Type.String, Type.Date, diff --git a/src/core/lib/hashCode.ts b/src/core/lib/hashCode.ts index 95bcb3e..bb52615 100644 --- a/src/core/lib/hashCode.ts +++ b/src/core/lib/hashCode.ts @@ -32,6 +32,6 @@ export function strToHashCode(str: string): number { /** * Convert UInt16 to a 2-character String. */ -export function hashCodeTo2CharStr(hashCode: number): string { +export function hashCodeToStr(hashCode: number): string { return String.fromCharCode(Math.floor(hashCode / 256)) + String.fromCharCode(hashCode % 256); } diff --git a/src/core/lib/peek.ts b/src/core/lib/peek.ts new file mode 100644 index 0000000..00614dc --- /dev/null +++ b/src/core/lib/peek.ts @@ -0,0 +1,20 @@ +import { hashCodeToStr } from "./hashCode"; + + +/** + * Read the header bytes of a buffer as a number. + * + * @throws {RangeError} if buffer size < 2 + */ +export function peekHeader(buffer: ArrayBuffer | ArrayBufferView): number { + return new DataView(buffer instanceof ArrayBuffer ? buffer : buffer.buffer).getUint16(0, false); +} + +/** + * Read the header bytes of a buffer as a string. + * + * @throws {RangeError} if buffer size < 2 + */ +export function peekHeaderStr(buffer: ArrayBuffer | ArrayBufferView): string { + return hashCodeToStr(peekHeader(buffer)); +} diff --git a/src/index.ts b/src/index.ts index c09e2d8..e01b878 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,23 +1,23 @@ import BinaryCoder from './core/BinaryCoder'; import { BinaryFormatHandler } from './core/BinaryFormatHandler'; -import { EncoderDefinition } from './core/Type'; +import { EncoderDefinition, optional } from './core/Type'; export { Type, - Optional, + optional, EncoderDefinition, FieldDefinition, InferredDecodedType, - ValueTypes + ValueTypes, } from './core/Type'; export * from './core/BinaryCoder'; export * from './core/BinaryFormatHandler'; export * from './core/MutableArrayBuffer'; -export * from './core/Field'; export * from './core/ReadState'; +export * from './core/lib/peek'; export * from './core/lib/float16'; export * from './core/lib/scalar'; export * as coders from './core/lib/coders'; @@ -28,12 +28,12 @@ export * as coders from './core/lib/coders'; * Processes incoming binary buffers and forwards data to the appropriate registered handler. * * @example - * let MyDecoder = decoder() + * const MyDecoder = decoder() * .on(MyFormat, data => { ... }); * * MyDecoder.processBuffer(rawBuffer); */ -export const decoder = (): BinaryFormatHandler => new BinaryFormatHandler(); +export const bufferParser = (): BinaryFormatHandler => new BinaryFormatHandler(); /** * Defines a format for encoding/decoding binary buffers. @@ -41,37 +41,36 @@ export const decoder = (): BinaryFormatHandler => new BinaryFormatHandler(); * Optionally customize the identifier, either as a 2-byte string, an unsigned integer (0 -> 65,535), or as `null` to disable entirely. * * @example - * let MyFormat = encoder({ ... }); - * let MyFormat = encoder('ab', { ... }); - * let MyFormat = encoder(1234, { ... }); - * let MyFormat = encoder(null, { ... }); + * const MyFormat = defineFormat({ ... }); + * const MyFormat = defineFormat('ab', { ... }); + * const MyFormat = defineFormat(1234, { ... }); + * const MyFormat = defineFormat(null, { ... }); */ -export function encoder(Id: IdType | null, def: T): BinaryCoder; +export function defineFormat(def: T): BinaryCoder; /** * Defines a format for encoding/decoding binary buffers. * * Optionally customize the identifier, either as a 2-byte string, an unsigned integer (0 -> 65,535), or as `null` to disable entirely. * * @example - * let MyFormat = encoder({ ... }); - * let MyFormat = encoder('ab', { ... }); - * let MyFormat = encoder(1234, { ... }); - * let MyFormat = encoder(null, { ... }); + * const MyFormat = defineFormat({ ... }); + * const MyFormat = defineFormat('ab', { ... }); + * const MyFormat = defineFormat(1234, { ... }); + * const MyFormat = defineFormat(null, { ... }); */ -export function encoder(def: T): BinaryCoder; -/** - * Defines a format for encoding/decoding binary buffers. - * - * Optionally customize the identifier, either as a 2-byte string, an unsigned integer (0 -> 65,535), or as `null` to disable entirely. - * - * @example - * let MyFormat = encoder({ ... }); - * let MyFormat = encoder('ab', { ... }); - * let MyFormat = encoder(1234, { ... }); - * let MyFormat = encoder(null, { ... }); - */ -export function encoder(a?: IdType | T, b?: T): BinaryCoder { +export function defineFormat(h: HeaderType | null, def: T): BinaryCoder; +// eslint-disable-next-line disable-autofix/jsdoc/require-jsdoc +export function defineFormat(a?: HeaderType | T, b?: T): BinaryCoder { return a !== null && typeof a === 'object' - ? new BinaryCoder(a as T) - : new BinaryCoder(b as T, a as IdType); + ? new BinaryCoder(a as T) + : new BinaryCoder(b as T, a as HeaderType); } + +/** @deprecated use `optional()` instead */ +export const Optional = optional; + +/** @deprecated use `defineFormat()` instead */ +export const encoder = defineFormat; + +/** @deprecated use `bufferParser()` instead */ +export const decoder = bufferParser; diff --git a/tests/BinaryCoder.test.ts b/tests/BinaryCoder.test.ts index d82fa43..349451d 100644 --- a/tests/BinaryCoder.test.ts +++ b/tests/BinaryCoder.test.ts @@ -1,9 +1,9 @@ import { BinaryCoder, Type, - Optional, Decoded, - encoder + optional, + defineFormat } from '../src/index'; describe('BinaryCoder', () => { @@ -11,7 +11,7 @@ describe('BinaryCoder', () => { a: Type.Int, b: [Type.Int], c: [{ - d: Optional(Type.String) + d: optional(Type.String) }], }); @@ -30,10 +30,10 @@ describe('BinaryCoder', () => { }; it('should encode all types', () => { - const MyCoder = encoder({ + const MyCoder = defineFormat({ myBinary: Type.Binary, myBoolean: Type.Boolean, - myBooleanTuple: Type.BooleanTuple, + myBools: Type.Bools, myUScalar: Type.UScalar, myScalar: Type.Scalar, myInt: Type.Int, @@ -43,7 +43,7 @@ describe('BinaryCoder', () => { myJSON: Type.JSON, myRegExp: Type.RegExp, myString: Type.String, - myOptional: Optional([Type.String]), + myOptional: optional([Type.String]), myObject: { myUInt: Type.UInt, myUInt16: Type.UInt16, @@ -55,18 +55,18 @@ describe('BinaryCoder', () => { myFloat64: Type.Float64, }] }, - myOptionalObject: Optional({ + myOptionalObject: optional({ myDate: Type.Date, - myBitmask16: Type.Bitmask16, - myBitmask32: Type.Bitmask32, - myBitmask8: Type.Bitmask8, + myBools16: Type.Bools16, + myBools32: Type.Bools32, + myBools8: Type.Bools8, }) }); const before = { myBinary: new TextEncoder().encode('binary').buffer, myBoolean: true, - myBooleanTuple: [false, true], + myBools: [false, true], myUScalar: 0.5, myScalar: -0.5, myInt: 1, @@ -99,14 +99,14 @@ describe('BinaryCoder', () => { }, myOptionalObject: { myDate: new Date(), - myBitmask8: [ + myBools8: [ true, false, true, false, true, false, true, false ], - myBitmask16: [ + myBools16: [ true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false ], - myBitmask32: [ + myBools32: [ true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, @@ -252,7 +252,7 @@ describe('BinaryCoder', () => { }); it('should throw TypeError when root object is optional', () => { - expect(() => new BinaryCoder(Optional({ a: Type.UInt }) as any)).toThrow(TypeError); + expect(() => new BinaryCoder(optional({ a: Type.UInt }) as any)).toThrow(TypeError); }); it('should throw TypeError when root object is unknown coder type', () => { @@ -265,15 +265,15 @@ describe('BinaryCoder', () => { objectArray: [{ str: Type.String, uint: Type.UInt8, - optionalObject: Optional({ + optionalObject: optional({ x: Type.Float, y: Type.Float }), boolean: Type.Boolean }], - optionalArray: Optional([Type.String]), - booleanTuple: Type.BooleanTuple, - bitmask8: Type.Bitmask8, + optionalArray: optional([Type.String]), + booleanTuple: Type.Bools, + bools8: Type.Bools8, }); const binary = Example.encode({ @@ -290,7 +290,7 @@ describe('BinaryCoder', () => { } ], booleanTuple: [true, false, true], - bitmask8: [false, false, true, false, false, false, false, true], + bools8: [false, false, true, false, false, false, false, true], }); expect(binary.byteLength).toBe(23); @@ -387,9 +387,9 @@ describe('transforms and validation', () => { it('should handle advanced case', () => { const MyCoder = new BinaryCoder({ id: Type.UInt, - names: Optional([Type.String]), + names: optional([Type.String]), dates: [Type.Date], - myOptionalObject: Optional({ + myOptionalObject: optional({ myDate: Type.Date, }), myObject: { @@ -503,7 +503,7 @@ describe('transforms and validation', () => { describe('BOOLEAN_ARRAY', () => { const MyCoder = new BinaryCoder({ name: Type.String, - coolBools: Type.BooleanTuple, + coolBools: Type.Bools, }); it('should encode less than 8', () => { @@ -557,7 +557,7 @@ describe('BOOLEAN_ARRAY', () => { describe('BITMASK_8', () => { const MyCoder = new BinaryCoder({ name: Type.String, - coolBools: Type.Bitmask8, + coolBools: Type.Bools8, }); it('should encode all booleans below the minimum allowed', () => { @@ -614,10 +614,10 @@ describe('Id', () => { }); }); -describe('Bitmask16', () => { +describe('Bools16', () => { const MyCoder = new BinaryCoder({ name: Type.String, - coolBools: Type.Bitmask16, + coolBools: Type.Bools16, }); it('should encode all booleans below the minimum allowed', () => { @@ -641,11 +641,11 @@ describe('Bitmask16', () => { }); }); -describe('Bitmask32', () => { +describe('Bools32', () => { const MyCoder = new BinaryCoder({ name: Type.String, - coolBools: Type.Bitmask32, - other: Optional(Type.String), + coolBools: Type.Bools32, + other: optional(Type.String), }); it('should encode all booleans below the minimum allowed', () => { diff --git a/tests/BinaryFormatHandler.test.ts b/tests/BinaryFormatHandler.test.ts index 75af8fe..4158e60 100644 --- a/tests/BinaryFormatHandler.test.ts +++ b/tests/BinaryFormatHandler.test.ts @@ -1,6 +1,6 @@ import { BinaryCoder, - BinaryCoderIdCollisionError, + FormatHeaderCollisionError, BinaryFormatHandler, Type, UnhandledBinaryDecodeError, @@ -101,13 +101,13 @@ describe('BinaryCoderInterpreter', () => { expect(() => binaryHandler.on(format, () => {})).toThrow(TypeError); }); - it('throws BinaryCoderIdCollisionError if registering the same format twice', () => { + it('throws FormatHeaderCollisionError if registering the same format twice', () => { const binaryHandler = decoder() .on(new BinaryCoder({ a: [Type.String] }), () => {}); const identicalFormat = new BinaryCoder({ a: [Type.String] }); - expect(() => binaryHandler.on(identicalFormat, () => {})).toThrow(BinaryCoderIdCollisionError); + expect(() => binaryHandler.on(identicalFormat, () => {})).toThrow(FormatHeaderCollisionError); }); }); });