improved viewers
This commit is contained in:
+76
@@ -0,0 +1,76 @@
|
||||
import { utf8DecodeJs } from "./utils/utf8";
|
||||
|
||||
const DEFAULT_MAX_KEY_LENGTH = 16;
|
||||
const DEFAULT_MAX_LENGTH_PER_KEY = 16;
|
||||
|
||||
export interface KeyDecoder {
|
||||
canBeCached(byteLength: number): boolean;
|
||||
decode(bytes: Uint8Array, inputOffset: number, byteLength: number): string;
|
||||
}
|
||||
interface KeyCacheRecord {
|
||||
readonly bytes: Uint8Array;
|
||||
readonly str: string;
|
||||
}
|
||||
|
||||
export class CachedKeyDecoder implements KeyDecoder {
|
||||
hit = 0;
|
||||
miss = 0;
|
||||
private readonly caches: Array<Array<KeyCacheRecord>>;
|
||||
|
||||
constructor(readonly maxKeyLength = DEFAULT_MAX_KEY_LENGTH, readonly maxLengthPerKey = DEFAULT_MAX_LENGTH_PER_KEY) {
|
||||
// avoid `new Array(N)`, which makes a sparse array,
|
||||
// because a sparse array is typically slower than a non-sparse array.
|
||||
this.caches = [];
|
||||
for (let i = 0; i < this.maxKeyLength; i++) {
|
||||
this.caches.push([]);
|
||||
}
|
||||
}
|
||||
|
||||
public canBeCached(byteLength: number): boolean {
|
||||
return byteLength > 0 && byteLength <= this.maxKeyLength;
|
||||
}
|
||||
|
||||
private find(bytes: Uint8Array, inputOffset: number, byteLength: number): string | null {
|
||||
const records = this.caches[byteLength - 1]!;
|
||||
|
||||
FIND_CHUNK: for (const record of records) {
|
||||
const recordBytes = record.bytes;
|
||||
|
||||
for (let j = 0; j < byteLength; j++) {
|
||||
if (recordBytes[j] !== bytes[inputOffset + j]) {
|
||||
continue FIND_CHUNK;
|
||||
}
|
||||
}
|
||||
return record.str;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private store(bytes: Uint8Array, value: string) {
|
||||
const records = this.caches[bytes.length - 1]!;
|
||||
const record: KeyCacheRecord = { bytes, str: value };
|
||||
|
||||
if (records.length >= this.maxLengthPerKey) {
|
||||
// `records` are full!
|
||||
// Set `record` to an arbitrary position.
|
||||
records[(Math.random() * records.length) | 0] = record;
|
||||
} else {
|
||||
records.push(record);
|
||||
}
|
||||
}
|
||||
|
||||
public decode(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
|
||||
const cachedValue = this.find(bytes, inputOffset, byteLength);
|
||||
if (cachedValue != null) {
|
||||
this.hit++;
|
||||
return cachedValue;
|
||||
}
|
||||
this.miss++;
|
||||
|
||||
const str = utf8DecodeJs(bytes, inputOffset, byteLength);
|
||||
// Ensure to copy a slice of bytes because the byte may be NodeJS Buffer and Buffer#slice() returns a reference to its internal ArrayBuffer.
|
||||
const slicedCopyOfBytes = Uint8Array.prototype.slice.call(bytes, inputOffset, inputOffset + byteLength);
|
||||
this.store(slicedCopyOfBytes, str);
|
||||
return str;
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
export class DecodeError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
|
||||
// fix the prototype chain in a cross-platform way
|
||||
const proto: typeof DecodeError.prototype = Object.create(DecodeError.prototype);
|
||||
Object.setPrototypeOf(this, proto);
|
||||
|
||||
Object.defineProperty(this, "name", {
|
||||
configurable: true,
|
||||
enumerable: false,
|
||||
value: DecodeError.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
+627
@@ -0,0 +1,627 @@
|
||||
import { prettyByte } from "./utils/prettyByte";
|
||||
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
|
||||
import { getInt64, getUint64, UINT32_MAX } from "./utils/int";
|
||||
import { utf8DecodeJs, TEXT_DECODER_THRESHOLD, utf8DecodeTD } from "./utils/utf8";
|
||||
import { createDataView, ensureUint8Array } from "./utils/typedArrays";
|
||||
import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder";
|
||||
import { DecodeError } from "./DecodeError";
|
||||
|
||||
const enum State {
|
||||
ARRAY,
|
||||
MAP_KEY,
|
||||
MAP_VALUE,
|
||||
}
|
||||
|
||||
type MapKeyType = string | number;
|
||||
|
||||
const isValidMapKeyType = (key: unknown): key is MapKeyType => {
|
||||
const keyType = typeof key;
|
||||
|
||||
return keyType === "string" || keyType === "number";
|
||||
};
|
||||
|
||||
type StackMapState = {
|
||||
type: State.MAP_KEY | State.MAP_VALUE;
|
||||
size: number;
|
||||
key: MapKeyType | null;
|
||||
readCount: number;
|
||||
map: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type StackArrayState = {
|
||||
type: State.ARRAY;
|
||||
size: number;
|
||||
array: Array<unknown>;
|
||||
position: number;
|
||||
};
|
||||
|
||||
type StackState = StackArrayState | StackMapState;
|
||||
|
||||
const HEAD_BYTE_REQUIRED = -1;
|
||||
|
||||
const EMPTY_VIEW = new DataView(new ArrayBuffer(0));
|
||||
const EMPTY_BYTES = new Uint8Array(EMPTY_VIEW.buffer);
|
||||
|
||||
// IE11: Hack to support IE11.
|
||||
// IE11: Drop this hack and just use RangeError when IE11 is obsolete.
|
||||
export const DataViewIndexOutOfBoundsError: typeof Error = (() => {
|
||||
try {
|
||||
// IE11: The spec says it should throw RangeError,
|
||||
// IE11: but in IE11 it throws TypeError.
|
||||
EMPTY_VIEW.getInt8(0);
|
||||
} catch (e: any) {
|
||||
return e.constructor;
|
||||
}
|
||||
throw new Error("never reached");
|
||||
})();
|
||||
|
||||
const MORE_DATA = new DataViewIndexOutOfBoundsError("Insufficient data");
|
||||
|
||||
const sharedCachedKeyDecoder = new CachedKeyDecoder();
|
||||
|
||||
export class Decoder<ContextType = undefined> {
|
||||
private totalPos = 0;
|
||||
private pos = 0;
|
||||
|
||||
private view = EMPTY_VIEW;
|
||||
private bytes = EMPTY_BYTES;
|
||||
private headByte = HEAD_BYTE_REQUIRED;
|
||||
private readonly stack: Array<StackState> = [];
|
||||
|
||||
public constructor(
|
||||
private readonly extensionCodec: ExtensionCodecType<ContextType> = ExtensionCodec.defaultCodec as any,
|
||||
private readonly context: ContextType = undefined as any,
|
||||
private readonly maxStrLength = UINT32_MAX,
|
||||
private readonly maxBinLength = UINT32_MAX,
|
||||
private readonly maxArrayLength = UINT32_MAX,
|
||||
private readonly maxMapLength = UINT32_MAX,
|
||||
private readonly maxExtLength = UINT32_MAX,
|
||||
private readonly keyDecoder: KeyDecoder | null = sharedCachedKeyDecoder,
|
||||
) {}
|
||||
|
||||
private reinitializeState() {
|
||||
this.totalPos = 0;
|
||||
this.headByte = HEAD_BYTE_REQUIRED;
|
||||
this.stack.length = 0;
|
||||
|
||||
// view, bytes, and pos will be re-initialized in setBuffer()
|
||||
}
|
||||
|
||||
private setBuffer(buffer: ArrayLike<number> | BufferSource): void {
|
||||
this.bytes = ensureUint8Array(buffer);
|
||||
this.view = createDataView(this.bytes);
|
||||
this.pos = 0;
|
||||
}
|
||||
|
||||
private appendBuffer(buffer: ArrayLike<number> | BufferSource) {
|
||||
if (this.headByte === HEAD_BYTE_REQUIRED && !this.hasRemaining(1)) {
|
||||
this.setBuffer(buffer);
|
||||
} else {
|
||||
const remainingData = this.bytes.subarray(this.pos);
|
||||
const newData = ensureUint8Array(buffer);
|
||||
|
||||
// concat remainingData + newData
|
||||
const newBuffer = new Uint8Array(remainingData.length + newData.length);
|
||||
newBuffer.set(remainingData);
|
||||
newBuffer.set(newData, remainingData.length);
|
||||
this.setBuffer(newBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
private hasRemaining(size: number) {
|
||||
return this.view.byteLength - this.pos >= size;
|
||||
}
|
||||
|
||||
private createExtraByteError(posToShow: number): Error {
|
||||
const { view, pos } = this;
|
||||
return new RangeError(`Extra ${view.byteLength - pos} of ${view.byteLength} byte(s) found at buffer[${posToShow}]`);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws {@link DecodeError}
|
||||
* @throws {@link RangeError}
|
||||
*/
|
||||
public decode(buffer: ArrayLike<number> | BufferSource): unknown {
|
||||
this.reinitializeState();
|
||||
this.setBuffer(buffer);
|
||||
|
||||
const object = this.doDecodeSync();
|
||||
if (this.hasRemaining(1)) {
|
||||
throw this.createExtraByteError(this.pos);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
|
||||
public *decodeMulti(buffer: ArrayLike<number> | BufferSource): Generator<unknown, void, unknown> {
|
||||
this.reinitializeState();
|
||||
this.setBuffer(buffer);
|
||||
|
||||
while (this.hasRemaining(1)) {
|
||||
yield this.doDecodeSync();
|
||||
}
|
||||
}
|
||||
|
||||
public async decodeAsync(stream: AsyncIterable<ArrayLike<number> | BufferSource>): Promise<unknown> {
|
||||
let decoded = false;
|
||||
let object: unknown;
|
||||
for await (const buffer of stream) {
|
||||
if (decoded) {
|
||||
throw this.createExtraByteError(this.totalPos);
|
||||
}
|
||||
|
||||
this.appendBuffer(buffer);
|
||||
|
||||
try {
|
||||
object = this.doDecodeSync();
|
||||
decoded = true;
|
||||
} catch (e) {
|
||||
if (!(e instanceof DataViewIndexOutOfBoundsError)) {
|
||||
throw e; // rethrow
|
||||
}
|
||||
// fallthrough
|
||||
}
|
||||
this.totalPos += this.pos;
|
||||
}
|
||||
|
||||
if (decoded) {
|
||||
if (this.hasRemaining(1)) {
|
||||
throw this.createExtraByteError(this.totalPos);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
|
||||
const { headByte, pos, totalPos } = this;
|
||||
throw new RangeError(
|
||||
`Insufficient data in parsing ${prettyByte(headByte)} at ${totalPos} (${pos} in the current buffer)`,
|
||||
);
|
||||
}
|
||||
|
||||
public decodeArrayStream(
|
||||
stream: AsyncIterable<ArrayLike<number> | BufferSource>,
|
||||
): AsyncGenerator<unknown, void, unknown> {
|
||||
return this.decodeMultiAsync(stream, true);
|
||||
}
|
||||
|
||||
public decodeStream(stream: AsyncIterable<ArrayLike<number> | BufferSource>): AsyncGenerator<unknown, void, unknown> {
|
||||
return this.decodeMultiAsync(stream, false);
|
||||
}
|
||||
|
||||
private async *decodeMultiAsync(stream: AsyncIterable<ArrayLike<number> | BufferSource>, isArray: boolean) {
|
||||
let isArrayHeaderRequired = isArray;
|
||||
let arrayItemsLeft = -1;
|
||||
|
||||
for await (const buffer of stream) {
|
||||
if (isArray && arrayItemsLeft === 0) {
|
||||
throw this.createExtraByteError(this.totalPos);
|
||||
}
|
||||
|
||||
this.appendBuffer(buffer);
|
||||
|
||||
if (isArrayHeaderRequired) {
|
||||
arrayItemsLeft = this.readArraySize();
|
||||
isArrayHeaderRequired = false;
|
||||
this.complete();
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
yield this.doDecodeSync();
|
||||
if (--arrayItemsLeft === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (!(e instanceof DataViewIndexOutOfBoundsError)) {
|
||||
throw e; // rethrow
|
||||
}
|
||||
// fallthrough
|
||||
}
|
||||
this.totalPos += this.pos;
|
||||
}
|
||||
}
|
||||
|
||||
private doDecodeSync(): unknown {
|
||||
DECODE: while (true) {
|
||||
const headByte = this.readHeadByte();
|
||||
let object: unknown;
|
||||
|
||||
if (headByte >= 0xe0) {
|
||||
// negative fixint (111x xxxx) 0xe0 - 0xff
|
||||
object = headByte - 0x100;
|
||||
} else if (headByte < 0xc0) {
|
||||
if (headByte < 0x80) {
|
||||
// positive fixint (0xxx xxxx) 0x00 - 0x7f
|
||||
object = headByte;
|
||||
} else if (headByte < 0x90) {
|
||||
// fixmap (1000 xxxx) 0x80 - 0x8f
|
||||
const size = headByte - 0x80;
|
||||
if (size !== 0) {
|
||||
this.pushMapState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = {};
|
||||
}
|
||||
} else if (headByte < 0xa0) {
|
||||
// fixarray (1001 xxxx) 0x90 - 0x9f
|
||||
const size = headByte - 0x90;
|
||||
if (size !== 0) {
|
||||
this.pushArrayState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = [];
|
||||
}
|
||||
} else {
|
||||
// fixstr (101x xxxx) 0xa0 - 0xbf
|
||||
const byteLength = headByte - 0xa0;
|
||||
object = this.decodeUtf8String(byteLength, 0);
|
||||
}
|
||||
} else if (headByte === 0xc0) {
|
||||
// nil
|
||||
object = null;
|
||||
} else if (headByte === 0xc2) {
|
||||
// false
|
||||
object = false;
|
||||
} else if (headByte === 0xc3) {
|
||||
// true
|
||||
object = true;
|
||||
} else if (headByte === 0xca) {
|
||||
// float 32
|
||||
object = this.readF32();
|
||||
} else if (headByte === 0xcb) {
|
||||
// float 64
|
||||
object = this.readF64();
|
||||
} else if (headByte === 0xcc) {
|
||||
// uint 8
|
||||
object = this.readU8();
|
||||
} else if (headByte === 0xcd) {
|
||||
// uint 16
|
||||
object = this.readU16();
|
||||
} else if (headByte === 0xce) {
|
||||
// uint 32
|
||||
object = this.readU32();
|
||||
} else if (headByte === 0xcf) {
|
||||
// uint 64
|
||||
object = this.readU64();
|
||||
} else if (headByte === 0xd0) {
|
||||
// int 8
|
||||
object = this.readI8();
|
||||
} else if (headByte === 0xd1) {
|
||||
// int 16
|
||||
object = this.readI16();
|
||||
} else if (headByte === 0xd2) {
|
||||
// int 32
|
||||
object = this.readI32();
|
||||
} else if (headByte === 0xd3) {
|
||||
// int 64
|
||||
object = this.readI64();
|
||||
} else if (headByte === 0xd9) {
|
||||
// str 8
|
||||
const byteLength = this.lookU8();
|
||||
object = this.decodeUtf8String(byteLength, 1);
|
||||
} else if (headByte === 0xda) {
|
||||
// str 16
|
||||
const byteLength = this.lookU16();
|
||||
object = this.decodeUtf8String(byteLength, 2);
|
||||
} else if (headByte === 0xdb) {
|
||||
// str 32
|
||||
const byteLength = this.lookU32();
|
||||
object = this.decodeUtf8String(byteLength, 4);
|
||||
} else if (headByte === 0xdc) {
|
||||
// array 16
|
||||
const size = this.readU16();
|
||||
if (size !== 0) {
|
||||
this.pushArrayState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = [];
|
||||
}
|
||||
} else if (headByte === 0xdd) {
|
||||
// array 32
|
||||
const size = this.readU32();
|
||||
if (size !== 0) {
|
||||
this.pushArrayState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = [];
|
||||
}
|
||||
} else if (headByte === 0xde) {
|
||||
// map 16
|
||||
const size = this.readU16();
|
||||
if (size !== 0) {
|
||||
this.pushMapState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = {};
|
||||
}
|
||||
} else if (headByte === 0xdf) {
|
||||
// map 32
|
||||
const size = this.readU32();
|
||||
if (size !== 0) {
|
||||
this.pushMapState(size);
|
||||
this.complete();
|
||||
continue DECODE;
|
||||
} else {
|
||||
object = {};
|
||||
}
|
||||
} else if (headByte === 0xc4) {
|
||||
// bin 8
|
||||
const size = this.lookU8();
|
||||
object = this.decodeBinary(size, 1);
|
||||
} else if (headByte === 0xc5) {
|
||||
// bin 16
|
||||
const size = this.lookU16();
|
||||
object = this.decodeBinary(size, 2);
|
||||
} else if (headByte === 0xc6) {
|
||||
// bin 32
|
||||
const size = this.lookU32();
|
||||
object = this.decodeBinary(size, 4);
|
||||
} else if (headByte === 0xd4) {
|
||||
// fixext 1
|
||||
object = this.decodeExtension(1, 0);
|
||||
} else if (headByte === 0xd5) {
|
||||
// fixext 2
|
||||
object = this.decodeExtension(2, 0);
|
||||
} else if (headByte === 0xd6) {
|
||||
// fixext 4
|
||||
object = this.decodeExtension(4, 0);
|
||||
} else if (headByte === 0xd7) {
|
||||
// fixext 8
|
||||
object = this.decodeExtension(8, 0);
|
||||
} else if (headByte === 0xd8) {
|
||||
// fixext 16
|
||||
object = this.decodeExtension(16, 0);
|
||||
} else if (headByte === 0xc7) {
|
||||
// ext 8
|
||||
const size = this.lookU8();
|
||||
object = this.decodeExtension(size, 1);
|
||||
} else if (headByte === 0xc8) {
|
||||
// ext 16
|
||||
const size = this.lookU16();
|
||||
object = this.decodeExtension(size, 2);
|
||||
} else if (headByte === 0xc9) {
|
||||
// ext 32
|
||||
const size = this.lookU32();
|
||||
object = this.decodeExtension(size, 4);
|
||||
} else {
|
||||
throw new DecodeError(`Unrecognized type byte: ${prettyByte(headByte)}`);
|
||||
}
|
||||
|
||||
this.complete();
|
||||
|
||||
const stack = this.stack;
|
||||
while (stack.length > 0) {
|
||||
// arrays and maps
|
||||
const state = stack[stack.length - 1]!;
|
||||
if (state.type === State.ARRAY) {
|
||||
state.array[state.position] = object;
|
||||
state.position++;
|
||||
if (state.position === state.size) {
|
||||
stack.pop();
|
||||
object = state.array;
|
||||
} else {
|
||||
continue DECODE;
|
||||
}
|
||||
} else if (state.type === State.MAP_KEY) {
|
||||
if (!isValidMapKeyType(object)) {
|
||||
throw new DecodeError("The type of key must be string or number but " + typeof object);
|
||||
}
|
||||
if (object === "__proto__") {
|
||||
throw new DecodeError("The key __proto__ is not allowed");
|
||||
}
|
||||
|
||||
state.key = object;
|
||||
state.type = State.MAP_VALUE;
|
||||
continue DECODE;
|
||||
} else {
|
||||
// it must be `state.type === State.MAP_VALUE` here
|
||||
|
||||
state.map[state.key!] = object;
|
||||
state.readCount++;
|
||||
|
||||
if (state.readCount === state.size) {
|
||||
stack.pop();
|
||||
object = state.map;
|
||||
} else {
|
||||
state.key = null;
|
||||
state.type = State.MAP_KEY;
|
||||
continue DECODE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return object;
|
||||
}
|
||||
}
|
||||
|
||||
private readHeadByte(): number {
|
||||
if (this.headByte === HEAD_BYTE_REQUIRED) {
|
||||
this.headByte = this.readU8();
|
||||
// console.log("headByte", prettyByte(this.headByte));
|
||||
}
|
||||
|
||||
return this.headByte;
|
||||
}
|
||||
|
||||
private complete(): void {
|
||||
this.headByte = HEAD_BYTE_REQUIRED;
|
||||
}
|
||||
|
||||
private readArraySize(): number {
|
||||
const headByte = this.readHeadByte();
|
||||
|
||||
switch (headByte) {
|
||||
case 0xdc:
|
||||
return this.readU16();
|
||||
case 0xdd:
|
||||
return this.readU32();
|
||||
default: {
|
||||
if (headByte < 0xa0) {
|
||||
return headByte - 0x90;
|
||||
} else {
|
||||
throw new DecodeError(`Unrecognized array type byte: ${prettyByte(headByte)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private pushMapState(size: number) {
|
||||
if (size > this.maxMapLength) {
|
||||
throw new DecodeError(`Max length exceeded: map length (${size}) > maxMapLengthLength (${this.maxMapLength})`);
|
||||
}
|
||||
|
||||
this.stack.push({
|
||||
type: State.MAP_KEY,
|
||||
size,
|
||||
key: null,
|
||||
readCount: 0,
|
||||
map: {},
|
||||
});
|
||||
}
|
||||
|
||||
private pushArrayState(size: number) {
|
||||
if (size > this.maxArrayLength) {
|
||||
throw new DecodeError(`Max length exceeded: array length (${size}) > maxArrayLength (${this.maxArrayLength})`);
|
||||
}
|
||||
|
||||
this.stack.push({
|
||||
type: State.ARRAY,
|
||||
size,
|
||||
array: new Array<unknown>(size),
|
||||
position: 0,
|
||||
});
|
||||
}
|
||||
|
||||
private decodeUtf8String(byteLength: number, headerOffset: number): string {
|
||||
if (byteLength > this.maxStrLength) {
|
||||
throw new DecodeError(
|
||||
`Max length exceeded: UTF-8 byte length (${byteLength}) > maxStrLength (${this.maxStrLength})`,
|
||||
);
|
||||
}
|
||||
|
||||
if (this.bytes.byteLength < this.pos + headerOffset + byteLength) {
|
||||
throw MORE_DATA;
|
||||
}
|
||||
|
||||
const offset = this.pos + headerOffset;
|
||||
let object: string;
|
||||
if (this.stateIsMapKey() && this.keyDecoder?.canBeCached(byteLength)) {
|
||||
object = this.keyDecoder.decode(this.bytes, offset, byteLength);
|
||||
} else if (byteLength > TEXT_DECODER_THRESHOLD) {
|
||||
object = utf8DecodeTD(this.bytes, offset, byteLength);
|
||||
} else {
|
||||
object = utf8DecodeJs(this.bytes, offset, byteLength);
|
||||
}
|
||||
this.pos += headerOffset + byteLength;
|
||||
return object;
|
||||
}
|
||||
|
||||
private stateIsMapKey(): boolean {
|
||||
if (this.stack.length > 0) {
|
||||
const state = this.stack[this.stack.length - 1]!;
|
||||
return state.type === State.MAP_KEY;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private decodeBinary(byteLength: number, headOffset: number): Uint8Array {
|
||||
if (byteLength > this.maxBinLength) {
|
||||
throw new DecodeError(`Max length exceeded: bin length (${byteLength}) > maxBinLength (${this.maxBinLength})`);
|
||||
}
|
||||
|
||||
if (!this.hasRemaining(byteLength + headOffset)) {
|
||||
throw MORE_DATA;
|
||||
}
|
||||
|
||||
const offset = this.pos + headOffset;
|
||||
const object = this.bytes.subarray(offset, offset + byteLength);
|
||||
this.pos += headOffset + byteLength;
|
||||
return object;
|
||||
}
|
||||
|
||||
private decodeExtension(size: number, headOffset: number): unknown {
|
||||
if (size > this.maxExtLength) {
|
||||
throw new DecodeError(`Max length exceeded: ext length (${size}) > maxExtLength (${this.maxExtLength})`);
|
||||
}
|
||||
|
||||
const extType = this.view.getInt8(this.pos + headOffset);
|
||||
const data = this.decodeBinary(size, headOffset + 1 /* extType */);
|
||||
return this.extensionCodec.decode(data, extType, this.context);
|
||||
}
|
||||
|
||||
private lookU8() {
|
||||
return this.view.getUint8(this.pos);
|
||||
}
|
||||
|
||||
private lookU16() {
|
||||
return this.view.getUint16(this.pos);
|
||||
}
|
||||
|
||||
private lookU32() {
|
||||
return this.view.getUint32(this.pos);
|
||||
}
|
||||
|
||||
private readU8(): number {
|
||||
const value = this.view.getUint8(this.pos);
|
||||
this.pos++;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readI8(): number {
|
||||
const value = this.view.getInt8(this.pos);
|
||||
this.pos++;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readU16(): number {
|
||||
const value = this.view.getUint16(this.pos);
|
||||
this.pos += 2;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readI16(): number {
|
||||
const value = this.view.getInt16(this.pos);
|
||||
this.pos += 2;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readU32(): number {
|
||||
const value = this.view.getUint32(this.pos);
|
||||
this.pos += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readI32(): number {
|
||||
const value = this.view.getInt32(this.pos);
|
||||
this.pos += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readU64(): number {
|
||||
const value = getUint64(this.view, this.pos);
|
||||
this.pos += 8;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readI64(): number {
|
||||
const value = getInt64(this.view, this.pos);
|
||||
this.pos += 8;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readF32() {
|
||||
const value = this.view.getFloat32(this.pos);
|
||||
this.pos += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
private readF64() {
|
||||
const value = this.view.getFloat64(this.pos);
|
||||
this.pos += 8;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
+412
@@ -0,0 +1,412 @@
|
||||
import { utf8EncodeJs, utf8Count, TEXT_ENCODER_THRESHOLD, utf8EncodeTE } from "./utils/utf8";
|
||||
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
|
||||
import { setInt64, setUint64 } from "./utils/int";
|
||||
import { ensureUint8Array } from "./utils/typedArrays";
|
||||
import type { ExtData } from "./ExtData";
|
||||
|
||||
export const DEFAULT_MAX_DEPTH = 100;
|
||||
export const DEFAULT_INITIAL_BUFFER_SIZE = 2048;
|
||||
|
||||
export class Encoder<ContextType = undefined> {
|
||||
private pos = 0;
|
||||
private view = new DataView(new ArrayBuffer(this.initialBufferSize));
|
||||
private bytes = new Uint8Array(this.view.buffer);
|
||||
|
||||
public constructor(
|
||||
private readonly extensionCodec: ExtensionCodecType<ContextType> = ExtensionCodec.defaultCodec as any,
|
||||
private readonly context: ContextType = undefined as any,
|
||||
private readonly maxDepth = DEFAULT_MAX_DEPTH,
|
||||
private readonly initialBufferSize = DEFAULT_INITIAL_BUFFER_SIZE,
|
||||
private readonly sortKeys = false,
|
||||
private readonly forceFloat32 = false,
|
||||
private readonly ignoreUndefined = false,
|
||||
private readonly forceIntegerToFloat = false,
|
||||
) {}
|
||||
|
||||
private reinitializeState() {
|
||||
this.pos = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* This is almost equivalent to {@link Encoder#encode}, but it returns an reference of the encoder's internal buffer and thus much faster than {@link Encoder#encode}.
|
||||
*
|
||||
* @returns Encodes the object and returns a shared reference the encoder's internal buffer.
|
||||
*/
|
||||
public encodeSharedRef(object: unknown): Uint8Array {
|
||||
this.reinitializeState();
|
||||
this.doEncode(object, 1);
|
||||
return this.bytes.subarray(0, this.pos);
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns Encodes the object and returns a copy of the encoder's internal buffer.
|
||||
*/
|
||||
public encode(object: unknown): Uint8Array {
|
||||
this.reinitializeState();
|
||||
this.doEncode(object, 1);
|
||||
return this.bytes.slice(0, this.pos);
|
||||
}
|
||||
|
||||
private doEncode(object: unknown, depth: number): void {
|
||||
if (depth > this.maxDepth) {
|
||||
throw new Error(`Too deep objects in depth ${depth}`);
|
||||
}
|
||||
|
||||
if (object == null) {
|
||||
this.encodeNil();
|
||||
} else if (typeof object === "boolean") {
|
||||
this.encodeBoolean(object);
|
||||
} else if (typeof object === "number") {
|
||||
this.encodeNumber(object);
|
||||
} else if (typeof object === "string") {
|
||||
this.encodeString(object);
|
||||
} else {
|
||||
this.encodeObject(object, depth);
|
||||
}
|
||||
}
|
||||
|
||||
private ensureBufferSizeToWrite(sizeToWrite: number) {
|
||||
const requiredSize = this.pos + sizeToWrite;
|
||||
|
||||
if (this.view.byteLength < requiredSize) {
|
||||
this.resizeBuffer(requiredSize * 2);
|
||||
}
|
||||
}
|
||||
|
||||
private resizeBuffer(newSize: number) {
|
||||
const newBuffer = new ArrayBuffer(newSize);
|
||||
const newBytes = new Uint8Array(newBuffer);
|
||||
const newView = new DataView(newBuffer);
|
||||
|
||||
newBytes.set(this.bytes);
|
||||
|
||||
this.view = newView;
|
||||
this.bytes = newBytes;
|
||||
}
|
||||
|
||||
private encodeNil() {
|
||||
this.writeU8(0xc0);
|
||||
}
|
||||
|
||||
private encodeBoolean(object: boolean) {
|
||||
if (object === false) {
|
||||
this.writeU8(0xc2);
|
||||
} else {
|
||||
this.writeU8(0xc3);
|
||||
}
|
||||
}
|
||||
private encodeNumber(object: number) {
|
||||
if (Number.isSafeInteger(object) && !this.forceIntegerToFloat) {
|
||||
if (object >= 0) {
|
||||
if (object < 0x80) {
|
||||
// positive fixint
|
||||
this.writeU8(object);
|
||||
} else if (object < 0x100) {
|
||||
// uint 8
|
||||
this.writeU8(0xcc);
|
||||
this.writeU8(object);
|
||||
} else if (object < 0x10000) {
|
||||
// uint 16
|
||||
this.writeU8(0xcd);
|
||||
this.writeU16(object);
|
||||
} else if (object < 0x100000000) {
|
||||
// uint 32
|
||||
this.writeU8(0xce);
|
||||
this.writeU32(object);
|
||||
} else {
|
||||
// uint 64
|
||||
this.writeU8(0xcf);
|
||||
this.writeU64(object);
|
||||
}
|
||||
} else {
|
||||
if (object >= -0x20) {
|
||||
// negative fixint
|
||||
this.writeU8(0xe0 | (object + 0x20));
|
||||
} else if (object >= -0x80) {
|
||||
// int 8
|
||||
this.writeU8(0xd0);
|
||||
this.writeI8(object);
|
||||
} else if (object >= -0x8000) {
|
||||
// int 16
|
||||
this.writeU8(0xd1);
|
||||
this.writeI16(object);
|
||||
} else if (object >= -0x80000000) {
|
||||
// int 32
|
||||
this.writeU8(0xd2);
|
||||
this.writeI32(object);
|
||||
} else {
|
||||
// int 64
|
||||
this.writeU8(0xd3);
|
||||
this.writeI64(object);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// non-integer numbers
|
||||
if (this.forceFloat32) {
|
||||
// float 32
|
||||
this.writeU8(0xca);
|
||||
this.writeF32(object);
|
||||
} else {
|
||||
// float 64
|
||||
this.writeU8(0xcb);
|
||||
this.writeF64(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private writeStringHeader(byteLength: number) {
|
||||
if (byteLength < 32) {
|
||||
// fixstr
|
||||
this.writeU8(0xa0 + byteLength);
|
||||
} else if (byteLength < 0x100) {
|
||||
// str 8
|
||||
this.writeU8(0xd9);
|
||||
this.writeU8(byteLength);
|
||||
} else if (byteLength < 0x10000) {
|
||||
// str 16
|
||||
this.writeU8(0xda);
|
||||
this.writeU16(byteLength);
|
||||
} else if (byteLength < 0x100000000) {
|
||||
// str 32
|
||||
this.writeU8(0xdb);
|
||||
this.writeU32(byteLength);
|
||||
} else {
|
||||
throw new Error(`Too long string: ${byteLength} bytes in UTF-8`);
|
||||
}
|
||||
}
|
||||
|
||||
private encodeString(object: string) {
|
||||
const maxHeaderSize = 1 + 4;
|
||||
const strLength = object.length;
|
||||
|
||||
if (strLength > TEXT_ENCODER_THRESHOLD) {
|
||||
const byteLength = utf8Count(object);
|
||||
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
|
||||
this.writeStringHeader(byteLength);
|
||||
utf8EncodeTE(object, this.bytes, this.pos);
|
||||
this.pos += byteLength;
|
||||
} else {
|
||||
const byteLength = utf8Count(object);
|
||||
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
|
||||
this.writeStringHeader(byteLength);
|
||||
utf8EncodeJs(object, this.bytes, this.pos);
|
||||
this.pos += byteLength;
|
||||
}
|
||||
}
|
||||
|
||||
private encodeObject(object: unknown, depth: number) {
|
||||
// try to encode objects with custom codec first of non-primitives
|
||||
const ext = this.extensionCodec.tryToEncode(object, this.context);
|
||||
if (ext != null) {
|
||||
this.encodeExtension(ext);
|
||||
} else if (Array.isArray(object)) {
|
||||
this.encodeArray(object, depth);
|
||||
} else if (ArrayBuffer.isView(object)) {
|
||||
this.encodeBinary(object);
|
||||
} else if (typeof object === "object") {
|
||||
this.encodeMap(object as Record<string, unknown>, depth);
|
||||
} else {
|
||||
// symbol, function and other special object come here unless extensionCodec handles them.
|
||||
throw new Error(`Unrecognized object: ${Object.prototype.toString.apply(object)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private encodeBinary(object: ArrayBufferView) {
|
||||
const size = object.byteLength;
|
||||
if (size < 0x100) {
|
||||
// bin 8
|
||||
this.writeU8(0xc4);
|
||||
this.writeU8(size);
|
||||
} else if (size < 0x10000) {
|
||||
// bin 16
|
||||
this.writeU8(0xc5);
|
||||
this.writeU16(size);
|
||||
} else if (size < 0x100000000) {
|
||||
// bin 32
|
||||
this.writeU8(0xc6);
|
||||
this.writeU32(size);
|
||||
} else {
|
||||
throw new Error(`Too large binary: ${size}`);
|
||||
}
|
||||
const bytes = ensureUint8Array(object);
|
||||
this.writeU8a(bytes);
|
||||
}
|
||||
|
||||
private encodeArray(object: Array<unknown>, depth: number) {
|
||||
const size = object.length;
|
||||
if (size < 16) {
|
||||
// fixarray
|
||||
this.writeU8(0x90 + size);
|
||||
} else if (size < 0x10000) {
|
||||
// array 16
|
||||
this.writeU8(0xdc);
|
||||
this.writeU16(size);
|
||||
} else if (size < 0x100000000) {
|
||||
// array 32
|
||||
this.writeU8(0xdd);
|
||||
this.writeU32(size);
|
||||
} else {
|
||||
throw new Error(`Too large array: ${size}`);
|
||||
}
|
||||
for (const item of object) {
|
||||
this.doEncode(item, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
private countWithoutUndefined(object: Record<string, unknown>, keys: ReadonlyArray<string>): number {
|
||||
let count = 0;
|
||||
|
||||
for (const key of keys) {
|
||||
if (object[key] !== undefined) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
private encodeMap(object: Record<string, unknown>, depth: number) {
|
||||
const keys = Object.keys(object);
|
||||
if (this.sortKeys) {
|
||||
keys.sort();
|
||||
}
|
||||
|
||||
const size = this.ignoreUndefined ? this.countWithoutUndefined(object, keys) : keys.length;
|
||||
|
||||
if (size < 16) {
|
||||
// fixmap
|
||||
this.writeU8(0x80 + size);
|
||||
} else if (size < 0x10000) {
|
||||
// map 16
|
||||
this.writeU8(0xde);
|
||||
this.writeU16(size);
|
||||
} else if (size < 0x100000000) {
|
||||
// map 32
|
||||
this.writeU8(0xdf);
|
||||
this.writeU32(size);
|
||||
} else {
|
||||
throw new Error(`Too large map object: ${size}`);
|
||||
}
|
||||
|
||||
for (const key of keys) {
|
||||
const value = object[key];
|
||||
|
||||
if (!(this.ignoreUndefined && value === undefined)) {
|
||||
this.encodeString(key);
|
||||
this.doEncode(value, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private encodeExtension(ext: ExtData) {
|
||||
const size = ext.data.length;
|
||||
if (size === 1) {
|
||||
// fixext 1
|
||||
this.writeU8(0xd4);
|
||||
} else if (size === 2) {
|
||||
// fixext 2
|
||||
this.writeU8(0xd5);
|
||||
} else if (size === 4) {
|
||||
// fixext 4
|
||||
this.writeU8(0xd6);
|
||||
} else if (size === 8) {
|
||||
// fixext 8
|
||||
this.writeU8(0xd7);
|
||||
} else if (size === 16) {
|
||||
// fixext 16
|
||||
this.writeU8(0xd8);
|
||||
} else if (size < 0x100) {
|
||||
// ext 8
|
||||
this.writeU8(0xc7);
|
||||
this.writeU8(size);
|
||||
} else if (size < 0x10000) {
|
||||
// ext 16
|
||||
this.writeU8(0xc8);
|
||||
this.writeU16(size);
|
||||
} else if (size < 0x100000000) {
|
||||
// ext 32
|
||||
this.writeU8(0xc9);
|
||||
this.writeU32(size);
|
||||
} else {
|
||||
throw new Error(`Too large extension object: ${size}`);
|
||||
}
|
||||
this.writeI8(ext.type);
|
||||
this.writeU8a(ext.data);
|
||||
}
|
||||
|
||||
private writeU8(value: number) {
|
||||
this.ensureBufferSizeToWrite(1);
|
||||
|
||||
this.view.setUint8(this.pos, value);
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
private writeU8a(values: ArrayLike<number>) {
|
||||
const size = values.length;
|
||||
this.ensureBufferSizeToWrite(size);
|
||||
|
||||
this.bytes.set(values, this.pos);
|
||||
this.pos += size;
|
||||
}
|
||||
|
||||
private writeI8(value: number) {
|
||||
this.ensureBufferSizeToWrite(1);
|
||||
|
||||
this.view.setInt8(this.pos, value);
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
private writeU16(value: number) {
|
||||
this.ensureBufferSizeToWrite(2);
|
||||
|
||||
this.view.setUint16(this.pos, value);
|
||||
this.pos += 2;
|
||||
}
|
||||
|
||||
private writeI16(value: number) {
|
||||
this.ensureBufferSizeToWrite(2);
|
||||
|
||||
this.view.setInt16(this.pos, value);
|
||||
this.pos += 2;
|
||||
}
|
||||
|
||||
private writeU32(value: number) {
|
||||
this.ensureBufferSizeToWrite(4);
|
||||
|
||||
this.view.setUint32(this.pos, value);
|
||||
this.pos += 4;
|
||||
}
|
||||
|
||||
private writeI32(value: number) {
|
||||
this.ensureBufferSizeToWrite(4);
|
||||
|
||||
this.view.setInt32(this.pos, value);
|
||||
this.pos += 4;
|
||||
}
|
||||
|
||||
private writeF32(value: number) {
|
||||
this.ensureBufferSizeToWrite(4);
|
||||
this.view.setFloat32(this.pos, value);
|
||||
this.pos += 4;
|
||||
}
|
||||
|
||||
private writeF64(value: number) {
|
||||
this.ensureBufferSizeToWrite(8);
|
||||
this.view.setFloat64(this.pos, value);
|
||||
this.pos += 8;
|
||||
}
|
||||
|
||||
private writeU64(value: number) {
|
||||
this.ensureBufferSizeToWrite(8);
|
||||
|
||||
setUint64(this.view, this.pos, value);
|
||||
this.pos += 8;
|
||||
}
|
||||
|
||||
private writeI64(value: number) {
|
||||
this.ensureBufferSizeToWrite(8);
|
||||
|
||||
setInt64(this.view, this.pos, value);
|
||||
this.pos += 8;
|
||||
}
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
/**
|
||||
* ExtData is used to handle Extension Types that are not registered to ExtensionCodec.
|
||||
*/
|
||||
export class ExtData {
|
||||
constructor(readonly type: number, readonly data: Uint8Array) {}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
// ExtensionCodec to handle MessagePack extensions
|
||||
|
||||
import { ExtData } from "./ExtData";
|
||||
import { timestampExtension } from "./timestamp";
|
||||
|
||||
export type ExtensionDecoderType<ContextType> = (
|
||||
data: Uint8Array,
|
||||
extensionType: number,
|
||||
context: ContextType,
|
||||
) => unknown;
|
||||
|
||||
export type ExtensionEncoderType<ContextType> = (input: unknown, context: ContextType) => Uint8Array | null;
|
||||
|
||||
// immutable interface to ExtensionCodec
|
||||
export type ExtensionCodecType<ContextType> = {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
__brand?: ContextType;
|
||||
tryToEncode(object: unknown, context: ContextType): ExtData | null;
|
||||
decode(data: Uint8Array, extType: number, context: ContextType): unknown;
|
||||
};
|
||||
|
||||
export class ExtensionCodec<ContextType = undefined> implements ExtensionCodecType<ContextType> {
|
||||
public static readonly defaultCodec: ExtensionCodecType<undefined> = new ExtensionCodec();
|
||||
|
||||
// ensures ExtensionCodecType<X> matches ExtensionCodec<X>
|
||||
// this will make type errors a lot more clear
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
__brand?: ContextType;
|
||||
|
||||
// built-in extensions
|
||||
private readonly builtInEncoders: Array<ExtensionEncoderType<ContextType> | undefined | null> = [];
|
||||
private readonly builtInDecoders: Array<ExtensionDecoderType<ContextType> | undefined | null> = [];
|
||||
|
||||
// custom extensions
|
||||
private readonly encoders: Array<ExtensionEncoderType<ContextType> | undefined | null> = [];
|
||||
private readonly decoders: Array<ExtensionDecoderType<ContextType> | undefined | null> = [];
|
||||
|
||||
public constructor() {
|
||||
this.register(timestampExtension);
|
||||
}
|
||||
|
||||
public register({
|
||||
type,
|
||||
encode,
|
||||
decode,
|
||||
}: {
|
||||
type: number;
|
||||
encode: ExtensionEncoderType<ContextType>;
|
||||
decode: ExtensionDecoderType<ContextType>;
|
||||
}): void {
|
||||
if (type >= 0) {
|
||||
// custom extensions
|
||||
this.encoders[type] = encode;
|
||||
this.decoders[type] = decode;
|
||||
} else {
|
||||
// built-in extensions
|
||||
const index = 1 + type;
|
||||
this.builtInEncoders[index] = encode;
|
||||
this.builtInDecoders[index] = decode;
|
||||
}
|
||||
}
|
||||
|
||||
public tryToEncode(object: unknown, context: ContextType): ExtData | null {
|
||||
// built-in extensions
|
||||
for (let i = 0; i < this.builtInEncoders.length; i++) {
|
||||
const encodeExt = this.builtInEncoders[i];
|
||||
if (encodeExt != null) {
|
||||
const data = encodeExt(object, context);
|
||||
if (data != null) {
|
||||
const type = -1 - i;
|
||||
return new ExtData(type, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// custom extensions
|
||||
for (let i = 0; i < this.encoders.length; i++) {
|
||||
const encodeExt = this.encoders[i];
|
||||
if (encodeExt != null) {
|
||||
const data = encodeExt(object, context);
|
||||
if (data != null) {
|
||||
const type = i;
|
||||
return new ExtData(type, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (object instanceof ExtData) {
|
||||
// to keep ExtData as is
|
||||
return object;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public decode(data: Uint8Array, type: number, context: ContextType): unknown {
|
||||
const decodeExt = type < 0 ? this.builtInDecoders[-1 - type] : this.decoders[type];
|
||||
if (decodeExt) {
|
||||
return decodeExt(data, type, context);
|
||||
} else {
|
||||
// decode() does not fail, returns ExtData instead.
|
||||
return new ExtData(type, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
/* eslint-disable @typescript-eslint/ban-types */
|
||||
|
||||
export type SplitTypes<T, U> = U extends T ? (Exclude<T, U> extends never ? T : Exclude<T, U>) : T;
|
||||
|
||||
export type SplitUndefined<T> = SplitTypes<T, undefined>;
|
||||
|
||||
export type ContextOf<ContextType> = ContextType extends undefined
|
||||
? {}
|
||||
: {
|
||||
/**
|
||||
* Custom user-defined data, read/writable
|
||||
*/
|
||||
context: ContextType;
|
||||
};
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
import { Decoder } from "./Decoder";
|
||||
import type { ExtensionCodecType } from "./ExtensionCodec";
|
||||
import type { ContextOf, SplitUndefined } from "./context";
|
||||
|
||||
export type DecodeOptions<ContextType = undefined> = Readonly<
|
||||
Partial<{
|
||||
extensionCodec: ExtensionCodecType<ContextType>;
|
||||
|
||||
/**
|
||||
* Maximum string length.
|
||||
*
|
||||
* Defaults to 4_294_967_295 (UINT32_MAX).
|
||||
*/
|
||||
maxStrLength: number;
|
||||
/**
|
||||
* Maximum binary length.
|
||||
*
|
||||
* Defaults to 4_294_967_295 (UINT32_MAX).
|
||||
*/
|
||||
maxBinLength: number;
|
||||
/**
|
||||
* Maximum array length.
|
||||
*
|
||||
* Defaults to 4_294_967_295 (UINT32_MAX).
|
||||
*/
|
||||
maxArrayLength: number;
|
||||
/**
|
||||
* Maximum map length.
|
||||
*
|
||||
* Defaults to 4_294_967_295 (UINT32_MAX).
|
||||
*/
|
||||
maxMapLength: number;
|
||||
/**
|
||||
* Maximum extension length.
|
||||
*
|
||||
* Defaults to 4_294_967_295 (UINT32_MAX).
|
||||
*/
|
||||
maxExtLength: number;
|
||||
}>
|
||||
> &
|
||||
ContextOf<ContextType>;
|
||||
|
||||
export const defaultDecodeOptions: DecodeOptions = {};
|
||||
|
||||
/**
|
||||
* It decodes a single MessagePack object in a buffer.
|
||||
*
|
||||
* This is a synchronous decoding function.
|
||||
* See other variants for asynchronous decoding: {@link decodeAsync()}, {@link decodeStream()}, or {@link decodeArrayStream()}.
|
||||
*
|
||||
* @throws {@link RangeError} if the buffer is incomplete, including the case where the buffer is empty.
|
||||
* @throws {@link DecodeError} if the buffer contains invalid data.
|
||||
*/
|
||||
export function decode<ContextType = undefined>(
|
||||
buffer: ArrayLike<number> | BufferSource,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): unknown {
|
||||
const decoder = new Decoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxStrLength,
|
||||
options.maxBinLength,
|
||||
options.maxArrayLength,
|
||||
options.maxMapLength,
|
||||
options.maxExtLength,
|
||||
);
|
||||
return decoder.decode(buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* It decodes multiple MessagePack objects in a buffer.
|
||||
* This is corresponding to {@link decodeMultiStream()}.
|
||||
*
|
||||
* @throws {@link RangeError} if the buffer is incomplete, including the case where the buffer is empty.
|
||||
* @throws {@link DecodeError} if the buffer contains invalid data.
|
||||
*/
|
||||
export function decodeMulti<ContextType = undefined>(
|
||||
buffer: ArrayLike<number> | BufferSource,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): Generator<unknown, void, unknown> {
|
||||
const decoder = new Decoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxStrLength,
|
||||
options.maxBinLength,
|
||||
options.maxArrayLength,
|
||||
options.maxMapLength,
|
||||
options.maxExtLength,
|
||||
);
|
||||
return decoder.decodeMulti(buffer);
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
import { Decoder } from "./Decoder";
|
||||
import { ensureAsyncIterable } from "./utils/stream";
|
||||
import { defaultDecodeOptions } from "./decode";
|
||||
import type { ReadableStreamLike } from "./utils/stream";
|
||||
import type { DecodeOptions } from "./decode";
|
||||
import type { SplitUndefined } from "./context";
|
||||
|
||||
/**
|
||||
* @throws {@link RangeError} if the buffer is incomplete, including the case where the buffer is empty.
|
||||
* @throws {@link DecodeError} if the buffer contains invalid data.
|
||||
*/
|
||||
export async function decodeAsync<ContextType>(
|
||||
streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): Promise<unknown> {
|
||||
const stream = ensureAsyncIterable(streamLike);
|
||||
|
||||
const decoder = new Decoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxStrLength,
|
||||
options.maxBinLength,
|
||||
options.maxArrayLength,
|
||||
options.maxMapLength,
|
||||
options.maxExtLength,
|
||||
);
|
||||
return decoder.decodeAsync(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws {@link RangeError} if the buffer is incomplete, including the case where the buffer is empty.
|
||||
* @throws {@link DecodeError} if the buffer contains invalid data.
|
||||
*/
|
||||
export function decodeArrayStream<ContextType>(
|
||||
streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): AsyncGenerator<unknown, void, unknown> {
|
||||
const stream = ensureAsyncIterable(streamLike);
|
||||
|
||||
const decoder = new Decoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxStrLength,
|
||||
options.maxBinLength,
|
||||
options.maxArrayLength,
|
||||
options.maxMapLength,
|
||||
options.maxExtLength,
|
||||
);
|
||||
|
||||
return decoder.decodeArrayStream(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws {@link RangeError} if the buffer is incomplete, including the case where the buffer is empty.
|
||||
* @throws {@link DecodeError} if the buffer contains invalid data.
|
||||
*/
|
||||
export function decodeMultiStream<ContextType>(
|
||||
streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): AsyncGenerator<unknown, void, unknown> {
|
||||
const stream = ensureAsyncIterable(streamLike);
|
||||
|
||||
const decoder = new Decoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxStrLength,
|
||||
options.maxBinLength,
|
||||
options.maxArrayLength,
|
||||
options.maxMapLength,
|
||||
options.maxExtLength,
|
||||
);
|
||||
|
||||
return decoder.decodeStream(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link decodeMultiStream()} instead.
|
||||
*/
|
||||
export function decodeStream<ContextType>(
|
||||
streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
|
||||
options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
|
||||
): AsyncGenerator<unknown, void, unknown> {
|
||||
return decodeMultiStream(streamLike, options);
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
import { Encoder } from "./Encoder";
|
||||
import type { ExtensionCodecType } from "./ExtensionCodec";
|
||||
import type { ContextOf, SplitUndefined } from "./context";
|
||||
|
||||
export type EncodeOptions<ContextType = undefined> = Partial<
|
||||
Readonly<{
|
||||
extensionCodec: ExtensionCodecType<ContextType>;
|
||||
|
||||
/**
|
||||
* The maximum depth in nested objects and arrays.
|
||||
*
|
||||
* Defaults to 100.
|
||||
*/
|
||||
maxDepth: number;
|
||||
|
||||
/**
|
||||
* The initial size of the internal buffer.
|
||||
*
|
||||
* Defaults to 2048.
|
||||
*/
|
||||
initialBufferSize: number;
|
||||
|
||||
/**
|
||||
* If `true`, the keys of an object is sorted. In other words, the encoded
|
||||
* binary is canonical and thus comparable to another encoded binary.
|
||||
*
|
||||
* Defaults to `false`. If enabled, it spends more time in encoding objects.
|
||||
*/
|
||||
sortKeys: boolean;
|
||||
/**
|
||||
* If `true`, non-integer numbers are encoded in float32, not in float64 (the default).
|
||||
*
|
||||
* Only use it if precisions don't matter.
|
||||
*
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
forceFloat32: boolean;
|
||||
|
||||
/**
|
||||
* If `true`, an object property with `undefined` value are ignored.
|
||||
* e.g. `{ foo: undefined }` will be encoded as `{}`, as `JSON.stringify()` does.
|
||||
*
|
||||
* Defaults to `false`. If enabled, it spends more time in encoding objects.
|
||||
*/
|
||||
ignoreUndefined: boolean;
|
||||
|
||||
/**
|
||||
* If `true`, integer numbers are encoded as floating point numbers,
|
||||
* with the `forceFloat32` option taken into account.
|
||||
*
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
forceIntegerToFloat: boolean;
|
||||
}>
|
||||
> &
|
||||
ContextOf<ContextType>;
|
||||
|
||||
const defaultEncodeOptions: EncodeOptions = {};
|
||||
|
||||
/**
|
||||
* It encodes `value` in the MessagePack format and
|
||||
* returns a byte buffer.
|
||||
*
|
||||
* The returned buffer is a slice of a larger `ArrayBuffer`, so you have to use its `#byteOffset` and `#byteLength` in order to convert it to another typed arrays including NodeJS `Buffer`.
|
||||
*/
|
||||
export function encode<ContextType = undefined>(
|
||||
value: unknown,
|
||||
options: EncodeOptions<SplitUndefined<ContextType>> = defaultEncodeOptions as any,
|
||||
): Uint8Array {
|
||||
const encoder = new Encoder(
|
||||
options.extensionCodec,
|
||||
(options as typeof options & { context: any }).context,
|
||||
options.maxDepth,
|
||||
options.initialBufferSize,
|
||||
options.sortKeys,
|
||||
options.forceFloat32,
|
||||
options.ignoreUndefined,
|
||||
options.forceIntegerToFloat,
|
||||
);
|
||||
return encoder.encodeSharedRef(value);
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
// Main Functions:
|
||||
|
||||
import { encode } from "./encode";
|
||||
export { encode };
|
||||
import type { EncodeOptions } from "./encode";
|
||||
export type { EncodeOptions };
|
||||
|
||||
import { decode, decodeMulti } from "./decode";
|
||||
export { decode, decodeMulti };
|
||||
import type { DecodeOptions } from "./decode";
|
||||
export { DecodeOptions };
|
||||
|
||||
import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync";
|
||||
export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream };
|
||||
|
||||
import { Decoder, DataViewIndexOutOfBoundsError } from "./Decoder";
|
||||
import { DecodeError } from "./DecodeError";
|
||||
export { Decoder, DecodeError, DataViewIndexOutOfBoundsError };
|
||||
|
||||
import { Encoder } from "./Encoder";
|
||||
export { Encoder };
|
||||
|
||||
// Utilitiies for Extension Types:
|
||||
|
||||
import { ExtensionCodec } from "./ExtensionCodec";
|
||||
export { ExtensionCodec };
|
||||
import type { ExtensionCodecType, ExtensionDecoderType, ExtensionEncoderType } from "./ExtensionCodec";
|
||||
export type { ExtensionCodecType, ExtensionDecoderType, ExtensionEncoderType };
|
||||
import { ExtData } from "./ExtData";
|
||||
export { ExtData };
|
||||
|
||||
import {
|
||||
EXT_TIMESTAMP,
|
||||
encodeDateToTimeSpec,
|
||||
encodeTimeSpecToTimestamp,
|
||||
decodeTimestampToTimeSpec,
|
||||
encodeTimestampExtension,
|
||||
decodeTimestampExtension,
|
||||
} from "./timestamp";
|
||||
export {
|
||||
EXT_TIMESTAMP,
|
||||
encodeDateToTimeSpec,
|
||||
encodeTimeSpecToTimestamp,
|
||||
decodeTimestampToTimeSpec,
|
||||
encodeTimestampExtension,
|
||||
decodeTimestampExtension,
|
||||
};
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
// https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type
|
||||
import { DecodeError } from "./DecodeError";
|
||||
import { getInt64, setInt64 } from "./utils/int";
|
||||
|
||||
export const EXT_TIMESTAMP = -1;
|
||||
|
||||
export type TimeSpec = {
|
||||
sec: number;
|
||||
nsec: number;
|
||||
};
|
||||
|
||||
const TIMESTAMP32_MAX_SEC = 0x100000000 - 1; // 32-bit unsigned int
|
||||
const TIMESTAMP64_MAX_SEC = 0x400000000 - 1; // 34-bit unsigned int
|
||||
|
||||
export function encodeTimeSpecToTimestamp({ sec, nsec }: TimeSpec): Uint8Array {
|
||||
if (sec >= 0 && nsec >= 0 && sec <= TIMESTAMP64_MAX_SEC) {
|
||||
// Here sec >= 0 && nsec >= 0
|
||||
if (nsec === 0 && sec <= TIMESTAMP32_MAX_SEC) {
|
||||
// timestamp 32 = { sec32 (unsigned) }
|
||||
const rv = new Uint8Array(4);
|
||||
const view = new DataView(rv.buffer);
|
||||
view.setUint32(0, sec);
|
||||
return rv;
|
||||
} else {
|
||||
// timestamp 64 = { nsec30 (unsigned), sec34 (unsigned) }
|
||||
const secHigh = sec / 0x100000000;
|
||||
const secLow = sec & 0xffffffff;
|
||||
const rv = new Uint8Array(8);
|
||||
const view = new DataView(rv.buffer);
|
||||
// nsec30 | secHigh2
|
||||
view.setUint32(0, (nsec << 2) | (secHigh & 0x3));
|
||||
// secLow32
|
||||
view.setUint32(4, secLow);
|
||||
return rv;
|
||||
}
|
||||
} else {
|
||||
// timestamp 96 = { nsec32 (unsigned), sec64 (signed) }
|
||||
const rv = new Uint8Array(12);
|
||||
const view = new DataView(rv.buffer);
|
||||
view.setUint32(0, nsec);
|
||||
setInt64(view, 4, sec);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
export function encodeDateToTimeSpec(date: Date): TimeSpec {
|
||||
const msec = date.getTime();
|
||||
const sec = Math.floor(msec / 1e3);
|
||||
const nsec = (msec - sec * 1e3) * 1e6;
|
||||
|
||||
// Normalizes { sec, nsec } to ensure nsec is unsigned.
|
||||
const nsecInSec = Math.floor(nsec / 1e9);
|
||||
return {
|
||||
sec: sec + nsecInSec,
|
||||
nsec: nsec - nsecInSec * 1e9,
|
||||
};
|
||||
}
|
||||
|
||||
export function encodeTimestampExtension(object: unknown): Uint8Array | null {
|
||||
if (object instanceof Date) {
|
||||
const timeSpec = encodeDateToTimeSpec(object);
|
||||
return encodeTimeSpecToTimestamp(timeSpec);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeTimestampToTimeSpec(data: Uint8Array): TimeSpec {
|
||||
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
|
||||
// data may be 32, 64, or 96 bits
|
||||
switch (data.byteLength) {
|
||||
case 4: {
|
||||
// timestamp 32 = { sec32 }
|
||||
const sec = view.getUint32(0);
|
||||
const nsec = 0;
|
||||
return { sec, nsec };
|
||||
}
|
||||
case 8: {
|
||||
// timestamp 64 = { nsec30, sec34 }
|
||||
const nsec30AndSecHigh2 = view.getUint32(0);
|
||||
const secLow32 = view.getUint32(4);
|
||||
const sec = (nsec30AndSecHigh2 & 0x3) * 0x100000000 + secLow32;
|
||||
const nsec = nsec30AndSecHigh2 >>> 2;
|
||||
return { sec, nsec };
|
||||
}
|
||||
case 12: {
|
||||
// timestamp 96 = { nsec32 (unsigned), sec64 (signed) }
|
||||
|
||||
const sec = getInt64(view, 4);
|
||||
const nsec = view.getUint32(0);
|
||||
return { sec, nsec };
|
||||
}
|
||||
default:
|
||||
throw new DecodeError(`Unrecognized data size for timestamp (expected 4, 8, or 12): ${data.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeTimestampExtension(data: Uint8Array): Date {
|
||||
const timeSpec = decodeTimestampToTimeSpec(data);
|
||||
return new Date(timeSpec.sec * 1e3 + timeSpec.nsec / 1e6);
|
||||
}
|
||||
|
||||
export const timestampExtension = {
|
||||
type: EXT_TIMESTAMP,
|
||||
encode: encodeTimestampExtension,
|
||||
decode: decodeTimestampExtension,
|
||||
};
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// Integer Utility
|
||||
|
||||
export const UINT32_MAX = 0xffff_ffff;
|
||||
|
||||
// DataView extension to handle int64 / uint64,
|
||||
// where the actual range is 53-bits integer (a.k.a. safe integer)
|
||||
|
||||
export function setUint64(view: DataView, offset: number, value: number): void {
|
||||
const high = value / 0x1_0000_0000;
|
||||
const low = value; // high bits are truncated by DataView
|
||||
view.setUint32(offset, high);
|
||||
view.setUint32(offset + 4, low);
|
||||
}
|
||||
|
||||
export function setInt64(view: DataView, offset: number, value: number): void {
|
||||
const high = Math.floor(value / 0x1_0000_0000);
|
||||
const low = value; // high bits are truncated by DataView
|
||||
view.setUint32(offset, high);
|
||||
view.setUint32(offset + 4, low);
|
||||
}
|
||||
|
||||
export function getInt64(view: DataView, offset: number): number {
|
||||
const high = view.getInt32(offset);
|
||||
const low = view.getUint32(offset + 4);
|
||||
return high * 0x1_0000_0000 + low;
|
||||
}
|
||||
|
||||
export function getUint64(view: DataView, offset: number): number {
|
||||
const high = view.getUint32(offset);
|
||||
const low = view.getUint32(offset + 4);
|
||||
return high * 0x1_0000_0000 + low;
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export function prettyByte(byte: number): string {
|
||||
return `${byte < 0 ? "-" : ""}0x${Math.abs(byte).toString(16).padStart(2, "0")}`;
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// utility for whatwg streams
|
||||
|
||||
// The living standard of whatwg streams says
|
||||
// ReadableStream is also AsyncIterable, but
|
||||
// as of June 2019, no browser implements it.
|
||||
// See https://streams.spec.whatwg.org/ for details
|
||||
export type ReadableStreamLike<T> = AsyncIterable<T> | ReadableStream<T>;
|
||||
|
||||
export function isAsyncIterable<T>(object: ReadableStreamLike<T>): object is AsyncIterable<T> {
|
||||
return (object as any)[Symbol.asyncIterator] != null;
|
||||
}
|
||||
|
||||
function assertNonNull<T>(value: T | null | undefined): asserts value is T {
|
||||
if (value == null) {
|
||||
throw new Error("Assertion Failure: value must not be null nor undefined");
|
||||
}
|
||||
}
|
||||
|
||||
export async function* asyncIterableFromStream<T>(stream: ReadableStream<T>): AsyncIterable<T> {
|
||||
const reader = stream.getReader();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
assertNonNull(value);
|
||||
yield value;
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureAsyncIterable<T>(streamLike: ReadableStreamLike<T>): AsyncIterable<T> {
|
||||
if (isAsyncIterable(streamLike)) {
|
||||
return streamLike;
|
||||
} else {
|
||||
return asyncIterableFromStream(streamLike);
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
export function ensureUint8Array(buffer: ArrayLike<number> | Uint8Array | ArrayBufferView | ArrayBuffer): Uint8Array {
|
||||
if (buffer instanceof Uint8Array) {
|
||||
return buffer;
|
||||
} else if (ArrayBuffer.isView(buffer)) {
|
||||
return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
||||
} else if (buffer instanceof ArrayBuffer) {
|
||||
return new Uint8Array(buffer);
|
||||
} else {
|
||||
// ArrayLike<number>
|
||||
return Uint8Array.from(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
export function createDataView(buffer: ArrayLike<number> | ArrayBufferView | ArrayBuffer): DataView {
|
||||
if (buffer instanceof ArrayBuffer) {
|
||||
return new DataView(buffer);
|
||||
}
|
||||
|
||||
const bufferView = ensureUint8Array(buffer);
|
||||
return new DataView(bufferView.buffer, bufferView.byteOffset, bufferView.byteLength);
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
|
||||
import { UINT32_MAX } from "./int";
|
||||
|
||||
const TEXT_ENCODING_AVAILABLE =
|
||||
(typeof process === "undefined" || process?.env?.["TEXT_ENCODING"] !== "never") &&
|
||||
typeof TextEncoder !== "undefined" &&
|
||||
typeof TextDecoder !== "undefined";
|
||||
|
||||
export function utf8Count(str: string): number {
|
||||
const strLength = str.length;
|
||||
|
||||
let byteLength = 0;
|
||||
let pos = 0;
|
||||
while (pos < strLength) {
|
||||
let value = str.charCodeAt(pos++);
|
||||
|
||||
if ((value & 0xffffff80) === 0) {
|
||||
// 1-byte
|
||||
byteLength++;
|
||||
continue;
|
||||
} else if ((value & 0xfffff800) === 0) {
|
||||
// 2-bytes
|
||||
byteLength += 2;
|
||||
} else {
|
||||
// handle surrogate pair
|
||||
if (value >= 0xd800 && value <= 0xdbff) {
|
||||
// high surrogate
|
||||
if (pos < strLength) {
|
||||
const extra = str.charCodeAt(pos);
|
||||
if ((extra & 0xfc00) === 0xdc00) {
|
||||
++pos;
|
||||
value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((value & 0xffff0000) === 0) {
|
||||
// 3-byte
|
||||
byteLength += 3;
|
||||
} else {
|
||||
// 4-byte
|
||||
byteLength += 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
return byteLength;
|
||||
}
|
||||
|
||||
export function utf8EncodeJs(str: string, output: Uint8Array, outputOffset: number): void {
|
||||
const strLength = str.length;
|
||||
let offset = outputOffset;
|
||||
let pos = 0;
|
||||
while (pos < strLength) {
|
||||
let value = str.charCodeAt(pos++);
|
||||
|
||||
if ((value & 0xffffff80) === 0) {
|
||||
// 1-byte
|
||||
output[offset++] = value;
|
||||
continue;
|
||||
} else if ((value & 0xfffff800) === 0) {
|
||||
// 2-bytes
|
||||
output[offset++] = ((value >> 6) & 0x1f) | 0xc0;
|
||||
} else {
|
||||
// handle surrogate pair
|
||||
if (value >= 0xd800 && value <= 0xdbff) {
|
||||
// high surrogate
|
||||
if (pos < strLength) {
|
||||
const extra = str.charCodeAt(pos);
|
||||
if ((extra & 0xfc00) === 0xdc00) {
|
||||
++pos;
|
||||
value = ((value & 0x3ff) << 10) + (extra & 0x3ff) + 0x10000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((value & 0xffff0000) === 0) {
|
||||
// 3-byte
|
||||
output[offset++] = ((value >> 12) & 0x0f) | 0xe0;
|
||||
output[offset++] = ((value >> 6) & 0x3f) | 0x80;
|
||||
} else {
|
||||
// 4-byte
|
||||
output[offset++] = ((value >> 18) & 0x07) | 0xf0;
|
||||
output[offset++] = ((value >> 12) & 0x3f) | 0x80;
|
||||
output[offset++] = ((value >> 6) & 0x3f) | 0x80;
|
||||
}
|
||||
}
|
||||
|
||||
output[offset++] = (value & 0x3f) | 0x80;
|
||||
}
|
||||
}
|
||||
|
||||
const sharedTextEncoder = TEXT_ENCODING_AVAILABLE ? new TextEncoder() : undefined;
|
||||
export const TEXT_ENCODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE
|
||||
? UINT32_MAX
|
||||
: typeof process !== "undefined" && process?.env?.["TEXT_ENCODING"] !== "force"
|
||||
? 200
|
||||
: 0;
|
||||
|
||||
function utf8EncodeTEencode(str: string, output: Uint8Array, outputOffset: number): void {
|
||||
output.set(sharedTextEncoder!.encode(str), outputOffset);
|
||||
}
|
||||
|
||||
function utf8EncodeTEencodeInto(str: string, output: Uint8Array, outputOffset: number): void {
|
||||
sharedTextEncoder!.encodeInto(str, output.subarray(outputOffset));
|
||||
}
|
||||
|
||||
export const utf8EncodeTE = sharedTextEncoder?.encodeInto ? utf8EncodeTEencodeInto : utf8EncodeTEencode;
|
||||
|
||||
const CHUNK_SIZE = 0x1_000;
|
||||
|
||||
export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
|
||||
let offset = inputOffset;
|
||||
const end = offset + byteLength;
|
||||
|
||||
const units: Array<number> = [];
|
||||
let result = "";
|
||||
while (offset < end) {
|
||||
const byte1 = bytes[offset++]!;
|
||||
if ((byte1 & 0x80) === 0) {
|
||||
// 1 byte
|
||||
units.push(byte1);
|
||||
} else if ((byte1 & 0xe0) === 0xc0) {
|
||||
// 2 bytes
|
||||
const byte2 = bytes[offset++]! & 0x3f;
|
||||
units.push(((byte1 & 0x1f) << 6) | byte2);
|
||||
} else if ((byte1 & 0xf0) === 0xe0) {
|
||||
// 3 bytes
|
||||
const byte2 = bytes[offset++]! & 0x3f;
|
||||
const byte3 = bytes[offset++]! & 0x3f;
|
||||
units.push(((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3);
|
||||
} else if ((byte1 & 0xf8) === 0xf0) {
|
||||
// 4 bytes
|
||||
const byte2 = bytes[offset++]! & 0x3f;
|
||||
const byte3 = bytes[offset++]! & 0x3f;
|
||||
const byte4 = bytes[offset++]! & 0x3f;
|
||||
let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
|
||||
if (unit > 0xffff) {
|
||||
unit -= 0x10000;
|
||||
units.push(((unit >>> 10) & 0x3ff) | 0xd800);
|
||||
unit = 0xdc00 | (unit & 0x3ff);
|
||||
}
|
||||
units.push(unit);
|
||||
} else {
|
||||
units.push(byte1);
|
||||
}
|
||||
|
||||
if (units.length >= CHUNK_SIZE) {
|
||||
result += String.fromCharCode(...units);
|
||||
units.length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (units.length > 0) {
|
||||
result += String.fromCharCode(...units);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
const sharedTextDecoder = TEXT_ENCODING_AVAILABLE ? new TextDecoder() : null;
|
||||
export const TEXT_DECODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE
|
||||
? UINT32_MAX
|
||||
: typeof process !== "undefined" && process?.env?.["TEXT_DECODER"] !== "force"
|
||||
? 200
|
||||
: 0;
|
||||
|
||||
export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
|
||||
const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength);
|
||||
return sharedTextDecoder!.decode(stringBytes);
|
||||
}
|
||||
Reference in New Issue
Block a user