progress on migrating to heex templates and font-icons

This commit is contained in:
Adam Piontek 2022-08-13 07:32:36 -04:00
commit 3eff955672
21793 changed files with 2161968 additions and 16895 deletions

View file

@ -0,0 +1,22 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class ArraySerializer {
serialize(array, { write }) {
write(array.length);
for (const item of array) write(item);
}
deserialize({ read }) {
const length = read();
const array = [];
for (let i = 0; i < length; i++) {
array.push(read());
}
return array;
}
}
module.exports = ArraySerializer;

View file

@ -0,0 +1,893 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const memoize = require("../util/memoize");
const SerializerMiddleware = require("./SerializerMiddleware");
/** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
/** @typedef {import("./types").PrimitiveSerializableType} PrimitiveSerializableType */
/* eslint-disable no-loop-func */
/*
Format:
File -> Section*
Section -> NullsSection |
BooleansSection |
F64NumbersSection |
I32NumbersSection |
I8NumbersSection |
ShortStringSection |
StringSection |
BufferSection |
NopSection
NullsSection ->
NullHeaderByte | Null2HeaderByte | Null3HeaderByte |
Nulls8HeaderByte 0xnn (n:count - 4) |
Nulls32HeaderByte n:ui32 (n:count - 260) |
BooleansSection -> TrueHeaderByte | FalseHeaderByte | BooleansSectionHeaderByte BooleansCountAndBitsByte
F64NumbersSection -> F64NumbersSectionHeaderByte f64*
I32NumbersSection -> I32NumbersSectionHeaderByte i32*
I8NumbersSection -> I8NumbersSectionHeaderByte i8*
ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
StringSection -> StringSectionHeaderByte i32:length utf8-byte*
BufferSection -> BufferSectionHeaderByte i32:length byte*
NopSection --> NopSectionHeaderByte
ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)
F64NumbersSectionHeaderByte -> 0b001n_nnnn (n:count - 1)
I32NumbersSectionHeaderByte -> 0b010n_nnnn (n:count - 1)
I8NumbersSectionHeaderByte -> 0b011n_nnnn (n:count - 1)
NullsSectionHeaderByte -> 0b0001_nnnn (n:count - 1)
BooleansCountAndBitsByte ->
0b0000_1xxx (count = 3) |
0b0001_xxxx (count = 4) |
0b001x_xxxx (count = 5) |
0b01xx_xxxx (count = 6) |
0b1nnn_nnnn (n:count - 7, 7 <= count <= 133)
0xff n:ui32 (n:count, 134 <= count < 2^32)
StringSectionHeaderByte -> 0b0000_1110
BufferSectionHeaderByte -> 0b0000_1111
NopSectionHeaderByte -> 0b0000_1011
FalseHeaderByte -> 0b0000_1100
TrueHeaderByte -> 0b0000_1101
RawNumber -> n (n <= 10)
*/
const LAZY_HEADER = 0x0b;
const TRUE_HEADER = 0x0c;
const FALSE_HEADER = 0x0d;
const BOOLEANS_HEADER = 0x0e;
const NULL_HEADER = 0x10;
const NULL2_HEADER = 0x11;
const NULL3_HEADER = 0x12;
const NULLS8_HEADER = 0x13;
const NULLS32_HEADER = 0x14;
const NULL_AND_I8_HEADER = 0x15;
const NULL_AND_I32_HEADER = 0x16;
const NULL_AND_TRUE_HEADER = 0x17;
const NULL_AND_FALSE_HEADER = 0x18;
const STRING_HEADER = 0x1e;
const BUFFER_HEADER = 0x1f;
const I8_HEADER = 0x60;
const I32_HEADER = 0x40;
const F64_HEADER = 0x20;
const SHORT_STRING_HEADER = 0x80;
/** Uplift high-order bits */
const NUMBERS_HEADER_MASK = 0xe0;
const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111
const HEADER_SIZE = 1;
const I8_SIZE = 1;
const I32_SIZE = 4;
const F64_SIZE = 8;
const MEASURE_START_OPERATION = Symbol("MEASURE_START_OPERATION");
const MEASURE_END_OPERATION = Symbol("MEASURE_END_OPERATION");
const identifyNumber = n => {
if (n === (n | 0)) {
if (n <= 127 && n >= -128) return 0;
if (n <= 2147483647 && n >= -2147483648) return 1;
}
return 2;
};
/**
* @typedef {PrimitiveSerializableType[]} DeserializedType
* @typedef {BufferSerializableType[]} SerializedType
* @extends {SerializerMiddleware<DeserializedType, SerializedType>}
*/
class BinaryMiddleware extends SerializerMiddleware {
static optimizeSerializedData(data) {
const result = [];
const temp = [];
const flush = () => {
if (temp.length > 0) {
if (temp.length === 1) {
result.push(temp[0]);
} else {
result.push(Buffer.concat(temp));
}
temp.length = 0;
}
};
for (const item of data) {
if (Buffer.isBuffer(item)) {
temp.push(item);
} else {
flush();
result.push(item);
}
}
flush();
return result;
}
/**
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType|Promise<SerializedType>} serialized data
*/
serialize(data, context) {
return this._serialize(data, context);
}
/**
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType} serialized data
*/
_serialize(data, context) {
/** @type {Buffer} */
let currentBuffer = null;
/** @type {Buffer} */
let leftOverBuffer = null;
let currentPosition = 0;
/** @type {BufferSerializableType[]} */
const buffers = [];
let buffersTotalLength = 0;
const allocate = (bytesNeeded, exact = false) => {
if (currentBuffer !== null) {
if (currentBuffer.length - currentPosition >= bytesNeeded) return;
flush();
}
if (leftOverBuffer && leftOverBuffer.length >= bytesNeeded) {
currentBuffer = leftOverBuffer;
leftOverBuffer = null;
} else {
currentBuffer = Buffer.allocUnsafe(
exact ? bytesNeeded : Math.max(bytesNeeded, buffersTotalLength, 1024)
);
}
};
const flush = () => {
if (currentBuffer !== null) {
buffers.push(currentBuffer.slice(0, currentPosition));
if (
!leftOverBuffer ||
leftOverBuffer.length < currentBuffer.length - currentPosition
)
leftOverBuffer = currentBuffer.slice(currentPosition);
currentBuffer = null;
buffersTotalLength += currentPosition;
currentPosition = 0;
}
};
const writeU8 = byte => {
currentBuffer.writeUInt8(byte, currentPosition++);
};
const writeU32 = ui32 => {
currentBuffer.writeUInt32LE(ui32, currentPosition);
currentPosition += 4;
};
const measureStack = [];
const measureStart = () => {
measureStack.push(buffers.length, currentPosition);
};
const measureEnd = () => {
const oldPos = measureStack.pop();
const buffersIndex = measureStack.pop();
let size = currentPosition - oldPos;
for (let i = buffersIndex; i < buffers.length; i++) {
size += buffers[i].length;
}
return size;
};
const serializeData = data => {
for (let i = 0; i < data.length; i++) {
const thing = data[i];
switch (typeof thing) {
case "function": {
if (!SerializerMiddleware.isLazy(thing))
throw new Error("Unexpected function " + thing);
/** @type {SerializedType | (() => SerializedType)} */
let serializedData = SerializerMiddleware.getLazySerializedValue(
thing
);
if (serializedData === undefined) {
if (SerializerMiddleware.isLazy(thing, this)) {
const data = this._serialize(thing(), context);
SerializerMiddleware.setLazySerializedValue(thing, data);
serializedData = data;
} else {
serializedData = SerializerMiddleware.serializeLazy(
thing,
data => this._serialize(data, context)
);
}
}
if (typeof serializedData === "function") {
flush();
buffers.push(serializedData);
} else {
const lengths = [];
for (const item of serializedData) {
let last;
if (typeof item === "function") {
lengths.push(0);
} else if (item.length === 0) {
// ignore
} else if (
lengths.length > 0 &&
(last = lengths[lengths.length - 1]) !== 0
) {
const remaining = 0xffffffff - last;
if (remaining >= item.length) {
lengths[lengths.length - 1] += item.length;
} else {
lengths.push(item.length - remaining);
lengths[lengths.length - 2] = 0xffffffff;
}
} else {
lengths.push(item.length);
}
}
allocate(5 + lengths.length * 4);
writeU8(LAZY_HEADER);
writeU32(lengths.length);
for (const l of lengths) {
writeU32(l);
}
for (const item of serializedData) {
flush();
buffers.push(item);
}
}
break;
}
case "string": {
const len = Buffer.byteLength(thing);
if (len >= 128 || len !== thing.length) {
allocate(len + HEADER_SIZE + I32_SIZE);
writeU8(STRING_HEADER);
writeU32(len);
currentBuffer.write(thing, currentPosition);
} else {
allocate(len + HEADER_SIZE);
writeU8(SHORT_STRING_HEADER | len);
currentBuffer.write(thing, currentPosition, "latin1");
}
currentPosition += len;
break;
}
case "number": {
const type = identifyNumber(thing);
if (type === 0 && thing >= 0 && thing <= 10) {
// shortcut for very small numbers
allocate(I8_SIZE);
writeU8(thing);
break;
}
/**
* amount of numbers to write
* @type {number}
*/
let n = 1;
for (; n < 32 && i + n < data.length; n++) {
const item = data[i + n];
if (typeof item !== "number") break;
if (identifyNumber(item) !== type) break;
}
switch (type) {
case 0:
allocate(HEADER_SIZE + I8_SIZE * n);
writeU8(I8_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt8(
/** @type {number} */ (data[i]),
currentPosition
);
currentPosition += I8_SIZE;
n--;
i++;
}
break;
case 1:
allocate(HEADER_SIZE + I32_SIZE * n);
writeU8(I32_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeInt32LE(
/** @type {number} */ (data[i]),
currentPosition
);
currentPosition += I32_SIZE;
n--;
i++;
}
break;
case 2:
allocate(HEADER_SIZE + F64_SIZE * n);
writeU8(F64_HEADER | (n - 1));
while (n > 0) {
currentBuffer.writeDoubleLE(
/** @type {number} */ (data[i]),
currentPosition
);
currentPosition += F64_SIZE;
n--;
i++;
}
break;
}
i--;
break;
}
case "boolean": {
let lastByte = thing === true ? 1 : 0;
const bytes = [];
let count = 1;
let n;
for (n = 1; n < 0xffffffff && i + n < data.length; n++) {
const item = data[i + n];
if (typeof item !== "boolean") break;
const pos = count & 0x7;
if (pos === 0) {
bytes.push(lastByte);
lastByte = item === true ? 1 : 0;
} else if (item === true) {
lastByte |= 1 << pos;
}
count++;
}
i += count - 1;
if (count === 1) {
allocate(HEADER_SIZE);
writeU8(lastByte === 1 ? TRUE_HEADER : FALSE_HEADER);
} else if (count === 2) {
allocate(HEADER_SIZE * 2);
writeU8(lastByte & 1 ? TRUE_HEADER : FALSE_HEADER);
writeU8(lastByte & 2 ? TRUE_HEADER : FALSE_HEADER);
} else if (count <= 6) {
allocate(HEADER_SIZE + I8_SIZE);
writeU8(BOOLEANS_HEADER);
writeU8((1 << count) | lastByte);
} else if (count <= 133) {
allocate(
HEADER_SIZE + I8_SIZE + I8_SIZE * bytes.length + I8_SIZE
);
writeU8(BOOLEANS_HEADER);
writeU8(0x80 | (count - 7));
for (const byte of bytes) writeU8(byte);
writeU8(lastByte);
} else {
allocate(
HEADER_SIZE +
I8_SIZE +
I32_SIZE +
I8_SIZE * bytes.length +
I8_SIZE
);
writeU8(BOOLEANS_HEADER);
writeU8(0xff);
writeU32(count);
for (const byte of bytes) writeU8(byte);
writeU8(lastByte);
}
break;
}
case "object": {
if (thing === null) {
let n;
for (n = 1; n < 0x100000104 && i + n < data.length; n++) {
const item = data[i + n];
if (item !== null) break;
}
i += n - 1;
if (n === 1) {
if (i + 1 < data.length) {
const next = data[i + 1];
if (next === true) {
allocate(HEADER_SIZE);
writeU8(NULL_AND_TRUE_HEADER);
i++;
} else if (next === false) {
allocate(HEADER_SIZE);
writeU8(NULL_AND_FALSE_HEADER);
i++;
} else if (typeof next === "number") {
const type = identifyNumber(next);
if (type === 0) {
allocate(HEADER_SIZE + I8_SIZE);
writeU8(NULL_AND_I8_HEADER);
currentBuffer.writeInt8(next, currentPosition);
currentPosition += I8_SIZE;
i++;
} else if (type === 1) {
allocate(HEADER_SIZE + I32_SIZE);
writeU8(NULL_AND_I32_HEADER);
currentBuffer.writeInt32LE(next, currentPosition);
currentPosition += I32_SIZE;
i++;
} else {
allocate(HEADER_SIZE);
writeU8(NULL_HEADER);
}
} else {
allocate(HEADER_SIZE);
writeU8(NULL_HEADER);
}
} else {
allocate(HEADER_SIZE);
writeU8(NULL_HEADER);
}
} else if (n === 2) {
allocate(HEADER_SIZE);
writeU8(NULL2_HEADER);
} else if (n === 3) {
allocate(HEADER_SIZE);
writeU8(NULL3_HEADER);
} else if (n < 260) {
allocate(HEADER_SIZE + I8_SIZE);
writeU8(NULLS8_HEADER);
writeU8(n - 4);
} else {
allocate(HEADER_SIZE + I32_SIZE);
writeU8(NULLS32_HEADER);
writeU32(n - 260);
}
} else if (Buffer.isBuffer(thing)) {
allocate(HEADER_SIZE + I32_SIZE, true);
writeU8(BUFFER_HEADER);
writeU32(thing.length);
flush();
buffers.push(thing);
}
break;
}
case "symbol": {
if (thing === MEASURE_START_OPERATION) {
measureStart();
} else if (thing === MEASURE_END_OPERATION) {
const size = measureEnd();
allocate(HEADER_SIZE + I32_SIZE);
writeU8(I32_HEADER);
currentBuffer.writeInt32LE(size, currentPosition);
currentPosition += I32_SIZE;
}
break;
}
}
}
};
serializeData(data);
flush();
return buffers;
}
/**
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType|Promise<DeserializedType>} deserialized data
*/
deserialize(data, context) {
return this._deserialize(data, context);
}
/**
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType} deserialized data
*/
_deserialize(data, context) {
let currentDataItem = 0;
let currentBuffer = data[0];
let currentIsBuffer = Buffer.isBuffer(currentBuffer);
let currentPosition = 0;
const checkOverflow = () => {
if (currentPosition >= currentBuffer.length) {
currentPosition = 0;
currentDataItem++;
currentBuffer =
currentDataItem < data.length ? data[currentDataItem] : null;
currentIsBuffer = Buffer.isBuffer(currentBuffer);
}
};
const isInCurrentBuffer = n => {
return currentIsBuffer && n + currentPosition <= currentBuffer.length;
};
/**
* Reads n bytes
* @param {number} n amount of bytes to read
* @returns {Buffer} buffer with bytes
*/
const read = n => {
if (!currentIsBuffer) {
throw new Error(
currentBuffer === null
? "Unexpected end of stream"
: "Unexpected lazy element in stream"
);
}
const rem = currentBuffer.length - currentPosition;
if (rem < n) {
return Buffer.concat([read(rem), read(n - rem)]);
}
const res = /** @type {Buffer} */ (currentBuffer).slice(
currentPosition,
currentPosition + n
);
currentPosition += n;
checkOverflow();
return res;
};
/**
* Reads up to n bytes
* @param {number} n amount of bytes to read
* @returns {Buffer} buffer with bytes
*/
const readUpTo = n => {
if (!currentIsBuffer) {
throw new Error(
currentBuffer === null
? "Unexpected end of stream"
: "Unexpected lazy element in stream"
);
}
const rem = currentBuffer.length - currentPosition;
if (rem < n) {
n = rem;
}
const res = /** @type {Buffer} */ (currentBuffer).slice(
currentPosition,
currentPosition + n
);
currentPosition += n;
checkOverflow();
return res;
};
const readU8 = () => {
if (!currentIsBuffer) {
throw new Error(
currentBuffer === null
? "Unexpected end of stream"
: "Unexpected lazy element in stream"
);
}
/**
* There is no need to check remaining buffer size here
* since {@link checkOverflow} guarantees at least one byte remaining
*/
const byte = /** @type {Buffer} */ (currentBuffer).readUInt8(
currentPosition
);
currentPosition += I8_SIZE;
checkOverflow();
return byte;
};
const readU32 = () => {
return read(I32_SIZE).readUInt32LE(0);
};
const readBits = (data, n) => {
let mask = 1;
while (n !== 0) {
result.push((data & mask) !== 0);
mask = mask << 1;
n--;
}
};
const dispatchTable = Array.from({ length: 256 }).map((_, header) => {
switch (header) {
case LAZY_HEADER:
return () => {
const count = readU32();
const lengths = Array.from({ length: count }).map(() => readU32());
const content = [];
for (let l of lengths) {
if (l === 0) {
if (typeof currentBuffer !== "function") {
throw new Error("Unexpected non-lazy element in stream");
}
content.push(currentBuffer);
currentDataItem++;
currentBuffer =
currentDataItem < data.length ? data[currentDataItem] : null;
currentIsBuffer = Buffer.isBuffer(currentBuffer);
} else {
do {
const buf = readUpTo(l);
l -= buf.length;
content.push(buf);
} while (l > 0);
}
}
result.push(
SerializerMiddleware.createLazy(
memoize(() => this._deserialize(content, context)),
this,
undefined,
content
)
);
};
case BUFFER_HEADER:
return () => {
const len = readU32();
result.push(read(len));
};
case TRUE_HEADER:
return () => result.push(true);
case FALSE_HEADER:
return () => result.push(false);
case NULL3_HEADER:
return () => result.push(null, null, null);
case NULL2_HEADER:
return () => result.push(null, null);
case NULL_HEADER:
return () => result.push(null);
case NULL_AND_TRUE_HEADER:
return () => result.push(null, true);
case NULL_AND_FALSE_HEADER:
return () => result.push(null, false);
case NULL_AND_I8_HEADER:
return () => {
if (currentIsBuffer) {
result.push(
null,
/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
);
currentPosition += I8_SIZE;
checkOverflow();
} else {
result.push(null, read(I8_SIZE).readInt8(0));
}
};
case NULL_AND_I32_HEADER:
return () => {
result.push(null);
if (isInCurrentBuffer(I32_SIZE)) {
result.push(
/** @type {Buffer} */ (currentBuffer).readInt32LE(
currentPosition
)
);
currentPosition += I32_SIZE;
checkOverflow();
} else {
result.push(read(I32_SIZE).readInt32LE(0));
}
};
case NULLS8_HEADER:
return () => {
const len = readU8() + 4;
for (let i = 0; i < len; i++) {
result.push(null);
}
};
case NULLS32_HEADER:
return () => {
const len = readU32() + 260;
for (let i = 0; i < len; i++) {
result.push(null);
}
};
case BOOLEANS_HEADER:
return () => {
const innerHeader = readU8();
if ((innerHeader & 0xf0) === 0) {
readBits(innerHeader, 3);
} else if ((innerHeader & 0xe0) === 0) {
readBits(innerHeader, 4);
} else if ((innerHeader & 0xc0) === 0) {
readBits(innerHeader, 5);
} else if ((innerHeader & 0x80) === 0) {
readBits(innerHeader, 6);
} else if (innerHeader !== 0xff) {
let count = (innerHeader & 0x7f) + 7;
while (count > 8) {
readBits(readU8(), 8);
count -= 8;
}
readBits(readU8(), count);
} else {
let count = readU32();
while (count > 8) {
readBits(readU8(), 8);
count -= 8;
}
readBits(readU8(), count);
}
};
case STRING_HEADER:
return () => {
const len = readU32();
if (isInCurrentBuffer(len)) {
result.push(
currentBuffer.toString(
undefined,
currentPosition,
currentPosition + len
)
);
currentPosition += len;
checkOverflow();
} else {
result.push(read(len).toString());
}
};
case SHORT_STRING_HEADER:
return () => result.push("");
case SHORT_STRING_HEADER | 1:
return () => {
if (currentIsBuffer) {
result.push(
currentBuffer.toString(
"latin1",
currentPosition,
currentPosition + 1
)
);
currentPosition++;
checkOverflow();
} else {
result.push(read(1).toString("latin1"));
}
};
case I8_HEADER:
return () => {
if (currentIsBuffer) {
result.push(
/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
);
currentPosition++;
checkOverflow();
} else {
result.push(read(1).readInt8(0));
}
};
default:
if (header <= 10) {
return () => result.push(header);
} else if ((header & SHORT_STRING_HEADER) === SHORT_STRING_HEADER) {
const len = header & SHORT_STRING_LENGTH_MASK;
return () => {
if (isInCurrentBuffer(len)) {
result.push(
currentBuffer.toString(
"latin1",
currentPosition,
currentPosition + len
)
);
currentPosition += len;
checkOverflow();
} else {
result.push(read(len).toString("latin1"));
}
};
} else if ((header & NUMBERS_HEADER_MASK) === F64_HEADER) {
const len = (header & NUMBERS_COUNT_MASK) + 1;
return () => {
const need = F64_SIZE * len;
if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
result.push(
/** @type {Buffer} */ (currentBuffer).readDoubleLE(
currentPosition
)
);
currentPosition += F64_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
result.push(buf.readDoubleLE(i * F64_SIZE));
}
}
};
} else if ((header & NUMBERS_HEADER_MASK) === I32_HEADER) {
const len = (header & NUMBERS_COUNT_MASK) + 1;
return () => {
const need = I32_SIZE * len;
if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
result.push(
/** @type {Buffer} */ (currentBuffer).readInt32LE(
currentPosition
)
);
currentPosition += I32_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
result.push(buf.readInt32LE(i * I32_SIZE));
}
}
};
} else if ((header & NUMBERS_HEADER_MASK) === I8_HEADER) {
const len = (header & NUMBERS_COUNT_MASK) + 1;
return () => {
const need = I8_SIZE * len;
if (isInCurrentBuffer(need)) {
for (let i = 0; i < len; i++) {
result.push(
/** @type {Buffer} */ (currentBuffer).readInt8(
currentPosition
)
);
currentPosition += I8_SIZE;
}
checkOverflow();
} else {
const buf = read(need);
for (let i = 0; i < len; i++) {
result.push(buf.readInt8(i * I8_SIZE));
}
}
};
} else {
return () => {
throw new Error(
`Unexpected header byte 0x${header.toString(16)}`
);
};
}
}
});
/** @type {DeserializedType} */
const result = [];
while (currentBuffer !== null) {
if (typeof currentBuffer === "function") {
result.push(
SerializerMiddleware.deserializeLazy(currentBuffer, data =>
this._deserialize(data, context)
)
);
currentDataItem++;
currentBuffer =
currentDataItem < data.length ? data[currentDataItem] : null;
currentIsBuffer = Buffer.isBuffer(currentBuffer);
} else {
const header = readU8();
dispatchTable[header]();
}
}
return result;
}
}
module.exports = BinaryMiddleware;
module.exports.MEASURE_START_OPERATION = MEASURE_START_OPERATION;
module.exports.MEASURE_END_OPERATION = MEASURE_END_OPERATION;

View file

@ -0,0 +1,16 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class DateObjectSerializer {
serialize(obj, { write }) {
write(obj.getTime());
}
deserialize({ read }) {
return new Date(read());
}
}
module.exports = DateObjectSerializer;

View file

@ -0,0 +1,27 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class ErrorObjectSerializer {
constructor(Type) {
this.Type = Type;
}
serialize(obj, { write }) {
write(obj.message);
write(obj.stack);
}
deserialize({ read }) {
const err = new this.Type();
err.message = read();
err.stack = read();
return err;
}
}
module.exports = ErrorObjectSerializer;

View file

@ -0,0 +1,511 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const { constants } = require("buffer");
const createHash = require("../util/createHash");
const { dirname, join, mkdirp } = require("../util/fs");
const memoize = require("../util/memoize");
const SerializerMiddleware = require("./SerializerMiddleware");
/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
/** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
/*
Format:
File -> Header Section*
Version -> u32
AmountOfSections -> u32
SectionSize -> i32 (if less than zero represents lazy value)
Header -> Version AmountOfSections SectionSize*
Buffer -> n bytes
Section -> Buffer
*/
// "wpc" + 1 in little-endian
const VERSION = 0x01637077;
const hashForName = buffers => {
const hash = createHash("md4");
for (const buf of buffers) hash.update(buf);
return /** @type {string} */ (hash.digest("hex"));
};
const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
? (buf, value, offset) => {
buf.writeBigUInt64LE(BigInt(value), offset);
}
: (buf, value, offset) => {
const low = value % 0x100000000;
const high = (value - low) / 0x100000000;
buf.writeUInt32LE(low, offset);
buf.writeUInt32LE(high, offset + 4);
};
const readUInt64LE = Buffer.prototype.readBigUInt64LE
? (buf, offset) => {
return Number(buf.readBigUInt64LE(offset));
}
: (buf, offset) => {
const low = buf.readUInt32LE(offset);
const high = buf.readUInt32LE(offset + 4);
return high * 0x100000000 + low;
};
/**
* @typedef {Object} SerializeResult
* @property {string | false} name
* @property {number} size
* @property {Promise=} backgroundJob
*/
/**
* @param {FileMiddleware} middleware this
* @param {BufferSerializableType[] | Promise<BufferSerializableType[]>} data data to be serialized
* @param {string | boolean} name file base name
* @param {function(string | false, Buffer[]): Promise} writeFile writes a file
* @returns {Promise<SerializeResult>} resulting file pointer and promise
*/
const serialize = async (middleware, data, name, writeFile) => {
/** @type {(Buffer[] | Buffer | SerializeResult | Promise<SerializeResult>)[]} */
const processedData = [];
/** @type {WeakMap<SerializeResult, function(): any | Promise<any>>} */
const resultToLazy = new WeakMap();
/** @type {Buffer[]} */
let lastBuffers = undefined;
for (const item of await data) {
if (typeof item === "function") {
if (!SerializerMiddleware.isLazy(item))
throw new Error("Unexpected function");
if (!SerializerMiddleware.isLazy(item, middleware)) {
throw new Error(
"Unexpected lazy value with non-this target (can't pass through lazy values)"
);
}
lastBuffers = undefined;
const serializedInfo = SerializerMiddleware.getLazySerializedValue(item);
if (serializedInfo) {
if (typeof serializedInfo === "function") {
throw new Error(
"Unexpected lazy value with non-this target (can't pass through lazy values)"
);
} else {
processedData.push(serializedInfo);
}
} else {
const content = item();
if (content) {
const options = SerializerMiddleware.getLazyOptions(item);
processedData.push(
serialize(
middleware,
content,
(options && options.name) || true,
writeFile
).then(result => {
/** @type {any} */ (item).options.size = result.size;
resultToLazy.set(result, item);
return result;
})
);
} else {
throw new Error(
"Unexpected falsy value returned by lazy value function"
);
}
}
} else if (item) {
if (lastBuffers) {
lastBuffers.push(item);
} else {
lastBuffers = [item];
processedData.push(lastBuffers);
}
} else {
throw new Error("Unexpected falsy value in items array");
}
}
/** @type {Promise<any>[]} */
const backgroundJobs = [];
const resolvedData = (
await Promise.all(
/** @type {Promise<Buffer[] | Buffer | SerializeResult>[]} */ (processedData)
)
).map(item => {
if (Array.isArray(item) || Buffer.isBuffer(item)) return item;
backgroundJobs.push(item.backgroundJob);
// create pointer buffer from size and name
const name = /** @type {string} */ (item.name);
const nameBuffer = Buffer.from(name);
const buf = Buffer.allocUnsafe(8 + nameBuffer.length);
writeUInt64LE(buf, item.size, 0);
nameBuffer.copy(buf, 8, 0);
const lazy = resultToLazy.get(item);
SerializerMiddleware.setLazySerializedValue(lazy, buf);
return buf;
});
const lengths = [];
for (const item of resolvedData) {
if (Array.isArray(item)) {
let l = 0;
for (const b of item) l += b.length;
while (l > 0x7fffffff) {
lengths.push(0x7fffffff);
l -= 0x7fffffff;
}
lengths.push(l);
} else if (item) {
lengths.push(-item.length);
} else {
throw new Error("Unexpected falsy value in resolved data " + item);
}
}
const header = Buffer.allocUnsafe(8 + lengths.length * 4);
header.writeUInt32LE(VERSION, 0);
header.writeUInt32LE(lengths.length, 4);
for (let i = 0; i < lengths.length; i++) {
header.writeInt32LE(lengths[i], 8 + i * 4);
}
const buf = [header];
for (const item of resolvedData) {
if (Array.isArray(item)) {
for (const b of item) buf.push(b);
} else if (item) {
buf.push(item);
}
}
if (name === true) {
name = hashForName(buf);
}
backgroundJobs.push(writeFile(name, buf));
let size = 0;
for (const b of buf) size += b.length;
return {
size,
name,
backgroundJob:
backgroundJobs.length === 1
? backgroundJobs[0]
: Promise.all(backgroundJobs)
};
};
/**
* @param {FileMiddleware} middleware this
* @param {string | false} name filename
* @param {function(string | false): Promise<Buffer[]>} readFile read content of a file
* @returns {Promise<BufferSerializableType[]>} deserialized data
*/
const deserialize = async (middleware, name, readFile) => {
const contents = await readFile(name);
if (contents.length === 0) throw new Error("Empty file " + name);
let contentsIndex = 0;
let contentItem = contents[0];
let contentItemLength = contentItem.length;
let contentPosition = 0;
if (contentItemLength === 0) throw new Error("Empty file " + name);
const nextContent = () => {
contentsIndex++;
contentItem = contents[contentsIndex];
contentItemLength = contentItem.length;
contentPosition = 0;
};
const ensureData = n => {
if (contentPosition === contentItemLength) {
nextContent();
}
while (contentItemLength - contentPosition < n) {
const remaining = contentItem.slice(contentPosition);
let lengthFromNext = n - remaining.length;
const buffers = [remaining];
for (let i = contentsIndex + 1; i < contents.length; i++) {
const l = contents[i].length;
if (l > lengthFromNext) {
buffers.push(contents[i].slice(0, lengthFromNext));
contents[i] = contents[i].slice(lengthFromNext);
lengthFromNext = 0;
break;
} else {
buffers.push(contents[i]);
contentsIndex = i;
lengthFromNext -= l;
}
}
if (lengthFromNext > 0) throw new Error("Unexpected end of data");
contentItem = Buffer.concat(buffers, n);
contentItemLength = n;
contentPosition = 0;
}
};
const readUInt32LE = () => {
ensureData(4);
const value = contentItem.readUInt32LE(contentPosition);
contentPosition += 4;
return value;
};
const readInt32LE = () => {
ensureData(4);
const value = contentItem.readInt32LE(contentPosition);
contentPosition += 4;
return value;
};
const readSlice = l => {
ensureData(l);
if (contentPosition === 0 && contentItemLength === l) {
const result = contentItem;
if (contentsIndex + 1 < contents.length) {
nextContent();
} else {
contentPosition = l;
}
return result;
}
const result = contentItem.slice(contentPosition, contentPosition + l);
contentPosition += l;
// we clone the buffer here to allow the original content to be garbage collected
return l * 2 < contentItem.buffer.byteLength ? Buffer.from(result) : result;
};
const version = readUInt32LE();
if (version !== VERSION) {
throw new Error("Invalid file version");
}
const sectionCount = readUInt32LE();
const lengths = [];
for (let i = 0; i < sectionCount; i++) {
lengths.push(readInt32LE());
}
const result = [];
for (let length of lengths) {
if (length < 0) {
const slice = readSlice(-length);
const size = Number(readUInt64LE(slice, 0));
const nameBuffer = slice.slice(8);
const name = nameBuffer.toString();
result.push(
SerializerMiddleware.createLazy(
memoize(() => deserialize(middleware, name, readFile)),
middleware,
{
name,
size
},
slice
)
);
} else {
if (contentPosition === contentItemLength) {
nextContent();
} else if (contentPosition !== 0) {
if (length <= contentItemLength - contentPosition) {
result.push(
contentItem.slice(contentPosition, contentPosition + length)
);
contentPosition += length;
length = 0;
} else {
result.push(contentItem.slice(contentPosition));
length -= contentItemLength - contentPosition;
contentPosition = contentItemLength;
}
} else {
if (length >= contentItemLength) {
result.push(contentItem);
length -= contentItemLength;
contentPosition = contentItemLength;
} else {
result.push(contentItem.slice(0, length));
contentPosition += length;
length = 0;
}
}
while (length > 0) {
nextContent();
if (length >= contentItemLength) {
result.push(contentItem);
length -= contentItemLength;
contentPosition = contentItemLength;
} else {
result.push(contentItem.slice(0, length));
contentPosition += length;
length = 0;
}
}
}
}
return result;
};
/**
* @typedef {BufferSerializableType[]} DeserializedType
* @typedef {true} SerializedType
* @extends {SerializerMiddleware<DeserializedType, SerializedType>}
*/
class FileMiddleware extends SerializerMiddleware {
/**
* @param {IntermediateFileSystem} fs filesystem
*/
constructor(fs) {
super();
this.fs = fs;
}
/**
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType|Promise<SerializedType>} serialized data
*/
serialize(data, context) {
const { filename, extension = "" } = context;
return new Promise((resolve, reject) => {
mkdirp(this.fs, dirname(this.fs, filename), err => {
if (err) return reject(err);
// It's important that we don't touch existing files during serialization
// because serialize may read existing files (when deserializing)
const allWrittenFiles = new Set();
const writeFile = async (name, content) => {
const file = name
? join(this.fs, filename, `../${name}${extension}`)
: filename;
await new Promise((resolve, reject) => {
const stream = this.fs.createWriteStream(file + "_");
for (const b of content) stream.write(b);
stream.end();
stream.on("error", err => reject(err));
stream.on("finish", () => resolve());
});
if (name) allWrittenFiles.add(file);
};
resolve(
serialize(this, data, false, writeFile).then(
async ({ backgroundJob }) => {
await backgroundJob;
// Rename the index file to disallow access during inconsistent file state
await new Promise(resolve =>
this.fs.rename(filename, filename + ".old", err => {
resolve();
})
);
// update all written files
await Promise.all(
Array.from(
allWrittenFiles,
file =>
new Promise((resolve, reject) => {
this.fs.rename(file + "_", file, err => {
if (err) return reject(err);
resolve();
});
})
)
);
// As final step automatically update the index file to have a consistent pack again
await new Promise(resolve => {
this.fs.rename(filename + "_", filename, err => {
if (err) return reject(err);
resolve();
});
});
return /** @type {true} */ (true);
}
)
);
});
});
}
/**
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType|Promise<DeserializedType>} deserialized data
*/
deserialize(data, context) {
const { filename, extension = "" } = context;
const readFile = name =>
new Promise((resolve, reject) => {
const file = name
? join(this.fs, filename, `../${name}${extension}`)
: filename;
this.fs.stat(file, (err, stats) => {
if (err) {
reject(err);
return;
}
let remaining = /** @type {number} */ (stats.size);
let currentBuffer;
let currentBufferUsed;
const buf = [];
this.fs.open(file, "r", (err, fd) => {
if (err) {
reject(err);
return;
}
const read = () => {
if (currentBuffer === undefined) {
currentBuffer = Buffer.allocUnsafeSlow(
Math.min(constants.MAX_LENGTH, remaining)
);
currentBufferUsed = 0;
}
let readBuffer = currentBuffer;
let readOffset = currentBufferUsed;
let readLength = currentBuffer.length - currentBufferUsed;
if (readOffset > 0x7fffffff) {
readBuffer = currentBuffer.slice(readOffset);
readOffset = 0;
}
if (readLength > 0x7fffffff) {
readLength = 0x7fffffff;
}
this.fs.read(
fd,
readBuffer,
readOffset,
readLength,
null,
(err, bytesRead) => {
if (err) {
this.fs.close(fd, () => {
reject(err);
});
return;
}
currentBufferUsed += bytesRead;
remaining -= bytesRead;
if (currentBufferUsed === currentBuffer.length) {
buf.push(currentBuffer);
currentBuffer = undefined;
if (remaining === 0) {
this.fs.close(fd, err => {
if (err) {
reject(err);
return;
}
resolve(buf);
});
return;
}
}
read();
}
);
};
read();
});
});
});
return deserialize(this, false, readFile);
}
}
module.exports = FileMiddleware;

View file

@ -0,0 +1,31 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class MapObjectSerializer {
serialize(obj, { write }) {
write(obj.size);
for (const key of obj.keys()) {
write(key);
}
for (const value of obj.values()) {
write(value);
}
}
deserialize({ read }) {
let size = read();
const map = new Map();
const keys = [];
for (let i = 0; i < size; i++) {
keys.push(read());
}
for (let i = 0; i < size; i++) {
map.set(keys[i], read());
}
return map;
}
}
module.exports = MapObjectSerializer;

View file

@ -0,0 +1,33 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class NullPrototypeObjectSerializer {
serialize(obj, { write }) {
const keys = Object.keys(obj);
for (const key of keys) {
write(key);
}
write(null);
for (const key of keys) {
write(obj[key]);
}
}
deserialize({ read }) {
const obj = Object.create(null);
const keys = [];
let key = read();
while (key !== null) {
keys.push(key);
key = read();
}
for (const key of keys) {
obj[key] = read();
}
return obj;
}
}
module.exports = NullPrototypeObjectSerializer;

View file

@ -0,0 +1,703 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const createHash = require("../util/createHash");
const ArraySerializer = require("./ArraySerializer");
const DateObjectSerializer = require("./DateObjectSerializer");
const ErrorObjectSerializer = require("./ErrorObjectSerializer");
const MapObjectSerializer = require("./MapObjectSerializer");
const NullPrototypeObjectSerializer = require("./NullPrototypeObjectSerializer");
const PlainObjectSerializer = require("./PlainObjectSerializer");
const RegExpObjectSerializer = require("./RegExpObjectSerializer");
const SerializerMiddleware = require("./SerializerMiddleware");
const SetObjectSerializer = require("./SetObjectSerializer");
/** @typedef {import("./types").ComplexSerializableType} ComplexSerializableType */
/** @typedef {import("./types").PrimitiveSerializableType} PrimitiveSerializableType */
/** @typedef {new (...params: any[]) => any} Constructor */
/*
Format:
File -> Section*
Section -> ObjectSection | ReferenceSection | EscapeSection | OtherSection
ObjectSection -> ESCAPE (
number:relativeOffset (number > 0) |
string:request (string|null):export
) Section:value* ESCAPE ESCAPE_END_OBJECT
ReferenceSection -> ESCAPE number:relativeOffset (number < 0)
EscapeSection -> ESCAPE ESCAPE_ESCAPE_VALUE (escaped value ESCAPE)
EscapeSection -> ESCAPE ESCAPE_UNDEFINED (escaped value ESCAPE)
OtherSection -> any (except ESCAPE)
Why using null as escape value?
Multiple null values can merged by the BinaryMiddleware, which makes it very efficient
Technically any value can be used.
*/
/**
* @typedef {Object} ObjectSerializerContext
* @property {function(any): void} write
*/
/**
* @typedef {Object} ObjectDeserializerContext
* @property {function(): any} read
*/
/**
* @typedef {Object} ObjectSerializer
* @property {function(any, ObjectSerializerContext): void} serialize
* @property {function(ObjectDeserializerContext): any} deserialize
*/
const setSetSize = (set, size) => {
let i = 0;
for (const item of set) {
if (i++ >= size) {
set.delete(item);
}
}
};
const setMapSize = (map, size) => {
let i = 0;
for (const item of map.keys()) {
if (i++ >= size) {
map.delete(item);
}
}
};
const toHash = buffer => {
const hash = createHash("md4");
hash.update(buffer);
return /** @type {string} */ (hash.digest("latin1"));
};
const ESCAPE = null;
const ESCAPE_ESCAPE_VALUE = null;
const ESCAPE_END_OBJECT = true;
const ESCAPE_UNDEFINED = false;
const CURRENT_VERSION = 2;
const serializers = new Map();
const serializerInversed = new Map();
const loadedRequests = new Set();
const NOT_SERIALIZABLE = {};
const jsTypes = new Map();
jsTypes.set(Object, new PlainObjectSerializer());
jsTypes.set(Array, new ArraySerializer());
jsTypes.set(null, new NullPrototypeObjectSerializer());
jsTypes.set(Map, new MapObjectSerializer());
jsTypes.set(Set, new SetObjectSerializer());
jsTypes.set(Date, new DateObjectSerializer());
jsTypes.set(RegExp, new RegExpObjectSerializer());
jsTypes.set(Error, new ErrorObjectSerializer(Error));
jsTypes.set(EvalError, new ErrorObjectSerializer(EvalError));
jsTypes.set(RangeError, new ErrorObjectSerializer(RangeError));
jsTypes.set(ReferenceError, new ErrorObjectSerializer(ReferenceError));
jsTypes.set(SyntaxError, new ErrorObjectSerializer(SyntaxError));
jsTypes.set(TypeError, new ErrorObjectSerializer(TypeError));
// If in a sandboxed environment (e. g. jest), this escapes the sandbox and registers
// real Object and Array types to. These types may occur in the wild too, e. g. when
// using Structured Clone in postMessage.
if (exports.constructor !== Object) {
const Obj = /** @type {typeof Object} */ (exports.constructor);
const Fn = /** @type {typeof Function} */ (Obj.constructor);
for (const [type, config] of Array.from(jsTypes)) {
if (type) {
const Type = new Fn(`return ${type.name};`)();
jsTypes.set(Type, config);
}
}
}
{
let i = 1;
for (const [type, serializer] of jsTypes) {
serializers.set(type, {
request: "",
name: i++,
serializer
});
}
}
for (const { request, name, serializer } of serializers.values()) {
serializerInversed.set(`${request}/${name}`, serializer);
}
/** @type {Map<RegExp, (request: string) => boolean>} */
const loaders = new Map();
/**
* @typedef {ComplexSerializableType[]} DeserializedType
* @typedef {PrimitiveSerializableType[]} SerializedType
* @extends {SerializerMiddleware<DeserializedType, SerializedType>}
*/
class ObjectMiddleware extends SerializerMiddleware {
constructor(extendContext) {
super();
this.extendContext = extendContext;
}
/**
* @param {RegExp} regExp RegExp for which the request is tested
* @param {function(string): boolean} loader loader to load the request, returns true when successful
* @returns {void}
*/
static registerLoader(regExp, loader) {
loaders.set(regExp, loader);
}
/**
* @param {Constructor} Constructor the constructor
* @param {string} request the request which will be required when deserializing
* @param {string} name the name to make multiple serializer unique when sharing a request
* @param {ObjectSerializer} serializer the serializer
* @returns {void}
*/
static register(Constructor, request, name, serializer) {
const key = request + "/" + name;
if (serializers.has(Constructor)) {
throw new Error(
`ObjectMiddleware.register: serializer for ${Constructor.name} is already registered`
);
}
if (serializerInversed.has(key)) {
throw new Error(
`ObjectMiddleware.register: serializer for ${key} is already registered`
);
}
serializers.set(Constructor, {
request,
name,
serializer
});
serializerInversed.set(key, serializer);
}
/**
* @param {Constructor} Constructor the constructor
* @returns {void}
*/
static registerNotSerializable(Constructor) {
if (serializers.has(Constructor)) {
throw new Error(
`ObjectMiddleware.registerNotSerializable: serializer for ${Constructor.name} is already registered`
);
}
serializers.set(Constructor, NOT_SERIALIZABLE);
}
static getSerializerFor(object) {
const proto = Object.getPrototypeOf(object);
let c;
if (proto === null) {
// Object created with Object.create(null)
c = null;
} else {
c = proto.constructor;
if (!c) {
throw new Error(
"Serialization of objects with prototype without valid constructor property not possible"
);
}
}
const config = serializers.get(c);
if (!config) throw new Error(`No serializer registered for ${c.name}`);
if (config === NOT_SERIALIZABLE) throw NOT_SERIALIZABLE;
return config;
}
static getDeserializerFor(request, name) {
const key = request + "/" + name;
const serializer = serializerInversed.get(key);
if (serializer === undefined) {
throw new Error(`No deserializer registered for ${key}`);
}
return serializer;
}
static _getDeserializerForWithoutError(request, name) {
const key = request + "/" + name;
const serializer = serializerInversed.get(key);
return serializer;
}
/**
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType|Promise<SerializedType>} serialized data
*/
serialize(data, context) {
/** @type {any[]} */
const result = [CURRENT_VERSION];
let currentPos = 0;
const referenceable = new Map();
const addReferenceable = item => {
referenceable.set(item, currentPos++);
};
const bufferDedupeMap = new Map();
const dedupeBuffer = buf => {
const len = buf.length;
const entry = bufferDedupeMap.get(len);
if (entry === undefined) {
bufferDedupeMap.set(len, buf);
return buf;
}
if (Buffer.isBuffer(entry)) {
if (len < 32) {
if (buf.equals(entry)) {
return entry;
}
bufferDedupeMap.set(len, [entry, buf]);
return buf;
} else {
const hash = toHash(entry);
const newMap = new Map();
newMap.set(hash, entry);
bufferDedupeMap.set(len, newMap);
const hashBuf = toHash(buf);
if (hash === hashBuf) {
return entry;
}
return buf;
}
} else if (Array.isArray(entry)) {
if (entry.length < 16) {
for (const item of entry) {
if (buf.equals(item)) {
return item;
}
}
entry.push(buf);
return buf;
} else {
const newMap = new Map();
const hash = toHash(buf);
let found;
for (const item of entry) {
const itemHash = toHash(item);
newMap.set(itemHash, item);
if (found === undefined && itemHash === hash) found = item;
}
bufferDedupeMap.set(len, newMap);
if (found === undefined) {
newMap.set(hash, buf);
return buf;
} else {
return found;
}
}
} else {
const hash = toHash(buf);
const item = entry.get(hash);
if (item !== undefined) {
return item;
}
entry.set(hash, buf);
return buf;
}
};
let currentPosTypeLookup = 0;
const objectTypeLookup = new Map();
const cycleStack = new Set();
const stackToString = item => {
const arr = Array.from(cycleStack);
arr.push(item);
return arr
.map(item => {
if (typeof item === "string") {
if (item.length > 100) {
return `String ${JSON.stringify(item.slice(0, 100)).slice(
0,
-1
)}..."`;
}
return `String ${JSON.stringify(item)}`;
}
try {
const { request, name } = ObjectMiddleware.getSerializerFor(item);
if (request) {
return `${request}${name ? `.${name}` : ""}`;
}
} catch (e) {
// ignore -> fallback
}
if (typeof item === "object" && item !== null) {
if (item.constructor) {
if (item.constructor === Object)
return `Object { ${Object.keys(item).join(", ")} }`;
if (item.constructor === Map) return `Map { ${item.size} items }`;
if (item.constructor === Array)
return `Array { ${item.length} items }`;
if (item.constructor === Set) return `Set { ${item.size} items }`;
if (item.constructor === RegExp) return item.toString();
return `${item.constructor.name}`;
}
return `Object [null prototype] { ${Object.keys(item).join(
", "
)} }`;
}
try {
return `${item}`;
} catch (e) {
return `(${e.message})`;
}
})
.join(" -> ");
};
let hasDebugInfoAttached;
const ctx = {
write(value, key) {
try {
process(value);
} catch (e) {
if (e !== NOT_SERIALIZABLE) {
if (hasDebugInfoAttached === undefined)
hasDebugInfoAttached = new WeakSet();
if (!hasDebugInfoAttached.has(e)) {
e.message += `\nwhile serializing ${stackToString(value)}`;
hasDebugInfoAttached.add(e);
}
}
throw e;
}
},
snapshot() {
return {
length: result.length,
cycleStackSize: cycleStack.size,
referenceableSize: referenceable.size,
currentPos,
objectTypeLookupSize: objectTypeLookup.size,
currentPosTypeLookup
};
},
rollback(snapshot) {
result.length = snapshot.length;
setSetSize(cycleStack, snapshot.cycleStackSize);
setMapSize(referenceable, snapshot.referenceableSize);
currentPos = snapshot.currentPos;
setMapSize(objectTypeLookup, snapshot.objectTypeLookupSize);
currentPosTypeLookup = snapshot.currentPosTypeLookup;
},
...context
};
this.extendContext(ctx);
const process = item => {
if (Buffer.isBuffer(item)) {
// check if we can emit a reference
const ref = referenceable.get(item);
if (ref !== undefined) {
result.push(ESCAPE, ref - currentPos);
return;
}
const alreadyUsedBuffer = dedupeBuffer(item);
if (alreadyUsedBuffer !== item) {
const ref = referenceable.get(alreadyUsedBuffer);
if (ref !== undefined) {
referenceable.set(item, ref);
result.push(ESCAPE, ref - currentPos);
return;
}
item = alreadyUsedBuffer;
}
addReferenceable(item);
result.push(item);
} else if (item === ESCAPE) {
result.push(ESCAPE, ESCAPE_ESCAPE_VALUE);
} else if (
typeof item === "object"
// We don't have to check for null as ESCAPE is null and this has been checked before
) {
// check if we can emit a reference
const ref = referenceable.get(item);
if (ref !== undefined) {
result.push(ESCAPE, ref - currentPos);
return;
}
if (cycleStack.has(item)) {
throw new Error(`Circular references can't be serialized`);
}
const { request, name, serializer } = ObjectMiddleware.getSerializerFor(
item
);
const key = `${request}/${name}`;
const lastIndex = objectTypeLookup.get(key);
if (lastIndex === undefined) {
objectTypeLookup.set(key, currentPosTypeLookup++);
result.push(ESCAPE, request, name);
} else {
result.push(ESCAPE, currentPosTypeLookup - lastIndex);
}
cycleStack.add(item);
try {
serializer.serialize(item, ctx);
} finally {
cycleStack.delete(item);
}
result.push(ESCAPE, ESCAPE_END_OBJECT);
addReferenceable(item);
} else if (typeof item === "string") {
if (item.length > 1) {
// short strings are shorter when not emitting a reference (this saves 1 byte per empty string)
// check if we can emit a reference
const ref = referenceable.get(item);
if (ref !== undefined) {
result.push(ESCAPE, ref - currentPos);
return;
}
addReferenceable(item);
}
if (item.length > 102400 && context.logger) {
context.logger.warn(
`Serializing big strings (${Math.round(
item.length / 1024
)}kiB) impacts deserialization performance (consider using Buffer instead and decode when needed)`
);
}
result.push(item);
} else if (typeof item === "function") {
if (!SerializerMiddleware.isLazy(item))
throw new Error("Unexpected function " + item);
/** @type {SerializedType} */
const serializedData = SerializerMiddleware.getLazySerializedValue(
item
);
if (serializedData !== undefined) {
if (typeof serializedData === "function") {
result.push(serializedData);
} else {
throw new Error("Not implemented");
}
} else if (SerializerMiddleware.isLazy(item, this)) {
throw new Error("Not implemented");
} else {
result.push(
SerializerMiddleware.serializeLazy(item, data =>
this.serialize([data], context)
)
);
}
} else if (item === undefined) {
result.push(ESCAPE, ESCAPE_UNDEFINED);
} else {
result.push(item);
}
};
try {
for (const item of data) {
process(item);
}
} catch (e) {
if (e === NOT_SERIALIZABLE) return null;
throw e;
}
return result;
}
/**
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType|Promise<DeserializedType>} deserialized data
*/
deserialize(data, context) {
let currentDataPos = 0;
const read = () => {
if (currentDataPos >= data.length)
throw new Error("Unexpected end of stream");
return data[currentDataPos++];
};
if (read() !== CURRENT_VERSION)
throw new Error("Version mismatch, serializer changed");
let currentPos = 0;
let referenceable = [];
const addReferenceable = item => {
referenceable.push(item);
currentPos++;
};
let currentPosTypeLookup = 0;
let objectTypeLookup = [];
const result = [];
const ctx = {
read() {
return decodeValue();
},
...context
};
this.extendContext(ctx);
const decodeValue = () => {
const item = read();
if (item === ESCAPE) {
const nextItem = read();
if (nextItem === ESCAPE_ESCAPE_VALUE) {
return ESCAPE;
} else if (nextItem === ESCAPE_UNDEFINED) {
return undefined;
} else if (nextItem === ESCAPE_END_OBJECT) {
throw new Error(
`Unexpected end of object at position ${currentDataPos - 1}`
);
} else if (typeof nextItem === "number" && nextItem < 0) {
// relative reference
return referenceable[currentPos + nextItem];
} else {
const request = nextItem;
let serializer;
if (typeof request === "number") {
serializer = objectTypeLookup[currentPosTypeLookup - request];
} else {
if (typeof request !== "string") {
throw new Error(
`Unexpected type (${typeof request}) of request ` +
`at position ${currentDataPos - 1}`
);
}
const name = read();
serializer = ObjectMiddleware._getDeserializerForWithoutError(
request,
name
);
if (serializer === undefined) {
if (request && !loadedRequests.has(request)) {
let loaded = false;
for (const [regExp, loader] of loaders) {
if (regExp.test(request)) {
if (loader(request)) {
loaded = true;
break;
}
}
}
if (!loaded) {
require(request);
}
loadedRequests.add(request);
}
serializer = ObjectMiddleware.getDeserializerFor(request, name);
}
objectTypeLookup.push(serializer);
currentPosTypeLookup++;
}
try {
const item = serializer.deserialize(ctx);
const end1 = read();
if (end1 !== ESCAPE) {
throw new Error("Expected end of object");
}
const end2 = read();
if (end2 !== ESCAPE_END_OBJECT) {
throw new Error("Expected end of object");
}
addReferenceable(item);
return item;
} catch (err) {
// As this is only for error handling, we omit creating a Map for
// faster access to this information, as this would affect performance
// in the good case
let serializerEntry;
for (const entry of serializers) {
if (entry[1].serializer === serializer) {
serializerEntry = entry;
break;
}
}
const name = !serializerEntry
? "unknown"
: !serializerEntry[1].request
? serializerEntry[0].name
: serializerEntry[1].name
? `${serializerEntry[1].request} ${serializerEntry[1].name}`
: serializerEntry[1].request;
err.message += `\n(during deserialization of ${name})`;
throw err;
}
}
} else if (typeof item === "string") {
if (item.length > 1) {
addReferenceable(item);
}
return item;
} else if (Buffer.isBuffer(item)) {
addReferenceable(item);
return item;
} else if (typeof item === "function") {
return SerializerMiddleware.deserializeLazy(
item,
data => this.deserialize(data, context)[0]
);
} else {
return item;
}
};
while (currentDataPos < data.length) {
result.push(decodeValue());
}
// Help the GC, as functions above might be cached in inline caches
referenceable = undefined;
objectTypeLookup = undefined;
data = undefined;
return result;
}
}
module.exports = ObjectMiddleware;
module.exports.NOT_SERIALIZABLE = NOT_SERIALIZABLE;

View file

@ -0,0 +1,71 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const cache = new WeakMap();
class ObjectStructure {
constructor(keys) {
this.keys = keys;
this.children = new Map();
}
getKeys() {
return this.keys;
}
key(key) {
const child = this.children.get(key);
if (child !== undefined) return child;
const newChild = new ObjectStructure(this.keys.concat(key));
this.children.set(key, newChild);
return newChild;
}
}
const getCachedKeys = (keys, cacheAssoc) => {
let root = cache.get(cacheAssoc);
if (root === undefined) {
root = new ObjectStructure([]);
cache.set(cacheAssoc, root);
}
let current = root;
for (const key of keys) {
current = current.key(key);
}
return current.getKeys();
};
class PlainObjectSerializer {
serialize(obj, { write }) {
const keys = Object.keys(obj);
if (keys.length > 1) {
write(getCachedKeys(keys, write));
for (const key of keys) {
write(obj[key]);
}
} else if (keys.length === 1) {
const key = keys[0];
write(key);
write(obj[key]);
} else {
write(null);
}
}
deserialize({ read }) {
const keys = read();
const obj = {};
if (Array.isArray(keys)) {
for (const key of keys) {
obj[key] = read();
}
} else if (keys !== null) {
obj[keys] = read();
}
return obj;
}
}
module.exports = PlainObjectSerializer;

View file

@ -0,0 +1,17 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class RegExpObjectSerializer {
serialize(obj, { write }) {
write(obj.source);
write(obj.flags);
}
deserialize({ read }) {
return new RegExp(read(), read());
}
}
module.exports = RegExpObjectSerializer;

View file

@ -0,0 +1,48 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class Serializer {
constructor(middlewares, context) {
this.serializeMiddlewares = middlewares.slice();
this.deserializeMiddlewares = middlewares.slice().reverse();
this.context = context;
}
serialize(obj, context) {
const ctx = { ...context, ...this.context };
let current = obj;
for (const middleware of this.serializeMiddlewares) {
if (current instanceof Promise) {
current = current.then(
data => data && middleware.serialize(data, context)
);
} else if (current) {
try {
current = middleware.serialize(current, ctx);
} catch (err) {
current = Promise.reject(err);
}
} else break;
}
return current;
}
deserialize(value, context) {
const ctx = { ...context, ...this.context };
/** @type {any} */
let current = value;
for (const middleware of this.deserializeMiddlewares) {
if (current instanceof Promise) {
current = current.then(data => middleware.deserialize(data, context));
} else {
current = middleware.deserialize(current, ctx);
}
}
return current;
}
}
module.exports = Serializer;

View file

@ -0,0 +1,131 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const memoize = require("../util/memoize");
const LAZY_TARGET = Symbol("lazy serialization target");
const LAZY_SERIALIZED_VALUE = Symbol("lazy serialization data");
/**
* @template DeserializedType
* @template SerializedType
*/
class SerializerMiddleware {
/* istanbul ignore next */
/**
* @abstract
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType|Promise<SerializedType>} serialized data
*/
serialize(data, context) {
const AbstractMethodError = require("../AbstractMethodError");
throw new AbstractMethodError();
}
/* istanbul ignore next */
/**
* @abstract
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType|Promise<DeserializedType>} deserialized data
*/
deserialize(data, context) {
const AbstractMethodError = require("../AbstractMethodError");
throw new AbstractMethodError();
}
/**
* @param {any | function(): Promise<any> | any} value contained value or function to value
* @param {SerializerMiddleware<any, any>} target target middleware
* @param {object=} options lazy options
* @param {any=} serializedValue serialized value
* @returns {function(): Promise<any> | any} lazy function
*/
static createLazy(value, target, options = {}, serializedValue) {
if (SerializerMiddleware.isLazy(value, target)) return value;
const fn = typeof value === "function" ? value : () => value;
fn[LAZY_TARGET] = target;
/** @type {any} */ (fn).options = options;
fn[LAZY_SERIALIZED_VALUE] = serializedValue;
return fn;
}
/**
* @param {function(): Promise<any> | any} fn lazy function
* @param {SerializerMiddleware<any, any>=} target target middleware
* @returns {boolean} true, when fn is a lazy function (optionally of that target)
*/
static isLazy(fn, target) {
if (typeof fn !== "function") return false;
const t = fn[LAZY_TARGET];
return target ? t === target : !!t;
}
/**
* @param {function(): Promise<any> | any} fn lazy function
* @returns {object} options
*/
static getLazyOptions(fn) {
if (typeof fn !== "function") return undefined;
return /** @type {any} */ (fn).options;
}
/**
* @param {function(): Promise<any> | any} fn lazy function
* @returns {any} serialized value
*/
static getLazySerializedValue(fn) {
if (typeof fn !== "function") return undefined;
return fn[LAZY_SERIALIZED_VALUE];
}
/**
* @param {function(): Promise<any> | any} fn lazy function
* @param {any} value serialized value
* @returns {void}
*/
static setLazySerializedValue(fn, value) {
fn[LAZY_SERIALIZED_VALUE] = value;
}
/**
* @param {function(): Promise<any> | any} lazy lazy function
* @param {function(any): Promise<any> | any} serialize serialize function
* @returns {function(): Promise<any> | any} new lazy
*/
static serializeLazy(lazy, serialize) {
const fn = memoize(() => {
const r = lazy();
if (r instanceof Promise) return r.then(data => data && serialize(data));
if (r) return serialize(r);
return null;
});
fn[LAZY_TARGET] = lazy[LAZY_TARGET];
/** @type {any} */ (fn).options = /** @type {any} */ (lazy).options;
lazy[LAZY_SERIALIZED_VALUE] = fn;
return fn;
}
/**
* @param {function(): Promise<any> | any} lazy lazy function
* @param {function(any): Promise<any> | any} deserialize deserialize function
* @returns {function(): Promise<any> | any} new lazy
*/
static deserializeLazy(lazy, deserialize) {
const fn = memoize(() => {
const r = lazy();
if (r instanceof Promise) return r.then(data => deserialize(data));
return deserialize(r);
});
fn[LAZY_TARGET] = lazy[LAZY_TARGET];
/** @type {any} */ (fn).options = /** @type {any} */ (lazy).options;
fn[LAZY_SERIALIZED_VALUE] = lazy;
return fn;
}
}
module.exports = SerializerMiddleware;

View file

@ -0,0 +1,24 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
class SetObjectSerializer {
serialize(obj, { write }) {
write(obj.size);
for (const value of obj) {
write(value);
}
}
deserialize({ read }) {
let size = read();
const set = new Set();
for (let i = 0; i < size; i++) {
set.add(read());
}
return set;
}
}
module.exports = SetObjectSerializer;

View file

@ -0,0 +1,34 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
const SerializerMiddleware = require("./SerializerMiddleware");
/**
* @typedef {any} DeserializedType
* @typedef {any[]} SerializedType
* @extends {SerializerMiddleware<any, any[]>}
*/
class SingleItemMiddleware extends SerializerMiddleware {
/**
* @param {DeserializedType} data data
* @param {Object} context context object
* @returns {SerializedType|Promise<SerializedType>} serialized data
*/
serialize(data, context) {
return [data];
}
/**
* @param {SerializedType} data data
* @param {Object} context context object
* @returns {DeserializedType|Promise<DeserializedType>} deserialized data
*/
deserialize(data, context) {
return data[0];
}
}
module.exports = SingleItemMiddleware;

View file

@ -0,0 +1,13 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
*/
"use strict";
/** @typedef {undefined|null|number|string|boolean|Buffer|Object|(() => ComplexSerializableType[] | Promise<ComplexSerializableType[]>)} ComplexSerializableType */
/** @typedef {undefined|null|number|string|boolean|Buffer|(() => PrimitiveSerializableType[] | Promise<PrimitiveSerializableType[]>)} PrimitiveSerializableType */
/** @typedef {Buffer|(() => BufferSerializableType[] | Promise<BufferSerializableType[]>)} BufferSerializableType */
module.exports = {};