From 6708160cec15ccd4a79979ed8ba79310fbd9285f Mon Sep 17 00:00:00 2001 From: Elizabeth Hunt Date: Tue, 5 Sep 2023 22:28:11 -0600 Subject: [PATCH] add compression to engine messages --- .gitmodules | 3 ++ client/src/network/MessagePublisher.ts | 4 +- client/src/network/MessageQueueReceiver.ts | 6 ++- engine/utils/coding.ts | 18 +++++-- engine/utils/compatto | 1 + engine/utils/dictionary.ts | 56 ++++++++++++++++++++++ server/src/network/MessagePublisher.ts | 4 +- server/src/server.ts | 10 ++-- 8 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 .gitmodules create mode 160000 engine/utils/compatto create mode 100644 engine/utils/dictionary.ts diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..57de596 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "engine/utils/compatto"] + path = engine/utils/compatto + url = https://github.com/macarie/compatto diff --git a/client/src/network/MessagePublisher.ts b/client/src/network/MessagePublisher.ts index de105c5..c0aec28 100644 --- a/client/src/network/MessagePublisher.ts +++ b/client/src/network/MessagePublisher.ts @@ -1,5 +1,5 @@ import type { Message, MessagePublisher } from '@engine/network'; -import { stringify } from '@engine/utils'; +import { serialize } from '@engine/utils'; export class ClientSocketMessagePublisher implements MessagePublisher { private socket: WebSocket; @@ -17,7 +17,7 @@ export class ClientSocketMessagePublisher implements MessagePublisher { public publish() { if (this.socket.readyState == WebSocket.OPEN) { this.messages.forEach((message: Message) => - this.socket.send(stringify(message)) + this.socket.send(serialize(message)) ); this.messages = []; } diff --git a/client/src/network/MessageQueueReceiver.ts b/client/src/network/MessageQueueReceiver.ts index 46ca5cc..cb74cf7 100644 --- a/client/src/network/MessageQueueReceiver.ts +++ b/client/src/network/MessageQueueReceiver.ts @@ -10,8 +10,10 @@ export class ClientSocketMessageQueueProvider implements MessageQueueProvider { this.messages = []; this.socket.addEventListener('message', (e) => { - const messages = parse(e.data); - this.messages = this.messages.concat(messages); + e.data.arrayBuffer().then((buffer) => { + const messages = parse(new Uint8Array(buffer)); + this.messages = this.messages.concat(messages); + }); }); } diff --git a/engine/utils/coding.ts b/engine/utils/coding.ts index 283844f..d0904fe 100644 --- a/engine/utils/coding.ts +++ b/engine/utils/coding.ts @@ -1,3 +1,8 @@ +import { compatto } from './compatto'; +import dictionary from './dictionary'; + +const { compress, decompress } = compatto({ dictionary }); + const replacer = (_key: any, value: any) => { if (value instanceof Map) { return { @@ -31,10 +36,17 @@ const reviver = (_key: any, value: any) => { }; // "deterministic" stringify -export const stringify = (obj: any) => { + +export const stringify = (obj: any): string => { return JSON.stringify(sortObj(obj), replacer); }; -export const parse = (str: string) => { - return JSON.parse(str, reviver) as unknown as T; +export const serialize = (obj: any): Uint8Array => { + //return new Uint8Array(new TextEncoder().encode(stringify(obj))); + return compress(stringify(obj)); +}; + +export const parse = (serialized: Uint8Array): T => { + //return JSON.parse(new TextDecoder().decode(serialized), reviver) as T; + return JSON.parse(decompress(serialized), reviver) as T; }; diff --git a/engine/utils/compatto b/engine/utils/compatto new file mode 160000 index 0000000..68628ac --- /dev/null +++ b/engine/utils/compatto @@ -0,0 +1 @@ +Subproject commit 68628ac6f173f50b81cc2d83a75de176484b2c7f diff --git a/engine/utils/dictionary.ts b/engine/utils/dictionary.ts new file mode 100644 index 0000000..5a19c6d --- /dev/null +++ b/engine/utils/dictionary.ts @@ -0,0 +1,56 @@ +// basically a list of common strings between network components to help in compression +// can't be longer than 254 "words" +export default [ + '{', + '}', + ',', + '"', + "'", + ',', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '0', + '.', + 'Map', + '[', + ']', + 'BoundingBox', + 'args', + 'height', + 'width', + 'name', + 'body', + 'Velocity', + 'Control', + 'Component', + 'NEW_ENTITIES', + 'REMOVE_ENTITIES', + 'UPDATE_ENTITIES', + 'NEW_INPUT', + 'REMOVE_INPUT', + 'boundingBox', + 'dTheta', + 'id', + 'Forces', + ':', + 'control', + 'controllableBy', + 'dimension', + 'rotation', + 'forces', + 'velocity', + 'center', + 'dCartesian', + 'x', + 'y', + '"body":[{"args":{', + 'controlVelocityComponent', + '"boundingBox":{"center":' +]; diff --git a/server/src/network/MessagePublisher.ts b/server/src/network/MessagePublisher.ts index 9c6011f..0b8340e 100644 --- a/server/src/network/MessagePublisher.ts +++ b/server/src/network/MessagePublisher.ts @@ -1,7 +1,7 @@ import { Message, MessagePublisher } from '@engine/network'; import { Server } from 'bun'; import { Constants } from '../constants'; -import { stringify } from '@engine/utils'; +import { serialize } from '@engine/utils'; export class ServerSocketMessagePublisher implements MessagePublisher { private server?: Server; @@ -23,7 +23,7 @@ export class ServerSocketMessagePublisher implements MessagePublisher { public publish() { if (this.messages.length) { - this.server?.publish(Constants.GAME_TOPIC, stringify(this.messages)); + this.server?.publish(Constants.GAME_TOPIC, serialize(this.messages)); this.messages = []; } diff --git a/server/src/server.ts b/server/src/server.ts index 251fd89..b99380b 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -10,11 +10,10 @@ import { Session, SessionManager } from './network'; -import { parse } from '@engine/utils'; +import { parse, serialize } from '@engine/utils'; import { Server, ServerWebSocket } from 'bun'; import { Input } from '@engine/systems'; import { Control, NetworkUpdateable } from '@engine/components'; -import { stringify } from '@engine/utils'; export class GameServer { private server?: Server; @@ -44,7 +43,8 @@ export class GameServer { websocket: { open: (ws) => this.openWebsocket(ws), message: (ws, msg) => { - if (typeof msg === 'string') this.websocketMessage(ws, msg); + if (typeof msg !== 'string') + this.websocketMessage(ws, new Uint8Array(msg)); }, close: (ws) => this.closeWebsocket(ws) } @@ -57,7 +57,7 @@ export class GameServer { private websocketMessage( websocket: ServerWebSocket, - message: string + message: Uint8Array ) { const receivedMessage = parse(message); receivedMessage.sessionData = websocket.data; @@ -117,7 +117,7 @@ export class GameServer { }) } ]; - websocket.sendText(stringify(addCurrentEntities)); + websocket.sendBinary(serialize(addCurrentEntities)); const addNewPlayer: Message = { type: MessageType.NEW_ENTITIES,