Socket Read - buffer is greater than read bytes

[lynxaegon]

const turbo = require('turbo-net');
const fs = require("fs");
const msgpack = require('msgpack-lite');

const server = turbo.createServer((socket) => {
	let contentLength = 0;
	let state = 0;

	socket.read(getBuffer(state, contentLength), function onRead(err, buf, read){
		console.log("reading...", state, contentLength);
		if(read > 0) {
			console.log("read bytes", read);
			if(state == 0){
				state = 1;

				contentLength = buf.readUInt32BE(0);
				if(contentLength > 5e+7){
					console.log("socket closed");
					socket.close();
					return;
				}
			} else {
				contentLength -= read;
				console.log("remaining", contentLength, buf.length);
				if(contentLength == 0){
					state = 0;
					buf = msgpack.decode(buf);
					const type = buf[0];
					const payload = buf[1];
					console.log(type, payload);
				}
			}
		}

		if(!socket)
		{
			console.log("failed to read from socket! Maybe disconnected?");
			return;
		}
		socket.read(getBuffer(state, contentLength), onRead.bind(this));
	}.bind(this));
});
server.listen(8888);
const client = turbo.connect(8888, "localhost");
let content = fs.readFileSync("10mb.txt");

write(client, content);

write(client, content);

write(client, content);

function getBuffer(state, length) {
	return state == 0 ? Buffer.allocUnsafe(4) : Buffer.allocUnsafe(length);
}

function write(client, data) {
	data = msgpack.encode(["test_content", data]);
	const header = Buffer.allocUnsafe(4);
	header.writeUInt32BE(data.length);
	client.write(header, header.length);
	client.write(data, data.length);
}

Let me explain what happens.
A server and a client are created, the client sends 3 packets of 10mb encoded with msgpack. Each packet has header and data (where data contains type and payload.
The header contains 4 bytes so on the server side we know the contentLength.

When the client receives data, he first reads 4 bytes for the contentLength and then uses the contentLength to alloc a Buffer of that size.

On the first read it returns the Buffer with all data (of the correct size and correct contents) but the read bytes are lower than the Buffer size and you have to process the rest of the bytes in order to get to the next packet, even though the rest of the bytes are invalid (or parts of the original Buffer)

Output:

# important part here

reading... 0 0
read bytes 4
reading... 1 10485779
read bytes 400628
remaining 10085151 10485779 ---------_> says it read only 400628 bytes, 
#but the buffer is valid and with all the data there (10485779 bytes) and 
#can be successfully be parsed by msgpack

#end of important part

reading... 1 10085151
read bytes 237100
remaining 9848051 10085151
reading... 1 9848051
read bytes 408716
remaining 9439335 9848051
reading... 1 9439335
read bytes 278060
remaining 9161275 9439335
reading... 1 9161275
read bytes 293976
remaining 8867299 9161275
reading... 1 8867299
read bytes 180692
remaining 8686607 8867299
reading... 1 8686607
read bytes 1113072
remaining 7573535 8686607
reading... 1 7573535
read bytes 458336
remaining 7115199 7573535
reading... 1 7115199
read bytes 1112968
remaining 6002231 7115199
reading... 1 6002231
read bytes 523820
remaining 5478411 6002231
reading... 1 5478411
read bytes 1113488
remaining 4364923 5478411
reading... 1 4364923
read bytes 589408
remaining 3775515 4364923
reading... 1 3775515
read bytes 1121680
remaining 2653835 3775515
reading... 1 2653835
read bytes 630472
remaining 2023363 2653835
reading... 1 2023363
read bytes 1121836
remaining 901527 2023363
reading... 1 901527
read bytes 727788
remaining 173739 901527
reading... 1 173739
read bytes 130656
remaining 43083 173739
reading... 1 43083
read bytes 43083
remaining 0 43083
undefined undefined
reading... 0 0
read bytes 4
reading... 1 10485779
read bytes 1071696
remaining 9414083 10485779
reading... 1 9414083
read bytes 712392
remaining 8701691 9414083
reading... 1 8701691
read bytes 1121316
remaining 7580375 8701691
reading... 1 7580375
read bytes 728724
remaining 6851651 7580375
reading... 1 6851651
read bytes 1121732
remaining 5729919 6851651
reading... 1 5729919
read bytes 728724
remaining 5001195 5729919
reading... 1 5001195
read bytes 1127948
remaining 3873247 5001195
reading... 1 3873247
read bytes 745056
remaining 3128191 3873247
reading... 1 3128191
read bytes 1121732
remaining 2006459 3128191
reading... 1 2006459
read bytes 728308
remaining 1278151 2006459
reading... 1 1278151
read bytes 1120692
remaining 157459 1278151
reading... 1 157459
read bytes 157459
remaining 0 157459
undefined undefined
reading... 0 0
read bytes 4
reading... 1 10485779
read bytes 957372
remaining 9528407 10485779
reading... 1 9528407
read bytes 696060
remaining 8832347 9528407
reading... 1 8832347
read bytes 1121524
remaining 7710823 8832347
reading... 1 7710823
read bytes 663396
remaining 7047427 7710823
reading... 1 7047427
read bytes 1120692
remaining 5926735 7047427
reading... 1 5926735
read bytes 663396
remaining 5263339 5926735
reading... 1 5263339
read bytes 1121524
remaining 4141815 5263339
reading... 1 4141815
read bytes 663396
remaining 3478419 4141815
reading... 1 3478419
read bytes 1120692
remaining 2357727 3478419
reading... 1 2357727
read bytes 663396
remaining 1694331 2357727
reading... 1 1694331
read bytes 1120692
remaining 573639 1694331
reading... 1 573639
read bytes 573639
remaining 0 573639
undefined undefined

Am i doing something wrong, or is this a bug?

[mafintosh]

Hmm maybe I'm misunderstading your question, but the read does not fill the buffer necessarily. That's why it returns how many bytes it read :)

[lynxaegon]

Im my case, it fills the whole buffer with the correct data, but the read variable says it only read part of it. Usually below 64kb.

I figured it out, i don't know why on the first read it the buffer is with complete data, but if i slice the buffer with the bytes read and concat when the total sum of the read bytes is equal to the contentLength, everything works correctly.

buffers.push(buf.slice(0, read));
buf = Buffer.concat(buffers);
buffers = [];

buf = msgpack.decode(buf);

Issue can be closed