Fix(ish) for unicode encoding in packet (#608)

* Fix-ish the unicode issues

* removing unneded spaces

* fixing documentation

* fixed linting

* fixing benchmarking of decode

Author: grahamjenson

engineio/payload/data_test.go

@@ -20,18 +20,18 @@ var tests = []struct {
 		{frame.String, packet.OPEN, []byte{}},
 	},
 	},
-	{true, []byte{0x00, 0x01, 0x03, 0xff, '4', 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd},
+	{true, []byte{0x00, 0x09, 0xff, '4', 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd},
 		[]Packet{
 			{frame.String, packet.MESSAGE, []byte("hello 你好")},
 		},
 	},
-	{true, []byte{0x01, 0x01, 0x03, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd}, []Packet{
+	{true, []byte{0x01, 0x09, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd}, []Packet{
 		{frame.Binary, packet.MESSAGE, []byte("hello 你好")},
 	},
 	},
 	{true, []byte{
 		0x01, 0x07, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', '\n',
-		0x00, 0x08, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
+		0x00, 0x04, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
 		0x00, 0x06, 0xff, '2', 'p', 'r', 'o', 'b', 'e',
 	}, []Packet{
 		{frame.Binary, packet.MESSAGE, []byte("hello\n")},
@@ -43,18 +43,45 @@ var tests = []struct {
 		{frame.String, packet.OPEN, []byte{}},
 	},
 	},
-	{false, []byte("13:4hello 你好"), []Packet{
+	{false, []byte("9:4hello 你好"), []Packet{
 		{frame.String, packet.MESSAGE, []byte("hello 你好")},
 	},
 	},
 	{false, []byte("18:b4aGVsbG8g5L2g5aW9"), []Packet{
 		{frame.Binary, packet.MESSAGE, []byte("hello 你好")},
 	},
 	},
-	{false, []byte("10:b4aGVsbG8K8:4你好\n6:2probe"), []Packet{
+	{false, []byte("10:b4aGVsbG8K4:4你好\n6:2probe"), []Packet{
 		{frame.Binary, packet.MESSAGE, []byte("hello\n")},
 		{frame.String, packet.MESSAGE, []byte("你好\n")},
 		{frame.String, packet.PING, []byte("probe")},
 	},
 	},
+	// ↓ is 3 bytes, JavaScript `.length` 1 See https://socket.io/docs/v4/engine-io-protocol/#from-v3-to-v4
+	{false, []byte("6:412↓453:41↓"), []Packet{
+		{frame.String, packet.MESSAGE, []byte("12↓45")},
+		{frame.String, packet.MESSAGE, []byte("1↓")},
+	},
+	},
+	// 🇩🇪 is 8 bytes, 2 unicode chars JavaScript `.length` 4
+	{false, []byte("6:4hello6:4🇩🇪a5:41234"), []Packet{
+		{frame.String, packet.MESSAGE, []byte("hello")},
+		{frame.String, packet.MESSAGE, []byte("🇩🇪a")},
+		{frame.String, packet.MESSAGE, []byte("1234")},
+	},
+	},
+	// € is 3 bytes, JavaScript `.length` 1
+	{false, []byte("2:4h3:4€a2:41"), []Packet{
+		{frame.String, packet.MESSAGE, []byte("h")},
+		{frame.String, packet.MESSAGE, []byte("€a")},
+		{frame.String, packet.MESSAGE, []byte("1")},
+	},
+	},
+	//👍 is 4 bytes, JavaScript `.length` 2
+	{false, []byte("2:4h4:4👍a2:41"), []Packet{
+		{frame.String, packet.MESSAGE, []byte("h")},
+		{frame.String, packet.MESSAGE, []byte("👍a")},
+		{frame.String, packet.MESSAGE, []byte("1")},
+	},
+	},
 }

engineio/payload/decoder.go

@@ -53,7 +53,24 @@ func (d *decoder) Read(p []byte) (int, error) {
 	if d.b64Reader != nil {
 		return d.b64Reader.Read(p)
 	}
-	return d.limitReader.Read(p)
+	dd, err := d.limitReader.Read(p)
+	unicodeCount := 0
+	for i := range p[:dd] {
+		b := p[i]
+		if b>>3 == 30 {
+			// starts with 11110 4 byte unicode char, probably 2 length in JS
+			unicodeCount = unicodeCount + 2
+		} else if b>>4 == 14 {
+			// starts with 1110 3 byte unicode char, probably 2 length in JS
+			unicodeCount = unicodeCount + 2
+		} else if b>>5 == 6 {
+			// starts with 110 2 byte unicode char, , probably 1 length in JS
+			unicodeCount = unicodeCount + 1
+		}
+	}
+
+	d.limitReader.N = d.limitReader.N + int64(unicodeCount)
+	return dd, err
 }
 
 func (d *decoder) Close() error {

engineio/payload/decoder_test.go

@@ -96,7 +96,7 @@ func TestDecoderNextReaderError(t *testing.T) {
 
 func BenchmarkStringDecoder(b *testing.B) {
 	feeder := fakeReaderFeeder{
-		data:          []byte("8:4你好\n6:2probe"),
+		data:          []byte("4:4你好\n6:2probe"),
 		supportBinary: false,
 	}
 	d := decoder{
@@ -148,7 +148,7 @@ func BenchmarkBinaryDecoder(b *testing.B) {
 	feeder := fakeReaderFeeder{
 		data: []byte{
 			0x01, 0x07, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', '\n',
-			0x00, 0x08, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
+			0x00, 0x04, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
 			0x00, 0x06, 0xff, '2', 'p', 'r', 'o', 'b', 'e',
 		},
 		supportBinary: true,

engineio/payload/encoder.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/base64"
 	"io"
+	"unicode/utf8"
 
 	"github.com/googollee/go-socket.io/engineio/frame"
 	"github.com/googollee/go-socket.io/engineio/packet"
@@ -56,6 +57,7 @@ func (e *encoder) Write(p []byte) (int, error) {
 	if e.b64Writer != nil {
 		return e.b64Writer.Write(p)
 	}
+	// Need to scan here and add unicode chars to the list
 	return e.frameCache.Write(p)
 }
 
@@ -92,16 +94,16 @@ func (e *encoder) Close() error {
 }
 
 func (e *encoder) writeTextHeader() error {
-	l := int64(e.frameCache.Len() + 1) // length for packet type
-	err := writeTextLen(l, &e.header)
+
+	err := writeTextLen(e.calcCodeUnitLength(), &e.header)
 	if err == nil {
 		err = e.header.WriteByte(e.pt.StringByte())
 	}
 	return err
 }
 
 func (e *encoder) writeB64Header() error {
-	l := int64(e.frameCache.Len() + 2) // length for 'b' and packet type
+	l := int64(utf8.RuneCount(e.frameCache.Bytes()) + 2) // length for 'b' and packet type
 	err := writeTextLen(l, &e.header)
 	if err == nil {
 		err = e.header.WriteByte('b')
@@ -112,8 +114,34 @@ func (e *encoder) writeB64Header() error {
 	return err
 }
 
+func (e *encoder) calcCodeUnitLength() int64 {
+	var l int64 = 1
+	var codeUnitSize int64
+	bytes := e.frameCache.Bytes()
+	for i := range bytes {
+		b := bytes[i]
+		if b>>3 == 30 {
+			// starts with 11110 4 byte unicode char, probably 2 length in JS
+			codeUnitSize = 2
+		} else if b>>4 == 14 {
+			// starts with 1110 3 byte unicode char, probably 1 length in JS
+			codeUnitSize = 1
+		} else if b>>5 == 6 {
+			// starts with 110 2 byte unicode char, , probably 1 length in JS
+			codeUnitSize = 1
+		} else if b>>6 == 2 {
+			// starts with 10 just unicode byte
+			codeUnitSize = 0
+		} else {
+			codeUnitSize = 1
+		}
+		l = l + codeUnitSize
+	}
+
+	return int64(l)
+}
 func (e *encoder) writeBinaryHeader() error {
-	l := int64(e.frameCache.Len() + 1) // length for packet type
+	l := int64(e.calcCodeUnitLength()) // length for packet type
 	b := e.pt.StringByte()
 	if e.ft == frame.Binary {
 		b = e.pt.BinaryByte()