MM-21012: Revamp websocket implementation (#16620)

* MM-21012: Revamp websocket implementation

We replace the old gorilla/websocket implementation with the
gobwas/ws library. The gorilla library was in maintenance mode
and had a high level API due to which we cannot use that for
situations where a large number of concurrent connections needs
to be supported.

The ws library is a very low-level library that allows us
to work with raw net.Conns. We make several improvements:

- We completely remove the reader goroutines, and instead
replace them with a manual epoll implementation which sends off
messages to be read when it receives any data on the connection.
This lets us scale to a much larger number of connections.
- The reader buffer is eliminated, because we directly read
from the connection now.

https://mattermost.atlassian.net/browse/MM-21012

```release-notes
Improved the websocket implementation by using epoll manually
to read from a websocket. As a result, the number of goroutines
is expected to go down by half.
```

* fix tests

* fix shadowing errors

* final changes

* windows support!

* Remove pointer to waitgroup

* Fix edge case

* Trigger CI

* Trigger CI

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/web_hub_test.go

@@ -4,13 +4,17 @@
 package app
 
 import (
+	"context"
+	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
-	"github.com/gorilla/websocket"
+	"github.com/gobwas/ws"
+	"github.com/gobwas/ws/wsutil"
 	goi18n "github.com/mattermost/go-i18n/i18n"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -22,16 +26,35 @@ import (
 
 func dummyWebsocketHandler(t *testing.T) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
-		upgrader := &websocket.Upgrader{
-			ReadBufferSize:  1024,
-			WriteBufferSize: 1024,
+		upgrader := ws.HTTPUpgrader{
+			Timeout: 5 * time.Second,
+		}
+
+		var hdr ws.Header
+		conn, _, _, err := upgrader.Upgrade(req, w)
+		rd := wsutil.Reader{
+			Source:          conn,
+			State:           ws.StateServerSide,
+			CheckUTF8:       true,
+			SkipHeaderCheck: true,
 		}
-		conn, err := upgrader.Upgrade(w, req, nil)
 		for err == nil {
-			_, _, err = conn.ReadMessage()
+			hdr, err = rd.NextFrame()
+			if err != nil {
+				continue
+			}
+			if hdr.OpCode.IsControl() {
+				continue
+			}
+			if hdr.OpCode&(ws.OpText|ws.OpBinary) == 0 {
+				err = rd.Discard()
+				continue
+			}
+
+			_, err = ioutil.ReadAll(&rd)
 		}
-		if _, ok := err.(*websocket.CloseError); !ok {
-			require.NoError(t, err)
+		if err != io.EOF {
+			require.Fail(t, "unexpected error:", err)
 		}
 	}
 }
@@ -42,8 +65,7 @@ func registerDummyWebConn(t *testing.T, a *App, addr net.Addr, userID string) *W
 	})
 	require.Nil(t, appErr)
 
-	d := websocket.Dialer{}
-	c, _, err := d.Dial("ws://"+addr.String()+"/ws", nil)
+	c, _, _, err := ws.Dial(context.Background(), "ws://"+addr.String()+"/ws")
 	require.NoError(t, err)
 
 	wc := a.NewWebConn(c, *session, goi18n.IdentityTfunc(), "en")
@@ -78,8 +100,7 @@ func TestHubStopRaceCondition(t *testing.T) {
 	s := httptest.NewServer(dummyWebsocketHandler(t))
 
 	th.App.HubStart()
-	wc1 := registerDummyWebConn(t, th.App, s.Listener.Addr(), th.BasicUser.Id)
-	defer wc1.Close()
+	registerDummyWebConn(t, th.App, s.Listener.Addr(), th.BasicUser.Id)
 
 	hub := th.App.Srv().hubs[0]
 	th.App.HubStop()