MM-39612: Make acquiring and removing connections atomic (#18982)

* MM-39612: Make acquiring and removing connections atomic

The reconnect phase of a websocket was split into two parts:
one where we check if a connection with a given connectionID
exists or not. And second, where we remove that connection
and insert the new connection again in the index.

This would lead to a race where it would be possible
for 2 concurrent requests for the same connectionID to go through
which would lead to separate goroutines working on the same dead queue.

We simplify this by removing the connection from the index
in the check connection stage itself. And then just add that
during register phase.

And to distinguish between a fresh and an old connection, we add
a new field called reuseCount.

While here, we also cleanup some old comments and add more
in some places.

https://mattermost.atlassian.net/browse/MM-39612

```release-note
NONE
```

* remove unused method

```release-note
NONE
```

* race test

```release-note
NONE
```

model/websocket_client.go

@@ -6,6 +6,7 @@ package model
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"sync/atomic"
 	"time"
@@ -65,10 +66,26 @@ func NewWebSocketClient(url, authToken string) (*WebSocketClient, error) {
 	return NewWebSocketClientWithDialer(websocket.DefaultDialer, url, authToken)
 }
 
+func NewReliableWebSocketClientWithDialer(dialer *websocket.Dialer, url, authToken, connID string, seqNo int, withAuthHeader bool) (*WebSocketClient, error) {
+	connectURL := url + APIURLSuffix + "/websocket" + fmt.Sprintf("?connection_id=%s&sequence_number=%d", connID, seqNo)
+	var header http.Header
+	if withAuthHeader {
+		header = http.Header{
+			"Authorization": []string{"Bearer " + authToken},
+		}
+	}
+
+	return makeClient(dialer, url, connectURL, authToken, header)
+}
+
 // NewWebSocketClientWithDialer constructs a new WebSocket client with convenience
 // methods for talking to the server using a custom dialer.
 func NewWebSocketClientWithDialer(dialer *websocket.Dialer, url, authToken string) (*WebSocketClient, error) {
-	conn, _, err := dialer.Dial(url+APIURLSuffix+"/websocket", nil)
+	return makeClient(dialer, url, url+APIURLSuffix+"/websocket", authToken, nil)
+}
+
+func makeClient(dialer *websocket.Dialer, url, connectURL, authToken string, header http.Header) (*WebSocketClient, error) {
+	conn, _, err := dialer.Dial(connectURL, header)
 	if err != nil {
 		return nil, NewAppError("NewWebSocketClient", "model.websocket_client.connect_fail.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
@@ -76,7 +93,7 @@ func NewWebSocketClientWithDialer(dialer *websocket.Dialer, url, authToken strin
 	client := &WebSocketClient{
 		URL:                url,
 		APIURL:             url + APIURLSuffix,
-		ConnectURL:         url + APIURLSuffix + "/websocket",
+		ConnectURL:         connectURL,
 		Conn:               conn,
 		AuthToken:          authToken,
 		Sequence:           1,