MM-39612: Make acquiring and removing connections atomic (#18982)

* MM-39612: Make acquiring and removing connections atomic

The reconnect phase of a websocket was split into two parts:
one where we check if a connection with a given connectionID
exists or not. And second, where we remove that connection
and insert the new connection again in the index.

This would lead to a race where it would be possible
for 2 concurrent requests for the same connectionID to go through
which would lead to separate goroutines working on the same dead queue.

We simplify this by removing the connection from the index
in the check connection stage itself. And then just add that
during register phase.

And to distinguish between a fresh and an old connection, we add
a new field called reuseCount.

While here, we also cleanup some old comments and add more
in some places.

https://mattermost.atlassian.net/browse/MM-39612

```release-note
NONE
```

* remove unused method

```release-note
NONE
```

* race test

```release-note
NONE
```

app/web_hub.go

@@ -432,7 +432,7 @@ func (h *Hub) Start() {
 				webSessionMessage.isRegistered <- isRegistered
 			case req := <-h.checkConn:
 				var res *CheckConnResult
-				conn := connIndex.GetInactiveByConnectionID(req.userID, req.connectionID)
+				conn := connIndex.RemoveInactiveByConnectionID(req.userID, req.connectionID)
 				if conn != nil {
 					res = &CheckConnResult{
 						ConnectionID:     req.connectionID,
@@ -440,20 +440,13 @@ func (h *Hub) Start() {
 						ActiveQueue:      conn.send,
 						DeadQueue:        conn.deadQueue,
 						DeadQueuePointer: conn.deadQueuePointer,
+						ReuseCount:       conn.reuseCount + 1,
 					}
 				}
 				req.result <- res
 			case <-ticker.C:
 				connIndex.RemoveInactiveConnections()
 			case webConn := <-h.register:
-				var oldConn *WebConn
-				if *h.srv.Config().ServiceSettings.EnableReliableWebSockets {
-					// Delete the old conn from connIndex if it exists.
-					oldConn = connIndex.RemoveInactiveByConnectionID(
-						webConn.GetSession().UserId,
-						webConn.GetConnectionID())
-				}
-
 				// Mark the current one as active.
 				// There is no need to check if it was inactive or not,
 				// we will anyways need to make it active.
@@ -462,8 +455,8 @@ func (h *Hub) Start() {
 				connIndex.Add(webConn)
 				atomic.StoreInt64(&h.connectionCount, int64(connIndex.AllActive()))
 
-				if webConn.IsAuthenticated() && oldConn == nil {
-					// The hello message should only be sent when the conn wasn't found.
+				if webConn.IsAuthenticated() && webConn.reuseCount == 0 {
+					// The hello message should only be sent when the reuseCount is 0.
 					// i.e in server restart, or long timeout, or fresh connection case.
 					// In case of seq number not found in dead queue, it is handled by
 					// the webconn write pump.
@@ -660,21 +653,6 @@ func (i *hubConnectionIndex) All() map[*WebConn]int {
 	return i.byConnection
 }
 
-// GetInactiveByConnectionID returns an inactive connection for the given
-// userID and connectionID.
-func (i *hubConnectionIndex) GetInactiveByConnectionID(userID, connectionID string) *WebConn {
-	// To handle empty sessions.
-	if userID == "" {
-		return nil
-	}
-	for _, conn := range i.ForUser(userID) {
-		if conn.GetConnectionID() == connectionID && !conn.active {
-			return conn
-		}
-	}
-	return nil
-}
-
 // RemoveInactiveByConnectionID removes an inactive connection for the given
 // userID and connectionID.
 func (i *hubConnectionIndex) RemoveInactiveByConnectionID(userID, connectionID string) *WebConn {