MM-62960: Improve webConn remove performance from hubConnectionIndex (#30178)

When we added iteration by channelID, this was a known tradeoff during that.
However, it has been observed that the regular connection removal function
creates considerable blocking of the processing loop, leading to high
CPU usage and API latencies.

To fix this, we add a reverse mapping of channelIDs to connections
and their positions in the slice. This helps us to remove the connection
from the slice without iteration.

Unfortunately, this still needs to iterate through all channelIDs
during invalidation of the channel member cache. However, the user
cache invalidation is not a regular activity. So it should be an acceptable
tradeoff to make.

https://mattermost.atlassian.net/browse/MM-62960

```release-note
A new config knob ServiceSettings.EnableWebHubChannelIteration which allows a user to control the performance of websocket broadcasting. By default, this setting is turned off. If it is turned on, it improves the websocket broadcasting performance at the expense of poor performance when users join/leave a channel. It is not recommended to turn it on unless you have atleast 200,000 concurrent users actively using MM.
```

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/app/platform/web_conn.go

@@ -27,6 +27,7 @@ import (
 	"github.com/mattermost/mattermost/server/public/shared/i18n"
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
 )
 
 const (
@@ -95,8 +96,10 @@ type WebConn struct {
 	UserId           string
 	PostedAck        bool
 
-	lastUserActivityAt int64
-	send               chan model.WebSocketMessage
+	allChannelMembers         map[string]string
+	lastAllChannelMembersTime int64
+	lastUserActivityAt        int64
+	send                      chan model.WebSocketMessage
 	// deadQueue behaves like a queue of a finite size
 	// which is used to store all messages that are sent via the websocket.
 	// It basically acts as the user-space socket buffer, and is used
@@ -758,6 +761,8 @@ func (wc *WebConn) drainDeadQueue(index int) error {
 
 // InvalidateCache resets all internal data of the WebConn.
 func (wc *WebConn) InvalidateCache() {
+	wc.allChannelMembers = nil
+	wc.lastAllChannelMembersTime = 0
 	wc.SetSession(nil)
 	wc.SetSessionExpiresAt(0)
 }
@@ -938,9 +943,36 @@ func (wc *WebConn) ShouldSendEvent(msg *model.WebSocketEvent) bool {
 			return false
 		}
 
-		// We don't need to do any further checks because this is already scoped
-		// to channel members from web_hub.
-		return true
+		if *wc.Platform.Config().ServiceSettings.EnableWebHubChannelIteration {
+			// We don't need to do any further checks because this is already scoped
+			// to channel members from web_hub.
+			return true
+		}
+
+		if model.GetMillis()-wc.lastAllChannelMembersTime > webConnMemberCacheTime {
+			wc.allChannelMembers = nil
+			wc.lastAllChannelMembersTime = 0
+		}
+
+		if wc.allChannelMembers == nil {
+			result, err := wc.Platform.Store.Channel().GetAllChannelMembersForUser(
+				sqlstore.RequestContextWithMaster(request.EmptyContext(wc.Platform.logger)),
+				wc.UserId,
+				false,
+				false,
+			)
+			if err != nil {
+				mlog.Error("webhub.shouldSendEvent.", mlog.Err(err))
+				return false
+			}
+			wc.allChannelMembers = result
+			wc.lastAllChannelMembersTime = model.GetMillis()
+		}
+
+		if _, ok := wc.allChannelMembers[chID]; ok {
+			return true
+		}
+		return false
 	}
 
 	// Only report events to users who are in the team for the event