diff --git a/api4/websocket.go b/api4/websocket.go index fc7339b2b3..2608b5ebbf 100644 --- a/api4/websocket.go +++ b/api4/websocket.go @@ -5,10 +5,17 @@ package api4 import ( "net/http" + "strconv" "github.com/gorilla/websocket" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/shared/mlog" +) + +const ( + connectionIDParam = "connection_id" + sequenceNumberParam = "sequence_number" ) func (api *API) InitWebSocket() { @@ -31,6 +38,51 @@ func connectWebSocket(c *Context, w http.ResponseWriter, r *http.Request) { wc := c.App.NewWebConn(ws, *c.App.Session(), c.App.T, "") + if *c.App.Config().ServiceSettings.EnableReliableWebSockets { + connID := r.URL.Query().Get(connectionIDParam) + if connID == "" { + // If not present, we assume client is not capable yet, or it's a fresh connection. + // We just create a new ID. + connID = model.NewId() + } else { + if !model.IsValidId(connID) { + mlog.Error("Invalid connection ID", mlog.String("id", connID)) + wc.WebSocket.Close() + return + } + // If present, we check if it's present in the connection manager. + // TODO: the connection manager internally should forward the request + // to the cluster if it does not have it. + // + // If the connection is not present, then we assume either timeout, + // or server restart. In that case, we set a new one. + // + // Now we get the sequence number + seqVal := r.URL.Query().Get(sequenceNumberParam) + if seqVal == "" { + // Sequence_number must be sent with connection id. + // A client must be either non-compliant or fully compliant. + mlog.Error("Sequence number not present in websocket request") + wc.WebSocket.Close() + return + } + seq, err := strconv.Atoi(seqVal) + if err != nil || seq < 0 { + mlog.Error("Invalid sequence number set in query param", + mlog.String("query", seqVal), + mlog.Err(err)) + wc.WebSocket.Close() + return + } + wc.Sequence = int64(seq) + // Now if there have been past entries to be back-filled, we do it. + // First we find the right sequence number point. + // We start consuming from dead queue first, and then move to active queue + } + // In case of fresh connection id, sequence number is already zero. + wc.SetConnectionID(connID) + } + if c.App.Session().UserId != "" { c.App.HubRegister(wc) } diff --git a/app/web_conn.go b/app/web_conn.go index c562426375..60f03124f2 100644 --- a/app/web_conn.go +++ b/app/web_conn.go @@ -49,6 +49,7 @@ type WebConn struct { send chan model.WebSocketMessage sessionToken atomic.Value session atomic.Value + connectionID atomic.Value endWritePump chan struct{} pumpFinished chan struct{} } @@ -118,6 +119,11 @@ func (wc *WebConn) SetSessionToken(v string) { wc.sessionToken.Store(v) } +// SetConnectionID sets the connection id of the connection. +func (wc *WebConn) SetConnectionID(id string) { + wc.connectionID.Store(id) +} + // GetSession returns the session of the connection. func (wc *WebConn) GetSession() *model.Session { return wc.session.Load().(*model.Session) @@ -147,6 +153,12 @@ func (wc *WebConn) Pump() { wc.App.HubUnregister(wc) close(wc.pumpFinished) + // TODO: + // Check if the channel is closed or not, + // if closed, then remove the entry from conn manager + // else + // take both channels, and store them in connection manager. + defer ReturnSessionToPool(wc.GetSession()) } @@ -195,8 +207,7 @@ func (wc *WebConn) writePump() { select { case msg, ok := <-wc.send: if !ok { - wc.WebSocket.SetWriteDeadline(time.Now().Add(writeWaitTime)) - wc.WebSocket.WriteMessage(websocket.CloseMessage, []byte{}) + wc.writeMessage(websocket.CloseMessage, []byte{}) return } @@ -250,18 +261,18 @@ func (wc *WebConn) writePump() { mlog.Warn("websocket.full", logData...) } - wc.WebSocket.SetWriteDeadline(time.Now().Add(writeWaitTime)) - if err := wc.WebSocket.WriteMessage(websocket.TextMessage, buf.Bytes()); err != nil { + if err := wc.writeMessage(websocket.TextMessage, buf.Bytes()); err != nil { wc.logSocketErr("websocket.send", err) return } + // TODO: move to dead queue + if wc.App.Metrics() != nil { wc.App.Metrics().IncrementWebSocketBroadcast(msg.EventType()) } case <-ticker.C: - wc.WebSocket.SetWriteDeadline(time.Now().Add(writeWaitTime)) - if err := wc.WebSocket.WriteMessage(websocket.PingMessage, []byte{}); err != nil { + if err := wc.writeMessage(websocket.PingMessage, []byte{}); err != nil { wc.logSocketErr("websocket.ticker", err) return } @@ -279,6 +290,13 @@ func (wc *WebConn) writePump() { } } +// writeMessage is a helper utility that wraps the write to the socket +// along with setting the write deadline. +func (wc *WebConn) writeMessage(msgType int, data []byte) error { + wc.WebSocket.SetWriteDeadline(time.Now().Add(writeWaitTime)) + return wc.WebSocket.WriteMessage(msgType, data) +} + // InvalidateCache resets all internal data of the WebConn. func (wc *WebConn) InvalidateCache() { wc.allChannelMembers = nil @@ -318,7 +336,11 @@ func (wc *WebConn) IsAuthenticated() bool { func (wc *WebConn) createHelloMessage() *model.WebSocketEvent { msg := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_HELLO, "", "", wc.UserId, nil) - msg.Add("server_version", fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, model.BuildNumber, wc.App.ClientConfigHash(), wc.App.Srv().License() != nil)) + msg.Add("server_version", fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion, + model.BuildNumber, + wc.App.ClientConfigHash(), + wc.App.Srv().License() != nil)) + msg.Add("connection_id", wc.connectionID.Load()) return msg } diff --git a/app/websocket_router.go b/app/websocket_router.go index 6bbb2539b0..bf0bc1d7e3 100644 --- a/app/websocket_router.go +++ b/app/websocket_router.go @@ -55,11 +55,12 @@ func (wr *WebSocketRouter) ServeWebSocket(conn *WebConn, r *model.WebSocketReque conn.WebSocket.Close() return } - conn.SetSession(session) conn.SetSessionToken(session.Token) conn.UserId = session.UserId + // TODO: Same logic to reconnect queue as api4/websocket.go + wr.app.HubRegister(conn) wr.app.Srv().Go(func() { diff --git a/model/config.go b/model/config.go index d0e1adab02..1a633a8f5b 100644 --- a/model/config.go +++ b/model/config.go @@ -373,6 +373,7 @@ type ServiceSettings struct { CollapsedThreads *string `access:"experimental"` ManagedResourcePaths *string `access:"environment,write_restrictable,cloud_restrictable"` EnableLegacySidebar *bool `access:"experimental"` + EnableReliableWebSockets *bool `access:"experimental"` // telemetry: none } func (s *ServiceSettings) SetDefaults(isUpdate bool) { @@ -818,6 +819,10 @@ func (s *ServiceSettings) SetDefaults(isUpdate bool) { if s.EnableLegacySidebar == nil { s.EnableLegacySidebar = NewBool(false) } + + if s.EnableReliableWebSockets == nil { + s.EnableReliableWebSockets = NewBool(false) + } } type ClusterSettings struct {