Webhub race condition (#9863)

* fix webconn close semantics

Avoid race conditions in WebConn on shutdown by closing channels to guarantee all readers are notified. Wrap this with sync.Once to avoid closing the channel more than once.

* web_hub_test.go

* webhub: fix race condition on shutdown

Ensure that if the webhub shuts down in the process of sending, the caller unblocks given that the webhub will no longer consume incoming events.

* panic if app shutdown takes >30 seconds

* simplify WebConn::Pump channel semantics too

app/web_hub_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/gorilla/websocket"
 	goi18n "github.com/nicksnyder/go-i18n/i18n"
@@ -60,3 +61,45 @@ func TestHubStopWithMultipleConnections(t *testing.T) {
 	defer wc2.Close()
 	defer wc3.Close()
 }
+
+// TestHubStopRaceCondition verifies that attempts to use the hub after it has shutdown does not
+// block the caller indefinitely.
+func TestHubStopRaceCondition(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	s := httptest.NewServer(http.HandlerFunc(dummyWebsocketHandler(t)))
+
+	th.App.HubStart()
+	wc1 := registerDummyWebConn(t, th.App, s.Listener.Addr(), th.BasicUser.Id)
+	defer wc1.Close()
+
+	hub := th.App.Srv.Hubs[0]
+	th.App.HubStop()
+	time.Sleep(5 * time.Second)
+
+	done := make(chan bool)
+	go func() {
+		wc4 := registerDummyWebConn(t, th.App, s.Listener.Addr(), th.BasicUser.Id)
+		wc5 := registerDummyWebConn(t, th.App, s.Listener.Addr(), th.BasicUser.Id)
+		hub.Register(wc4)
+		hub.Register(wc5)
+
+		hub.UpdateActivity("userId", "sessionToken", 0)
+
+		for i := 0; i <= BROADCAST_QUEUE_SIZE; i++ {
+			hub.Broadcast(&model.WebSocketEvent{})
+		}
+
+		hub.InvalidateUser("userId")
+		hub.Unregister(wc4)
+		hub.Unregister(wc5)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(15 * time.Second):
+		t.Fatalf("hub call did not return within 15 seconds after stop")
+	}
+}