PLT-2899 adding clustering of app servers (#3682)

* PLT-2899 adding clustering of app servers

* PLT-2899 base framework

* PLT-2899 HA backend

* PLT-2899 Fixing config file

* PLT-2899 adding config syncing

* PLT-2899 set System console to readonly when clustering enabled.

* PLT-2899 Fixing publish API

* PLT-2899 fixing strings

Makefile

@@ -182,18 +182,22 @@ ifeq ($(BUILD_ENTERPRISE_READY),true)
 	$(GO) test $(GOFLAGS) -run=$(TESTS) -covermode=count -c ./enterprise/compliance && ./compliance.test -test.v -test.timeout=120s -test.coverprofile=ccompliance.out || exit 1
 	$(GO) test $(GOFLAGS) -run=$(TESTS) -covermode=count -c ./enterprise/emoji && ./emoji.test -test.v -test.timeout=120s -test.coverprofile=cemoji.out || exit 1
 	$(GO) test $(GOFLAGS) -run=$(TESTS) -covermode=count -c ./enterprise/saml && ./saml.test -test.v -test.timeout=60s -test.coverprofile=csaml.out || exit 1
+	$(GO) test $(GOFLAGS) -run=$(TESTS) -covermode=count -c ./enterprise/cluster && ./cluster.test -test.v -test.timeout=60s -test.coverprofile=ccluster.out || exit 1
 	$(GO) test $(GOFLAGS) -run=$(TESTS) -covermode=count -c ./enterprise/account_migration && ./account_migration.test -test.v -test.timeout=60s -test.coverprofile=caccount_migration.out || exit 1
 
 	tail -n +2 cldap.out >> ecover.out
 	tail -n +2 ccompliance.out >> ecover.out
 	tail -n +2 cemoji.out >> ecover.out
 	tail -n +2 csaml.out >> ecover.out
+	tail -n +2 ccluster.out >> ecover.out
 	tail -n +2 caccount_migration.out >> ecover.out
-	rm -f cldap.out ccompliance.out cemoji.out csaml.out caccount_migration.out
+	rm -f cldap.out ccompliance.out cemoji.out csaml.out ccluster.out caccount_migration.out
+
 	rm -r ldap.test
 	rm -r compliance.test
 	rm -r emoji.test
 	rm -r saml.test
+	rm -r cluster.test
 	rm -r account_migration.test
 	rm -f config/*.crt
 	rm -f config/*.key

api/admin.go

@@ -46,6 +46,7 @@ func InitAdmin() {
 	BaseRoutes.Admin.Handle("/add_certificate", ApiAdminSystemRequired(addCertificate)).Methods("POST")
 	BaseRoutes.Admin.Handle("/remove_certificate", ApiAdminSystemRequired(removeCertificate)).Methods("POST")
 	BaseRoutes.Admin.Handle("/saml_cert_status", ApiAdminSystemRequired(samlCertificateStatus)).Methods("GET")
+	BaseRoutes.Admin.Handle("/cluster_status", ApiAdminSystemRequired(getClusterStatus)).Methods("GET")
 }
 
 func getLogs(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -54,13 +55,32 @@ func getLogs(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	lines, err := GetLogs()
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	if einterfaces.GetClusterInterface() != nil {
+		clines, err := einterfaces.GetClusterInterface().GetLogs()
+		if err != nil {
+			c.Err = err
+			return
+		}
+
+		lines = append(lines, clines...)
+	}
+
+	w.Write([]byte(model.ArrayToJson(lines)))
+}
+
+func GetLogs() ([]string, *model.AppError) {
 	var lines []string
 
 	if utils.Cfg.LogSettings.EnableFile {
-
 		file, err := os.Open(utils.GetLogFileLocation(utils.Cfg.LogSettings.FileLocation))
 		if err != nil {
-			c.Err = model.NewLocAppError("getLogs", "api.admin.file_read_error", nil, err.Error())
+			return nil, model.NewLocAppError("getLogs", "api.admin.file_read_error", nil, err.Error())
 		}
 
 		defer file.Close()
@@ -73,7 +93,21 @@ func getLogs(c *Context, w http.ResponseWriter, r *http.Request) {
 		lines = append(lines, "")
 	}
 
-	w.Write([]byte(model.ArrayToJson(lines)))
+	return lines, nil
+}
+
+func getClusterStatus(c *Context, w http.ResponseWriter, r *http.Request) {
+
+	if !c.HasSystemAdminPermissions("getClusterStatus") {
+		return
+	}
+
+	infos := make([]*model.ClusterInfo, 0)
+	if einterfaces.GetClusterInterface() != nil {
+		infos = einterfaces.GetClusterInterface().GetClusterInfos()
+	}
+
+	w.Write([]byte(model.ClusterInfosToJson(infos)))
 }
 
 func getAllAudits(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -150,11 +184,26 @@ func saveConfig(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if *utils.Cfg.ClusterSettings.Enable {
+		c.Err = model.NewLocAppError("saveConfig", "ent.cluster.save_config.error", nil, "")
+		return
+	}
+
 	c.LogAudit("")
 
+	//oldCfg := utils.Cfg
 	utils.SaveConfig(utils.CfgFileName, cfg)
 	utils.LoadConfig(utils.CfgFileName)
 
+	// Future feature is to sync the configuration files
+	// if einterfaces.GetClusterInterface() != nil {
+	// 	err := einterfaces.GetClusterInterface().ConfigChanged(cfg, oldCfg, true)
+	// 	if err != nil {
+	// 		c.Err = err
+	// 		return
+	// 	}
+	// }
+
 	rdata := map[string]string{}
 	rdata["status"] = "OK"
 	w.Write([]byte(model.MapToJson(rdata)))

api/admin_test.go

@@ -25,6 +25,18 @@ func TestGetLogs(t *testing.T) {
 	}
 }
 
+func TestGetClusterInfos(t *testing.T) {
+	th := Setup().InitSystemAdmin().InitBasic()
+
+	if _, err := th.BasicClient.GetClusterStatus(); err == nil {
+		t.Fatal("Shouldn't have permissions")
+	}
+
+	if _, err := th.SystemAdminClient.GetClusterStatus(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestGetAllAudits(t *testing.T) {
 	th := Setup().InitBasic().InitSystemAdmin()
 

api/context.go

@@ -14,13 +14,13 @@ import (
 	"github.com/gorilla/mux"
 	goi18n "github.com/nicksnyder/go-i18n/i18n"
 
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/store"
 	"github.com/mattermost/platform/utils"
 )
 
 var sessionCache *utils.Cache = utils.NewLru(model.SESSION_CACHE_SIZE)
-var statusCache *utils.Cache = utils.NewLru(model.STATUS_CACHE_SIZE)
 
 var allowedMethods []string = []string{
 	"POST",
@@ -148,7 +148,10 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.Header().Set(model.HEADER_REQUEST_ID, c.RequestId)
-	w.Header().Set(model.HEADER_VERSION_ID, fmt.Sprintf("%v.%v", model.CurrentVersion, utils.CfgLastModified))
+	w.Header().Set(model.HEADER_VERSION_ID, fmt.Sprintf("%v.%v", model.CurrentVersion, utils.CfgHash))
+	if einterfaces.GetClusterInterface() != nil {
+		w.Header().Set(model.HEADER_CLUSTER_ID, einterfaces.GetClusterInterface().GetClusterId())
+	}
 
 	// Instruct the browser not to display us in an iframe unless is the same origin for anti-clickjacking
 	if !h.isApi {
@@ -554,6 +557,10 @@ func RemoveAllSessionsForUserId(userId string) {
 			}
 		}
 	}
+
+	if einterfaces.GetClusterInterface() != nil {
+		einterfaces.GetClusterInterface().RemoveAllSessionsForUserId(userId)
+	}
 }
 
 func AddSessionToCache(session *model.Session) {

api/general.go

@@ -69,7 +69,6 @@ func ping(c *Context, w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]string)
 	m["version"] = model.CurrentVersion
 	m["server_time"] = fmt.Sprintf("%v", model.GetMillis())
-	m["node_id"] = ""
 	w.Write([]byte(model.MapToJson(m)))
 }
 

api/status.go

@@ -7,11 +7,23 @@ import (
 	"net/http"
 
 	l4g "github.com/alecthomas/log4go"
+
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/store"
 	"github.com/mattermost/platform/utils"
 )
 
+var statusCache *utils.Cache = utils.NewLru(model.STATUS_CACHE_SIZE)
+
+func AddStatusCache(status *model.Status) {
+	statusCache.Add(status.UserId, status)
+
+	if einterfaces.GetClusterInterface() != nil {
+		einterfaces.GetClusterInterface().UpdateStatus(status)
+	}
+}
+
 func InitStatus() {
 	l4g.Debug(utils.T("api.status.init.debug"))
 
@@ -69,7 +81,7 @@ func SetStatusOnline(userId string, sessionId string) {
 		status.LastActivityAt = model.GetMillis()
 	}
 
-	statusCache.Add(status.UserId, status)
+	AddStatusCache(status)
 
 	achan := Srv.Store.Session().UpdateLastActivityAt(sessionId, model.GetMillis())
 
@@ -98,7 +110,7 @@ func SetStatusOnline(userId string, sessionId string) {
 func SetStatusOffline(userId string) {
 	status := &model.Status{userId, model.STATUS_OFFLINE, model.GetMillis()}
 
-	statusCache.Add(status.UserId, status)
+	AddStatusCache(status)
 
 	if result := <-Srv.Store.Status().SaveOrUpdate(status); result.Err != nil {
 		l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err)
@@ -125,7 +137,7 @@ func SetStatusAwayIfNeeded(userId string) {
 
 	status.Status = model.STATUS_AWAY
 
-	statusCache.Add(status.UserId, status)
+	AddStatusCache(status)
 
 	if result := <-Srv.Store.Status().SaveOrUpdate(status); result.Err != nil {
 		l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err)

api/web_hub.go

@@ -5,6 +5,8 @@ package api
 
 import (
 	l4g "github.com/alecthomas/log4go"
+
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 )
@@ -31,14 +33,30 @@ var hub = &Hub{
 
 func Publish(message *model.WebSocketEvent) {
 	hub.Broadcast(message)
+
+	if einterfaces.GetClusterInterface() != nil {
+		einterfaces.GetClusterInterface().Publish(message)
+	}
+}
+
+func PublishSkipClusterSend(message *model.WebSocketEvent) {
+	hub.Broadcast(message)
 }
 
 func InvalidateCacheForUser(userId string) {
 	hub.invalidateUser <- userId
+
+	if einterfaces.GetClusterInterface() != nil {
+		einterfaces.GetClusterInterface().InvalidateCacheForUser(userId)
+	}
 }
 
 func InvalidateCacheForChannel(channelId string) {
 	hub.invalidateChannel <- channelId
+
+	if einterfaces.GetClusterInterface() != nil {
+		einterfaces.GetClusterInterface().InvalidateCacheForChannel(channelId)
+	}
 }
 
 func (h *Hub) Register(webConn *WebConn) {

config/config.json

@@ -211,5 +211,10 @@
         "AppDownloadLink": "https://about.mattermost.com/downloads/",
         "AndroidAppDownloadLink": "https://about.mattermost.com/mattermost-android-app/",
         "IosAppDownloadLink": "https://about.mattermost.com/mattermost-ios-app/"
+    },
+    "ClusterSettings": {
+        "Enable": false,
+        "InterNodeListenAddress": ":8075",
+        "InterNodeUrls": []
     }
 }

einterfaces/cluster.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package einterfaces
+
+import (
+	"github.com/mattermost/platform/model"
+)
+
+type ClusterInterface interface {
+	StartInterNodeCommunication()
+	StopInterNodeCommunication()
+	GetClusterInfos() []*model.ClusterInfo
+	RemoveAllSessionsForUserId(userId string)
+	InvalidateCacheForUser(userId string)
+	InvalidateCacheForChannel(channelId string)
+	Publish(event *model.WebSocketEvent)
+	UpdateStatus(status *model.Status)
+	GetLogs() ([]string, *model.AppError)
+	GetClusterId() string
+	ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError
+}
+
+var theClusterInterface ClusterInterface
+
+func RegisterClusterInterface(newInterface ClusterInterface) {
+	theClusterInterface = newInterface
+}
+
+func GetClusterInterface() ClusterInterface {
+	return theClusterInterface
+}

i18n/en.json

@@ -2203,6 +2203,10 @@
     "id": "ent.brand.save_brand_image.too_large.app_error",
     "translation": "Unable to open image. Image is too large."
   },
+  {
+    "id": "ent.cluster.licence_disable.app_error",
+    "translation": "Clustering functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license."
+  },
   {
     "id": "ent.compliance.licence_disable.app_error",
     "translation": "Compliance functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license."
@@ -4678,5 +4682,45 @@
   {
     "id": "web.watcher_fail.error",
     "translation": "Failed to add directory to watcher %v"
+  },
+  {
+    "id": "ent.cluster.starting.info",
+    "translation": "Cluster internode communication is listening on %v with hostname=%v id=%v"
+  },
+  {
+    "id": "ent.cluster.save_config.error",
+    "translation": "System Console is set to read-only when High Availability is enabled."
+  },
+  {
+    "id": "ent.cluster.config_changed.info",
+    "translation": "Cluster configuration has changed for id=%v.  Attempting to restart cluster service.  To ensure the cluster is configured correctly you should not rely on this restart because we detected a core configuration change."
+  },
+  {
+    "id": "ent.cluster.stopping.info",
+    "translation": "Cluster internode communication is stopping on %v with hostname=%v id=%v"
+  },
+  {
+    "id": "ent.cluster.ping_failed.info",
+    "translation": "Cluster ping failed with hostname=%v on=%v with id=%v"
+  },
+  {
+    "id": "ent.cluster.ping_success.info",
+    "translation": "Cluster ping successful with hostname=%v on=%v with id=%v self=%v"
+  },
+  {
+    "id": "ent.cluster.incompatibile.warn",
+    "translation": "Potential incompatibile version detected for clustering with %v"
+  },
+  {
+    "id": "ent.cluster.incompatibile_config.warn",
+    "translation": "Potential incompatibile config detected for clustering with %v"
+  },
+  {
+    "id": "ent.cluster.debug_fail.debug",
+    "translation": "Cluster send failed at `%v` detail=%v, extra=%v, retry number=%v"
+  },
+  {
+    "id": "ent.cluster.final_fail.error",
+    "translation": "Cluster send final fail at `%v` detail=%v, extra=%v, retry number=%v"
   }
 ]

mattermost.go

@@ -149,12 +149,20 @@ func main() {
 			complianceI.StartComplianceDailyJob()
 		}
 
+		if einterfaces.GetClusterInterface() != nil {
+			einterfaces.GetClusterInterface().StartInterNodeCommunication()
+		}
+
 		// wait for kill signal before attempting to gracefully shutdown
 		// the running service
 		c := make(chan os.Signal)
 		signal.Notify(c, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
 		<-c
 
+		if einterfaces.GetClusterInterface() != nil {
+			einterfaces.GetClusterInterface().StopInterNodeCommunication()
+		}
+
 		api.StopServer()
 	}
 }

model/client.go

@@ -20,6 +20,7 @@ import (
 const (
 	HEADER_REQUEST_ID         = "X-Request-ID"
 	HEADER_VERSION_ID         = "X-Version-ID"
+	HEADER_CLUSTER_ID         = "X-Cluster-ID"
 	HEADER_ETAG_SERVER        = "ETag"
 	HEADER_ETAG_CLIENT        = "If-None-Match"
 	HEADER_FORWARDED          = "X-Forwarded-For"
@@ -808,6 +809,15 @@ func (c *Client) GetLogs() (*Result, *AppError) {
 	}
 }
 
+func (c *Client) GetClusterStatus() ([]*ClusterInfo, *AppError) {
+	if r, err := c.DoApiGet("/admin/cluster_status", "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return ClusterInfosFromJson(r.Body), nil
+	}
+}
+
 func (c *Client) GetAllAudits() (*Result, *AppError) {
 	if r, err := c.DoApiGet("/admin/audits", "", ""); err != nil {
 		return nil, err

model/cluster_info.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"encoding/json"
+	"io"
+)
+
+type ClusterInfo struct {
+	Id                 string `json:"id"`
+	Version            string `json:"version"`
+	ConfigHash         string `json:"config_hash"`
+	InterNodeUrl       string `json:"internode_url"`
+	Hostname           string `json:"hostname"`
+	LastSuccessfulPing int64  `json:"last_ping"`
+	IsAlive            bool   `json:"is_alive"`
+}
+
+func (me *ClusterInfo) ToJson() string {
+	b, err := json.Marshal(me)
+	if err != nil {
+		return ""
+	} else {
+		return string(b)
+	}
+}
+
+func ClusterInfoFromJson(data io.Reader) *ClusterInfo {
+	decoder := json.NewDecoder(data)
+	var me ClusterInfo
+	err := decoder.Decode(&me)
+	if err == nil {
+		return &me
+	} else {
+		return nil
+	}
+}
+
+func (me *ClusterInfo) HaveEstablishedInitialContact() bool {
+	if me.Id != "" {
+		return true
+	}
+
+	return false
+}
+
+func ClusterInfosToJson(objmap []*ClusterInfo) string {
+	if b, err := json.Marshal(objmap); err != nil {
+		return ""
+	} else {
+		return string(b)
+	}
+}
+
+func ClusterInfosFromJson(data io.Reader) []*ClusterInfo {
+	decoder := json.NewDecoder(data)
+
+	var objmap []*ClusterInfo
+	if err := decoder.Decode(&objmap); err != nil {
+		return make([]*ClusterInfo, 0)
+	} else {
+		return objmap
+	}
+}

model/cluster_info_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestClusterInfoJson(t *testing.T) {
+	cluster := ClusterInfo{Id: NewId(), InterNodeUrl: NewId(), Hostname: NewId()}
+	json := cluster.ToJson()
+	result := ClusterInfoFromJson(strings.NewReader(json))
+
+	if cluster.Id != result.Id {
+		t.Fatal("Ids do not match")
+	}
+}
+
+func TestClusterInfosJson(t *testing.T) {
+	cluster := ClusterInfo{Id: NewId(), InterNodeUrl: NewId(), Hostname: NewId()}
+	clusterInfos := make([]*ClusterInfo, 1)
+	clusterInfos[0] = &cluster
+	json := ClusterInfosToJson(clusterInfos)
+	result := ClusterInfosFromJson(strings.NewReader(json))
+
+	if clusterInfos[0].Id != result[0].Id {
+		t.Fatal("Ids do not match")
+	}
+
+}

model/config.go

@@ -80,6 +80,12 @@ type ServiceSettings struct {
 	RestrictCustomEmojiCreation       *string
 }
 
+type ClusterSettings struct {
+	Enable                 *bool
+	InterNodeListenAddress *string
+	InterNodeUrls          []string
+}
+
 type SSOSettings struct {
 	Enable          bool
 	Secret          string
@@ -297,6 +303,7 @@ type Config struct {
 	LocalizationSettings LocalizationSettings
 	SamlSettings         SamlSettings
 	NativeAppSettings    NativeAppSettings
+	ClusterSettings      ClusterSettings
 }
 
 func (o *Config) ToJson() string {
@@ -707,6 +714,20 @@ func (o *Config) SetDefaults() {
 		*o.ServiceSettings.RestrictCustomEmojiCreation = RESTRICT_EMOJI_CREATION_ALL
 	}
 
+	if o.ClusterSettings.InterNodeListenAddress == nil {
+		o.ClusterSettings.InterNodeListenAddress = new(string)
+		*o.ClusterSettings.InterNodeListenAddress = ":8075"
+	}
+
+	if o.ClusterSettings.Enable == nil {
+		o.ClusterSettings.Enable = new(bool)
+		*o.ClusterSettings.Enable = false
+	}
+
+	if o.ClusterSettings.InterNodeUrls == nil {
+		o.ClusterSettings.InterNodeUrls = []string{}
+	}
+
 	if o.ComplianceSettings.Enable == nil {
 		o.ComplianceSettings.Enable = new(bool)
 		*o.ComplianceSettings.Enable = false

model/license.go

@@ -38,6 +38,7 @@ type Features struct {
 	GoogleSSO            *bool `json:"google_sso"`
 	Office365SSO         *bool `json:"office365_sso"`
 	Compliance           *bool `json:"compliance"`
+	Cluster              *bool `json:"cluster"`
 	CustomBrand          *bool `json:"custom_brand"`
 	MHPNS                *bool `json:"mhpns"`
 	SAML                 *bool `json:"saml"`
@@ -81,6 +82,11 @@ func (f *Features) SetDefaults() {
 		*f.Compliance = *f.FutureFeatures
 	}
 
+	if f.Cluster == nil {
+		f.Cluster = new(bool)
+		*f.Cluster = *f.FutureFeatures
+	}
+
 	if f.CustomBrand == nil {
 		f.CustomBrand = new(bool)
 		*f.CustomBrand = *f.FutureFeatures

store/sql_user_store.go

@@ -502,7 +502,7 @@ func (s SqlUserStore) GetEtagForDirectProfiles(userId string) StoreChannel {
 			result.Data = fmt.Sprintf("%v.%v.0.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
 		} else {
 			allIds := strings.Join(ids, "")
-			result.Data = fmt.Sprintf("%v.%v.%v.%v.%v", model.CurrentVersion, md5.Sum([]byte(allIds)), len(ids), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
+			result.Data = fmt.Sprintf("%v.%x.%v.%v.%v", model.CurrentVersion, md5.Sum([]byte(allIds)), len(ids), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
 		}
 
 		storeChannel <- result

utils/config.go

@@ -4,6 +4,7 @@
 package utils
 
 import (
+	"crypto/md5"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -26,7 +27,7 @@ const (
 
 var Cfg *model.Config = &model.Config{}
 var CfgDiagnosticId = ""
-var CfgLastModified int64 = 0
+var CfgHash = ""
 var CfgFileName string = ""
 var ClientCfg map[string]string = map[string]string{}
 
@@ -157,11 +158,10 @@ func LoadConfig(fileName string) {
 			map[string]interface{}{"Filename": fileName, "Error": err.Error()}))
 	}
 
-	if info, err := file.Stat(); err != nil {
+	if _, err := file.Stat(); err != nil {
 		panic(T("utils.config.load_config.getting.panic",
 			map[string]interface{}{"Filename": fileName, "Error": err.Error()}))
 	} else {
-		CfgLastModified = info.ModTime().Unix()
 		CfgFileName = fileName
 	}
 
@@ -185,6 +185,7 @@ func LoadConfig(fileName string) {
 	}
 
 	Cfg = &config
+	CfgHash = fmt.Sprintf("%x", md5.Sum([]byte(Cfg.ToJson())))
 	RegenerateClientConfig()
 
 	// Actions that need to run every time the config is loaded
@@ -298,6 +299,10 @@ func getClientConfig(c *model.Config) map[string]string {
 			props["SamlLoginButtonText"] = *c.SamlSettings.LoginButtonText
 		}
 
+		if *License.Features.Cluster {
+			props["EnableCluster"] = strconv.FormatBool(*c.ClusterSettings.Enable)
+		}
+
 		if *License.Features.GoogleSSO {
 			props["EnableSignUpWithGoogle"] = strconv.FormatBool(c.GoogleSettings.Enable)
 		}

utils/license.go

@@ -122,6 +122,7 @@ func getClientLicense(l *model.License) map[string]string {
 		props["LDAP"] = strconv.FormatBool(*l.Features.LDAP)
 		props["MFA"] = strconv.FormatBool(*l.Features.MFA)
 		props["SAML"] = strconv.FormatBool(*l.Features.SAML)
+		props["Cluster"] = strconv.FormatBool(*l.Features.Cluster)
 		props["GoogleSSO"] = strconv.FormatBool(*l.Features.GoogleSSO)
 		props["Office365SSO"] = strconv.FormatBool(*l.Features.Office365SSO)
 		props["Compliance"] = strconv.FormatBool(*l.Features.Compliance)

webapp/client/client.jsx

@@ -4,6 +4,7 @@
 import request from 'superagent';
 
 const HEADER_X_VERSION_ID = 'x-version-id';
+const HEADER_X_CLUSTER_ID = 'x-cluster-id';
 const HEADER_TOKEN = 'token';
 const HEADER_BEARER = 'BEARER';
 const HEADER_AUTH = 'Authorization';
@@ -12,6 +13,7 @@ export default class Client {
     constructor() {
         this.teamId = '';
         this.serverVersion = '';
+        this.clusterId = '';
         this.logToConsole = false;
         this.useToken = false;
         this.token = '';
@@ -152,6 +154,11 @@ export default class Client {
             if (res.header[HEADER_X_VERSION_ID]) {
                 this.serverVersion = res.header[HEADER_X_VERSION_ID];
             }
+
+            this.clusterId = res.header[HEADER_X_CLUSTER_ID];
+            if (res.header[HEADER_X_CLUSTER_ID]) {
+                this.clusterId = res.header[HEADER_X_CLUSTER_ID];
+            }
         }
 
         if (err) {
@@ -295,6 +302,15 @@ export default class Client {
             end(this.handleResponse.bind(this, 'getLogs', success, error));
     }
 
+    getClusterStatus(success, error) {
+        return request.
+            get(`${this.getAdminRoute()}/cluster_status`).
+            set(this.defaultHeaders).
+            type('application/json').
+            accept('application/json').
+            end(this.handleResponse.bind(this, 'getClusterStatus', success, error));
+    }
+
     getServerAudits(success, error) {
         return request.
             get(`${this.getAdminRoute()}/audits`).

webapp/components/admin_console/admin_sidebar.jsx

@@ -178,6 +178,7 @@ export default class AdminSidebar extends React.Component {
         let oauthSettings = null;
         let ldapSettings = null;
         let samlSettings = null;
+        let clusterSettings = null;
         let complianceSettings = null;
 
         let license = null;
@@ -213,6 +214,20 @@ export default class AdminSidebar extends React.Component {
                 );
             }
 
+            if (global.window.mm_license.Cluster === 'true') {
+                clusterSettings = (
+                    <AdminSidebarSection
+                        name='cluster'
+                        title={
+                            <FormattedMessage
+                                id='admin.sidebar.cluster'
+                                defaultMessage='High Availability'
+                            />
+                        }
+                    />
+                );
+            }
+
             if (global.window.mm_license.SAML === 'true') {
                 samlSettings = (
                     <AdminSidebarSection
@@ -656,6 +671,7 @@ export default class AdminSidebar extends React.Component {
                                         />
                                     }
                                 />
+                                {clusterSettings}
                             </AdminSidebarSection>
                         </AdminSidebarCategory>
                         {this.renderTeams()}

webapp/components/admin_console/cluster_settings.jsx

@@ -0,0 +1,188 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+
+import AdminSettings from './admin_settings.jsx';
+import BooleanSetting from './boolean_setting.jsx';
+import TextSetting from './text_setting.jsx';
+
+import {FormattedMessage, FormattedHTMLMessage} from 'react-intl';
+import SettingsGroup from './settings_group.jsx';
+import ClusterTableContainer from './cluster_table_container.jsx';
+
+import AdminStore from 'stores/admin_store.jsx';
+import * as Utils from 'utils/utils.jsx';
+
+export default class ClusterSettings extends AdminSettings {
+    constructor(props) {
+        super(props);
+
+        this.getConfigFromState = this.getConfigFromState.bind(this);
+        this.renderSettings = this.renderSettings.bind(this);
+    }
+
+    getConfigFromState(config) {
+        config.ClusterSettings.Enable = this.state.enable;
+        config.ClusterSettings.InterNodeListenAddress = this.state.interNodeListenAddress;
+
+        config.ClusterSettings.InterNodeUrls = this.state.interNodeUrls.split(',');
+        config.ClusterSettings.InterNodeUrls = config.ClusterSettings.InterNodeUrls.map((url) => {
+            return url.trim();
+        });
+
+        if (config.ClusterSettings.InterNodeUrls.length === 1 && config.ClusterSettings.InterNodeUrls[0] === '') {
+            config.ClusterSettings.InterNodeUrls = [];
+        }
+
+        return config;
+    }
+
+    getStateFromConfig(config) {
+        const settings = config.ClusterSettings;
+
+        return {
+            enable: settings.Enable,
+            interNodeUrls: settings.InterNodeUrls.join(', '),
+            interNodeListenAddress: settings.InterNodeListenAddress,
+            showWarning: false
+        };
+    }
+
+    renderTitle() {
+        return (
+            <h3>
+                <FormattedMessage
+                    id='admin.advance.cluster'
+                    defaultMessage='High Availability'
+                />
+            </h3>
+        );
+    }
+
+    overrideHandleChange = (id, value) => {
+        this.setState({
+            showWarning: true
+        });
+
+        this.handleChange(id, value);
+    }
+
+    renderSettings() {
+        const licenseEnabled = global.window.mm_license.IsLicensed === 'true' && global.window.mm_license.Cluster === 'true';
+        if (!licenseEnabled) {
+            return null;
+        }
+
+        var configLoadedFromCluster = null;
+
+        if (AdminStore.getClusterId()) {
+            configLoadedFromCluster = (
+                <div
+                    style={{marginBottom: '10px'}}
+                    className='alert alert-warning'
+                >
+                    <i className='fa fa-warning'></i>
+                    <FormattedHTMLMessage
+                        id='admin.cluster.loadedFrom'
+                        defaultMessage='This configuration file was loaded from Node ID {clusterId}. Please see the Troubleshooting Guide in our <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> if you are accessing the System Console through a load balancer and experiencing issues.'
+                        values={{
+                            clusterId: AdminStore.getClusterId()
+                        }}
+                    />
+                </div>
+            );
+        }
+
+        var warning = null;
+        if (this.state.showWarning) {
+            warning = (
+                <div
+                    style={{marginBottom: '10px'}}
+                    className='alert alert-warning'
+                >
+                    <i className='fa fa-warning'></i>
+                    <FormattedMessage
+                        id='admin.cluster.should_not_change'
+                        defaultMessage='WARNING: These settings may not sync with the other servers in the cluster. High Availability inter-node communication will not start until you modify the config.json to be identical on all servers and restart Mattermost. Please see the <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> on how to add or remove a server from the cluster. If you are accessing the System Console through a load balancer and experiencing issues, please see the Troubleshooting Guide in our <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a>.'
+                    />
+                </div>
+            );
+        }
+
+        var clusterTableContainer = null;
+        if (this.state.enable) {
+            clusterTableContainer = (<ClusterTableContainer/>);
+        }
+
+        return (
+            <SettingsGroup>
+                {configLoadedFromCluster}
+                {clusterTableContainer}
+                <p>
+                    <FormattedMessage
+                        id='admin.cluster.noteDescription'
+                        defaultMessage='Changing properties in this section will require a server restart before taking effect. When High Availability mode is enabled, the System Console is set to read-only and can only be changed from the configuration file.'
+                    />
+                </p>
+                {warning}
+                <BooleanSetting
+                    id='enable'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.enableTitle'
+                            defaultMessage='Enable High Availability Mode:'
+                        />
+                    }
+                    helpText={
+                        <FormattedHTMLMessage
+                            id='admin.cluster.enableDescription'
+                            defaultMessage='When true, Mattermost will run in High Availability mode. Please see <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> to learn more about configuring High Availability for Mattermost.'
+                        />
+                    }
+                    value={this.state.enable}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+                <TextSetting
+                    id='interNodeListenAddress'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeListenAddressTitle'
+                            defaultMessage='Inter-Node Listen Address:'
+                        />
+                    }
+                    placeholder={Utils.localizeMessage('admin.cluster.interNodeListenAddressEx', 'Ex ":8075"')}
+                    helpText={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeListenAddressDesc'
+                            defaultMessage='The address the server will listen on for communicating with other servers.'
+                        />
+                    }
+                    value={this.state.interNodeListenAddress}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+                <TextSetting
+                    id='interNodeUrls'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeUrlsTitle'
+                            defaultMessage='Inter-Node URLs:'
+                        />
+                    }
+                    placeholder={Utils.localizeMessage('admin.cluster.interNodeUrlsEx', 'Ex "http://10.10.10.30, http://10.10.10.31"')}
+                    helpText={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeUrlsDesc'
+                            defaultMessage='The internal/private URLs of all the Mattermost servers separated by commas.'
+                        />
+                    }
+                    value={this.state.interNodeUrls}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+            </SettingsGroup>
+        );
+    }
+}

webapp/components/admin_console/cluster_table.jsx

@@ -0,0 +1,179 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+
+import {FormattedMessage} from 'react-intl';
+import * as Utils from 'utils/utils.jsx';
+
+import statusGreen from 'images/status_green.png';
+import statusRed from 'images/status_red.png';
+
+export default class ClusterTable extends React.Component {
+    static propTypes = {
+        clusterInfos: React.PropTypes.array.isRequired,
+        reload: React.PropTypes.func.isRequired
+    }
+
+    render() {
+        var versionMismatch = (
+            <img
+                className='cluster-status'
+                src={statusGreen}
+            />
+        );
+
+        var configMismatch = (
+            <img
+                className='cluster-status'
+                src={statusGreen}
+            />
+        );
+
+        var version = '';
+        var configHash = '';
+
+        if (this.props.clusterInfos.length) {
+            version = this.props.clusterInfos[0].version;
+            configHash = this.props.clusterInfos[0].config_hash;
+        }
+
+        this.props.clusterInfos.map((clusterInfo) => {
+            if (clusterInfo.version !== version) {
+                versionMismatch = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            if (clusterInfo.config_hash !== configHash) {
+                configMismatch = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            return null;
+        });
+
+        var items = this.props.clusterInfos.map((clusterInfo) => {
+            var status = null;
+
+            if (clusterInfo.hostname === '') {
+                clusterInfo.hostname = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.version === '') {
+                clusterInfo.version = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.config_hash === '') {
+                clusterInfo.config_hash = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.id === '') {
+                clusterInfo.id = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.is_alive) {
+                status = (
+                    <img
+                        className='cluster-status'
+                        src={statusGreen}
+                    />
+                );
+            } else {
+                status = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            return (
+                <tr key={clusterInfo.id}>
+                    <td style={{whiteSpace: 'nowrap'}}>{status}</td>
+                    <td style={{whiteSpace: 'nowrap'}}>{clusterInfo.hostname}</td>
+                    <td style={{whiteSpace: 'nowrap'}}>{versionMismatch} {clusterInfo.version}</td>
+                    <td style={{whiteSpace: 'nowrap'}}><div className='config-hash'>{configMismatch} {clusterInfo.config_hash}</div></td>
+                    <td style={{whiteSpace: 'nowrap'}}>{clusterInfo.internode_url}</td>
+                    <td style={{whiteSpace: 'nowrap'}}><div className='config-hash'>{clusterInfo.id}</div></td>
+                </tr>
+            );
+        });
+
+        return (
+            <div
+                className='cluster-panel__table'
+                style={{
+                    margin: '10px',
+                    marginBottom: '30px'
+                }}
+            >
+                <div className='text-right'>
+                    <button
+                        type='submit'
+                        className='btn btn-link'
+                        onClick={this.props.reload}
+                    >
+                        <i className='fa fa-refresh'></i>
+                        <FormattedMessage
+                            id='admin.cluster.status_table.reload'
+                            defaultMessage=' Reload Cluster Status'
+                        />
+                    </button>
+                </div>
+                <table className='table'>
+                    <thead>
+                        <tr>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.status'
+                                    defaultMessage='Status'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.hostname'
+                                    defaultMessage='Hostname'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.version'
+                                    defaultMessage='Version'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.config_hash'
+                                    defaultMessage='Config File MD5'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.url'
+                                    defaultMessage='Inter-Node URL'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.id'
+                                    defaultMessage='Node ID'
+                                />
+                            </th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {items}
+                    </tbody>
+                </table>
+            </div>
+        );
+    }
+}

webapp/components/admin_console/cluster_table_container.jsx

@@ -0,0 +1,71 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+import ClusterTable from './cluster_table.jsx';
+import LoadingScreen from '../loading_screen.jsx';
+import Client from 'client/web_client.jsx';
+import * as AsyncClient from 'utils/async_client.jsx';
+
+export default class ClusterTableContainer extends React.Component {
+    constructor(props) {
+        super(props);
+
+        this.interval = null;
+
+        this.state = {
+            clusterInfos: null
+        };
+    }
+
+    load = () => {
+        Client.getClusterStatus(
+            (data) => {
+                this.setState({
+                    clusterInfos: data
+                });
+            },
+            (err) => {
+                AsyncClient.dispatchError(err, 'getClusterStatus');
+            }
+        );
+    }
+
+    componentWillMount() {
+        this.load();
+
+        // reload the cluster status every 15 seconds
+        this.interval = setInterval(this.load, 15000);
+    }
+
+    componentWillUnmount() {
+        if (this.interval) {
+            clearInterval(this.interval);
+        }
+    }
+
+    reload = (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+
+        this.setState({
+            clusterInfos: null
+        });
+
+        this.load();
+    }
+
+    render() {
+        if (this.state.clusterInfos == null) {
+            return (<LoadingScreen/>);
+        }
+
+        return (
+            <ClusterTable
+                clusterInfos={this.state.clusterInfos}
+                reload={this.reload}
+            />
+        );
+    }
+}

webapp/i18n/en.json

@@ -569,6 +569,24 @@
   "admin.saml.usernameAttrTitle": "Username Attribute:",
   "admin.saml.verifyDescription": "When true, Mattermost verifies that the signature sent from the SAML Response matches the Service Provider Login URL",
   "admin.saml.verifyTitle": "Verify Signature:",
+  "admin.cluster.loadedFrom": "This configuration file was loaded from Node ID {clusterId}. Please see the Troubleshooting Guide in our <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> if you are accessing the System Console through a load balancer and experiencing issues.",
+  "admin.cluster.should_not_change": "WARNING: These settings may not sync with the other servers in the cluster. High Availability inter-node communication will not start until you modify the config.json to be identical on all servers and restart Mattermost. Please see the <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> on how to add or remove a server from the cluster. If you are accessing the System Console through a load balancer and experiencing issues, please see the Troubleshooting Guide in our <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a>.",
+  "admin.cluster.noteDescription": "Changing properties in this section will require a server restart before taking effect. When High Availability mode is enabled, the System Console is set to read-only and can only be changed from the configuration file.",
+  "admin.cluster.enableTitle": "Enable High Availability Mode:",
+  "admin.cluster.enableDescription": "When true, Mattermost will run in High Availability mode. Please see <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> to learn more about configuring High Availability for Mattermost.",
+  "admin.cluster.interNodeListenAddressTitle": "Inter-Node Listen Address:",
+  "admin.cluster.interNodeListenAddressEx": "Ex \":8075\"",
+  "admin.cluster.interNodeListenAddressDesc": "The address the server will listen on for communicating with other servers.",
+  "admin.cluster.interNodeUrlsTitle": "Inter-Node URLs:",
+  "admin.cluster.interNodeUrlsEx": "Ex \"http://10.10.10.30, http://10.10.10.31\"",
+  "admin.cluster.interNodeUrlsDesc": "The internal/private URLs of all the Mattermost servers separated by commas.",
+  "admin.cluster.status_table.reload": " Reload Cluster Status",
+  "admin.cluster.status_table.status": "Status",
+  "admin.cluster.status_table.hostname": "Hostname",
+  "admin.cluster.status_table.version": "Version",
+  "admin.cluster.status_table.config_hash": "Config File MD5",
+  "admin.cluster.status_table.url": "Inter-Node URL",
+  "admin.cluster.status_table.id": "Node ID",
   "admin.save": "Save",
   "admin.saving": "Saving Config...",
   "admin.security.connection": "Connections",
@@ -668,6 +686,7 @@
   "admin.sidebar.reports": "REPORTING",
   "admin.sidebar.rmTeamSidebar": "Remove team from sidebar menu",
   "admin.sidebar.saml": "SAML",
+  "admin.sidebar.cluster": "High Availability",
   "admin.sidebar.security": "Security",
   "admin.sidebar.sessions": "Sessions",
   "admin.sidebar.settings": "SETTINGS",

webapp/routes/route_admin_console.jsx

@@ -17,6 +17,7 @@ import GitLabSettings from 'components/admin_console/gitlab_settings.jsx';
 import OAuthSettings from 'components/admin_console/oauth_settings.jsx';
 import LdapSettings from 'components/admin_console/ldap_settings.jsx';
 import SamlSettings from 'components/admin_console/saml_settings.jsx';
+import ClusterSettings from 'components/admin_console/cluster_settings.jsx';
 import SignupSettings from 'components/admin_console/signup_settings.jsx';
 import PasswordSettings from 'components/admin_console/password_settings.jsx';
 import PublicLinkSettings from 'components/admin_console/public_link_settings.jsx';
@@ -191,6 +192,10 @@ export default (
                 path='developer'
                 component={DeveloperSettings}
             />
+            <Route
+                path='cluster'
+                component={ClusterSettings}
+            />
         </Route>
         <Route path='team'>
             <Redirect

webapp/sass/routes/_admin-console.scss

@@ -432,3 +432,16 @@
 .recycle-db {
     margin-top: 50px !important;
 }
+
+.cluster-status {
+    width: 24px;
+    height: 24px;
+}
+
+.config-hash {
+    width: 130px;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}
+

webapp/sass/routes/_compliance.scss

@@ -1,7 +1,8 @@
 @charset 'UTF-8';
 
 .compliance-panel__table,
-.audit-panel__table {
+.audit-panel__table,
+.cluster-panel__table {
     background-color: $white;
     border: 1px solid $border-gray;
     margin-top: 10px;

webapp/stores/admin_store.jsx

@@ -22,6 +22,7 @@ class AdminStoreClass extends EventEmitter {
         this.logs = null;
         this.audits = null;
         this.config = null;
+        this.clusterId = null;
         this.teams = {};
         this.complianceReports = null;
     }
@@ -86,6 +87,14 @@ class AdminStoreClass extends EventEmitter {
         this.removeListener(ALL_TEAMS_EVENT, callback);
     }
 
+    getClusterId() {
+        return this.clusterId;
+    }
+
+    saveClusterId(clusterId) {
+        this.clusterId = clusterId;
+    }
+
     getLogs() {
         return this.logs;
     }
@@ -163,6 +172,7 @@ AdminStoreClass.dispatchToken = AppDispatcher.register((payload) => {
         break;
     case ActionTypes.RECEIVED_CONFIG:
         AdminStore.saveConfig(action.config);
+        AdminStore.saveClusterId(action.clusterId);
         AdminStore.emitConfigChange();
         break;
     case ActionTypes.RECEIVED_ALL_TEAMS:

webapp/utils/async_client.jsx

@@ -453,7 +453,8 @@ export function getConfig(success, error) {
 
             AppDispatcher.handleServerAction({
                 type: ActionTypes.RECEIVED_CONFIG,
-                config: data
+                config: data,
+                clusterId: Client.clusterId
             });
 
             if (success) {