PLT-2899 adding clustering of app servers (#3682)

* PLT-2899 adding clustering of app servers

* PLT-2899 base framework

* PLT-2899 HA backend

* PLT-2899 Fixing config file

* PLT-2899 adding config syncing

* PLT-2899 set System console to readonly when clustering enabled.

* PLT-2899 Fixing publish API

* PLT-2899 fixing strings

webapp/client/client.jsx

@@ -4,6 +4,7 @@
 import request from 'superagent';
 
 const HEADER_X_VERSION_ID = 'x-version-id';
+const HEADER_X_CLUSTER_ID = 'x-cluster-id';
 const HEADER_TOKEN = 'token';
 const HEADER_BEARER = 'BEARER';
 const HEADER_AUTH = 'Authorization';
@@ -12,6 +13,7 @@ export default class Client {
     constructor() {
         this.teamId = '';
         this.serverVersion = '';
+        this.clusterId = '';
         this.logToConsole = false;
         this.useToken = false;
         this.token = '';
@@ -152,6 +154,11 @@ export default class Client {
             if (res.header[HEADER_X_VERSION_ID]) {
                 this.serverVersion = res.header[HEADER_X_VERSION_ID];
             }
+
+            this.clusterId = res.header[HEADER_X_CLUSTER_ID];
+            if (res.header[HEADER_X_CLUSTER_ID]) {
+                this.clusterId = res.header[HEADER_X_CLUSTER_ID];
+            }
         }
 
         if (err) {
@@ -295,6 +302,15 @@ export default class Client {
             end(this.handleResponse.bind(this, 'getLogs', success, error));
     }
 
+    getClusterStatus(success, error) {
+        return request.
+            get(`${this.getAdminRoute()}/cluster_status`).
+            set(this.defaultHeaders).
+            type('application/json').
+            accept('application/json').
+            end(this.handleResponse.bind(this, 'getClusterStatus', success, error));
+    }
+
     getServerAudits(success, error) {
         return request.
             get(`${this.getAdminRoute()}/audits`).

webapp/components/admin_console/admin_sidebar.jsx

@@ -178,6 +178,7 @@ export default class AdminSidebar extends React.Component {
         let oauthSettings = null;
         let ldapSettings = null;
         let samlSettings = null;
+        let clusterSettings = null;
         let complianceSettings = null;
 
         let license = null;
@@ -213,6 +214,20 @@ export default class AdminSidebar extends React.Component {
                 );
             }
 
+            if (global.window.mm_license.Cluster === 'true') {
+                clusterSettings = (
+                    <AdminSidebarSection
+                        name='cluster'
+                        title={
+                            <FormattedMessage
+                                id='admin.sidebar.cluster'
+                                defaultMessage='High Availability'
+                            />
+                        }
+                    />
+                );
+            }
+
             if (global.window.mm_license.SAML === 'true') {
                 samlSettings = (
                     <AdminSidebarSection
@@ -656,6 +671,7 @@ export default class AdminSidebar extends React.Component {
                                         />
                                     }
                                 />
+                                {clusterSettings}
                             </AdminSidebarSection>
                         </AdminSidebarCategory>
                         {this.renderTeams()}

webapp/components/admin_console/cluster_settings.jsx

@@ -0,0 +1,188 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+
+import AdminSettings from './admin_settings.jsx';
+import BooleanSetting from './boolean_setting.jsx';
+import TextSetting from './text_setting.jsx';
+
+import {FormattedMessage, FormattedHTMLMessage} from 'react-intl';
+import SettingsGroup from './settings_group.jsx';
+import ClusterTableContainer from './cluster_table_container.jsx';
+
+import AdminStore from 'stores/admin_store.jsx';
+import * as Utils from 'utils/utils.jsx';
+
+export default class ClusterSettings extends AdminSettings {
+    constructor(props) {
+        super(props);
+
+        this.getConfigFromState = this.getConfigFromState.bind(this);
+        this.renderSettings = this.renderSettings.bind(this);
+    }
+
+    getConfigFromState(config) {
+        config.ClusterSettings.Enable = this.state.enable;
+        config.ClusterSettings.InterNodeListenAddress = this.state.interNodeListenAddress;
+
+        config.ClusterSettings.InterNodeUrls = this.state.interNodeUrls.split(',');
+        config.ClusterSettings.InterNodeUrls = config.ClusterSettings.InterNodeUrls.map((url) => {
+            return url.trim();
+        });
+
+        if (config.ClusterSettings.InterNodeUrls.length === 1 && config.ClusterSettings.InterNodeUrls[0] === '') {
+            config.ClusterSettings.InterNodeUrls = [];
+        }
+
+        return config;
+    }
+
+    getStateFromConfig(config) {
+        const settings = config.ClusterSettings;
+
+        return {
+            enable: settings.Enable,
+            interNodeUrls: settings.InterNodeUrls.join(', '),
+            interNodeListenAddress: settings.InterNodeListenAddress,
+            showWarning: false
+        };
+    }
+
+    renderTitle() {
+        return (
+            <h3>
+                <FormattedMessage
+                    id='admin.advance.cluster'
+                    defaultMessage='High Availability'
+                />
+            </h3>
+        );
+    }
+
+    overrideHandleChange = (id, value) => {
+        this.setState({
+            showWarning: true
+        });
+
+        this.handleChange(id, value);
+    }
+
+    renderSettings() {
+        const licenseEnabled = global.window.mm_license.IsLicensed === 'true' && global.window.mm_license.Cluster === 'true';
+        if (!licenseEnabled) {
+            return null;
+        }
+
+        var configLoadedFromCluster = null;
+
+        if (AdminStore.getClusterId()) {
+            configLoadedFromCluster = (
+                <div
+                    style={{marginBottom: '10px'}}
+                    className='alert alert-warning'
+                >
+                    <i className='fa fa-warning'></i>
+                    <FormattedHTMLMessage
+                        id='admin.cluster.loadedFrom'
+                        defaultMessage='This configuration file was loaded from Node ID {clusterId}. Please see the Troubleshooting Guide in our <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> if you are accessing the System Console through a load balancer and experiencing issues.'
+                        values={{
+                            clusterId: AdminStore.getClusterId()
+                        }}
+                    />
+                </div>
+            );
+        }
+
+        var warning = null;
+        if (this.state.showWarning) {
+            warning = (
+                <div
+                    style={{marginBottom: '10px'}}
+                    className='alert alert-warning'
+                >
+                    <i className='fa fa-warning'></i>
+                    <FormattedMessage
+                        id='admin.cluster.should_not_change'
+                        defaultMessage='WARNING: These settings may not sync with the other servers in the cluster. High Availability inter-node communication will not start until you modify the config.json to be identical on all servers and restart Mattermost. Please see the <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> on how to add or remove a server from the cluster. If you are accessing the System Console through a load balancer and experiencing issues, please see the Troubleshooting Guide in our <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a>.'
+                    />
+                </div>
+            );
+        }
+
+        var clusterTableContainer = null;
+        if (this.state.enable) {
+            clusterTableContainer = (<ClusterTableContainer/>);
+        }
+
+        return (
+            <SettingsGroup>
+                {configLoadedFromCluster}
+                {clusterTableContainer}
+                <p>
+                    <FormattedMessage
+                        id='admin.cluster.noteDescription'
+                        defaultMessage='Changing properties in this section will require a server restart before taking effect. When High Availability mode is enabled, the System Console is set to read-only and can only be changed from the configuration file.'
+                    />
+                </p>
+                {warning}
+                <BooleanSetting
+                    id='enable'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.enableTitle'
+                            defaultMessage='Enable High Availability Mode:'
+                        />
+                    }
+                    helpText={
+                        <FormattedHTMLMessage
+                            id='admin.cluster.enableDescription'
+                            defaultMessage='When true, Mattermost will run in High Availability mode. Please see <a href="http://docs.mattermost.com/deployment/cluster.html" target="_blank">documentation</a> to learn more about configuring High Availability for Mattermost.'
+                        />
+                    }
+                    value={this.state.enable}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+                <TextSetting
+                    id='interNodeListenAddress'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeListenAddressTitle'
+                            defaultMessage='Inter-Node Listen Address:'
+                        />
+                    }
+                    placeholder={Utils.localizeMessage('admin.cluster.interNodeListenAddressEx', 'Ex ":8075"')}
+                    helpText={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeListenAddressDesc'
+                            defaultMessage='The address the server will listen on for communicating with other servers.'
+                        />
+                    }
+                    value={this.state.interNodeListenAddress}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+                <TextSetting
+                    id='interNodeUrls'
+                    label={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeUrlsTitle'
+                            defaultMessage='Inter-Node URLs:'
+                        />
+                    }
+                    placeholder={Utils.localizeMessage('admin.cluster.interNodeUrlsEx', 'Ex "http://10.10.10.30, http://10.10.10.31"')}
+                    helpText={
+                        <FormattedMessage
+                            id='admin.cluster.interNodeUrlsDesc'
+                            defaultMessage='The internal/private URLs of all the Mattermost servers separated by commas.'
+                        />
+                    }
+                    value={this.state.interNodeUrls}
+                    onChange={this.overrideHandleChange}
+                    disabled={true}
+                />
+            </SettingsGroup>
+        );
+    }
+}

webapp/components/admin_console/cluster_table.jsx

@@ -0,0 +1,179 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+
+import {FormattedMessage} from 'react-intl';
+import * as Utils from 'utils/utils.jsx';
+
+import statusGreen from 'images/status_green.png';
+import statusRed from 'images/status_red.png';
+
+export default class ClusterTable extends React.Component {
+    static propTypes = {
+        clusterInfos: React.PropTypes.array.isRequired,
+        reload: React.PropTypes.func.isRequired
+    }
+
+    render() {
+        var versionMismatch = (
+            <img
+                className='cluster-status'
+                src={statusGreen}
+            />
+        );
+
+        var configMismatch = (
+            <img
+                className='cluster-status'
+                src={statusGreen}
+            />
+        );
+
+        var version = '';
+        var configHash = '';
+
+        if (this.props.clusterInfos.length) {
+            version = this.props.clusterInfos[0].version;
+            configHash = this.props.clusterInfos[0].config_hash;
+        }
+
+        this.props.clusterInfos.map((clusterInfo) => {
+            if (clusterInfo.version !== version) {
+                versionMismatch = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            if (clusterInfo.config_hash !== configHash) {
+                configMismatch = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            return null;
+        });
+
+        var items = this.props.clusterInfos.map((clusterInfo) => {
+            var status = null;
+
+            if (clusterInfo.hostname === '') {
+                clusterInfo.hostname = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.version === '') {
+                clusterInfo.version = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.config_hash === '') {
+                clusterInfo.config_hash = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.id === '') {
+                clusterInfo.id = Utils.localizeMessage('admin.cluster.unknown', 'unknown');
+            }
+
+            if (clusterInfo.is_alive) {
+                status = (
+                    <img
+                        className='cluster-status'
+                        src={statusGreen}
+                    />
+                );
+            } else {
+                status = (
+                    <img
+                        className='cluster-status'
+                        src={statusRed}
+                    />
+                );
+            }
+
+            return (
+                <tr key={clusterInfo.id}>
+                    <td style={{whiteSpace: 'nowrap'}}>{status}</td>
+                    <td style={{whiteSpace: 'nowrap'}}>{clusterInfo.hostname}</td>
+                    <td style={{whiteSpace: 'nowrap'}}>{versionMismatch} {clusterInfo.version}</td>
+                    <td style={{whiteSpace: 'nowrap'}}><div className='config-hash'>{configMismatch} {clusterInfo.config_hash}</div></td>
+                    <td style={{whiteSpace: 'nowrap'}}>{clusterInfo.internode_url}</td>
+                    <td style={{whiteSpace: 'nowrap'}}><div className='config-hash'>{clusterInfo.id}</div></td>
+                </tr>
+            );
+        });
+
+        return (
+            <div
+                className='cluster-panel__table'
+                style={{
+                    margin: '10px',
+                    marginBottom: '30px'
+                }}
+            >
+                <div className='text-right'>
+                    <button
+                        type='submit'
+                        className='btn btn-link'
+                        onClick={this.props.reload}
+                    >
+                        <i className='fa fa-refresh'></i>
+                        <FormattedMessage
+                            id='admin.cluster.status_table.reload'
+                            defaultMessage=' Reload Cluster Status'
+                        />
+                    </button>
+                </div>
+                <table className='table'>
+                    <thead>
+                        <tr>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.status'
+                                    defaultMessage='Status'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.hostname'
+                                    defaultMessage='Hostname'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.version'
+                                    defaultMessage='Version'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.config_hash'
+                                    defaultMessage='Config File MD5'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.url'
+                                    defaultMessage='Inter-Node URL'
+                                />
+                            </th>
+                            <th>
+                                <FormattedMessage
+                                    id='admin.cluster.status_table.id'
+                                    defaultMessage='Node ID'
+                                />
+                            </th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {items}
+                    </tbody>
+                </table>
+            </div>
+        );
+    }
+}

webapp/components/admin_console/cluster_table_container.jsx

@@ -0,0 +1,71 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+import React from 'react';
+import ClusterTable from './cluster_table.jsx';
+import LoadingScreen from '../loading_screen.jsx';
+import Client from 'client/web_client.jsx';
+import * as AsyncClient from 'utils/async_client.jsx';
+
+export default class ClusterTableContainer extends React.Component {
+    constructor(props) {
+        super(props);
+
+        this.interval = null;
+
+        this.state = {
+            clusterInfos: null
+        };
+    }
+
+    load = () => {
+        Client.getClusterStatus(
+            (data) => {
+                this.setState({
+                    clusterInfos: data
+                });
+            },
+            (err) => {
+                AsyncClient.dispatchError(err, 'getClusterStatus');
+            }
+        );
+    }
+
+    componentWillMount() {
+        this.load();
+
+        // reload the cluster status every 15 seconds
+        this.interval = setInterval(this.load, 15000);
+    }
+
+    componentWillUnmount() {
+        if (this.interval) {
+            clearInterval(this.interval);
+        }
+    }
+
+    reload = (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+
+        this.setState({
+            clusterInfos: null
+        });
+
+        this.load();
+    }
+
+    render() {
+        if (this.state.clusterInfos == null) {
+            return (<LoadingScreen/>);
+        }
+
+        return (
+            <ClusterTable
+                clusterInfos={this.state.clusterInfos}
+                reload={this.reload}
+            />
+        );
+    }
+}

webapp/i18n/en.json

@@ -569,6 +569,24 @@
   "admin.saml.usernameAttrTitle": "Username Attribute:",
   "admin.saml.verifyDescription": "When true, Mattermost verifies that the signature sent from the SAML Response matches the Service Provider Login URL",
   "admin.saml.verifyTitle": "Verify Signature:",
+  "admin.cluster.loadedFrom": "This configuration file was loaded from Node ID {clusterId}. Please see the Troubleshooting Guide in our <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> if you are accessing the System Console through a load balancer and experiencing issues.",
+  "admin.cluster.should_not_change": "WARNING: These settings may not sync with the other servers in the cluster. High Availability inter-node communication will not start until you modify the config.json to be identical on all servers and restart Mattermost. Please see the <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> on how to add or remove a server from the cluster. If you are accessing the System Console through a load balancer and experiencing issues, please see the Troubleshooting Guide in our <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a>.",
+  "admin.cluster.noteDescription": "Changing properties in this section will require a server restart before taking effect. When High Availability mode is enabled, the System Console is set to read-only and can only be changed from the configuration file.",
+  "admin.cluster.enableTitle": "Enable High Availability Mode:",
+  "admin.cluster.enableDescription": "When true, Mattermost will run in High Availability mode. Please see <a href=\"http://docs.mattermost.com/deployment/cluster.html\" target=\"_blank\">documentation</a> to learn more about configuring High Availability for Mattermost.",
+  "admin.cluster.interNodeListenAddressTitle": "Inter-Node Listen Address:",
+  "admin.cluster.interNodeListenAddressEx": "Ex \":8075\"",
+  "admin.cluster.interNodeListenAddressDesc": "The address the server will listen on for communicating with other servers.",
+  "admin.cluster.interNodeUrlsTitle": "Inter-Node URLs:",
+  "admin.cluster.interNodeUrlsEx": "Ex \"http://10.10.10.30, http://10.10.10.31\"",
+  "admin.cluster.interNodeUrlsDesc": "The internal/private URLs of all the Mattermost servers separated by commas.",
+  "admin.cluster.status_table.reload": " Reload Cluster Status",
+  "admin.cluster.status_table.status": "Status",
+  "admin.cluster.status_table.hostname": "Hostname",
+  "admin.cluster.status_table.version": "Version",
+  "admin.cluster.status_table.config_hash": "Config File MD5",
+  "admin.cluster.status_table.url": "Inter-Node URL",
+  "admin.cluster.status_table.id": "Node ID",
   "admin.save": "Save",
   "admin.saving": "Saving Config...",
   "admin.security.connection": "Connections",
@@ -668,6 +686,7 @@
   "admin.sidebar.reports": "REPORTING",
   "admin.sidebar.rmTeamSidebar": "Remove team from sidebar menu",
   "admin.sidebar.saml": "SAML",
+  "admin.sidebar.cluster": "High Availability",
   "admin.sidebar.security": "Security",
   "admin.sidebar.sessions": "Sessions",
   "admin.sidebar.settings": "SETTINGS",

webapp/routes/route_admin_console.jsx

@@ -17,6 +17,7 @@ import GitLabSettings from 'components/admin_console/gitlab_settings.jsx';
 import OAuthSettings from 'components/admin_console/oauth_settings.jsx';
 import LdapSettings from 'components/admin_console/ldap_settings.jsx';
 import SamlSettings from 'components/admin_console/saml_settings.jsx';
+import ClusterSettings from 'components/admin_console/cluster_settings.jsx';
 import SignupSettings from 'components/admin_console/signup_settings.jsx';
 import PasswordSettings from 'components/admin_console/password_settings.jsx';
 import PublicLinkSettings from 'components/admin_console/public_link_settings.jsx';
@@ -191,6 +192,10 @@ export default (
                 path='developer'
                 component={DeveloperSettings}
             />
+            <Route
+                path='cluster'
+                component={ClusterSettings}
+            />
         </Route>
         <Route path='team'>
             <Redirect

webapp/sass/routes/_admin-console.scss

@@ -432,3 +432,16 @@
 .recycle-db {
     margin-top: 50px !important;
 }
+
+.cluster-status {
+    width: 24px;
+    height: 24px;
+}
+
+.config-hash {
+    width: 130px;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}
+

webapp/sass/routes/_compliance.scss

@@ -1,7 +1,8 @@
 @charset 'UTF-8';
 
 .compliance-panel__table,
-.audit-panel__table {
+.audit-panel__table,
+.cluster-panel__table {
     background-color: $white;
     border: 1px solid $border-gray;
     margin-top: 10px;

webapp/stores/admin_store.jsx

@@ -22,6 +22,7 @@ class AdminStoreClass extends EventEmitter {
         this.logs = null;
         this.audits = null;
         this.config = null;
+        this.clusterId = null;
         this.teams = {};
         this.complianceReports = null;
     }
@@ -86,6 +87,14 @@ class AdminStoreClass extends EventEmitter {
         this.removeListener(ALL_TEAMS_EVENT, callback);
     }
 
+    getClusterId() {
+        return this.clusterId;
+    }
+
+    saveClusterId(clusterId) {
+        this.clusterId = clusterId;
+    }
+
     getLogs() {
         return this.logs;
     }
@@ -163,6 +172,7 @@ AdminStoreClass.dispatchToken = AppDispatcher.register((payload) => {
         break;
     case ActionTypes.RECEIVED_CONFIG:
         AdminStore.saveConfig(action.config);
+        AdminStore.saveClusterId(action.clusterId);
         AdminStore.emitConfigChange();
         break;
     case ActionTypes.RECEIVED_ALL_TEAMS:

webapp/utils/async_client.jsx

@@ -453,7 +453,8 @@ export function getConfig(success, error) {
 
             AppDispatcher.handleServerAction({
                 type: ActionTypes.RECEIVED_CONFIG,
-                config: data
+                config: data,
+                clusterId: Client.clusterId
             });
 
             if (success) {