Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

webapp/platform/client/src/client4.test.ts

@@ -0,0 +1,92 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import nock from 'nock';
+
+import Client4, {ClientError, HEADER_X_VERSION_ID} from './client4';
+import {TelemetryHandler} from './telemetry';
+
+describe('Client4', () => {
+    beforeAll(() => {
+        if (!nock.isActive()) {
+            nock.activate();
+        }
+    });
+
+    afterAll(() => {
+        nock.restore();
+    });
+
+    describe('doFetchWithResponse', () => {
+        test('serverVersion should be set from response header', async () => {
+            const client = new Client4();
+            client.setUrl('http://mattermost.example.com');
+
+            expect(client.serverVersion).toEqual('');
+
+            nock(client.getBaseRoute()).
+                get('/users/me').
+                reply(200, '{}', {[HEADER_X_VERSION_ID]: '5.0.0.5.0.0.abc123'});
+
+            await client.getMe();
+
+            expect(client.serverVersion).toEqual('5.0.0.5.0.0.abc123');
+
+            nock(client.getBaseRoute()).
+                get('/users/me').
+                reply(200, '{}', {[HEADER_X_VERSION_ID]: '5.3.0.5.3.0.abc123'});
+
+            await client.getMe();
+
+            expect(client.serverVersion).toEqual('5.3.0.5.3.0.abc123');
+        });
+    });
+
+    describe('fetchWithGraphQL', () => {
+        test('Should have correct graphql url', async () => {
+            const client = new Client4();
+            client.setUrl('http://mattermost.example.com');
+
+            expect(client.getGraphQLUrl()).toEqual('http://mattermost.example.com/api/v5/graphql');
+        });
+    });
+});
+
+describe('ClientError', () => {
+    test('standard fields should be enumerable', () => {
+        const error = new ClientError('https://example.com', {
+            message: 'This is a message',
+            server_error_id: 'test.app_error',
+            status_code: 418,
+            url: 'https://example.com/api/v4/error',
+        });
+
+        const copy = {...error};
+
+        expect(copy.message).toEqual(error.message);
+        expect(copy.server_error_id).toEqual(error.server_error_id);
+        expect(copy.status_code).toEqual(error.status_code);
+        expect(copy.url).toEqual(error.url);
+    });
+});
+
+describe('trackEvent', () => {
+    class TestTelemetryHandler implements TelemetryHandler {
+        trackEvent = jest.fn();
+        pageVisited = jest.fn();
+    }
+
+    test('should call the attached RudderTelemetryHandler, if one is attached to Client4', () => {
+        const client = new Client4();
+        client.setUrl('http://mattermost.example.com');
+
+        expect(() => client.trackEvent('test', 'onClick')).not.toThrowError();
+
+        const handler = new TestTelemetryHandler();
+
+        client.setTelemetryHandler(handler);
+        client.trackEvent('test', 'onClick');
+
+        expect(handler.trackEvent).toHaveBeenCalledTimes(1);
+    });
+});

webapp/platform/client/src/errors.test.ts

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import Client4 from './client4';
+
+import {cleanUrlForLogging} from './errors';
+
+describe('cleanUrlForLogging', () => {
+    const baseUrl = 'https://mattermost.example.com/subpath';
+
+    const client = new Client4();
+    client.setUrl(baseUrl);
+
+    const testCases = [{
+        name: 'should remove server URL',
+        input: client.getUserRoute('me'),
+        expected: `${client.urlVersion}/users/me`,
+    }, {
+        name: 'should filter user IDs',
+        input: client.getUserRoute('1234'),
+        expected: `${client.urlVersion}/users/<filtered>`,
+    }, {
+        name: 'should filter email addresses',
+        input: `${client.getUsersRoute()}/email/test@example.com`,
+        expected: `${client.urlVersion}/users/email/<filtered>`,
+    }, {
+        name: 'should filter query parameters',
+        input: `${client.getUserRoute('me')}?foo=bar`,
+        expected: `${client.urlVersion}/users/me?<filtered>`,
+    }];
+
+    for (const testCase of testCases) {
+        test(testCase.name, () => {
+            expect(cleanUrlForLogging(baseUrl, testCase.input)).toEqual(testCase.expected);
+        });
+    }
+});

webapp/platform/client/src/errors.ts

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Given a URL from an API request, return a URL that has any parts removed that are either sensitive or that would
+// prevent properly grouping the messages in Sentry.
+export function cleanUrlForLogging(baseUrl: string, apiUrl: string): string {
+    let url = apiUrl;
+
+    // Trim the host name
+    url = url.substring(baseUrl.length);
+
+    // Filter the query string
+    const index = url.indexOf('?');
+    if (index !== -1) {
+        url = url.substring(0, index);
+    }
+
+    // A non-exhaustive whitelist to exclude parts of the URL that are unimportant (eg IDs) or may be sentsitive
+    // (eg email addresses). We prefer filtering out fields that aren't recognized because there should generally
+    // be enough left over for debugging.
+    //
+    // Note that new API routes don't need to be added here since this shouldn't be happening for newly added routes.
+    const whitelist = [
+        'api', 'v4', 'users', 'teams', 'scheme', 'name', 'members', 'channels', 'posts', 'reactions', 'commands',
+        'files', 'preferences', 'hooks', 'incoming', 'outgoing', 'oauth', 'apps', 'emoji', 'brand', 'image',
+        'data_retention', 'jobs', 'plugins', 'roles', 'system', 'timezones', 'schemes', 'redirect_location', 'patch',
+        'mfa', 'password', 'reset', 'send', 'active', 'verify', 'terms_of_service', 'login', 'logout', 'ids',
+        'usernames', 'me', 'username', 'email', 'default', 'sessions', 'revoke', 'all', 'audits', 'device', 'status',
+        'search', 'switch', 'authorized', 'authorize', 'deauthorize', 'tokens', 'disable', 'enable', 'exists', 'unread',
+        'invite', 'batch', 'stats', 'import', 'schemeRoles', 'direct', 'group', 'convert', 'view', 'search_autocomplete',
+        'thread', 'info', 'flagged', 'pinned', 'pin', 'unpin', 'opengraph', 'actions', 'thumbnail', 'preview', 'link',
+        'delete', 'logs', 'ping', 'config', 'client', 'license', 'websocket', 'webrtc', 'token', 'regen_token',
+        'autocomplete', 'execute', 'regen_secret', 'policy', 'type', 'cancel', 'reload', 'environment', 's3_test', 'file',
+        'caches', 'invalidate', 'database', 'recycle', 'compliance', 'reports', 'cluster', 'ldap', 'test', 'sync', 'saml',
+        'certificate', 'public', 'private', 'idp', 'elasticsearch', 'purge_indexes', 'analytics', 'old', 'webapp', 'fake',
+    ];
+
+    url = url.split('/').map((part) => {
+        if (part !== '' && whitelist.indexOf(part) === -1) {
+            return '<filtered>';
+        }
+
+        return part;
+    }).join('/');
+
+    if (index !== -1) {
+        // Add this on afterwards since it wouldn't pass the whitelist
+        url += '?<filtered>';
+    }
+
+    return url;
+}

webapp/platform/client/src/helpers.test.ts

@@ -0,0 +1,18 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+import {buildQueryString} from './helpers';
+
+describe('Helpers', () => {
+    test.each([
+        [{}, ''],
+        [{a: 1}, '?a=1'],
+        [{a: 1, b: 'str'}, '?a=1&b=str'],
+        [{a: 1, b: 'str', c: undefined}, '?a=1&b=str'],
+        [{a: 1, b: 'str', c: 0}, '?a=1&b=str&c=0'],
+        [{a: 1, b: 'str', c: ''}, '?a=1&b=str&c='],
+        [{a: 1, b: undefined, c: 'str'}, '?a=1&c=str'],
+    ])('buildQueryString with %o should return %s', (params, expected) => {
+        expect(buildQueryString(params)).toEqual(expected);
+    });
+});

webapp/platform/client/src/helpers.ts

@@ -0,0 +1,17 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+export function buildQueryString(parameters: Record<string, any>): string {
+    const keys = Object.keys(parameters);
+    if (keys.length === 0) {
+        return '';
+    }
+
+    const queryParams = Object.entries(parameters).
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        filter(([_, value]) => value !== undefined).
+        map(([key, value]) => `${key}=${encodeURIComponent(value)}`).
+        join('&');
+
+    return queryParams.length > 0 ? `?${queryParams}` : '';
+}

webapp/platform/client/src/index.ts

@@ -0,0 +1,13 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+export {
+    default as Client4,
+    ClientError,
+    DEFAULT_LIMIT_AFTER,
+    DEFAULT_LIMIT_BEFORE,
+} from './client4';
+
+export type {TelemetryHandler} from './telemetry';
+export type {WebSocketMessage} from './websocket';
+export {default as WebSocketClient} from './websocket';

webapp/platform/client/src/telemetry.ts

@@ -0,0 +1,7 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+export interface TelemetryHandler {
+    trackEvent: (userId: string, userRoles: string, category: string, event: string, props?: any) => void;
+    pageVisited: (userId: string, userRoles: string, category: string, name: string) => void;
+}

webapp/platform/client/src/websocket.ts

@@ -0,0 +1,410 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+const MAX_WEBSOCKET_FAILS = 7;
+const MIN_WEBSOCKET_RETRY_TIME = 3000; // 3 sec
+const MAX_WEBSOCKET_RETRY_TIME = 300000; // 5 mins
+const JITTER_RANGE = 2000; // 2 sec
+
+const WEBSOCKET_HELLO = 'hello';
+
+export type MessageListener = (msg: WebSocketMessage) => void;
+export type FirstConnectListener = () => void;
+export type ReconnectListener = () => void;
+export type MissedMessageListener = () => void;
+export type ErrorListener = (event: Event) => void;
+export type CloseListener = (connectFailCount: number) => void;
+
+export default class WebSocketClient {
+    private conn: WebSocket | null;
+    private connectionUrl: string | null;
+
+    // responseSequence is the number to track a response sent
+    // via the websocket. A response will always have the same sequence number
+    // as the request.
+    private responseSequence: number;
+
+    // serverSequence is the incrementing sequence number from the
+    // server-sent event stream.
+    private serverSequence: number;
+    private connectFailCount: number;
+    private responseCallbacks: {[x: number]: ((msg: any) => void)};
+
+    /**
+     * @deprecated Use messageListeners instead
+     */
+    private eventCallback: MessageListener | null = null;
+
+    /**
+     * @deprecated Use firstConnectListeners instead
+     */
+    private firstConnectCallback: FirstConnectListener | null = null;
+
+    /**
+     * @deprecated Use reconnectListeners instead
+     */
+    private reconnectCallback: ReconnectListener | null = null;
+
+    /**
+     * @deprecated Use missedMessageListeners instead
+     */
+    private missedEventCallback: MissedMessageListener | null = null;
+
+    /**
+     * @deprecated Use errorListeners instead
+     */
+    private errorCallback: ErrorListener | null = null;
+
+    /**
+     * @deprecated Use closeListeners instead
+     */
+    private closeCallback: CloseListener | null = null;
+
+    private messageListeners = new Set<MessageListener>();
+    private firstConnectListeners = new Set<FirstConnectListener>();
+    private reconnectListeners = new Set<ReconnectListener>();
+    private missedMessageListeners = new Set<MissedMessageListener>();
+    private errorListeners = new Set<ErrorListener>();
+    private closeListeners = new Set<CloseListener>();
+
+    private connectionId: string | null;
+
+    constructor() {
+        this.conn = null;
+        this.connectionUrl = null;
+        this.responseSequence = 1;
+        this.serverSequence = 0;
+        this.connectFailCount = 0;
+        this.responseCallbacks = {};
+        this.connectionId = '';
+    }
+
+    // on connect, only send auth cookie and blank state.
+    // on hello, get the connectionID and store it.
+    // on reconnect, send cookie, connectionID, sequence number.
+    initialize(connectionUrl = this.connectionUrl, token?: string) {
+        if (this.conn) {
+            return;
+        }
+
+        if (connectionUrl == null) {
+            console.log('websocket must have connection url'); //eslint-disable-line no-console
+            return;
+        }
+
+        if (this.connectFailCount === 0) {
+            console.log('websocket connecting to ' + connectionUrl); //eslint-disable-line no-console
+        }
+
+        // Add connection id, and last_sequence_number to the query param.
+        // We cannot use a cookie because it will bleed across tabs.
+        // We cannot also send it as part of the auth_challenge, because the session cookie is already sent with the request.
+        this.conn = new WebSocket(`${connectionUrl}?connection_id=${this.connectionId}&sequence_number=${this.serverSequence}`);
+        this.connectionUrl = connectionUrl;
+
+        this.conn.onopen = () => {
+            if (token) {
+                this.sendMessage('authentication_challenge', {token});
+            }
+
+            if (this.connectFailCount > 0) {
+                console.log('websocket re-established connection'); //eslint-disable-line no-console
+
+                this.reconnectCallback?.();
+                this.reconnectListeners.forEach((listener) => listener());
+            } else if (this.firstConnectCallback || this.firstConnectListeners.size > 0) {
+                this.firstConnectCallback?.();
+                this.firstConnectListeners.forEach((listener) => listener());
+            }
+
+            this.connectFailCount = 0;
+        };
+
+        this.conn.onclose = () => {
+            this.conn = null;
+            this.responseSequence = 1;
+
+            if (this.connectFailCount === 0) {
+                console.log('websocket closed'); //eslint-disable-line no-console
+            }
+
+            this.connectFailCount++;
+
+            this.closeCallback?.(this.connectFailCount);
+            this.closeListeners.forEach((listener) => listener(this.connectFailCount));
+
+            let retryTime = MIN_WEBSOCKET_RETRY_TIME;
+
+            // If we've failed a bunch of connections then start backing off
+            if (this.connectFailCount > MAX_WEBSOCKET_FAILS) {
+                retryTime = MIN_WEBSOCKET_RETRY_TIME * this.connectFailCount * this.connectFailCount;
+                if (retryTime > MAX_WEBSOCKET_RETRY_TIME) {
+                    retryTime = MAX_WEBSOCKET_RETRY_TIME;
+                }
+            }
+
+            // Applying jitter to avoid thundering herd problems.
+            retryTime += Math.random() * JITTER_RANGE;
+
+            setTimeout(
+                () => {
+                    this.initialize(connectionUrl, token);
+                },
+                retryTime,
+            );
+        };
+
+        this.conn.onerror = (evt) => {
+            if (this.connectFailCount <= 1) {
+                console.log('websocket error'); //eslint-disable-line no-console
+                console.log(evt); //eslint-disable-line no-console
+            }
+
+            this.errorCallback?.(evt);
+            this.errorListeners.forEach((listener) => listener(evt));
+        };
+
+        this.conn.onmessage = (evt) => {
+            const msg = JSON.parse(evt.data);
+            if (msg.seq_reply) {
+                // This indicates a reply to a websocket request.
+                // We ignore sequence number validation of message responses
+                // and only focus on the purely server side event stream.
+                if (msg.error) {
+                    console.log(msg); //eslint-disable-line no-console
+                }
+
+                if (this.responseCallbacks[msg.seq_reply]) {
+                    this.responseCallbacks[msg.seq_reply](msg);
+                    Reflect.deleteProperty(this.responseCallbacks, msg.seq_reply);
+                }
+            } else if (this.eventCallback || this.messageListeners.size > 0) {
+                // We check the hello packet, which is always the first packet in a stream.
+                if (msg.event === WEBSOCKET_HELLO && (this.missedEventCallback || this.missedMessageListeners.size > 0)) {
+                    console.log('got connection id ', msg.data.connection_id); //eslint-disable-line no-console
+                    // If we already have a connectionId present, and server sends a different one,
+                    // that means it's either a long timeout, or server restart, or sequence number is not found.
+                    // Then we do the sync calls, and reset sequence number to 0.
+                    if (this.connectionId !== '' && this.connectionId !== msg.data.connection_id) {
+                        console.log('long timeout, or server restart, or sequence number is not found.'); //eslint-disable-line no-console
+
+                        this.missedEventCallback?.();
+                        this.missedMessageListeners.forEach((listener) => listener());
+
+                        this.serverSequence = 0;
+                    }
+
+                    // If it's a fresh connection, we have to set the connectionId regardless.
+                    // And if it's an existing connection, setting it again is harmless, and keeps the code simple.
+                    this.connectionId = msg.data.connection_id;
+                }
+
+                // Now we check for sequence number, and if it does not match,
+                // we just disconnect and reconnect.
+                if (msg.seq !== this.serverSequence) {
+                    console.log('missed websocket event, act_seq=' + msg.seq + ' exp_seq=' + this.serverSequence); //eslint-disable-line no-console
+                    // We are not calling this.close() because we need to auto-restart.
+                    this.connectFailCount = 0;
+                    this.responseSequence = 1;
+                    this.conn?.close(); // Will auto-reconnect after MIN_WEBSOCKET_RETRY_TIME.
+                    return;
+                }
+                this.serverSequence = msg.seq + 1;
+
+                this.eventCallback?.(msg);
+                this.messageListeners.forEach((listener) => listener(msg));
+            }
+        };
+    }
+
+    /**
+     * @deprecated Use addMessageListener instead
+     */
+    setEventCallback(callback: MessageListener) {
+        this.eventCallback = callback;
+    }
+
+    addMessageListener(listener: MessageListener) {
+        this.messageListeners.add(listener);
+
+        if (this.messageListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.messageListeners.size} message listeners registered`);
+        }
+    }
+
+    removeMessageListener(listener: MessageListener) {
+        this.messageListeners.delete(listener);
+    }
+
+    /**
+     * @deprecated Use addFirstConnectListener instead
+     */
+    setFirstConnectCallback(callback: FirstConnectListener) {
+        this.firstConnectCallback = callback;
+    }
+
+    addFirstConnectListener(listener: FirstConnectListener) {
+        this.firstConnectListeners.add(listener);
+
+        if (this.firstConnectListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.firstConnectListeners.size} first connect listeners registered`);
+        }
+    }
+
+    removeFirstConnectListener(listener: FirstConnectListener) {
+        this.firstConnectListeners.delete(listener);
+    }
+
+    /**
+     * @deprecated Use addReconnectListener instead
+     */
+    setReconnectCallback(callback: ReconnectListener) {
+        this.reconnectCallback = callback;
+    }
+
+    addReconnectListener(listener: ReconnectListener) {
+        this.reconnectListeners.add(listener);
+
+        if (this.reconnectListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.reconnectListeners.size} reconnect listeners registered`);
+        }
+    }
+
+    removeReconnectListener(listener: ReconnectListener) {
+        this.reconnectListeners.delete(listener);
+    }
+
+    /**
+     * @deprecated Use addMissedMessageListener instead
+     */
+    setMissedEventCallback(callback: MissedMessageListener) {
+        this.missedEventCallback = callback;
+    }
+
+    addMissedMessageListener(listener: MissedMessageListener) {
+        this.missedMessageListeners.add(listener);
+
+        if (this.missedMessageListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.missedMessageListeners.size} missed message listeners registered`);
+        }
+    }
+
+    removeMissedMessageListener(listener: MissedMessageListener) {
+        this.missedMessageListeners.delete(listener);
+    }
+
+    /**
+     * @deprecated Use addErrorListener instead
+     */
+    setErrorCallback(callback: ErrorListener) {
+        this.errorCallback = callback;
+    }
+
+    addErrorListener(listener: ErrorListener) {
+        this.errorListeners.add(listener);
+
+        if (this.errorListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.errorListeners.size} error listeners registered`);
+        }
+    }
+
+    removeErrorListener(listener: ErrorListener) {
+        this.errorListeners.delete(listener);
+    }
+
+    /**
+     * @deprecated Use addCloseListener instead
+     */
+    setCloseCallback(callback: CloseListener) {
+        this.closeCallback = callback;
+    }
+
+    addCloseListener(listener: CloseListener) {
+        this.closeListeners.add(listener);
+
+        if (this.closeListeners.size > 5) {
+            // eslint-disable-next-line no-console
+            console.warn(`WebSocketClient has ${this.closeListeners.size} close listeners registered`);
+        }
+    }
+
+    removeCloseListener(listener: CloseListener) {
+        this.closeListeners.delete(listener);
+    }
+
+    close() {
+        this.connectFailCount = 0;
+        this.responseSequence = 1;
+        if (this.conn && this.conn.readyState === WebSocket.OPEN) {
+            this.conn.onclose = () => {};
+            this.conn.close();
+            this.conn = null;
+            console.log('websocket closed'); //eslint-disable-line no-console
+        }
+    }
+
+    sendMessage(action: string, data: any, responseCallback?: () => void) {
+        const msg = {
+            action,
+            seq: this.responseSequence++,
+            data,
+        };
+
+        if (responseCallback) {
+            this.responseCallbacks[msg.seq] = responseCallback;
+        }
+
+        if (this.conn && this.conn.readyState === WebSocket.OPEN) {
+            this.conn.send(JSON.stringify(msg));
+        } else if (!this.conn || this.conn.readyState === WebSocket.CLOSED) {
+            this.conn = null;
+            this.initialize();
+        }
+    }
+
+    userTyping(channelId: string, parentId: string, callback?: () => void) {
+        const data = {
+            channel_id: channelId,
+            parent_id: parentId,
+        };
+        this.sendMessage('user_typing', data, callback);
+    }
+
+    userUpdateActiveStatus(userIsActive: boolean, manual: boolean, callback?: () => void) {
+        const data = {
+            user_is_active: userIsActive,
+            manual,
+        };
+        this.sendMessage('user_update_active_status', data, callback);
+    }
+
+    getStatuses(callback?: () => void) {
+        this.sendMessage('get_statuses', null, callback);
+    }
+
+    getStatusesByIds(userIds: string[], callback?: () => void) {
+        const data = {
+            user_ids: userIds,
+        };
+        this.sendMessage('get_statuses_by_ids', data, callback);
+    }
+}
+
+export type WebSocketBroadcast = {
+    omit_users: Record<string, boolean>;
+    user_id: string;
+    channel_id: string;
+    team_id: string;
+}
+
+export type WebSocketMessage<T = any> = {
+    event: string;
+    data: T;
+    broadcast: WebSocketBroadcast;
+    seq: number;
+}