Rebuild channel index (#26080)

* WIP

* Added rebuild channels index functionality

* Added rough logic to send message to all sysadmins

* WIP

* WIP

* WIP

* Cleanup

* i18n fix

* reading through all pages of system admins

* Fixed webapp style

* i18n fix

* Added help text

* i18n fix

* i18n update

* Updated system console button action

* Updated snapshots

* some cleanup

* Updated snapshot

* Update server/channels/app/server.go

Co-authored-by: Daniel Espino García <larkox@gmail.com>

* fixed typo

* Refactoring to improve readibility

* moved index check to API later during config update

* Added some docs

* Updated get system bot

---------

Co-authored-by: Daniel Espino García <larkox@gmail.com>

webapp/channels/src/actions/admin_actions.jsx

@@ -3,6 +3,7 @@
 
 import * as AdminActions from 'mattermost-redux/actions/admin';
 import {bindClientFunc} from 'mattermost-redux/actions/helpers';
+import {createJob} from 'mattermost-redux/actions/jobs';
 import * as TeamActions from 'mattermost-redux/actions/teams';
 import * as UserActions from 'mattermost-redux/actions/users';
 import {Client4} from 'mattermost-redux/client';
@@ -12,7 +13,7 @@ import {trackEvent} from 'actions/telemetry_actions.jsx';
 import {getOnNavigationConfirmed} from 'selectors/views/admin';
 import store from 'stores/redux_store';
 
-import {ActionTypes} from 'utils/constants';
+import {ActionTypes, JobTypes} from 'utils/constants';
 
 const dispatch = store.dispatch;
 
@@ -316,8 +317,8 @@ export async function testS3Connection(success, error) {
     }
 }
 
-export async function elasticsearchPurgeIndexes(success, error) {
-    const {data, error: err} = await dispatch(AdminActions.purgeElasticsearchIndexes());
+export async function elasticsearchPurgeIndexes(success, error, indexes) {
+    const {data, error: err} = await dispatch(AdminActions.purgeElasticsearchIndexes(indexes));
     if (data && success) {
         success(data);
     } else if (err && error) {
@@ -325,6 +326,31 @@ export async function elasticsearchPurgeIndexes(success, error) {
     }
 }
 
+export async function jobCreate(success, error, job) {
+    const {data, error: err} = await dispatch(createJob(job));
+    if (data && success) {
+        success(data);
+    } else if (err && error) {
+        error({id: err.server_error_id, ...err});
+    }
+}
+
+export async function rebuildChannelsIndex(success, error) {
+    await elasticsearchPurgeIndexes(undefined, error, ['channels']);
+    const job = {
+        type: JobTypes.ELASTICSEARCH_POST_INDEXING,
+        data: {
+            index_posts: 'false',
+            index_users: 'false',
+            index_files: 'false',
+            index_channels: 'true',
+            sub_type: 'channels_index_rebuild',
+        },
+    };
+    await jobCreate(undefined, error, job);
+    success();
+}
+
 export async function blevePurgeIndexes(success, error) {
     const purgeBleveIndexes = bindClientFunc({
         clientFunc: Client4.purgeBleveIndexes,

webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap

@@ -338,6 +338,50 @@ exports[`components/ElasticSearchSettings should match snapshot, disabled 1`] =
           </div>
         </div>
       </div>
+      <RequestButton
+        buttonText={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="Rebuild Channels Index"
+            id="admin.elasticsearch.rebuildChannelsIndex.title"
+          />
+        }
+        disabled={true}
+        errorMessage={
+          Object {
+            "defaultMessage": "Failed to trigger channels index rebuild job: {error}",
+            "id": "admin.elasticsearch.rebuildIndexSuccessfully.error",
+          }
+        }
+        helpText={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="This purges the channels index and re-indexes all channels in the database, from oldest to newest. Channel autocomplete is available during indexing but search results may be incomplete until the indexing job is complete.
+<b>Note- Please ensure no other indexing job is in progress in the table above.</b>"
+            id="admin.elasticsearch.rebuildChannelsIndex.helpText"
+            values={
+              Object {
+                "b": [Function],
+              }
+            }
+          />
+        }
+        id="rebuildChannelsIndexButton"
+        includeDetailedError={false}
+        label={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="Rebuild Channels Index"
+            id="admin.elasticsearch.rebuildChannelsIndex.title"
+          />
+        }
+        requestAction={[MockFunction]}
+        saveNeeded={false}
+        showSuccessMessage={true}
+        successMessage={
+          Object {
+            "defaultMessage": "Channels index rebuild job triggered successfully.",
+            "id": "admin.elasticsearch.rebuildIndexSuccessfully.success",
+          }
+        }
+      />
       <RequestButton
         buttonText={
           <Memo(MemoizedFormattedMessage)
@@ -843,6 +887,50 @@ exports[`components/ElasticSearchSettings should match snapshot, enabled 1`] = `
           </div>
         </div>
       </div>
+      <RequestButton
+        buttonText={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="Rebuild Channels Index"
+            id="admin.elasticsearch.rebuildChannelsIndex.title"
+          />
+        }
+        disabled={false}
+        errorMessage={
+          Object {
+            "defaultMessage": "Failed to trigger channels index rebuild job: {error}",
+            "id": "admin.elasticsearch.rebuildIndexSuccessfully.error",
+          }
+        }
+        helpText={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="This purges the channels index and re-indexes all channels in the database, from oldest to newest. Channel autocomplete is available during indexing but search results may be incomplete until the indexing job is complete.
+<b>Note- Please ensure no other indexing job is in progress in the table above.</b>"
+            id="admin.elasticsearch.rebuildChannelsIndex.helpText"
+            values={
+              Object {
+                "b": [Function],
+              }
+            }
+          />
+        }
+        id="rebuildChannelsIndexButton"
+        includeDetailedError={false}
+        label={
+          <Memo(MemoizedFormattedMessage)
+            defaultMessage="Rebuild Channels Index"
+            id="admin.elasticsearch.rebuildChannelsIndex.title"
+          />
+        }
+        requestAction={[MockFunction]}
+        saveNeeded={false}
+        showSuccessMessage={true}
+        successMessage={
+          Object {
+            "defaultMessage": "Channels index rebuild job triggered successfully.",
+            "id": "admin.elasticsearch.rebuildIndexSuccessfully.success",
+          }
+        }
+      />
       <RequestButton
         buttonText={
           <Memo(MemoizedFormattedMessage)

webapp/channels/src/components/admin_console/elasticsearch_settings.test.tsx

@@ -12,6 +12,7 @@ import SaveButton from 'components/save_button';
 jest.mock('actions/admin_actions.jsx', () => {
     return {
         elasticsearchPurgeIndexes: jest.fn(),
+        rebuildChannelsIndex: jest.fn(),
         elasticsearchTest: (config: AdminConfig, success: () => void) => success(),
     };
 });

webapp/channels/src/components/admin_console/elasticsearch_settings.tsx

@@ -8,7 +8,7 @@ import {FormattedMessage, defineMessage, defineMessages} from 'react-intl';
 import type {AdminConfig} from '@mattermost/types/config';
 import type {Job, JobType} from '@mattermost/types/jobs';
 
-import {elasticsearchPurgeIndexes, elasticsearchTest} from 'actions/admin_actions.jsx';
+import {elasticsearchPurgeIndexes, elasticsearchTest, rebuildChannelsIndex} from 'actions/admin_actions.jsx';
 
 import ExternalLink from 'components/external_link';
 
@@ -62,6 +62,9 @@ export const messages = defineMessages({
     elasticsearch_test_button: {id: 'admin.elasticsearch.elasticsearch_test_button', defaultMessage: 'Test Connection'},
     bulkIndexingTitle: {id: 'admin.elasticsearch.bulkIndexingTitle', defaultMessage: 'Bulk Indexing:'},
     help: {id: 'admin.elasticsearch.createJob.help', defaultMessage: 'All users, channels and posts in the database will be indexed from oldest to newest. Elasticsearch is available during indexing but search results may be incomplete until the indexing job is complete.'},
+    rebuildChannelsIndexTitle: {id: 'admin.elasticsearch.rebuildChannelsIndexTitle', defaultMessage: 'Rebuild Channels Index'},
+    rebuildChannelIndexHelpText: {id: 'admin.elasticsearch.rebuildChannelsIndex.helpText', defaultMessage: 'This purges the channels index and re-indexes all channels in the database, from oldest to newest. Channel autocomplete is available during indexing but search results may be incomplete until the indexing job is complete.\n<b>Note- Please ensure no other indexing job is in progress in the table above.</b>'},
+    rebuildChannelsIndexButtonText: {id: 'admin.elasticsearch.rebuildChannelsIndex.title', defaultMessage: 'Rebuild Channels Index'},
     purgeIndexesHelpText: {id: 'admin.elasticsearch.purgeIndexesHelpText', defaultMessage: 'Purging will entirely remove the indexes on the Elasticsearch server. Search results may be incomplete until a bulk index of the existing database is rebuilt.'},
     purgeIndexesButton: {id: 'admin.elasticsearch.purgeIndexesButton', defaultMessage: 'Purge Index'},
     label: {id: 'admin.elasticsearch.purgeIndexesButton.label', defaultMessage: 'Purge Indexes:'},
@@ -197,8 +200,22 @@ export default class ElasticsearchSettings extends AdminSettings<Props, State> {
     };
 
     getExtraInfo(job: Job) {
+        let jobSubType = null;
+        if (job.data?.sub_type === 'channels_index_rebuild') {
+            jobSubType = (
+                <span>
+                    {'. '}
+                    <FormattedMessage
+                        id='admin.elasticsearch.channelIndexRebuildJobTitle'
+                        defaultMessage='Channels index rebuild job.'
+                    />
+                </span>
+            );
+        }
+
+        let jobProgress = null;
         if (job.status === JobStatuses.IN_PROGRESS) {
-            return (
+            jobProgress = (
                 <FormattedMessage
                     id='admin.elasticsearch.percentComplete'
                     defaultMessage='{percent}% Complete'
@@ -207,7 +224,7 @@ export default class ElasticsearchSettings extends AdminSettings<Props, State> {
             );
         }
 
-        return null;
+        return (<span>{jobProgress}{jobSubType}</span>);
     }
 
     renderTitle() {
@@ -400,6 +417,29 @@ export default class ElasticsearchSettings extends AdminSettings<Props, State> {
                         </div>
                     </div>
                 </div>
+                <RequestButton
+                    id='rebuildChannelsIndexButton'
+                    requestAction={rebuildChannelsIndex}
+                    helpText={
+                        <FormattedMessage
+                            {...messages.rebuildChannelIndexHelpText}
+                            values={{
+                                b: (chunks: React.ReactNode) => (<b>{chunks}</b>),
+                            }}
+                        />
+                    }
+                    buttonText={<FormattedMessage {...messages.rebuildChannelsIndexButtonText}/>}
+                    successMessage={defineMessage({
+                        id: 'admin.elasticsearch.rebuildIndexSuccessfully.success',
+                        defaultMessage: 'Channels index rebuild job triggered successfully.',
+                    })}
+                    errorMessage={defineMessage({
+                        id: 'admin.elasticsearch.rebuildIndexSuccessfully.error',
+                        defaultMessage: 'Failed to trigger channels index rebuild job: {error}',
+                    })}
+                    disabled={!this.state.canPurgeAndIndex || this.props.isDisabled!}
+                    label={<FormattedMessage {...messages.rebuildChannelsIndexButtonText}/>}
+                />
                 <RequestButton
                     id='purgeIndexesSection'
                     requestAction={elasticsearchPurgeIndexes}

webapp/channels/src/components/admin_console/jobs/table.tsx

@@ -29,6 +29,8 @@ export type Props = {
     className?: string;
     hideJobCreateButton?: boolean;
     createJobButtonText: React.ReactNode;
+    hideTable?: boolean;
+    jobData?: any;
     actions: {
         getJobsByType: (jobType: JobType) => void;
         cancelJob: (jobId: string) => Promise<ActionResult>;
@@ -75,6 +77,7 @@ class JobTable extends React.PureComponent<Props> {
         e.preventDefault();
         const job = {
             type: this.props.jobType,
+            data: this.props.jobData,
         };
 
         await this.props.actions.createJob(job);
@@ -131,53 +134,56 @@ class JobTable extends React.PureComponent<Props> {
                         {this.props.createJobHelpText}
                     </div>
                 </div>
-                <div className='job-table__table'>
-                    <table
-                        className='table'
-                        data-testid='jobTable'
-                    >
-                        <thead>
-                            <tr>
-                                <th className='cancel-button-field'/>
-                                <th>
-                                    <FormattedMessage
-                                        id='admin.jobTable.headerStatus'
-                                        defaultMessage='Status'
-                                    />
-                                </th>
-                                {showFilesColumn &&
+                {
+                    !this.props.hideTable &&
+                    <div className='job-table__table'>
+                        <table
+                            className='table'
+                            data-testid='jobTable'
+                        >
+                            <thead>
+                                <tr>
+                                    <th className='cancel-button-field'/>
+                                    <th>
+                                        <FormattedMessage
+                                            id='admin.jobTable.headerStatus'
+                                            defaultMessage='Status'
+                                        />
+                                    </th>
+                                    {showFilesColumn &&
                                     <th>
                                         <FormattedMessage
                                             id='admin.jobTable.headerFiles'
                                             defaultMessage='Files'
                                         />
                                     </th>
-                                }
-                                <th>
-                                    <FormattedMessage
-                                        id='admin.jobTable.headerFinishAt'
-                                        defaultMessage='Finish Time'
-                                    />
-                                </th>
-                                <th>
-                                    <FormattedMessage
-                                        id='admin.jobTable.headerRunTime'
-                                        defaultMessage='Run Time'
-                                    />
-                                </th>
-                                <th colSpan={3}>
-                                    <FormattedMessage
-                                        id='admin.jobTable.headerExtraInfo'
-                                        defaultMessage='Details'
-                                    />
-                                </th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                            {items}
-                        </tbody>
-                    </table>
-                </div>
+                                    }
+                                    <th>
+                                        <FormattedMessage
+                                            id='admin.jobTable.headerFinishAt'
+                                            defaultMessage='Finish Time'
+                                        />
+                                    </th>
+                                    <th>
+                                        <FormattedMessage
+                                            id='admin.jobTable.headerRunTime'
+                                            defaultMessage='Run Time'
+                                        />
+                                    </th>
+                                    <th colSpan={3}>
+                                        <FormattedMessage
+                                            id='admin.jobTable.headerExtraInfo'
+                                            defaultMessage='Details'
+                                        />
+                                    </th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {items}
+                            </tbody>
+                        </table>
+                    </div>
+                }
             </div>
         );
     }

webapp/channels/src/i18n/en.json

@@ -799,6 +799,7 @@
   "admin.elasticsearch.caDescription": "(Optional) Custom Certificate Authority certificates for the Elasticsearch server. Leave this empty to use the default CAs from the operating system.",
   "admin.elasticsearch.caExample": "E.g.: \"./elasticsearch/ca.pem\"",
   "admin.elasticsearch.caTitle": "CA path:",
+  "admin.elasticsearch.channelIndexRebuildJobTitle": "Channels index rebuild job.",
   "admin.elasticsearch.clientCertDescription": "(Optional) The client certificate for the connection to the Elasticsearch server in the PEM format.",
   "admin.elasticsearch.clientCertExample": "E.g.: \"./elasticsearch/client-cert.pem\"",
   "admin.elasticsearch.clientCertTitle": "Client Certificate path:",
@@ -829,6 +830,11 @@
   "admin.elasticsearch.purgeIndexesButton.label": "Purge Indexes:",
   "admin.elasticsearch.purgeIndexesButton.success": "Indexes purged successfully.",
   "admin.elasticsearch.purgeIndexesHelpText": "Purging will entirely remove the indexes on the Elasticsearch server. Search results may be incomplete until a bulk index of the existing database is rebuilt.",
+  "admin.elasticsearch.rebuildChannelsIndex.helpText": "This purges the channels index and re-indexes all channels in the database, from oldest to newest. Channel autocomplete is available during indexing but search results may be incomplete until the indexing job is complete.\n\n<b>Note- Please ensure no other indexing job is in progress in the table above.</b>",
+  "admin.elasticsearch.rebuildChannelsIndex.title": "Rebuild Channels Index",
+  "admin.elasticsearch.rebuildChannelsIndexTitle": "Rebuild Channels Index",
+  "admin.elasticsearch.rebuildIndexSuccessfully.error": "Failed to trigger channels index rebuild job.",
+  "admin.elasticsearch.rebuildIndexSuccessfully.success": "Channels index rebuild job triggered successfully.",
   "admin.elasticsearch.skipTLSVerificationDescription": "When true, Mattermost will not require the Elasticsearch certificate to be signed by a trusted Certificate Authority.",
   "admin.elasticsearch.skipTLSVerificationTitle": "Skip TLS Verification:",
   "admin.elasticsearch.sniffDescription": "When true, sniffing finds and connects to all data nodes in your cluster automatically.",

webapp/channels/src/packages/mattermost-redux/src/actions/admin.ts

@@ -358,9 +358,12 @@ export function testElasticsearch(config?: AdminConfig) {
     });
 }
 
-export function purgeElasticsearchIndexes() {
+export function purgeElasticsearchIndexes(indexes?: string[]) {
     return bindClientFunc({
         clientFunc: Client4.purgeElasticsearchIndexes,
+        params: [
+            indexes,
+        ],
     });
 }