Add the API for search files (#15605)

* Adding search files api

* Fixing golangci-lint

* Adding bulk-indexing and improving a bit the name indexing for bleve and elasticsearch

* Add content extraction config migration

* Fixing a problem with document extraction

* Unapplying certain changes moved to other PR

* Fixing tests

* Making extract content app migration a private method

* Addressing PR review comments

* Addressing PR review comments

* Adding feature flag

* Removing debug string

* Fixing imports

* Fixing linting errors

* Do not migrate the config if the feature flag is not enabled

* Fix tests

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/app_iface.go

@@ -894,6 +894,7 @@ type AppIface interface {
 	SearchChannelsUserNotIn(teamID string, userID string, term string) (*model.ChannelList, *model.AppError)
 	SearchEmoji(name string, prefixOnly bool, limit int) ([]*model.Emoji, *model.AppError)
 	SearchEngine() *searchengine.Broker
+	SearchFilesInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.FileInfoList, *model.AppError)
 	SearchGroupChannels(userID, term string) (*model.ChannelList, *model.AppError)
 	SearchPostsInTeam(teamID string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError)
 	SearchPostsInTeamForUser(terms string, userID string, teamID string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError)

app/file.go

@@ -36,6 +36,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/plugin"
+	"github.com/mattermost/mattermost-server/v5/services/docextractor"
 	"github.com/mattermost/mattermost-server/v5/services/filesstore"
 	"github.com/mattermost/mattermost-server/v5/store"
 	"github.com/mattermost/mattermost-server/v5/utils"
@@ -771,6 +772,28 @@ func (a *App) UploadFileX(channelID, name string, input io.Reader,
 		}
 	}
 
+	if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch {
+		infoCopy := *t.fileinfo
+		a.Srv().Go(func() {
+			file, aerr := a.FileReader(t.fileinfo.Path)
+			if aerr != nil {
+				mlog.Error("Failed to open file for extract file content", mlog.Err(aerr))
+				return
+			}
+			defer file.Close()
+			text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{
+				ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
+			})
+			if err != nil {
+				mlog.Error("Failed to extract file content", mlog.Err(err))
+				return
+			}
+			if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil {
+				mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr))
+			}
+		})
+	}
+
 	return t.fileinfo, nil
 }
 
@@ -1020,6 +1043,28 @@ func (a *App) DoUploadFileExpectModification(now time.Time, rawTeamId string, ra
 		}
 	}
 
+	if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch {
+		infoCopy := *info
+		a.Srv().Go(func() {
+			file, aerr := a.FileReader(infoCopy.Path)
+			if aerr != nil {
+				mlog.Error("Failed to open file for extract file content", mlog.Err(aerr))
+				return
+			}
+			defer file.Close()
+			text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{
+				ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
+			})
+			if err != nil {
+				mlog.Error("Failed to extract file content", mlog.Err(err))
+				return
+			}
+			if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil {
+				mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr))
+			}
+		})
+	}
+
 	return info, data, nil
 }
 
@@ -1308,3 +1353,49 @@ func populateZipfile(w *zip.Writer, fileDatas []model.FileData) error {
 	}
 	return nil
 }
+
+func (a *App) SearchFilesInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.FileInfoList, *model.AppError) {
+	paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
+	includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
+
+	if !*a.Config().ServiceSettings.EnableFileSearch {
+		return nil, model.NewAppError("SearchFilesInTeamForUser", "store.sql_file_info.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
+	}
+
+	finalParamsList := []*model.SearchParams{}
+
+	for _, params := range paramsList {
+		params.OrTerms = isOrSearch
+		params.IncludeDeletedChannels = includeDeleted
+		// Don't allow users to search for "*"
+		if params.Terms != "*" {
+			// Convert channel names to channel IDs
+			params.InChannels = a.convertChannelNamesToChannelIds(params.InChannels, userId, teamId, includeDeletedChannels)
+			params.ExcludedChannels = a.convertChannelNamesToChannelIds(params.ExcludedChannels, userId, teamId, includeDeletedChannels)
+
+			// Convert usernames to user IDs
+			params.FromUsers = a.convertUserNameToUserIds(params.FromUsers)
+			params.ExcludedUsers = a.convertUserNameToUserIds(params.ExcludedUsers)
+
+			finalParamsList = append(finalParamsList, params)
+		}
+	}
+
+	// If the processed search params are empty, return empty search results.
+	if len(finalParamsList) == 0 {
+		return model.NewFileInfoList(), nil
+	}
+
+	fileInfoSearchResults, nErr := a.Srv().Store.FileInfo().Search(finalParamsList, userId, teamId, page, perPage)
+	if nErr != nil {
+		var appErr *model.AppError
+		switch {
+		case errors.As(nErr, &appErr):
+			return nil, appErr
+		default:
+			return nil, model.NewAppError("SearchPostsInTeamForUser", "app.post.search.app_error", nil, nErr.Error(), http.StatusInternalServerError)
+		}
+	}
+
+	return fileInfoSearchResults, nil
+}

app/file_test.go

@@ -13,11 +13,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v5/model"
-	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
-	"github.com/mattermost/mattermost-server/v5/services/filesstore/mocks"
+	filesStoreMocks "github.com/mattermost/mattermost-server/v5/services/filesstore/mocks"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine/mocks"
 	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
 )
 
@@ -277,7 +278,7 @@ func TestCreateZipFileAndAddFiles(t *testing.T) {
 	th := Setup(t)
 	defer th.TearDown()
 
-	mockBackend := mocks.FileBackend{}
+	mockBackend := filesStoreMocks.FileBackend{}
 	mockBackend.On("WriteFile", mock.Anything, "directory-to-heaven/zip-file-name-to-heaven.zip").Return(int64(666), errors.New("only those who dare to fail greatly can ever achieve greatly"))
 
 	err := th.App.CreateZipFileAndAddFiles(&mockBackend, []model.FileData{}, "zip-file-name-to-heaven.zip", "directory-to-heaven")
@@ -285,7 +286,7 @@ func TestCreateZipFileAndAddFiles(t *testing.T) {
 	require.NotNil(t, err)
 	require.Equal(t, err.Error(), "only those who dare to fail greatly can ever achieve greatly")
 
-	mockBackend = mocks.FileBackend{}
+	mockBackend = filesStoreMocks.FileBackend{}
 	mockBackend.On("WriteFile", mock.Anything, "directory-to-heaven/zip-file-name-to-heaven.zip").Return(int64(666), nil)
 	err = th.App.CreateZipFileAndAddFiles(&mockBackend, []model.FileData{}, "zip-file-name-to-heaven.zip", "directory-to-heaven")
 	require.NoError(t, err)
@@ -350,3 +351,196 @@ func createDummyImage() *image.RGBA {
 	lowerRightCorner := image.Point{width, height}
 	return image.NewRGBA(image.Rectangle{upperLeftCorner, lowerRightCorner})
 }
+
+func TestSearchFilesInTeamForUser(t *testing.T) {
+	perPage := 5
+	searchTerm := "searchTerm"
+
+	setup := func(t *testing.T, enableElasticsearch bool) (*TestHelper, []*model.FileInfo) {
+		th := Setup(t).InitBasic()
+
+		fileInfos := make([]*model.FileInfo, 7)
+		for i := 0; i < cap(fileInfos); i++ {
+			fileInfo, err := th.App.Srv().Store.FileInfo().Save(&model.FileInfo{
+				CreatorId: th.BasicUser.Id,
+				PostId:    th.BasicPost.Id,
+				Name:      searchTerm,
+				Path:      searchTerm,
+				Extension: "jpg",
+				MimeType:  "image/jpeg",
+			})
+			time.Sleep(1 * time.Millisecond)
+
+			require.Nil(t, err)
+
+			fileInfos[i] = fileInfo
+		}
+
+		if enableElasticsearch {
+			th.App.Srv().SetLicense(model.NewTestLicense("elastic_search"))
+
+			th.App.UpdateConfig(func(cfg *model.Config) {
+				*cfg.ElasticsearchSettings.EnableIndexing = true
+				*cfg.ElasticsearchSettings.EnableSearching = true
+			})
+		} else {
+			th.App.UpdateConfig(func(cfg *model.Config) {
+				*cfg.ElasticsearchSettings.EnableSearching = false
+			})
+		}
+
+		return th, fileInfos
+	}
+
+	t.Run("should return everything as first page of fileInfos from database", func(t *testing.T) {
+		th, fileInfos := setup(t, false)
+		defer th.TearDown()
+
+		page := 0
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		require.NotNil(t, results)
+		assert.Equal(t, []string{
+			fileInfos[6].Id,
+			fileInfos[5].Id,
+			fileInfos[4].Id,
+			fileInfos[3].Id,
+			fileInfos[2].Id,
+			fileInfos[1].Id,
+			fileInfos[0].Id,
+		}, results.Order)
+	})
+
+	t.Run("should not return later pages of fileInfos from database", func(t *testing.T) {
+		th, _ := setup(t, false)
+		defer th.TearDown()
+
+		page := 1
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		require.NotNil(t, results)
+		assert.Equal(t, []string{}, results.Order)
+	})
+
+	t.Run("should return first page of fileInfos from ElasticSearch", func(t *testing.T) {
+		th, fileInfos := setup(t, true)
+		defer th.TearDown()
+
+		page := 0
+		resultsPage := []string{
+			fileInfos[6].Id,
+			fileInfos[5].Id,
+			fileInfos[4].Id,
+			fileInfos[3].Id,
+			fileInfos[2].Id,
+		}
+
+		es := &mocks.SearchEngineInterface{}
+		es.On("SearchFiles", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil)
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		require.NotNil(t, results)
+		assert.Equal(t, resultsPage, results.Order)
+		es.AssertExpectations(t)
+	})
+
+	t.Run("should return later pages of fileInfos from ElasticSearch", func(t *testing.T) {
+		th, fileInfos := setup(t, true)
+		defer th.TearDown()
+
+		page := 1
+		resultsPage := []string{
+			fileInfos[1].Id,
+			fileInfos[0].Id,
+		}
+
+		es := &mocks.SearchEngineInterface{}
+		es.On("SearchFiles", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil)
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		require.NotNil(t, results)
+		assert.Equal(t, resultsPage, results.Order)
+		es.AssertExpectations(t)
+	})
+
+	t.Run("should fall back to database if ElasticSearch fails on first page", func(t *testing.T) {
+		th, fileInfos := setup(t, true)
+		defer th.TearDown()
+
+		page := 0
+
+		es := &mocks.SearchEngineInterface{}
+		es.On("SearchFiles", mock.Anything, mock.Anything, page, perPage).Return(nil, &model.AppError{})
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		require.NotNil(t, results)
+		assert.Equal(t, []string{
+			fileInfos[6].Id,
+			fileInfos[5].Id,
+			fileInfos[4].Id,
+			fileInfos[3].Id,
+			fileInfos[2].Id,
+			fileInfos[1].Id,
+			fileInfos[0].Id,
+		}, results.Order)
+		es.AssertExpectations(t)
+	})
+
+	t.Run("should return nothing if ElasticSearch fails on later pages", func(t *testing.T) {
+		th, _ := setup(t, true)
+		defer th.TearDown()
+
+		page := 1
+
+		es := &mocks.SearchEngineInterface{}
+		es.On("SearchFiles", mock.Anything, mock.Anything, page, perPage).Return(nil, &model.AppError{})
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
+
+		results, err := th.App.SearchFilesInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
+
+		require.Nil(t, err)
+		assert.Equal(t, []string{}, results.Order)
+		es.AssertExpectations(t)
+	})
+}

app/migrations.go

@@ -15,6 +15,8 @@ import (
 const EmojisPermissionsMigrationKey = "EmojisPermissionsMigrationComplete"
 const GuestRolesCreationMigrationKey = "GuestRolesCreationMigrationComplete"
 const SystemConsoleRolesCreationMigrationKey = "SystemConsoleRolesCreationMigrationComplete"
+const ContentExtractionConfigMigrationKey = "ContentExtractionConfigMigrationComplete"
+const usersLimitToAutoEnableContentExtraction = 500
 
 // This function migrates the default built in roles from code/config to the database.
 func (a *App) DoAdvancedPermissionsMigration() {
@@ -283,6 +285,35 @@ func (a *App) DoSystemConsoleRolesCreationMigration() {
 	}
 }
 
+func (a *App) doContentExtractionConfigMigration() {
+	if !a.Config().FeatureFlags.FilesSearch {
+		return
+	}
+	// If the migration is already marked as completed, don't do it again.
+	if _, err := a.Srv().Store.System().GetByName(ContentExtractionConfigMigrationKey); err == nil {
+		return
+	}
+
+	if usersCount, err := a.Srv().Store.User().Count(model.UserCountOptions{}); err != nil {
+		mlog.Critical("Failed to get the users count for migrating the content extraction, using default value", mlog.Err(err))
+	} else {
+		if usersCount < usersLimitToAutoEnableContentExtraction {
+			a.UpdateConfig(func(config *model.Config) {
+				config.FileSettings.ExtractContent = model.NewBool(true)
+			})
+		}
+	}
+
+	system := model.System{
+		Name:  ContentExtractionConfigMigrationKey,
+		Value: "true",
+	}
+
+	if err := a.Srv().Store.System().Save(&system); err != nil {
+		mlog.Critical("Failed to mark content extraction config migration as completed.", mlog.Err(err))
+	}
+}
+
 func (a *App) DoAppMigrations() {
 	a.DoAdvancedPermissionsMigration()
 	a.DoEmojisPermissionsMigration()
@@ -294,4 +325,5 @@ func (a *App) DoAppMigrations() {
 	if err != nil {
 		mlog.Critical("(app.App).DoPermissionsMigrations failed", mlog.Err(err))
 	}
+	a.doContentExtractionConfigMigration()
 }

app/opentracing/opentracing_layer.go

@@ -12956,6 +12956,28 @@ func (a *OpenTracingAppLayer) SearchEngine() *searchengine.Broker {
 	return resultVar0
 }
 
+func (a *OpenTracingAppLayer) SearchFilesInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page int, perPage int) (*model.FileInfoList, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchFilesInTeamForUser")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.SearchFilesInTeamForUser(terms, userId, teamId, isOrSearch, includeDeletedChannels, timeZoneOffset, page, perPage)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) SearchGroupChannels(userID string, term string) (*model.ChannelList, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchGroupChannels")

app/upload.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/plugin"
+	"github.com/mattermost/mattermost-server/v5/services/docextractor"
 	"github.com/mattermost/mattermost-server/v5/store"
 )
 
@@ -295,6 +296,22 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 		}
 	}
 
+	if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch {
+		infoCopy := *info
+		a.Srv().Go(func() {
+			text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{
+				ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
+			})
+			if err != nil {
+				mlog.Error("Failed to extract file content", mlog.Err(err))
+				return
+			}
+			if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil {
+				mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr))
+			}
+		})
+	}
+
 	// delete upload session
 	if storeErr := a.Srv().Store.UploadSession().Delete(us.Id); storeErr != nil {
 		mlog.Warn("Failed to delete UploadSession", mlog.Err(storeErr))