Fixing problem with read-replica on indexing files (#17460)

store/opentracinglayer/opentracinglayer.go

@@ -3146,6 +3146,24 @@ func (s *OpenTracingLayerFileInfoStore) GetForUser(userID string) ([]*model.File
 	return result, err
 }
 
+func (s *OpenTracingLayerFileInfoStore) GetFromMaster(id string) (*model.FileInfo, error) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetFromMaster")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	result, err := s.FileInfoStore.GetFromMaster(id)
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return result, err
+}
+
 func (s *OpenTracingLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetWithOptions")

store/retrylayer/retrylayer.go

@@ -3370,6 +3370,26 @@ func (s *RetryLayerFileInfoStore) GetForUser(userID string) ([]*model.FileInfo,
 
 }
 
+func (s *RetryLayerFileInfoStore) GetFromMaster(id string) (*model.FileInfo, error) {
+
+	tries := 0
+	for {
+		result, err := s.FileInfoStore.GetFromMaster(id)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) {
 
 	tries := 0

store/searchlayer/file_info_layer.go

@@ -105,7 +105,7 @@ func (s SearchFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error)
 func (s SearchFileInfoStore) SetContent(fileID, content string) error {
 	err := s.FileInfoStore.SetContent(fileID, content)
 	if err == nil {
-		nfile, err2 := s.FileInfoStore.Get(fileID)
+		nfile, err2 := s.FileInfoStore.GetFromMaster(fileID)
 		if err2 == nil {
 			nfile.Content = content
 			s.indexFile(nfile)
@@ -117,7 +117,7 @@ func (s SearchFileInfoStore) SetContent(fileID, content string) error {
 func (s SearchFileInfoStore) AttachToPost(fileId, postId, creatorId string) error {
 	err := s.FileInfoStore.AttachToPost(fileId, postId, creatorId)
 	if err == nil {
-		nFileInfo, err2 := s.FileInfoStore.Get(fileId)
+		nFileInfo, err2 := s.FileInfoStore.GetFromMaster(fileId)
 		if err2 == nil {
 			s.indexFile(nFileInfo)
 		}

store/sqlstore/file_info_store.go

@@ -138,7 +138,7 @@ func (fs SqlFileInfoStore) Upsert(info *model.FileInfo) (*model.FileInfo, error)
 	return info, nil
 }
 
-func (fs SqlFileInfoStore) Get(id string) (*model.FileInfo, error) {
+func (fs SqlFileInfoStore) get(id string, fromMaster bool) (*model.FileInfo, error) {
 	info := &model.FileInfo{}
 
 	query := fs.getQueryBuilder().
@@ -152,7 +152,12 @@ func (fs SqlFileInfoStore) Get(id string) (*model.FileInfo, error) {
 		return nil, errors.Wrap(err, "file_info_tosql")
 	}
 
-	if err := fs.GetReplica().SelectOne(info, queryString, args...); err != nil {
+	db := fs.GetReplica()
+	if fromMaster {
+		db = fs.GetMaster()
+	}
+
+	if err := db.SelectOne(info, queryString, args...); err != nil {
 		if err == sql.ErrNoRows {
 			return nil, store.NewErrNotFound("FileInfo", id)
 		}
@@ -161,6 +166,14 @@ func (fs SqlFileInfoStore) Get(id string) (*model.FileInfo, error) {
 	return info, nil
 }
 
+func (fs SqlFileInfoStore) Get(id string) (*model.FileInfo, error) {
+	return fs.get(id, false)
+}
+
+func (fs SqlFileInfoStore) GetFromMaster(id string) (*model.FileInfo, error) {
+	return fs.get(id, true)
+}
+
 func (fs SqlFileInfoStore) GetWithOptions(page, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) {
 	if perPage < 0 {
 		return nil, store.NewErrLimitExceeded("perPage", perPage, "value used in pagination while getting FileInfos")

store/store.go

@@ -586,6 +586,7 @@ type FileInfoStore interface {
 	Save(info *model.FileInfo) (*model.FileInfo, error)
 	Upsert(info *model.FileInfo) (*model.FileInfo, error)
 	Get(id string) (*model.FileInfo, error)
+	GetFromMaster(id string) (*model.FileInfo, error)
 	GetByIds(ids []string) ([]*model.FileInfo, error)
 	GetByPath(path string) (*model.FileInfo, error)
 	GetForPost(postID string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error)

store/storetest/mocks/FileInfoStore.go

@@ -213,6 +213,29 @@ func (_m *FileInfoStore) GetForUser(userID string) ([]*model.FileInfo, error) {
 	return r0, r1
 }
 
+// GetFromMaster provides a mock function with given fields: id
+func (_m *FileInfoStore) GetFromMaster(id string) (*model.FileInfo, error) {
+	ret := _m.Called(id)
+
+	var r0 *model.FileInfo
+	if rf, ok := ret.Get(0).(func(string) *model.FileInfo); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.FileInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // GetWithOptions provides a mock function with given fields: page, perPage, opt
 func (_m *FileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) {
 	ret := _m.Called(page, perPage, opt)

store/timerlayer/timerlayer.go

@@ -2882,6 +2882,22 @@ func (s *TimerLayerFileInfoStore) GetForUser(userID string) ([]*model.FileInfo,
 	return result, err
 }
 
+func (s *TimerLayerFileInfoStore) GetFromMaster(id string) (*model.FileInfo, error) {
+	start := timemodule.Now()
+
+	result, err := s.FileInfoStore.GetFromMaster(id)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetFromMaster", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) {
 	start := timemodule.Now()