From f6c934d7e0da2927d9c0ae1a9c7c97956fce891d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Dieguez Date: Thu, 18 Jun 2020 14:34:23 +0200 Subject: [PATCH] Create GetOldestEntityCreationTime method (#14515) This method will be used by the ES index jobs in order to get the first timestamp to be used as the starting point when doing indexing tasks --- i18n/en.json | 8 ++++++-- store/opentracing_layer.go | 18 ++++++++++++++++++ store/sqlstore/post_store.go | 23 +++++++++++++++++++++++ store/store.go | 1 + store/storetest/mocks/PostStore.go | 23 +++++++++++++++++++++++ store/timer_layer.go | 16 ++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 91c1f8e898..e44b0e6c2a 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -4228,7 +4228,7 @@ }, { "id": "ent.data_retention.channel_member_history_batch.internal_error", - "translation": "Failed to purge records." + "translation": "" }, { "id": "ent.data_retention.generic.license.error", @@ -4408,7 +4408,7 @@ }, { "id": "ent.get_users_in_channel_during", - "translation": "Failed to get users in channel during specified time period." + "translation": "" }, { "id": "ent.id_loaded.license_disable.app_error", @@ -6706,6 +6706,10 @@ "id": "store.sql_post.get_flagged_posts.app_error", "translation": "Unable to get the flagged posts." }, + { + "id": "store.sql_post.get_oldest_entity_creation_time.app_error", + "translation": "Unable to get the oldest entitiy creation time." + }, { "id": "store.sql_post.get_parents_posts.app_error", "translation": "Unable to get the parent post for the channel." diff --git a/store/opentracing_layer.go b/store/opentracing_layer.go index 523dc7fd4b..511969a319 100644 --- a/store/opentracing_layer.go +++ b/store/opentracing_layer.go @@ -4667,6 +4667,24 @@ func (s *OpenTracingLayerPostStore) GetOldest() (*model.Post, *model.AppError) { return resultVar0, resultVar1 } +func (s *OpenTracingLayerPostStore) GetOldestEntityCreationTime() (int64, *model.AppError) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.GetOldestEntityCreationTime") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + resultVar0, resultVar1 := s.PostStore.GetOldestEntityCreationTime() + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (s *OpenTracingLayerPostStore) GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.GetParentsForExportAfter") diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 3ef45c76e9..6557f0ad00 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -1779,3 +1779,26 @@ func (s *SqlPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams return model.MakePostSearchResults(posts, nil), nil } + +func (s *SqlPostStore) GetOldestEntityCreationTime() (int64, *model.AppError) { + query := s.getQueryBuilder().Select("MIN(min_createat) min_createat"). + Suffix(`FROM ( + (SELECT MIN(createat) min_createat FROM Posts) + UNION + (SELECT MIN(createat) min_createat FROM Users) + UNION + (SELECT MIN(createat) min_createat FROM Channels) + ) entities`) + queryString, _, err := query.ToSql() + if err != nil { + return -1, model.NewAppError("SqlPostStore.GetOldestEntityCreationTime", + "store.sql_post.get_oldest_entity_creation_time.app_error", nil, err.Error(), http.StatusInternalServerError) + } + row := s.GetReplica().Db.QueryRow(queryString) + var oldest int64 + if err := row.Scan(&oldest); err != nil { + return -1, model.NewAppError("SqlPostStore.GetOldestEntityCreationTime", + "store.sql_post.get_oldest_entity_creation_time.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return oldest, nil +} diff --git a/store/store.go b/store/store.go index cc5d1747f8..fba1734a11 100644 --- a/store/store.go +++ b/store/store.go @@ -270,6 +270,7 @@ type PostStore interface { GetRepliesForExport(parentId string) ([]*model.ReplyForExport, *model.AppError) GetDirectPostParentsForExportAfter(limit int, afterId string) ([]*model.DirectPostForExport, *model.AppError) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) + GetOldestEntityCreationTime() (int64, *model.AppError) } type UserStore interface { diff --git a/store/storetest/mocks/PostStore.go b/store/storetest/mocks/PostStore.go index 39ad2e446d..88e92fa0d2 100644 --- a/store/storetest/mocks/PostStore.go +++ b/store/storetest/mocks/PostStore.go @@ -286,6 +286,29 @@ func (_m *PostStore) GetOldest() (*model.Post, *model.AppError) { return r0, r1 } +// GetOldestEntityCreationTime provides a mock function with given fields: +func (_m *PostStore) GetOldestEntityCreationTime() (int64, *model.AppError) { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func() *model.AppError); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + // GetParentsForExportAfter provides a mock function with given fields: limit, afterId func (_m *PostStore) GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) { ret := _m.Called(limit, afterId) diff --git a/store/timer_layer.go b/store/timer_layer.go index 5f234c5f00..4fabf78518 100644 --- a/store/timer_layer.go +++ b/store/timer_layer.go @@ -4237,6 +4237,22 @@ func (s *TimerLayerPostStore) GetOldest() (*model.Post, *model.AppError) { return resultVar0, resultVar1 } +func (s *TimerLayerPostStore) GetOldestEntityCreationTime() (int64, *model.AppError) { + start := timemodule.Now() + + resultVar0, resultVar1 := s.PostStore.GetOldestEntityCreationTime() + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if resultVar1 == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("PostStore.GetOldestEntityCreationTime", success, elapsed) + } + return resultVar0, resultVar1 +} + func (s *TimerLayerPostStore) GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) { start := timemodule.Now()