From 2cd83d2f8d898cc624b3649a1e590900eff0d6b1 Mon Sep 17 00:00:00 2001 From: Shivashis Padhi Date: Mon, 20 Jun 2022 19:57:17 +0530 Subject: [PATCH] [MM-44084] Feature: Top threads insights (#20195) * Add route endpoints, model, store functions, and tests for top threads * Run make store-layers * Make the following changes - Fix top user threads query - Fix passing parameters in api4/insights.go to handler in app - Add top user threads test * Add post-message, user_id, participants information to insights results * model.TopThread.UserID -> model.TopThread.UserId, for compatibility with MySQL * Rename name -> channel_name * Add user information to response * Link post in response, filter out deleted root posts from top threads * Handle thread delete cases, add app tests for threads insights * lint: fix typo * lint: rename asserts * lint: require.nil -> require.NoError * Add integration tests for thread insights * Add embeds and images to top posts * Add license checks for top threads endpoints * Query users in batch to populate post-creator * Make the following changes - Add license to test server in api4/ - Add tests for threads insights - top team threads shouldn't include threads from other teams, DMs - Test duration constraint - Pagination testing for top threads in model/insights_test.go * Add i18n-extract * i18n fixes * Add username, nickname to user_information * Hide message, user_id, post_id, reply_count in depth=1 of top threads response * Fix tests using response.reply_count to use response.post.reply_count Co-authored-by: Mattermod --- api4/insights.go | 114 +++++++ api4/insights_test.go | 226 ++++++++++++++ app/app_iface.go | 2 + app/opentracing/opentracing_layer.go | 44 +++ app/post.go | 44 +++ app/post_test.go | 189 ++++++++++++ i18n/en.json | 18 +- model/client4.go | 35 +++ model/insights.go | 41 +++ model/insights_test.go | 38 +++ store/opentracinglayer/opentracinglayer.go | 36 +++ store/retrylayer/retrylayer.go | 42 +++ store/sqlstore/thread_store.go | 206 +++++++++++++ store/store.go | 6 +- store/storetest/mocks/ThreadStore.go | 46 +++ store/storetest/thread_store.go | 343 +++++++++++++++++++++ store/timerlayer/timerlayer.go | 32 ++ 17 files changed, 1460 insertions(+), 2 deletions(-) diff --git a/api4/insights.go b/api4/insights.go index 3bcedecb70..ba49bf5a34 100644 --- a/api4/insights.go +++ b/api4/insights.go @@ -20,6 +20,10 @@ func (api *API) InitInsights() { // Channels api.BaseRoutes.InsightsForTeam.Handle("/channels", api.APISessionRequired(minimumProfessionalLicense(rejectGuests(getTopChannelsForTeamSince)))).Methods("GET") api.BaseRoutes.InsightsForUser.Handle("/channels", api.APISessionRequired(minimumProfessionalLicense(rejectGuests(getTopChannelsForUserSince)))).Methods("GET") + + // Threads + api.BaseRoutes.InsightsForTeam.Handle("/threads", api.APISessionRequired(requireLicense(getTopThreadsForTeamSince))).Methods("GET") + api.BaseRoutes.InsightsForUser.Handle("/threads", api.APISessionRequired(requireLicense(getTopThreadsForUserSince))).Methods("GET") } // Top Reactions @@ -227,6 +231,116 @@ func getTopChannelsForUserSince(c *Context, w http.ResponseWriter, r *http.Reque w.Write(js) } +// Top Threads +func getTopThreadsForTeamSince(c *Context, w http.ResponseWriter, r *http.Request) { + c.RequireTeamId() + if c.Err != nil { + return + } + + team, err := c.App.GetTeam(c.Params.TeamId) + if err != nil { + c.Err = err + return + } + + // license check + lic := c.App.Srv().License() + if lic.SkuShortName != model.LicenseShortSkuProfessional && lic.SkuShortName != model.LicenseShortSkuEnterprise { + c.Err = model.NewAppError("", "api.insights.license_error", nil, "", http.StatusNotImplemented) + return + } + + // restrict guests and users with no access to team + user, err := c.App.GetUser(c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } + + if !c.App.SessionHasPermissionToTeam(*c.AppContext.Session(), team.Id, model.PermissionViewTeam) || user.IsGuest() { + c.SetPermissionError(model.PermissionViewTeam) + return + } + + startTime := model.StartOfDayForTimeRange(c.Params.TimeRange, user.GetTimezoneLocation()) + + topThreads, err := c.App.GetTopThreadsForTeamSince(c.Params.TeamId, c.AppContext.Session().UserId, &model.InsightsOpts{ + StartUnixMilli: startTime.UnixMilli(), + Page: c.Params.Page, + PerPage: c.Params.PerPage, + }) + if err != nil { + c.Err = err + return + } + + js, jsonErr := json.Marshal(topThreads) + if jsonErr != nil { + c.Err = model.NewAppError("getTopThreadsForTeamSince", "api.marshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + return + } + + w.Write(js) +} + +func getTopThreadsForUserSince(c *Context, w http.ResponseWriter, r *http.Request) { + c.Params.TeamId = r.URL.Query().Get("team_id") + + // restrict guests and users with no access to team + user, err := c.App.GetUser(c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } + // TeamId is an optional parameter + if c.Params.TeamId != "" { + if !model.IsValidId(c.Params.TeamId) { + c.SetInvalidURLParam("team_id") + return + } + + team, teamErr := c.App.GetTeam(c.Params.TeamId) + if teamErr != nil { + c.Err = teamErr + return + } + + // license check + lic := c.App.Srv().License() + if lic.SkuShortName != model.LicenseShortSkuProfessional && lic.SkuShortName != model.LicenseShortSkuEnterprise { + c.Err = model.NewAppError("", "api.insights.license_error", nil, "", http.StatusNotImplemented) + return + } + + if !c.App.SessionHasPermissionToTeam(*c.AppContext.Session(), team.Id, model.PermissionViewTeam) || user.IsGuest() { + c.SetPermissionError(model.PermissionViewTeam) + return + } + } + + startTime := model.StartOfDayForTimeRange(c.Params.TimeRange, user.GetTimezoneLocation()) + + topThreads, err := c.App.GetTopThreadsForUserSince(c.Params.TeamId, c.AppContext.Session().UserId, &model.InsightsOpts{ + StartUnixMilli: startTime.UnixMilli(), + Page: c.Params.Page, + PerPage: c.Params.PerPage, + }) + + if err != nil { + c.Err = err + return + } + + js, jsonErr := json.Marshal(topThreads) + if jsonErr != nil { + c.Err = model.NewAppError("getTopThreadsForUserSince", "api.marshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + return + } + + w.Write(js) +} + // postCountByDurationViewModel expects a list of channels that are pre-authorized for the given user to view. func postCountByDurationViewModel(app app.AppIface, topChannelList *model.TopChannelList, startTime *time.Time, timeRange string, userID *string, location *time.Location) (model.ChannelPostCountByDuration, *model.AppError) { if len(topChannelList.Items) == 0 { diff --git a/api4/insights_test.go b/api4/insights_test.go index 58457e8b38..9c090c1a37 100644 --- a/api4/insights_test.go +++ b/api4/insights_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -610,3 +611,228 @@ func TestGetTopChannelsForUserSince(t *testing.T) { CheckForbiddenStatus(t, resp) }) } + +func TestGetTopThreadsForTeamSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional)) + + th.ConfigStore.SetReadOnlyFF(false) + defer th.ConfigStore.SetReadOnlyFF(true) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.FeatureFlags.InsightsEnabled = true }) + + th.LoginBasic() + client := th.Client + + // create a public channel, a private channel + + channelPublic := th.BasicChannel + channelPrivate := th.BasicPrivateChannel + th.App.AddUserToChannel(th.BasicUser, channelPublic, false) + th.App.AddUserToChannel(th.BasicUser, channelPrivate, false) + th.App.AddUserToChannel(th.BasicUser2, channelPublic, false) + th.App.RemoveUserFromChannel(th.Context, th.BasicUser2.Id, th.BasicUser.Id, channelPrivate) + + // create two threads: one in public channel, one in private + // post in public channel has both users interacting, post in private only has user1 interacting + + rootPostPublicChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPublic.Id, + Message: "root post pub", + }, channelPublic, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPublic.Id, + RootId: rootPostPublicChannel.Id, + Message: "reply post 1", + }, channelPublic, false, true) + require.Nil(t, appErr) + + rootPostPrivateChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + Message: "root post priv", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 1", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 2", + }, channelPrivate, false, true) + + require.Nil(t, appErr) + + // get top threads for team, as user 1 and user 2 + // user 1, 2 should see both threads + + topTeamThreadsByUser1, _, _ := client.GetTopThreadsForTeamSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser1.Items, 2) + require.Equal(t, topTeamThreadsByUser1.Items[0].Post.Id, rootPostPrivateChannel.Id) + require.Equal(t, topTeamThreadsByUser1.Items[1].Post.Id, rootPostPublicChannel.Id) + + client.Logout() + + th.LoginBasic2() + + client = th.Client + + topTeamThreadsByUser2, _, _ := client.GetTopThreadsForTeamSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser2.Items, 1) + require.Equal(t, topTeamThreadsByUser2.Items[0].Post.Id, rootPostPublicChannel.Id) + + // add user2 to private channel and it can see 2 top threads. + th.AddUserToChannel(th.BasicUser2, channelPrivate) + topTeamThreadsByUser2IncludingPrivate, _, _ := client.GetTopThreadsForTeamSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser2IncludingPrivate.Items, 2) +} + +func TestGetTopThreadsForUserSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional)) + + th.ConfigStore.SetReadOnlyFF(false) + defer th.ConfigStore.SetReadOnlyFF(true) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.FeatureFlags.InsightsEnabled = true }) + + th.LoginBasic() + client := th.Client + + // create a public channel, a private channel + + channelPublic := th.BasicChannel + channelPrivate := th.BasicPrivateChannel + th.App.AddUserToChannel(th.BasicUser, channelPublic, false) + th.App.AddUserToChannel(th.BasicUser, channelPrivate, false) + th.App.AddUserToChannel(th.BasicUser2, channelPublic, false) + + // create two threads: one in public channel, one in private + // post in public channel has both users interacting, post in private only has user1 interacting + + rootPostPublicChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPublic.Id, + Message: "root post pub", + }, channelPublic, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPublic.Id, + RootId: rootPostPublicChannel.Id, + Message: "reply post 1", + }, channelPublic, false, true) + require.Nil(t, appErr) + + rootPostPrivateChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + Message: "root post priv", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 1", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 2", + }, channelPrivate, false, true) + + require.Nil(t, appErr) + + // get top threads for user, as user 1 and user 2 + // user 1 should see both threads, while user 2 should see only thread in public channel + // (even if user2 is in the private channel it hasn't interacted with the thread there.) + + topUser1Threads, _, _ := client.GetTopThreadsForUserSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topUser1Threads.Items, 2) + require.Equal(t, topUser1Threads.Items[0].Post.Id, rootPostPrivateChannel.Id) + require.Equal(t, topUser1Threads.Items[0].Post.ReplyCount, int64(2)) + require.Equal(t, topUser1Threads.Items[1].Post.Id, rootPostPublicChannel.Id) + require.Contains(t, topUser1Threads.Items[1].Participants, th.BasicUser2.Id) + require.Equal(t, topUser1Threads.Items[1].Post.ReplyCount, int64(1)) + + client.Logout() + + th.LoginBasic2() + + client = th.Client + + topUser2Threads, _, _ := client.GetTopThreadsForUserSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topUser2Threads.Items, 1) + require.Equal(t, topUser2Threads.Items[0].Post.Id, rootPostPublicChannel.Id) + require.Equal(t, topUser2Threads.Items[0].Post.ReplyCount, int64(1)) + + // deleting the root post results in the thread not making it to top threads list + _, appErr = th.App.DeletePost(rootPostPublicChannel.Id, th.BasicUser.Id) + require.Nil(t, appErr) + + client.Logout() + + th.LoginBasic() + + client = th.Client + + topUser1ThreadsAfterPost1Delete, _, _ := client.GetTopThreadsForUserSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topUser1ThreadsAfterPost1Delete.Items, 1) + + client.Logout() + + th.LoginBasic2() + + client = th.Client + + // reply with user2 in thread2. deleting that reply, shouldn't give any top thread for user2 if the user2 unsubscribes to the thread after deleting the comment + replyPostUser2InPrivate, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 3", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + topUser2ThreadsAfterPrivateReply, _, _ := client.GetTopThreadsForUserSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topUser2ThreadsAfterPrivateReply.Items, 1) + + // deleting reply, and unfollowing thread + _, appErr = th.App.DeletePost(replyPostUser2InPrivate.Id, th.BasicUser2.Id) + require.Nil(t, appErr) + // unfollow thread + _, err := th.App.Srv().Store.Thread().MaintainMembership(th.BasicUser2.Id, rootPostPrivateChannel.Id, store.ThreadMembershipOpts{ + Following: false, + UpdateFollowing: true, + }) + require.NoError(t, err) + + topUser2ThreadsAfterPrivateReplyDelete, _, _ := client.GetTopThreadsForUserSince(th.BasicTeam.Id, model.TimeRangeToday, 0, 10) + require.Nil(t, appErr) + require.Len(t, topUser2ThreadsAfterPrivateReplyDelete.Items, 0) +} diff --git a/app/app_iface.go b/app/app_iface.go index 4355db7376..bd422c5de9 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -777,6 +777,8 @@ type AppIface interface { GetTopChannelsForUserSince(userID, teamID string, opts *model.InsightsOpts) (*model.TopChannelList, *model.AppError) GetTopReactionsForTeamSince(teamID string, userID string, opts *model.InsightsOpts) (*model.TopReactionList, *model.AppError) GetTopReactionsForUserSince(userID string, teamID string, opts *model.InsightsOpts) (*model.TopReactionList, *model.AppError) + GetTopThreadsForTeamSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) + GetTopThreadsForUserSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) GetUploadSession(uploadId string) (*model.UploadSession, *model.AppError) GetUploadSessionsForUser(userID string) ([]*model.UploadSession, *model.AppError) GetUser(userID string) (*model.User, *model.AppError) diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 4c521dc204..c2417e813f 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -9860,6 +9860,50 @@ func (a *OpenTracingAppLayer) GetTopReactionsForUserSince(userID string, teamID return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetTopThreadsForTeamSince(teamID string, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetTopThreadsForTeamSince") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetTopThreadsForTeamSince(teamID, userID, opts) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + +func (a *OpenTracingAppLayer) GetTopThreadsForUserSince(teamID string, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetTopThreadsForUserSince") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetTopThreadsForUserSince(teamID, userID, opts) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetTotalUsersStats(viewRestrictions *model.ViewUsersRestrictions) (*model.UsersStats, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetTotalUsersStats") diff --git a/app/post.go b/app/post.go index 83ddf25bd7..217fdf2d19 100644 --- a/app/post.go +++ b/app/post.go @@ -1717,3 +1717,47 @@ func (a *App) GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) { return posts, nil } + +func (a *App) GetTopThreadsForTeamSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) { + if !a.Config().FeatureFlags.InsightsEnabled { + return nil, model.NewAppError("GetTopChannelsForTeamSince", "app.insights.feature_disabled", nil, "", http.StatusNotImplemented) + } + + topThreads, err := a.Srv().Store.Thread().GetTopThreadsForTeamSince(teamID, userID, opts.StartUnixMilli, opts.Page*opts.PerPage, opts.PerPage) + if err != nil { + return nil, model.NewAppError("GetTopChannelsForTeamSince", "app.post.get_top_threads_for_team_since.app_error", nil, err.Error(), http.StatusInternalServerError) + } + topThreadsWithEmbedAndImage, err := includeEmbedsAndImages(a, topThreads, userID) + if err != nil { + return nil, model.NewAppError("GetTopChannelsForTeamSince", "app.post.get_top_threads_for_team_since.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return topThreadsWithEmbedAndImage, nil +} + +func (a *App) GetTopThreadsForUserSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) { + if !a.Config().FeatureFlags.InsightsEnabled { + return nil, model.NewAppError("GetTopChannelsForTeamSince", "app.insights.feature_disabled", nil, "", http.StatusNotImplemented) + } + + topThreads, err := a.Srv().Store.Thread().GetTopThreadsForUserSince(teamID, userID, opts.StartUnixMilli, opts.Page*opts.PerPage, opts.PerPage) + if err != nil { + return nil, model.NewAppError("GetTopChannelsForTeamSince", "app.post.get_top_threads_for_team_since.app_error", nil, err.Error(), http.StatusInternalServerError) + } + topThreadsWithEmbedAndImage, err := includeEmbedsAndImages(a, topThreads, userID) + if err != nil { + return nil, model.NewAppError("GetTopChannelsForUserSince", "app.post.get_top_threads_for_user_since.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return topThreadsWithEmbedAndImage, nil +} + +func includeEmbedsAndImages(a *App, topThreadList *model.TopThreadList, userID string) (*model.TopThreadList, error) { + for _, topThread := range topThreadList.Items { + topThread.Post = a.PreparePostForClientWithEmbedsAndImages(topThread.Post, false, false) + sanitizedPost, err := a.SanitizePostMetadataForUser(topThread.Post, userID) + if err != nil { + return nil, err + } + topThread.Post = sanitizedPost + } + return topThreadList, nil +} diff --git a/app/post_test.go b/app/post_test.go index da87cc4539..5fe4156016 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -2817,3 +2817,192 @@ func TestShouldNotRefollowOnOthersReply(t *testing.T) { require.Nil(t, err) require.True(t, m.Following) } + +func TestGetTopThreadsForTeamSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + + th.Server.configStore.SetReadOnlyFF(false) + defer th.Server.configStore.SetReadOnlyFF(true) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.FeatureFlags.InsightsEnabled = true }) + + // create a public channel, a private channel + channelPublic := th.CreateChannel(th.BasicTeam) + channelPrivate := th.CreatePrivateChannel(th.BasicTeam) + th.AddUserToChannel(th.BasicUser, channelPublic) + th.AddUserToChannel(th.BasicUser, channelPrivate) + th.AddUserToChannel(th.BasicUser2, channelPublic) + + // create two threads: one in public channel, one in private with only basicUser1 + + rootPostPublicChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPublic.Id, + Message: "root post", + }, channelPublic, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPublic.Id, + RootId: rootPostPublicChannel.Id, + Message: fmt.Sprintf("@%s", th.BasicUser2.Username), + }, channelPublic, false, true) + require.Nil(t, appErr) + + rootPostPrivateChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + Message: "root post", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: fmt.Sprintf("@%s", th.BasicUser2.Username), + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: fmt.Sprintf("@%s", th.BasicUser2.Username), + }, channelPrivate, false, true) + + require.Nil(t, appErr) + + // get top threads for team, as user 1 and user 2 + // user 1 should see both threads, while user 2 should see only thread in public channel. + + topTeamThreadsByUser1, appErr := th.App.GetTopThreadsForTeamSince(th.BasicTeam.Id, th.BasicUser.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser1.Items, 2) + require.Equal(t, topTeamThreadsByUser1.Items[0].Post.Id, rootPostPrivateChannel.Id) + require.Equal(t, topTeamThreadsByUser1.Items[1].Post.Id, rootPostPublicChannel.Id) + + topTeamThreadsByUser2, appErr := th.App.GetTopThreadsForTeamSince(th.BasicTeam.Id, th.BasicUser2.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser2.Items, 1) + require.Equal(t, topTeamThreadsByUser2.Items[0].Post.Id, rootPostPublicChannel.Id) + + // add user2 to private channel and it can see 2 top threads. + th.AddUserToChannel(th.BasicUser2, channelPrivate) + topTeamThreadsByUser2IncludingPrivate, appErr := th.App.GetTopThreadsForTeamSince(th.BasicTeam.Id, th.BasicUser2.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topTeamThreadsByUser2IncludingPrivate.Items, 2) +} +func TestGetTopThreadsForUserSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + + th.Server.configStore.SetReadOnlyFF(false) + defer th.Server.configStore.SetReadOnlyFF(true) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.FeatureFlags.InsightsEnabled = true }) + + // create a public channel, a private channel + channelPublic := th.CreateChannel(th.BasicTeam) + channelPrivate := th.CreatePrivateChannel(th.BasicTeam) + th.AddUserToChannel(th.BasicUser, channelPublic) + th.AddUserToChannel(th.BasicUser, channelPrivate) + th.AddUserToChannel(th.BasicUser2, channelPublic) + th.AddUserToChannel(th.BasicUser2, channelPrivate) + + // create two threads: one in public channel, one in private + // post in public channel has both users interacting, post in private only has user1 interacting + + rootPostPublicChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPublic.Id, + Message: "root post pub", + }, channelPublic, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPublic.Id, + RootId: rootPostPublicChannel.Id, + Message: "reply post 1", + }, channelPublic, false, true) + require.Nil(t, appErr) + + rootPostPrivateChannel, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + Message: "root post priv", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 1", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + _, appErr = th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 2", + }, channelPrivate, false, true) + + require.Nil(t, appErr) + + // get top threads for user, as user 1 and user 2 + // user 1 should see both threads, while user 2 should see only thread in public channel + // (even if user2 is in the private channel it hasn't interacted with the thread there.) + + topUser1Threads, appErr := th.App.GetTopThreadsForUserSince(th.BasicTeam.Id, th.BasicUser.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topUser1Threads.Items, 2) + require.Equal(t, topUser1Threads.Items[0].Post.Id, rootPostPrivateChannel.Id) + require.Equal(t, topUser1Threads.Items[0].ReplyCount, int64(2)) + require.Equal(t, topUser1Threads.Items[1].Post.Id, rootPostPublicChannel.Id) + require.Contains(t, topUser1Threads.Items[1].Participants, th.BasicUser2.Id) + require.Equal(t, topUser1Threads.Items[1].ReplyCount, int64(1)) + + topUser2Threads, appErr := th.App.GetTopThreadsForUserSince(th.BasicTeam.Id, th.BasicUser2.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topUser2Threads.Items, 1) + require.Equal(t, topUser2Threads.Items[0].Post.Id, rootPostPublicChannel.Id) + require.Equal(t, topUser2Threads.Items[0].ReplyCount, int64(1)) + + // deleting the root post results in the thread not making it to top threads list + _, appErr = th.App.DeletePost(rootPostPublicChannel.Id, th.BasicUser.Id) + require.Nil(t, appErr) + + topUser1ThreadsAfterPost1Delete, appErr := th.App.GetTopThreadsForUserSince(th.BasicTeam.Id, th.BasicUser.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topUser1ThreadsAfterPost1Delete.Items, 1) + + // reply with user2 in thread2. deleting that reply, shouldn't give any top thread for user2 if the user2 unsubscribes to the thread after deleting the comment + replyPostUser2InPrivate, appErr := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser2.Id, + ChannelId: channelPrivate.Id, + RootId: rootPostPrivateChannel.Id, + Message: "reply post 3", + }, channelPrivate, false, true) + require.Nil(t, appErr) + + topUser2ThreadsAfterPrivateReply, appErr := th.App.GetTopThreadsForUserSince(th.BasicTeam.Id, th.BasicUser2.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topUser2ThreadsAfterPrivateReply.Items, 1) + + // deleting reply, and unfollowing thread + _, appErr = th.App.DeletePost(replyPostUser2InPrivate.Id, th.BasicUser2.Id) + require.Nil(t, appErr) + // unfollow thread + _, err := th.App.Srv().Store.Thread().MaintainMembership(th.BasicUser2.Id, rootPostPrivateChannel.Id, store.ThreadMembershipOpts{ + Following: false, + UpdateFollowing: true, + }) + require.NoError(t, err) + + topUser2ThreadsAfterPrivateReplyDelete, appErr := th.App.GetTopThreadsForUserSince(th.BasicTeam.Id, th.BasicUser2.Id, &model.InsightsOpts{StartUnixMilli: 200, PerPage: 100}) + require.Nil(t, appErr) + require.Len(t, topUser2ThreadsAfterPrivateReplyDelete.Items, 0) +} diff --git a/i18n/en.json b/i18n/en.json index 04d1dbf921..6e4b2d3bd1 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1907,7 +1907,11 @@ }, { "id": "api.insights.feature_disabled", - "translation": " " + "translation": "Insights is behind a feature flag which is not enabled." + }, + { + "id": "api.insights.license_error", + "translation": "Your license doesn't support Insights feature." }, { "id": "api.invalid_channel", @@ -5455,6 +5459,10 @@ "id": "app.insert_error", "translation": "insert error" }, + { + "id": "app.insights.feature_disabled", + "translation": "Insights feature is disabled." + }, { "id": "app.install_integration.reached_max_limit.error", "translation": "You've reached the max limit of {{.NumIntegrations}} enabled integrations. To install unlimited integrations, upgrade to one of our paid plans." @@ -5879,6 +5887,14 @@ "id": "app.post.get_root_posts.app_error", "translation": "Unable to get the posts for the channel." }, + { + "id": "app.post.get_top_threads_for_team_since.app_error", + "translation": "Unable to get top threads for team." + }, + { + "id": "app.post.get_top_threads_for_user_since.app_error", + "translation": "Unable to get top threads for user." + }, { "id": "app.post.marshal.app_error", "translation": "Failed to marshal post." diff --git a/model/client4.go b/model/client4.go index fda171b781..5c072765dd 100644 --- a/model/client4.go +++ b/model/client4.go @@ -4172,6 +4172,41 @@ func (c *Client4) DoPostActionWithCookie(postId, actionId, selected, cookieStr s return BuildResponse(r), nil } +// GetTopThreadsForTeamSince will return an ordered list of the top channels in a given team. +func (c *Client4) GetTopThreadsForTeamSince(teamId string, timeRange string, page int, perPage int) (*TopThreadList, *Response, error) { + query := fmt.Sprintf("?time_range=%v&page=%v&per_page=%v", timeRange, page, perPage) + r, err := c.DoAPIGet(c.teamRoute(teamId)+"/top/threads"+query, "") + if err != nil { + return nil, BuildResponse(r), err + } + defer closeBody(r) + var topThreads *TopThreadList + if jsonErr := json.NewDecoder(r.Body).Decode(&topThreads); jsonErr != nil { + return nil, nil, NewAppError("GetTopThreadsForTeamSince", "api.unmarshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + } + return topThreads, BuildResponse(r), nil +} + +// GetTopThreadsForUserSince will return an ordered list of your top channels in a given team. +func (c *Client4) GetTopThreadsForUserSince(teamId string, timeRange string, page int, perPage int) (*TopThreadList, *Response, error) { + query := fmt.Sprintf("?time_range=%v&page=%v&per_page=%v", timeRange, page, perPage) + + if teamId != "" { + query += fmt.Sprintf("&team_id=%v", teamId) + } + + r, err := c.DoAPIGet(c.usersRoute()+"/me/top/threads"+query, "") + if err != nil { + return nil, BuildResponse(r), err + } + defer closeBody(r) + var topThreads *TopThreadList + if jsonErr := json.NewDecoder(r.Body).Decode(&topThreads); jsonErr != nil { + return nil, nil, NewAppError("GetTopThreadsForUserSince", "api.unmarshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + } + return topThreads, BuildResponse(r), nil +} + // OpenInteractiveDialog sends a WebSocket event to a user's clients to // open interactive dialogs, based on the provided trigger ID and other // provided data. Used with interactive message buttons, menus and diff --git a/model/insights.go b/model/insights.go index 54db801170..dbd1720bc7 100644 --- a/model/insights.go +++ b/model/insights.go @@ -63,6 +63,33 @@ type TopChannel struct { MessageCount int64 `json:"message_count"` } +// Top Threads +type TopThreadList struct { + InsightsListData + Items []*TopThread `json:"items"` +} + +type TopThread struct { + PostId string `json:"-"` + ReplyCount int64 `json:"-"` + ChannelId string `json:"channel_id"` + DisplayName string `json:"channel_display_name"` + Name string `json:"channel_name"` + Participants StringArray `json:"participants"` + UserId string `json:"-"` + UserInformation *InsightUserInformation `json:"user_information"` + Post *Post `json:"post"` +} + +type InsightUserInformation struct { + Id string `json:"id"` + LastPictureUpdate int64 `json:"last_picture_update"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + NickName string `json:"nickname"` + Username string `json:"username"` +} + type DurationPostCount struct { ChannelID string `db:"channelid"` // Duration is an ISO8601 date string representing either a day or a day and hour (ex. "2022-05-26" or "2022-05-26T14"). @@ -202,3 +229,17 @@ func GetTopChannelListWithPagination(channels []*TopChannel, limit int) *TopChan return &TopChannelList{InsightsListData: InsightsListData{HasNext: hasNext}, Items: channels} } + +// GetTopThreadListWithPagination adds a rank to each item in the given list of TopThread and checks if there is +// another page that can be fetched based on the given limit and offset. The given list of TopThread is assumed to be +// sorted by ReplyCount(score). Returns a TopThreadList. +func GetTopThreadListWithPagination(threads []*TopThread, limit int) *TopThreadList { + // Add pagination support + var hasNext bool + if (limit != 0) && (len(threads) == limit+1) { + hasNext = true + threads = threads[:len(threads)-1] + } + + return &TopThreadList{InsightsListData: InsightsListData{HasNext: hasNext}, Items: threads} +} diff --git a/model/insights_test.go b/model/insights_test.go index 55a9359d09..671e0443fb 100644 --- a/model/insights_test.go +++ b/model/insights_test.go @@ -82,3 +82,41 @@ func TestGetTopChannelListWithPagination(t *testing.T) { }) } } + +func TestGetTopThreadListWithPagination(t *testing.T) { + threads := []*TopThread{ + {PostId: NewId(), ReplyCount: 100}, + {PostId: NewId(), ReplyCount: 80}, + {PostId: NewId(), ReplyCount: 90}, + {PostId: NewId(), ReplyCount: 76}, + {PostId: NewId(), ReplyCount: 43}, + {PostId: NewId(), ReplyCount: 2}, + {PostId: NewId(), ReplyCount: 1}, + } + hasNextTT := []struct { + Description string + Limit int + Offset int + Expected *TopThreadList + }{ + { + Description: "has one page", + Limit: len(threads), + Offset: 0, + Expected: &TopThreadList{InsightsListData: InsightsListData{HasNext: false}, Items: threads}, + }, + { + Description: "has more than one page", + Limit: len(threads) - 1, + Offset: 0, + Expected: &TopThreadList{InsightsListData: InsightsListData{HasNext: true}, Items: threads}, + }, + } + + for _, test := range hasNextTT { + t.Run(test.Description, func(t *testing.T) { + actual := GetTopThreadListWithPagination(threads, test.Limit) + assert.Equal(t, test.Expected.HasNext, actual.HasNext) + }) + } +} diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 4a7b743ab1..93fd26ad2a 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -9646,6 +9646,42 @@ func (s *OpenTracingLayerThreadStore) GetThreadsForUser(userId string, teamID st return result, err } +func (s *OpenTracingLayerThreadStore) GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ThreadStore.GetTopThreadsForTeamSince") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.ThreadStore.GetTopThreadsForTeamSince(teamID, userID, since, offset, limit) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + +func (s *OpenTracingLayerThreadStore) GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ThreadStore.GetTopThreadsForUserSince") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.ThreadStore.GetTopThreadsForUserSince(teamID, userID, since, offset, limit) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerThreadStore) GetTotalThreads(userId string, teamID string, opts model.GetUserThreadsOpts) (int64, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ThreadStore.GetTotalThreads") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 182381cfb8..0f09d785f9 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -11024,6 +11024,48 @@ func (s *RetryLayerThreadStore) GetThreadsForUser(userId string, teamID string, } +func (s *RetryLayerThreadStore) GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + + tries := 0 + for { + result, err := s.ThreadStore.GetTopThreadsForTeamSince(teamID, userID, since, offset, limit) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + +func (s *RetryLayerThreadStore) GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + + tries := 0 + for { + result, err := s.ThreadStore.GetTopThreadsForUserSince(teamID, userID, since, offset, limit) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerThreadStore) GetTotalThreads(userId string, teamID string, opts model.GetUserThreadsOpts) (int64, error) { tries := 0 diff --git a/store/sqlstore/thread_store.go b/store/sqlstore/thread_store.go index 9d34b0ae5d..038d0264eb 100644 --- a/store/sqlstore/thread_store.go +++ b/store/sqlstore/thread_store.go @@ -935,3 +935,209 @@ func (s *SqlThreadStore) GetThreadUnreadReplyCount(threadMembership *model.Threa return unreadReplies, nil } + +// Top threads in all public channels and private channels userID is a member of. Returns a list of threads ranked by interactions. +func (s *SqlThreadStore) GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + var args []interface{} + query := `select + threads_list.PostId, + threads_list.ReplyCount, + threads_list.ChannelId, + threads_list.DisplayName, + threads_list.Name, + threads_list.Participants, + p.UserId + from(( + SELECT + t.PostId, + t.ReplyCount, + t.ChannelId, + t.Participants, + c.DisplayName, + c.Name + FROM + Threads t + LEFT JOIN PublicChannels c ON t.ChannelId = c.Id + WHERE + t.threaddeleteat IS NULL + AND t.LastReplyAt > ? + AND c.TeamId = ? + GROUP BY + t.PostId, + c.DisplayName, + c.Name, + t.Participants + ) + UNION + ALL ( + SELECT + t.PostId, + t.ReplyCount, + t.ChannelId, + t.Participants, + c.DisplayName, + c.Name + FROM + Threads t + LEFT JOIN ChannelMembers cm ON t.ChannelId = cm.ChannelId + LEFT JOIN Channels c ON t.ChannelId = c.Id + WHERE + t.threaddeleteat IS NULL + AND cm.UserId = ? + AND c.Type = 'P' + AND c.TeamId = ? + AND t.LastReplyAt > ? + GROUP BY + t.PostId, + c.DisplayName, + c.Name, + t.Participants + )) as threads_list + LEFT JOIN Posts as p on p.Id = threads_list.PostId + ORDER BY ReplyCount DESC + limit ? offset ?` + + args = append(args, since, teamID, userID, teamID, since, limit+1, offset) + + topThreads := make([]*model.TopThread, 0) + err := s.GetReplicaX().Select(&topThreads, query, args...) + if err != nil { + return nil, errors.Wrapf(err, "failed to get top threads=%s", teamID) + } + topThreads, err = postProcessTopThreads(topThreads, s, teamID) + if err != nil { + return nil, err + } + return model.GetTopThreadListWithPagination(topThreads, limit), nil +} + +func (s *SqlThreadStore) GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + var args []interface{} + + // gets all threads within the team which user follows. + query := `select + threads_list.PostId, + threads_list.ReplyCount, + threads_list.ChannelId, + threads_list.DisplayName, + threads_list.Name, + threads_list.Participants, + p.UserId + from(( + SELECT + t.PostId, + t.ReplyCount, + t.ChannelId, + t.Participants, + c.DisplayName, + c.Name + FROM + Threads t + LEFT JOIN PublicChannels c ON t.ChannelId = c.Id + LEFT JOIN ThreadMemberships as tm on t.PostId = tm.PostId + WHERE + t.threaddeleteat IS NULL + AND t.LastReplyAt > ? + AND c.TeamId = ? + AND tm.UserId = ? + AND tm.Following = TRUE + GROUP BY + t.PostId, + c.DisplayName, + c.Name, + t.Participants + ) + UNION + ALL ( + SELECT + t.PostId, + t.ReplyCount, + t.ChannelId, + t.Participants, + c.DisplayName, + c.Name + FROM + Threads t + LEFT JOIN ChannelMembers cm ON t.ChannelId = cm.ChannelId + LEFT JOIN Channels c ON t.ChannelId = c.Id + LEFT JOIN ThreadMemberships as tm on t.PostId = tm.PostId + WHERE + cm.UserId = ? + AND c.Type = 'P' + AND c.TeamId = ? + AND t.threaddeleteat IS NULL + AND t.LastReplyAt > ? + AND tm.UserId = ? + AND tm.Following = TRUE + GROUP BY + t.PostId, + c.DisplayName, + c.Name, + t.Participants + )) as threads_list + LEFT JOIN Posts as p on p.Id = threads_list.PostId + ORDER BY ReplyCount DESC + limit ? offset ?` + + args = append(args, since, teamID, userID, userID, teamID, since, userID, limit+1, offset) + + topThreads := make([]*model.TopThread, 0) + err := s.GetReplicaX().Select(&topThreads, query, args...) + if err != nil { + return nil, errors.Wrapf(err, "failed to get top threads=%s", teamID) + } + topThreads, err = postProcessTopThreads(topThreads, s, teamID) + if err != nil { + return nil, err + } + return model.GetTopThreadListWithPagination(topThreads, limit), nil +} + +func userContains(userIDs []string, searchedUserID string) bool { + for _, userID := range userIDs { + if userID == searchedUserID { + return true + } + } + return false +} + +func postProcessTopThreads(topThreads []*model.TopThread, s *SqlThreadStore, teamID string) ([]*model.TopThread, error) { + // create list of userIDs + var userIDs []string + for _, topThread := range topThreads { + userID := topThread.UserId + if !userContains(userIDs, userID) { + userIDs = append(userIDs, userID) + } + } + + usersMap := map[string]*model.User{} + + users, err := s.User().GetProfileByIds(context.Background(), userIDs, &store.UserGetByIdsOpts{}, true) + if err != nil { + return nil, errors.Wrapf(err, "failed to get users for top threads in team=%s", teamID) + } + for _, user := range users { + usersMap[user.Id] = user + } + + // resolve user, root post for each top thread + for _, topThread := range topThreads { + postCreator := usersMap[topThread.UserId] + topThread.UserInformation = &model.InsightUserInformation{ + Id: postCreator.Id, + LastPictureUpdate: postCreator.LastPictureUpdate, + FirstName: postCreator.FirstName, + LastName: postCreator.LastName, + Username: postCreator.Username, + NickName: postCreator.Nickname, + } + post, err := s.Post().GetSingle(topThread.PostId, false) + if err != nil { + return nil, errors.Wrapf(err, "failed to get extended post for post id=%s", topThread.PostId) + } + topThread.Post = post + } + return topThreads, nil +} diff --git a/store/store.go b/store/store.go index 619f8936bd..14490aa9c0 100644 --- a/store/store.go +++ b/store/store.go @@ -290,7 +290,7 @@ type ChannelStore interface { // GetTeamForChannel returns the team for a given channelID. GetTeamForChannel(channelID string) (*model.Team, error) - // Insights + // Insights - channels GetTopChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopChannelList, error) GetTopChannelsForUserSince(userID string, teamID string, since int64, offset int, limit int) (*model.TopChannelList, error) PostCountsByDuration(channelIDs []string, sinceUnixMillis int64, userID *string, duration model.PostCountGrouping, groupingLocation *time.Location) ([]*model.DurationPostCount, error) @@ -331,6 +331,10 @@ type ThreadStore interface { PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) DeleteOrphanedRows(limit int) (deleted int64, err error) GetThreadUnreadReplyCount(threadMembership *model.ThreadMembership) (int64, error) + + // Insights - threads + GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) + GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) } type PostStore interface { diff --git a/store/storetest/mocks/ThreadStore.go b/store/storetest/mocks/ThreadStore.go index b3372cb7ee..0830bf73ff 100644 --- a/store/storetest/mocks/ThreadStore.go +++ b/store/storetest/mocks/ThreadStore.go @@ -255,6 +255,52 @@ func (_m *ThreadStore) GetThreadsForUser(userId string, teamID string, opts mode return r0, r1 } +// GetTopThreadsForTeamSince provides a mock function with given fields: teamID, userID, since, offset, limit +func (_m *ThreadStore) GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + ret := _m.Called(teamID, userID, since, offset, limit) + + var r0 *model.TopThreadList + if rf, ok := ret.Get(0).(func(string, string, int64, int, int) *model.TopThreadList); ok { + r0 = rf(teamID, userID, since, offset, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.TopThreadList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string, int64, int, int) error); ok { + r1 = rf(teamID, userID, since, offset, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTopThreadsForUserSince provides a mock function with given fields: teamID, userID, since, offset, limit +func (_m *ThreadStore) GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + ret := _m.Called(teamID, userID, since, offset, limit) + + var r0 *model.TopThreadList + if rf, ok := ret.Get(0).(func(string, string, int64, int, int) *model.TopThreadList); ok { + r0 = rf(teamID, userID, since, offset, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.TopThreadList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string, int64, int, int) error); ok { + r1 = rf(teamID, userID, since, offset, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTotalThreads provides a mock function with given fields: userId, teamID, opts func (_m *ThreadStore) GetTotalThreads(userId string, teamID string, opts model.GetUserThreadsOpts) (int64, error) { ret := _m.Called(userId, teamID, opts) diff --git a/store/storetest/thread_store.go b/store/storetest/thread_store.go index 66fc9de00f..e438ed8516 100644 --- a/store/storetest/thread_store.go +++ b/store/storetest/thread_store.go @@ -27,6 +27,7 @@ func TestThreadStore(t *testing.T, ss store.Store, s SqlStore) { t.Run("GetTeamsUnreadForUser", func(t *testing.T) { testGetTeamsUnreadForUser(t, ss) }) t.Run("GetVarious", func(t *testing.T) { testVarious(t, ss) }) t.Run("MarkAllAsReadByChannels", func(t *testing.T) { testMarkAllAsReadByChannels(t, ss) }) + t.Run("GetTopThreads", func(t *testing.T) { testGetTopThreads(t, ss) }) } func testThreadStorePopulation(t *testing.T, ss store.Store) { @@ -1226,3 +1227,345 @@ func testMarkAllAsReadByChannels(t *testing.T, ss store.Store) { assertThreadReplyCount(t, userBID, 0) }) } + +func testGetTopThreads(t *testing.T, ss store.Store) { + // create two users + u1 := model.User{ + Email: MakeEmail(), + Username: model.NewId(), + } + + _, err := ss.User().Save(&u1) + require.NoError(t, err) + + u2 := model.User{ + Email: MakeEmail(), + Username: model.NewId(), + } + + _, err = ss.User().Save(&u2) + require.NoError(t, err) + + u3 := model.User{ + Email: MakeEmail(), + Username: model.NewId(), + } + + _, err = ss.User().Save(&u3) + require.NoError(t, err) + + t.Run("test get top team threads", func(t *testing.T) { + const limit = 10 + team, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + channel, err := ss.Channel().Save(&model.Channel{ + TeamId: team.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + post1, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u1.Id, + }) + require.NoError(t, err) + post2, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u2.Id, + }) + require.NoError(t, err) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + + threadStoreCreateReply(t, ss, channel.Id, post2.Id, post1.UserId, 2000) + + // get top threads + topThreadsInTeam, err := ss.Thread().GetTopThreadsForTeamSince(team.Id, model.NewId(), 12, 0, limit) + require.NoError(t, err) + // require length of top threads to be 2 + require.Len(t, topThreadsInTeam.Items, 2) + + // require first element to be post1 with 2 replyCount=2 + require.Equal(t, topThreadsInTeam.Items[0].PostId, post1.Id) + require.Equal(t, topThreadsInTeam.Items[0].UserId, post1.UserId) + require.Equal(t, topThreadsInTeam.Items[0].UserInformation.Id, post1.UserId) + require.Equal(t, topThreadsInTeam.Items[0].Post.ReplyCount, int64(2)) + require.Equal(t, topThreadsInTeam.Items[0].Post.Message, post1.Message) + // require second element to be post2 with 2 replyCount=2 + require.Equal(t, topThreadsInTeam.Items[1].PostId, post2.Id) + require.Equal(t, topThreadsInTeam.Items[1].Post.ReplyCount, int64(1)) + require.Equal(t, topThreadsInTeam.Items[1].UserId, post2.UserId) + require.Equal(t, topThreadsInTeam.Items[1].UserInformation.Id, post2.UserId) + require.Equal(t, topThreadsInTeam.Items[1].Post.Message, post2.Message) + + // require topThreads[i].Post is not null + require.Equal(t, topThreadsInTeam.Items[0].Post.Id, post1.Id) + require.Equal(t, topThreadsInTeam.Items[1].Post.Id, post2.Id) + }) + t.Run("test get top user threads", func(t *testing.T) { + const limit = 10 + team, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + channel, err := ss.Channel().Save(&model.Channel{ + TeamId: team.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + post1, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u1.Id, + }) + require.NoError(t, err) + post2, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u2.Id, + }) + require.NoError(t, err) + post3, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u3.Id, + }) + require.NoError(t, err) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + + threadStoreCreateReply(t, ss, channel.Id, post2.Id, post2.UserId, 2000) + threadStoreCreateReply(t, ss, channel.Id, post2.Id, post2.UserId, 2000) + threadStoreCreateReply(t, ss, channel.Id, post3.Id, post3.UserId, 2000) + opts := store.ThreadMembershipOpts{ + Following: true, + IncrementMentions: false, + UpdateFollowing: true, + UpdateViewedTimestamp: false, + UpdateParticipants: false, + } + + // create threadmemberships entries. + _, err = ss.Thread().MaintainMembership(post1.UserId, post1.Id, opts) + require.NoError(t, err) + _, err = ss.Thread().MaintainMembership(post2.UserId, post2.Id, opts) + require.NoError(t, err) + _, err = ss.Thread().MaintainMembership(post2.UserId, post3.Id, opts) + require.NoError(t, err) + + // get top threads by user + topThreadsByUser1, err := ss.Thread().GetTopThreadsForUserSince(team.Id, post1.UserId, 12, 0, limit) + require.NoError(t, err) + topThreadsByUser2, err := ss.Thread().GetTopThreadsForUserSince(team.Id, post2.UserId, 12, 0, limit) + require.NoError(t, err) + // require length of top threads by users to be 1,2 respectively + require.Len(t, topThreadsByUser1.Items, 1) + require.Len(t, topThreadsByUser2.Items, 2) + + // require first element of topThreadsByUser1 to be post1 with 2 replyCount=2 + require.Equal(t, topThreadsByUser1.Items[0].PostId, post1.Id) + require.Equal(t, topThreadsByUser1.Items[0].Post.ReplyCount, int64(2)) + require.Equal(t, topThreadsByUser1.Items[0].Post.Message, post1.Message) + require.Equal(t, topThreadsByUser1.Items[0].UserId, post1.UserId) + require.Equal(t, topThreadsByUser1.Items[0].UserInformation.Id, post1.UserId) + // require elements of topThreadsByUser2 to be post2 and post3 respectively + require.Equal(t, topThreadsByUser2.Items[0].PostId, post2.Id) + require.Equal(t, topThreadsByUser2.Items[0].Post.ReplyCount, int64(2)) + require.Equal(t, topThreadsByUser2.Items[0].Post.Message, post2.Message) + require.Equal(t, topThreadsByUser2.Items[0].UserId, post2.UserId) + require.Equal(t, topThreadsByUser2.Items[0].UserInformation.Id, post2.UserId) + + require.Equal(t, topThreadsByUser2.Items[1].PostId, post3.Id) + require.Equal(t, topThreadsByUser2.Items[1].Post.ReplyCount, int64(1)) + require.Equal(t, topThreadsByUser2.Items[1].Post.Message, post3.Message) + require.Equal(t, topThreadsByUser2.Items[1].UserId, post3.UserId) + require.Equal(t, topThreadsByUser2.Items[1].UserInformation.Id, post3.UserId) + + // require topThreads[i].Post is not null + require.Equal(t, topThreadsByUser1.Items[0].Post.Id, post1.Id) + require.Equal(t, topThreadsByUser2.Items[1].Post.Id, post3.Id) + }) + t.Run("test get top threads only from given teamid", func(t *testing.T) { + const limit = 10 + team1, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + team2, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + channel1, err := ss.Channel().Save(&model.Channel{ + TeamId: team1.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + channel2, err := ss.Channel().Save(&model.Channel{ + TeamId: team2.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + post1, err := ss.Post().Save(&model.Post{ + ChannelId: channel1.Id, + UserId: u1.Id, + }) + require.NoError(t, err) + post2, err := ss.Post().Save(&model.Post{ + ChannelId: channel2.Id, + UserId: u2.Id, + }) + require.NoError(t, err) + threadStoreCreateReply(t, ss, channel1.Id, post1.Id, post1.UserId, 2000) + threadStoreCreateReply(t, ss, channel1.Id, post1.Id, post1.UserId, 2000) + + threadStoreCreateReply(t, ss, channel2.Id, post2.Id, post2.UserId, 2000) + + // assert that getting top threads from teamid 1 doesn't have post1.Id + + topThreadsTeam2, err := ss.Thread().GetTopThreadsForTeamSince(team2.Id, u1.Id, 12, 0, limit) + require.NoError(t, err) + require.Len(t, topThreadsTeam2.Items, 1) + require.Equal(t, topThreadsTeam2.Items[0].Post.Id, post2.Id) + }) + t.Run("test get top threads only from non-direct channels", func(t *testing.T) { + const limit = 10 + team1, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + channel1, err := ss.Channel().CreateDirectChannel(&u1, &u2) + require.NoError(t, err) + + channel2, err := ss.Channel().Save(&model.Channel{ + TeamId: team1.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + post1, err := ss.Post().Save(&model.Post{ + ChannelId: channel1.Id, + UserId: u1.Id, + }) + require.NoError(t, err) + post2, err := ss.Post().Save(&model.Post{ + ChannelId: channel2.Id, + UserId: u2.Id, + }) + require.NoError(t, err) + threadStoreCreateReply(t, ss, channel1.Id, post1.Id, post1.UserId, 2000) + threadStoreCreateReply(t, ss, channel1.Id, post1.Id, post1.UserId, 2000) + + threadStoreCreateReply(t, ss, channel2.Id, post2.Id, u1.Id, 2000) + + opts := store.ThreadMembershipOpts{ + Following: true, + IncrementMentions: false, + UpdateFollowing: true, + UpdateViewedTimestamp: false, + UpdateParticipants: false, + } + + // create threadmemberships entries. + _, err = ss.Thread().MaintainMembership(u1.Id, post1.Id, opts) + require.NoError(t, err) + _, err = ss.Thread().MaintainMembership(u1.Id, post2.Id, opts) + require.NoError(t, err) + _, err = ss.Thread().MaintainMembership(u2.Id, post1.Id, opts) + require.NoError(t, err) + _, err = ss.Thread().MaintainMembership(u2.Id, post2.Id, opts) + require.NoError(t, err) + + // assert that getting top threads from teamid 1 doesn't have DMs + + topThreadsTeam1, err := ss.Thread().GetTopThreadsForTeamSince(team1.Id, u1.Id, 12, 0, limit) + require.NoError(t, err) + require.Len(t, topThreadsTeam1.Items, 1) + require.Equal(t, topThreadsTeam1.Items[0].Post.Id, post2.Id) + + // assert that getting top threads from user 1 doesn't contain dm threads. + topUserThreads, err := ss.Thread().GetTopThreadsForUserSince(team1.Id, u1.Id, 12, 0, limit) + require.NoError(t, err) + require.Len(t, topUserThreads.Items, 1) + require.Equal(t, topUserThreads.Items[0].Post.Id, post2.Id) + }) + t.Run("test get top threads doesn't exceed duration", func(t *testing.T) { + const limit = 10 + team, err := ss.Team().Save(&model.Team{ + DisplayName: "DisplayName", + Name: "team" + model.NewId(), + Email: MakeEmail(), + Type: model.TeamOpen, + }) + require.NoError(t, err) + channel, err := ss.Channel().Save(&model.Channel{ + TeamId: team.Id, + DisplayName: "DisplayName", + Name: "channel" + model.NewId(), + Type: model.ChannelTypeOpen, + }, -1) + require.NoError(t, err) + + post1, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u1.Id, + }) + require.NoError(t, err) + // post 2 has replies after 10 ms unix time. + post2, err := ss.Post().Save(&model.Post{ + ChannelId: channel.Id, + UserId: u2.Id, + CreateAt: 1, + }) + require.NoError(t, err) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + threadStoreCreateReply(t, ss, channel.Id, post1.Id, post1.UserId, 2000) + + threadStoreCreateReply(t, ss, channel.Id, post2.Id, post1.UserId, 10) + + // get top threads + topThreadsInTeamNewer, err := ss.Thread().GetTopThreadsForTeamSince(team.Id, model.NewId(), 12, 0, limit) + require.NoError(t, err) + // require length of top threads to be 2 + require.Len(t, topThreadsInTeamNewer.Items, 1) + + // require first element to be post1 with 2 replyCount=2 + require.Equal(t, topThreadsInTeamNewer.Items[0].PostId, post1.Id) + + // get top threads + topThreadsInTeamOlder, err := ss.Thread().GetTopThreadsForTeamSince(team.Id, model.NewId(), 9, 0, limit) + require.NoError(t, err) + // require length of top threads to be 2 + require.Len(t, topThreadsInTeamOlder.Items, 2) + + // require first element to be post1 with 2 replyCount=2 + require.Equal(t, topThreadsInTeamOlder.Items[1].PostId, post2.Id) + }) + +} diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 010c4a61b5..e34512b464 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -8679,6 +8679,38 @@ func (s *TimerLayerThreadStore) GetThreadsForUser(userId string, teamID string, return result, err } +func (s *TimerLayerThreadStore) GetTopThreadsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + start := time.Now() + + result, err := s.ThreadStore.GetTopThreadsForTeamSince(teamID, userID, since, offset, limit) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("ThreadStore.GetTopThreadsForTeamSince", success, elapsed) + } + return result, err +} + +func (s *TimerLayerThreadStore) GetTopThreadsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopThreadList, error) { + start := time.Now() + + result, err := s.ThreadStore.GetTopThreadsForUserSince(teamID, userID, since, offset, limit) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("ThreadStore.GetTopThreadsForUserSince", success, elapsed) + } + return result, err +} + func (s *TimerLayerThreadStore) GetTotalThreads(userId string, teamID string, opts model.GetUserThreadsOpts) (int64, error) { start := time.Now()