From 55b3961b98faf67e3e24c3dfdf793fc9a571c0c9 Mon Sep 17 00:00:00 2001 From: Martin Kraft Date: Thu, 11 Aug 2022 10:38:52 -0400 Subject: [PATCH 1/2] MM-45120: Track team join times. Add API to retrieve new team members since a given time. (#20708) * MM-45120: Starts tracking team join time. Adds API to retrieve team members who joined after a given time. * MM-45120: Updates json casing to match model.User. --- api4/insights.go | 49 +++++ api4/insights_test.go | 83 ++++++++ app/app_iface.go | 1 + app/import_functions.go | 1 + app/opentracing/opentracing_layer.go | 22 ++ app/slashcommands/auto_users.go | 2 +- app/team.go | 13 ++ app/team_test.go | 190 ++++++++++++++++++ app/teams/teams.go | 1 + db/migrations/migrations.list | 4 + ...00092_add_createat_to_teammembers.down.sql | 14 ++ .../000092_add_createat_to_teammembers.up.sql | 29 +++ ...000092_add_createat_to_teamembers.down.sql | 1 + .../000092_add_createat_to_teamembers.up.sql | 2 + i18n/en.json | 4 + model/client4.go | 14 ++ model/insights.go | 28 ++- model/team_member.go | 2 + model/team_member_serial_gen.go | 34 +++- store/opentracinglayer/opentracinglayer.go | 18 ++ store/retrylayer/retrylayer.go | 21 ++ store/sqlstore/team_store.go | 45 ++++- store/sqlstore/user_store.go | 2 +- store/store.go | 2 + store/storetest/mocks/TeamStore.go | 30 +++ store/storetest/team_store.go | 15 ++ store/timerlayer/timerlayer.go | 16 ++ 27 files changed, 630 insertions(+), 13 deletions(-) create mode 100644 db/migrations/mysql/000092_add_createat_to_teammembers.down.sql create mode 100644 db/migrations/mysql/000092_add_createat_to_teammembers.up.sql create mode 100644 db/migrations/postgres/000092_add_createat_to_teamembers.down.sql create mode 100644 db/migrations/postgres/000092_add_createat_to_teamembers.up.sql diff --git a/api4/insights.go b/api4/insights.go index ef06e54afe..4e127eb19a 100644 --- a/api4/insights.go +++ b/api4/insights.go @@ -23,6 +23,9 @@ func (api *API) InitInsights() { // Threads api.BaseRoutes.InsightsForTeam.Handle("/threads", api.APISessionRequired(requireLicense(getTopThreadsForTeamSince))).Methods("GET") api.BaseRoutes.InsightsForUser.Handle("/threads", api.APISessionRequired(requireLicense(getTopThreadsForUserSince))).Methods("GET") + + // New teammembers + api.BaseRoutes.InsightsForTeam.Handle("/team_members", api.APISessionRequired(minimumProfessionalLicense(rejectGuests(getNewTeamMembersSince)))).Methods("GET") } // Top Reactions @@ -357,3 +360,49 @@ func postCountByDurationViewModel(c *Context, topChannelList *model.TopChannelLi } return model.ToDailyPostCountViewModel(postCountsByDay, startTime, model.TimeRangeToNumberDays(timeRange), channelIDs), nil } + +func getNewTeamMembersSince(c *Context, w http.ResponseWriter, r *http.Request) { + c.RequireTeamId() + if c.Err != nil { + return + } + + team, err := c.App.GetTeam(c.Params.TeamId) + if err != nil { + c.Err = err + return + } + + if !c.App.SessionHasPermissionToTeam(*c.AppContext.Session(), team.Id, model.PermissionViewTeam) { + c.SetPermissionError(model.PermissionViewTeam) + return + } + + user, err := c.App.GetUser(c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } + loc := user.GetTimezoneLocation() + startTime := model.StartOfDayForTimeRange(c.Params.TimeRange, loc) + + ntms, count, err := c.App.GetNewTeamMembersSince(c.AppContext, c.Params.TeamId, &model.InsightsOpts{ + StartUnixMilli: startTime.UnixMilli(), + Page: c.Params.Page, + PerPage: c.Params.PerPage, + }) + if err != nil { + c.Err = err + return + } + + ntms.TotalCount = count + + js, jsonErr := json.Marshal(ntms) + if jsonErr != nil { + c.Err = model.NewAppError("getNewTeamembersForTeamSince", "api.marshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + return + } + + w.Write(js) +} diff --git a/api4/insights_test.go b/api4/insights_test.go index 38f23367d7..59c5a848fc 100644 --- a/api4/insights_test.go +++ b/api4/insights_test.go @@ -816,3 +816,86 @@ func TestGetTopThreadsForUserSince(t *testing.T) { require.Nil(t, appErr) require.Len(t, topUser2ThreadsAfterPrivateReplyDelete.Items, 0) } + +func TestNewTeamMembersSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + + th.LoginBasic() + + team := th.CreateTeam() + + t.Run("accepts only starter or professional license skus", func(t *testing.T) { + _, resp, _ := th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + CheckNotImplementedStatus(t, resp) + + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuE10)) + _, resp, _ = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + CheckNotImplementedStatus(t, resp) + + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuE20)) + _, resp, _ = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + CheckNotImplementedStatus(t, resp) + + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional)) + _, resp, err := th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + require.NoError(t, err) + CheckOKStatus(t, resp) + + th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuEnterprise)) + _, resp, err = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + require.NoError(t, err) + CheckOKStatus(t, resp) + }) + + t.Run("rejects guests", func(t *testing.T) { + _, resp, err := th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + require.NoError(t, err) + CheckOKStatus(t, resp) + + th.App.DemoteUserToGuest(th.Context, th.BasicUser) + defer th.App.PromoteGuestToUser(th.Context, th.BasicUser, "") + + _, resp, _ = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 5) + CheckNotImplementedStatus(t, resp) + }) + + t.Run("implements pagination", func(t *testing.T) { + // check the first page of results + list, resp, err := th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 2) + require.NoError(t, err) + CheckOKStatus(t, resp) + + require.Equal(t, int(list.TotalCount), 1) + require.Len(t, list.Items, 1) + require.False(t, list.HasNext) + + // check the 2nd page + list, resp, err = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 1, 2) + require.NoError(t, err) + CheckOKStatus(t, resp) + + require.GreaterOrEqual(t, len(list.Items), 0) + + // add a few new team members and re-test the pagination + user := th.CreateUser() + _, appErr := th.App.AddTeamMember(th.Context, team.Id, th.BasicUser2.Id) + require.Nil(t, appErr) + _, appErr = th.App.AddTeamMember(th.Context, team.Id, user.Id) + require.Nil(t, appErr) + + list, resp, err = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 0, 2) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Equal(t, 3, int(list.TotalCount)) + require.Len(t, list.Items, 2) + require.True(t, list.HasNext) + + list, resp, err = th.Client.GetNewTeamMembersSince(team.Id, model.TimeRangeToday, 1, 2) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Equal(t, int(list.TotalCount), 3) + require.Len(t, list.Items, 1) + require.False(t, list.HasNext) + }) +} diff --git a/app/app_iface.go b/app/app_iface.go index 7b54096e61..2ba6a8f60e 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -670,6 +670,7 @@ type AppIface interface { GetMemberCountsByGroup(ctx context.Context, channelID string, includeTimezones bool) ([]*model.ChannelMemberCountByGroup, *model.AppError) GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string GetMultipleEmojiByName(names []string) ([]*model.Emoji, *model.AppError) + GetNewTeamMembersSince(c request.CTX, teamID string, opts *model.InsightsOpts) (*model.NewTeamMembersList, int64, *model.AppError) GetNewUsersForTeamPage(teamID string, page, perPage int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError) GetNextPostIdFromPostList(postList *model.PostList, collapsedThreads bool) string GetNotificationNameFormat(user *model.User) string diff --git a/app/import_functions.go b/app/import_functions.go index f84af9e500..63fad7c0f3 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -805,6 +805,7 @@ func (a *App) importUserTeams(c request.CTX, user *model.User, data *[]UserTeamI SchemeGuest: user.IsGuest(), SchemeUser: !user.IsGuest(), SchemeAdmin: team.Email == user.Email && !user.IsGuest(), + CreateAt: model.GetMillis(), } if !user.IsGuest() { var userShouldBeAdmin bool diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 9b071c3ccc..26a4f42af2 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -7045,6 +7045,28 @@ func (a *OpenTracingAppLayer) GetMultipleEmojiByName(names []string) ([]*model.E return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetNewTeamMembersSince(c request.CTX, teamID string, opts *model.InsightsOpts) (*model.NewTeamMembersList, int64, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetNewTeamMembersSince") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1, resultVar2 := a.app.GetNewTeamMembersSince(c, teamID, opts) + + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1, resultVar2 +} + func (a *OpenTracingAppLayer) GetNewUsersForTeamPage(teamID string, page int, perPage int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetNewUsersForTeamPage") diff --git a/app/slashcommands/auto_users.go b/app/slashcommands/auto_users.go index f7a9c588e0..6a63beb866 100644 --- a/app/slashcommands/auto_users.go +++ b/app/slashcommands/auto_users.go @@ -59,7 +59,7 @@ func CreateBasicUser(a *app.App, client *model.Client4) error { if err != nil { return model.NewAppError("CreateBasicUser", "app.user.verify_email.app_error", nil, err.Error(), http.StatusInternalServerError) } - if _, nErr := a.Srv().Store.Team().SaveMember(&model.TeamMember{TeamId: basicteam.Id, UserId: ruser.Id}, *a.Config().TeamSettings.MaxUsersPerTeam); nErr != nil { + if _, nErr := a.Srv().Store.Team().SaveMember(&model.TeamMember{TeamId: basicteam.Id, UserId: ruser.Id, CreateAt: model.GetMillis()}, *a.Config().TeamSettings.MaxUsersPerTeam); nErr != nil { var appErr *model.AppError var conflictErr *store.ErrConflict var limitExceededErr *store.ErrLimitExceeded diff --git a/app/team.go b/app/team.go index bf9425c145..d5f42770a1 100644 --- a/app/team.go +++ b/app/team.go @@ -2133,3 +2133,16 @@ func (a *App) ClearTeamMembersCache(teamID string) error { } return nil } + +func (a *App) GetNewTeamMembersSince(c request.CTX, teamID string, opts *model.InsightsOpts) (*model.NewTeamMembersList, int64, *model.AppError) { + if !a.Config().FeatureFlags.InsightsEnabled { + return nil, 0, model.NewAppError("GetNewTeamMembersSince", "app.insights.feature_disabled", nil, "", http.StatusNotImplemented) + } + + ntms, count, err := a.Srv().Store.Team().GetNewTeamMembersSince(teamID, opts.StartUnixMilli, opts.Page*opts.PerPage, opts.PerPage) + if err != nil { + return nil, 0, model.NewAppError("GetNewTeamMembersSince", "app.post.get_new_team_members_since.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return ntms, count, nil +} diff --git a/app/team_test.go b/app/team_test.go index 3938aaacff..2d17d62673 100644 --- a/app/team_test.go +++ b/app/team_test.go @@ -1594,3 +1594,193 @@ func TestInviteGuestsToChannelsGracefully(t *testing.T) { require.NotNil(t, res[0].Error) }) } + +func TestGetNewTeamMembersSince(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + + team := th.CreateTeam() + + t.Run("counts team members", func(t *testing.T) { + var originalExpectedCount int64 + var newTeamMemberJoinTime int64 + var anotherUser *model.User + + t.Run("since time 0", func(t *testing.T) { + teamMembers, err := th.App.Srv().Store.Team().GetMembers(team.Id, 0, 1000, nil) + require.NoError(t, err) + originalExpectedCount = int64(len(teamMembers)) + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, originalExpectedCount, actualCount) + }) + + t.Run("after a new team member was added", func(t *testing.T) { + anotherUser = th.CreateUser() + newTeamMember, appErr := th.App.JoinUserToTeam(th.Context, team, anotherUser, "") + newTeamMemberJoinTime = newTeamMember.CreateAt + require.Nil(t, appErr) + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, originalExpectedCount+1, actualCount) + }) + + t.Run("after a team member was added to a different team, ensuring the wrong team's member count isn't incremented", func(t *testing.T) { + anotherUser2 := th.CreateUser() + anotherTeam := th.CreateTeam() + _, appErr := th.App.JoinUserToTeam(th.Context, anotherTeam, anotherUser2, "") + require.Nil(t, appErr) + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, originalExpectedCount+1, actualCount) + }) + + t.Run("since a given time", func(t *testing.T) { + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: newTeamMemberJoinTime, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, int64(1), actualCount) + }) + + t.Run("after a team member was removed", func(t *testing.T) { + th.RemoveUserFromTeam(anotherUser, team) + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, originalExpectedCount, actualCount) + }) + + t.Run("after a user was deactivated", func(t *testing.T) { + _, appErr := th.App.JoinUserToTeam(th.Context, team, anotherUser, "") + require.Nil(t, appErr) + _, beforeCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + _, appErr = th.App.UpdateActive(th.Context, anotherUser, false) + defer th.App.UpdateActive(th.Context, anotherUser, true) + require.Nil(t, appErr) + _, afterCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, beforeCount-1, afterCount) + }) + + t.Run("after a user was permanently deleted", func(t *testing.T) { + _, beforeCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + appErr = th.App.PermanentDeleteUser(th.Context, anotherUser) + require.Nil(t, appErr) + _, afterCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, beforeCount-1, afterCount) + }) + + t.Run("exclude bots", func(t *testing.T) { + user := th.CreateUser() + _, appErr := th.App.ConvertUserToBot(user) + require.Nil(t, appErr) + _, appErr = th.App.JoinUserToTeam(th.Context, team, user, "") + require.Nil(t, appErr) + _, actualCount, appErr := th.App.GetNewTeamMembersSince(th.Context, team.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Equal(t, originalExpectedCount, actualCount) + }) + }) + + t.Run("returns the correct team members", func(t *testing.T) { + var originalExpectedMembers []*model.TeamMember + var newTeamMemberJoinTime int64 + var anotherUser *model.User + + uIDs := func(members []*model.TeamMember) []string { + ids := []string{} + for _, member := range members { + ids = append(ids, member.UserId) + } + return ids + } + + nUIDs := func(members []*model.NewTeamMember) []string { + ids := []string{} + for _, member := range members { + ids = append(ids, member.Id) + } + return ids + } + + t.Run("since time 0", func(t *testing.T) { + var err error + originalExpectedMembers, err = th.App.Srv().Store.Team().GetMembers(th.BasicTeam.Id, 0, 1000, nil) + require.NoError(t, err) + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.ElementsMatch(t, uIDs(originalExpectedMembers), nUIDs(actualMembersList.Items)) + }) + + t.Run("after a new team member was added", func(t *testing.T) { + anotherUser = th.CreateUser() + newTeamMember, appErr := th.App.JoinUserToTeam(th.Context, th.BasicTeam, anotherUser, "") + newTeamMemberJoinTime = newTeamMember.CreateAt + require.Nil(t, appErr) + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.ElementsMatch(t, append(uIDs(originalExpectedMembers), anotherUser.Id), nUIDs(actualMembersList.Items)) + }) + + t.Run("after a team member was added to a different team, ensuring the wrong team's member count isn't incremented", func(t *testing.T) { + anotherUser2 := th.CreateUser() + anotherTeam := th.CreateTeam() + _, appErr := th.App.JoinUserToTeam(th.Context, anotherTeam, anotherUser2, "") + require.Nil(t, appErr) + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.ElementsMatch(t, append(uIDs(originalExpectedMembers), anotherUser.Id), nUIDs(actualMembersList.Items)) + }) + + t.Run("since a given time", func(t *testing.T) { + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: newTeamMemberJoinTime, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Len(t, actualMembersList.Items, 1) + require.Equal(t, anotherUser.Id, actualMembersList.Items[0].Id) + }) + + t.Run("after a team member was removed", func(t *testing.T) { + th.RemoveUserFromTeam(anotherUser, th.BasicTeam) + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.ElementsMatch(t, uIDs(originalExpectedMembers), nUIDs(actualMembersList.Items)) + }) + + t.Run("after a user was deactivated", func(t *testing.T) { + _, appErr := th.App.JoinUserToTeam(th.Context, th.BasicTeam, anotherUser, "") + require.Nil(t, appErr) + beforeMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Contains(t, nUIDs(beforeMembersList.Items), anotherUser.Id) + _, appErr = th.App.UpdateActive(th.Context, anotherUser, false) + defer th.App.UpdateActive(th.Context, anotherUser, true) + require.Nil(t, appErr) + afterMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.NotContains(t, nUIDs(afterMembersList.Items), anotherUser.Id) + }) + + t.Run("after a user was permanently deleted", func(t *testing.T) { + beforeMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.Contains(t, nUIDs(beforeMembersList.Items), anotherUser.Id) + appErr = th.App.PermanentDeleteUser(th.Context, anotherUser) + require.Nil(t, appErr) + afterMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.NotContains(t, nUIDs(afterMembersList.Items), anotherUser.Id) + }) + + t.Run("exclude bots", func(t *testing.T) { + user := th.CreateUser() + _, appErr := th.App.ConvertUserToBot(user) + require.Nil(t, appErr) + _, appErr = th.App.JoinUserToTeam(th.Context, th.BasicTeam, user, "") + require.Nil(t, appErr) + actualMembersList, _, appErr := th.App.GetNewTeamMembersSince(th.Context, th.BasicTeam.Id, &model.InsightsOpts{StartUnixMilli: 0, Page: 0, PerPage: 1000}) + require.Nil(t, appErr) + require.ElementsMatch(t, uIDs(originalExpectedMembers), nUIDs(actualMembersList.Items)) + }) + }) +} diff --git a/app/teams/teams.go b/app/teams/teams.go index a143274a30..db9cd7c90a 100644 --- a/app/teams/teams.go +++ b/app/teams/teams.go @@ -141,6 +141,7 @@ func (ts *TeamService) JoinUserToTeam(team *model.Team, user *model.User) (*mode UserId: user.Id, SchemeGuest: user.IsGuest(), SchemeUser: !user.IsGuest(), + CreateAt: model.GetMillis(), } if !user.IsGuest() { diff --git a/db/migrations/migrations.list b/db/migrations/migrations.list index 1caa7a23a7..a0b64dcc01 100644 --- a/db/migrations/migrations.list +++ b/db/migrations/migrations.list @@ -182,6 +182,8 @@ db/migrations/mysql/000090_create_enums.down.sql db/migrations/mysql/000090_create_enums.up.sql db/migrations/mysql/000091_create_post_reminder.down.sql db/migrations/mysql/000091_create_post_reminder.up.sql +db/migrations/mysql/000092_add_createat_to_teammembers.down.sql +db/migrations/mysql/000092_add_createat_to_teammembers.up.sql db/migrations/postgres/000001_create_teams.down.sql db/migrations/postgres/000001_create_teams.up.sql db/migrations/postgres/000002_create_team_members.down.sql @@ -364,3 +366,5 @@ db/migrations/postgres/000090_create_enums.down.sql db/migrations/postgres/000090_create_enums.up.sql db/migrations/postgres/000091_create_post_reminder.down.sql db/migrations/postgres/000091_create_post_reminder.up.sql +db/migrations/postgres/000092_add_createat_to_teamembers.down.sql +db/migrations/postgres/000092_add_createat_to_teamembers.up.sql diff --git a/db/migrations/mysql/000092_add_createat_to_teammembers.down.sql b/db/migrations/mysql/000092_add_createat_to_teammembers.down.sql new file mode 100644 index 0000000000..6bec06a01f --- /dev/null +++ b/db/migrations/mysql/000092_add_createat_to_teammembers.down.sql @@ -0,0 +1,14 @@ +SET @preparedStatement = (SELECT IF( + EXISTS( + SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS + WHERE table_name = 'TeamMembers' + AND table_schema = DATABASE() + AND column_name = 'CreateAt' + ) > 0, + 'ALTER TABLE Reactions DROP COLUMN CreateAt;', + 'SELECT 1;' +)); + +PREPARE removeColumnIfExists FROM @preparedStatement; +EXECUTE removeColumnIfExists; +DEALLOCATE PREPARE removeColumnIfExists; diff --git a/db/migrations/mysql/000092_add_createat_to_teammembers.up.sql b/db/migrations/mysql/000092_add_createat_to_teammembers.up.sql new file mode 100644 index 0000000000..193391e63d --- /dev/null +++ b/db/migrations/mysql/000092_add_createat_to_teammembers.up.sql @@ -0,0 +1,29 @@ +SET @preparedStatement = (SELECT IF( + NOT EXISTS( + SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = 'TeamMembers' + AND table_schema = DATABASE() + AND column_name = 'CreateAt' + ), + 'ALTER TABLE TeamMembers ADD COLUMN CreateAt bigint DEFAULT 0;', + 'SELECT 1;' +)); + +PREPARE addColumnIfNotExists FROM @preparedStatement; +EXECUTE addColumnIfNotExists; +DEALLOCATE PREPARE addColumnIfNotExists; + +SET @preparedStatement = (SELECT IF( + ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE table_name = 'TeamMembers' + AND table_schema = DATABASE() + AND index_name = 'idx_teammembers_create_at' + ) > 0, + 'SELECT 1', + 'CREATE INDEX idx_teammembers_createat ON TeamMembers(CreateAt);' +)); + +PREPARE createIndexIfNotExists FROM @preparedStatement; +EXECUTE createIndexIfNotExists; +DEALLOCATE PREPARE createIndexIfNotExists; diff --git a/db/migrations/postgres/000092_add_createat_to_teamembers.down.sql b/db/migrations/postgres/000092_add_createat_to_teamembers.down.sql new file mode 100644 index 0000000000..d83e17fd3a --- /dev/null +++ b/db/migrations/postgres/000092_add_createat_to_teamembers.down.sql @@ -0,0 +1 @@ +ALTER TABLE teammembers DROP COLUMN IF EXISTS createat; diff --git a/db/migrations/postgres/000092_add_createat_to_teamembers.up.sql b/db/migrations/postgres/000092_add_createat_to_teamembers.up.sql new file mode 100644 index 0000000000..56a49e502c --- /dev/null +++ b/db/migrations/postgres/000092_add_createat_to_teamembers.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE teammembers ADD COLUMN IF NOT EXISTS createat bigint DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_teammembers_createat on teammembers (createat); diff --git a/i18n/en.json b/i18n/en.json index 980c0ffe45..4e138e82a6 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -5835,6 +5835,10 @@ "id": "app.post.get_flagged_posts.app_error", "translation": "Unable to get the flagged posts." }, + { + "id": "app.post.get_new_team_members_since.app_error", + "translation": " " + }, { "id": "app.post.get_post_after_time.app_error", "translation": "Unable to get post after time bound." diff --git a/model/client4.go b/model/client4.go index b00694fee4..0b2fc3a6bd 100644 --- a/model/client4.go +++ b/model/client4.go @@ -8354,3 +8354,17 @@ func (c *Client4) GetIntegrationsUsage() (*IntegrationsUsage, *Response, error) err = json.NewDecoder(r.Body).Decode(&usage) return usage, BuildResponse(r), err } + +func (c *Client4) GetNewTeamMembersSince(teamID string, timeRange string, page int, perPage int) (*NewTeamMembersList, *Response, error) { + query := fmt.Sprintf("?time_range=%v&page=%v&per_page=%v", timeRange, page, perPage) + r, err := c.DoAPIGet(c.teamRoute(teamID)+"/top/team_members"+query, "") + if err != nil { + return nil, BuildResponse(r), err + } + defer closeBody(r) + var newTeamMembersList *NewTeamMembersList + if jsonErr := json.NewDecoder(r.Body).Decode(&newTeamMembersList); jsonErr != nil { + return nil, nil, NewAppError("GetNewTeamMembersSince", "api.unmarshal_error", nil, jsonErr.Error(), http.StatusInternalServerError) + } + return newTeamMembersList, BuildResponse(r), nil +} diff --git a/model/insights.go b/model/insights.go index dbd1720bc7..b2a91a9bd4 100644 --- a/model/insights.go +++ b/model/insights.go @@ -90,9 +90,25 @@ type InsightUserInformation struct { Username string `json:"username"` } +type NewTeamMembersList struct { + InsightsListData + Items []*NewTeamMember `json:"items"` + TotalCount int64 `json:"total_count"` +} + +type NewTeamMember struct { + Id string `json:"id"` + Username string `json:"username"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Position string `json:"position"` + Nickname string `json:"nickname"` + CreateAt int64 `json:"create_at"` +} + type DurationPostCount struct { ChannelID string `db:"channelid"` - // Duration is an ISO8601 date string representing either a day or a day and hour (ex. "2022-05-26" or "2022-05-26T14"). + // Duration is an ISO8601 date string. Duration string `db:"duration"` PostCount int `db:"postcount"` } @@ -243,3 +259,13 @@ func GetTopThreadListWithPagination(threads []*TopThread, limit int) *TopThreadL return &TopThreadList{InsightsListData: InsightsListData{HasNext: hasNext}, Items: threads} } + +func GetNewTeamMembersListWithPagination(teamMembers []*NewTeamMember, limit int) *NewTeamMembersList { + var hasNext bool + if (limit != 0) && (len(teamMembers) == limit+1) { + hasNext = true + teamMembers = teamMembers[:len(teamMembers)-1] + } + + return &NewTeamMembersList{InsightsListData: InsightsListData{HasNext: hasNext}, Items: teamMembers} +} diff --git a/model/team_member.go b/model/team_member.go index bbd317369f..d58e888dcd 100644 --- a/model/team_member.go +++ b/model/team_member.go @@ -25,6 +25,7 @@ type TeamMember struct { SchemeUser bool `json:"scheme_user"` SchemeAdmin bool `json:"scheme_admin"` ExplicitRoles string `json:"explicit_roles"` + CreateAt int64 `json:"-"` } func (o *TeamMember) Auditable() map[string]interface{} { @@ -37,6 +38,7 @@ func (o *TeamMember) Auditable() map[string]interface{} { "scheme_user": o.SchemeUser, "scheme_admin": o.SchemeAdmin, "explicit_roles": o.ExplicitRoles, + "create_at": o.CreateAt, } } diff --git a/model/team_member_serial_gen.go b/model/team_member_serial_gen.go index 044a608a0c..9cf1d23101 100644 --- a/model/team_member_serial_gen.go +++ b/model/team_member_serial_gen.go @@ -17,8 +17,8 @@ func (z *TeamMember) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err) return } - if zb0001 != 8 { - err = msgp.ArrayError{Wanted: 8, Got: zb0001} + if zb0001 != 9 { + err = msgp.ArrayError{Wanted: 9, Got: zb0001} return } z.TeamId, err = dc.ReadString() @@ -61,13 +61,18 @@ func (z *TeamMember) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "ExplicitRoles") return } + z.CreateAt, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "CreateAt") + return + } return } // EncodeMsg implements msgp.Encodable func (z *TeamMember) EncodeMsg(en *msgp.Writer) (err error) { - // array header, size 8 - err = en.Append(0x98) + // array header, size 9 + err = en.Append(0x99) if err != nil { return } @@ -111,14 +116,19 @@ func (z *TeamMember) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ExplicitRoles") return } + err = en.WriteInt64(z.CreateAt) + if err != nil { + err = msgp.WrapError(err, "CreateAt") + return + } return } // MarshalMsg implements msgp.Marshaler func (z *TeamMember) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // array header, size 8 - o = append(o, 0x98) + // array header, size 9 + o = append(o, 0x99) o = msgp.AppendString(o, z.TeamId) o = msgp.AppendString(o, z.UserId) o = msgp.AppendString(o, z.Roles) @@ -127,6 +137,7 @@ func (z *TeamMember) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.AppendBool(o, z.SchemeUser) o = msgp.AppendBool(o, z.SchemeAdmin) o = msgp.AppendString(o, z.ExplicitRoles) + o = msgp.AppendInt64(o, z.CreateAt) return } @@ -138,8 +149,8 @@ func (z *TeamMember) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0001 != 8 { - err = msgp.ArrayError{Wanted: 8, Got: zb0001} + if zb0001 != 9 { + err = msgp.ArrayError{Wanted: 9, Got: zb0001} return } z.TeamId, bts, err = msgp.ReadStringBytes(bts) @@ -182,12 +193,17 @@ func (z *TeamMember) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "ExplicitRoles") return } + z.CreateAt, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreateAt") + return + } o = bts return } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *TeamMember) Msgsize() (s int) { - s = 1 + msgp.StringPrefixSize + len(z.TeamId) + msgp.StringPrefixSize + len(z.UserId) + msgp.StringPrefixSize + len(z.Roles) + msgp.Int64Size + msgp.BoolSize + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.ExplicitRoles) + s = 1 + msgp.StringPrefixSize + len(z.TeamId) + msgp.StringPrefixSize + len(z.UserId) + msgp.StringPrefixSize + len(z.Roles) + msgp.Int64Size + msgp.BoolSize + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.ExplicitRoles) + msgp.Int64Size return } diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index ee89078792..e4dc543a8c 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -8967,6 +8967,24 @@ func (s *OpenTracingLayerTeamStore) GetMembersByIds(teamID string, userIds []str return result, err } +func (s *OpenTracingLayerTeamStore) GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "TeamStore.GetNewTeamMembersSince") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, resultVar1, err := s.TeamStore.GetNewTeamMembersSince(teamID, since, offset, limit) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, resultVar1, err +} + func (s *OpenTracingLayerTeamStore) GetTeamMembersForExport(userID string) ([]*model.TeamMemberForExport, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "TeamStore.GetTeamMembersForExport") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index ebd66004f5..8888c13772 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -10241,6 +10241,27 @@ func (s *RetryLayerTeamStore) GetMembersByIds(teamID string, userIds []string, r } +func (s *RetryLayerTeamStore) GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) { + + tries := 0 + for { + result, resultVar1, err := s.TeamStore.GetNewTeamMembersSince(teamID, since, offset, limit) + if err == nil { + return result, resultVar1, nil + } + if !isRepeatableError(err) { + return result, resultVar1, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, resultVar1, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerTeamStore) GetTeamMembersForExport(userID string) ([]*model.TeamMemberForExport, error) { tries := 0 diff --git a/store/sqlstore/team_store.go b/store/sqlstore/team_store.go index 5473781720..10018bd3a8 100644 --- a/store/sqlstore/team_store.go +++ b/store/sqlstore/team_store.go @@ -35,6 +35,7 @@ type teamMember struct { SchemeUser sql.NullBool SchemeAdmin sql.NullBool SchemeGuest sql.NullBool + CreateAt int64 } func NewTeamMemberFromModel(tm *model.TeamMember) *teamMember { @@ -46,6 +47,7 @@ func NewTeamMemberFromModel(tm *model.TeamMember) *teamMember { SchemeGuest: sql.NullBool{Valid: true, Bool: tm.SchemeGuest}, SchemeUser: sql.NullBool{Valid: true, Bool: tm.SchemeUser}, SchemeAdmin: sql.NullBool{Valid: true, Bool: tm.SchemeAdmin}, + CreateAt: tm.CreateAt, } } @@ -60,12 +62,13 @@ type teamMemberWithSchemeRoles struct { TeamSchemeDefaultGuestRole sql.NullString TeamSchemeDefaultUserRole sql.NullString TeamSchemeDefaultAdminRole sql.NullString + CreateAt int64 } type teamMemberWithSchemeRolesList []teamMemberWithSchemeRoles func teamMemberSliceColumns() []string { - return []string{"TeamId", "UserId", "Roles", "DeleteAt", "SchemeUser", "SchemeAdmin", "SchemeGuest"} + return []string{"TeamId", "UserId", "Roles", "DeleteAt", "SchemeUser", "SchemeAdmin", "SchemeGuest", "CreateAt"} } func teamMemberToSlice(member *model.TeamMember) []any { @@ -77,6 +80,7 @@ func teamMemberToSlice(member *model.TeamMember) []any { resultSlice = append(resultSlice, member.SchemeUser) resultSlice = append(resultSlice, member.SchemeAdmin) resultSlice = append(resultSlice, member.SchemeGuest) + resultSlice = append(resultSlice, member.CreateAt) return resultSlice } @@ -187,6 +191,7 @@ func (db teamMemberWithSchemeRoles) ToModel() *model.TeamMember { SchemeUser: rolesResult.schemeUser, SchemeAdmin: rolesResult.schemeAdmin, ExplicitRoles: strings.Join(rolesResult.explicitRoles, " "), + CreateAt: db.CreateAt, } return tm } @@ -1646,3 +1651,41 @@ func (s SqlTeamStore) GroupSyncedTeamCount() (int64, error) { return count, nil } + +func (s SqlTeamStore) GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) { + builderF := func(selectClause string) sq.SelectBuilder { + return s.getQueryBuilder(). + Select(selectClause). + From("TeamMembers"). + Join("Users ON Users.id = TeamMembers.userid"). + LeftJoin("Bots ON Bots.userid = Users.id"). + Where(sq.GtOrEq{"TeamMembers.createat": since}). + Where(sq.Eq{"TeamMembers.deleteat": 0, "teamid": teamID, "Users.deleteat": 0, "Bots.userid": nil}) + } + + countBuilder := builderF("count(*)") + query, args, err := countBuilder.ToSql() + if err != nil { + return nil, 0, errors.Wrap(err, "team_tosql") + } + var totalCount int64 + err = s.GetReplicaX().Get(&totalCount, query, args...) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to count team members since") + } + + newTeamMembersBuilder := builderF("Users.Id, Users.Username, Users.FirstName, Users.LastName, Users.Position, TeamMembers.CreateAt, Users.Nickname"). + Limit(uint64(limit + 1)). + Offset(uint64(offset)) + query, args, err = newTeamMembersBuilder.ToSql() + if err != nil { + return nil, 0, errors.Wrap(err, "team_tosql") + } + var ntms []*model.NewTeamMember + err = s.GetReplicaX().Select(&ntms, query, args...) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to get team members since") + } + + return model.GetNewTeamMembersListWithPagination(ntms, limit), totalCount, nil +} diff --git a/store/sqlstore/user_store.go b/store/sqlstore/user_store.go index 5a4502a12e..f080872ab7 100644 --- a/store/sqlstore/user_store.go +++ b/store/sqlstore/user_store.go @@ -1765,7 +1765,7 @@ func (us SqlUserStore) GetUsersBatchForIndexing(startTime int64, startFileID str teamMembers := []*model.TeamMember{} teamMembersQuery, args, _ := us.getQueryBuilder(). - Select("TeamId, UserId, Roles, DeleteAt, (SchemeGuest IS NOT NULL AND SchemeGuest) as SchemeGuest, SchemeUser, SchemeAdmin"). + Select("TeamId, UserId, Roles, DeleteAt, CreateAt, (SchemeGuest IS NOT NULL AND SchemeGuest) as SchemeGuest, SchemeUser, SchemeAdmin"). From("TeamMembers"). Where(sq.Eq{"UserId": userIds, "DeleteAt": 0}). ToSql() diff --git a/store/store.go b/store/store.go index 911c06191f..af76210f4f 100644 --- a/store/store.go +++ b/store/store.go @@ -166,6 +166,8 @@ type TeamStore interface { // GetCommonTeamIDsForTwoUsers returns the intersection of all the teams to which the specified // users belong. GetCommonTeamIDsForTwoUsers(userID, otherUserID string) ([]string, error) + + GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) } type ChannelStore interface { diff --git a/store/storetest/mocks/TeamStore.go b/store/storetest/mocks/TeamStore.go index 7a74b0b8f4..6076693d3f 100644 --- a/store/storetest/mocks/TeamStore.go +++ b/store/storetest/mocks/TeamStore.go @@ -489,6 +489,36 @@ func (_m *TeamStore) GetMembersByIds(teamID string, userIds []string, restrictio return r0, r1 } +// GetNewTeamMembersSince provides a mock function with given fields: teamID, since, offset, limit +func (_m *TeamStore) GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) { + ret := _m.Called(teamID, since, offset, limit) + + var r0 *model.NewTeamMembersList + if rf, ok := ret.Get(0).(func(string, int64, int, int) *model.NewTeamMembersList); ok { + r0 = rf(teamID, since, offset, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.NewTeamMembersList) + } + } + + var r1 int64 + if rf, ok := ret.Get(1).(func(string, int64, int, int) int64); ok { + r1 = rf(teamID, since, offset, limit) + } else { + r1 = ret.Get(1).(int64) + } + + var r2 error + if rf, ok := ret.Get(2).(func(string, int64, int, int) error); ok { + r2 = rf(teamID, since, offset, limit) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // GetTeamMembersForExport provides a mock function with given fields: userID func (_m *TeamStore) GetTeamMembersForExport(userID string) ([]*model.TeamMemberForExport, error) { ret := _m.Called(userID) diff --git a/store/storetest/team_store.go b/store/storetest/team_store.go index ad9bcb4f6a..8dc65416be 100644 --- a/store/storetest/team_store.go +++ b/store/storetest/team_store.go @@ -73,6 +73,7 @@ func TestTeamStore(t *testing.T, ss store.Store) { t.Run("GetTeamMembersForExport", func(t *testing.T) { testTeamStoreGetTeamMembersForExport(t, ss) }) t.Run("GetTeamsForUserWithPagination", func(t *testing.T) { testTeamMembersWithPagination(t, ss) }) t.Run("GroupSyncedTeamCount", func(t *testing.T) { testGroupSyncedTeamCount(t, ss) }) + t.Run("GetNewTeamMembersSince", func(t *testing.T) { testGetNewTeamMembersSince(t, ss) }) } func testTeamStoreSave(t *testing.T, ss store.Store) { @@ -3618,3 +3619,17 @@ func testGroupSyncedTeamCount(t *testing.T, ss store.Store) { require.NoError(t, err) require.GreaterOrEqual(t, countAfter, count+1) } + +func testGetNewTeamMembersSince(t *testing.T, ss store.Store) { + team, err := ss.Team().Save(&model.Team{ + DisplayName: NewTestId(), + Name: NewTestId(), + Email: MakeEmail(), + Type: model.TeamInvite, + GroupConstrained: model.NewBool(true), + }) + require.NoError(t, err) + + _, _, err = ss.Team().GetNewTeamMembersSince(team.Id, 0, 0, 1000) + require.NoError(t, err) +} diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 3066b77559..367770da9d 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -8072,6 +8072,22 @@ func (s *TimerLayerTeamStore) GetMembersByIds(teamID string, userIds []string, r return result, err } +func (s *TimerLayerTeamStore) GetNewTeamMembersSince(teamID string, since int64, offset int, limit int) (*model.NewTeamMembersList, int64, error) { + start := time.Now() + + result, resultVar1, err := s.TeamStore.GetNewTeamMembersSince(teamID, since, offset, limit) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("TeamStore.GetNewTeamMembersSince", success, elapsed) + } + return result, resultVar1, err +} + func (s *TimerLayerTeamStore) GetTeamMembersForExport(userID string) ([]*model.TeamMemberForExport, error) { start := time.Now() From 1ef24221e3b2cfb0172e4843d007dc2bfb431543 Mon Sep 17 00:00:00 2001 From: Akis Maziotis <36594156+phoinixgrr@users.noreply.github.com> Date: Thu, 11 Aug 2022 18:16:59 +0300 Subject: [PATCH 2/2] [feat] CI essential multi-product architecture changes (#20783) * [feat] essential multi-product architecture changes This takes care of aligning mattermost-server CircleCI pipeline to the mutlti-product architecture. Specifically the following are performed: - seperation of mattermost-server and mattermost-webapp clone/setup stages - introduction of clone/setup stage for focalboard - change to using cimg/base image instaed of mattermost/mattermost-build-webapp when cloning/setup mattermost-server - introduction of `configure-multirepo` stage, in which multi-module workspaces (golang) configuration is being performed for mattermost-server and focalboard - introduction of parallel build of focalboard along with mattermost-server - reorganisation of step dependencies, to accomodate `configure-multirepo` step Ticket: https://mattermost.atlassian.net/browse/CLD-3801 Signed-off-by: Akis Maziotis --- .circleci/config.yml | 179 +++++++++++++++++++++++++++++++------------ 1 file changed, 132 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5063a7b3f3..baccb14dd4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,14 +14,24 @@ executors: COMPOSE_PROJECT_NAME: "circleci" jobs: - setup: + setup-mattermost-server: working_directory: ~/mattermost/mattermost-server docker: - - image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d + - image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2 resource_class: xlarge # Use `--retry-all-errors` instead of `until` after curl version >= 7.71.0; `retry` will not work, since it only retries on transient errors, 403 is not one of them. steps: - checkout + - persist_to_workspace: + root: ~/mattermost + paths: + - mattermost-server + setup-mattermost-webapp: + working_directory: ~/mattermost/mattermost-webapp + docker: + - image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d + resource_class: xlarge + steps: - run: | cd ../ git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git @@ -29,10 +39,8 @@ jobs: git checkout $CIRCLE_BRANCH || git checkout master export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) echo "$WEBAPP_GIT_COMMIT" - FILE_DIST=dist.tar.gz curl --version - runtime="2 minute" endtime=$(date -ud "$runtime" +%s) while [[ $(date -u +%s) -le $endtime ]]; do @@ -53,7 +61,22 @@ jobs: root: ~/mattermost paths: - mattermost-webapp - - mattermost-server + setup-focalboard: + working_directory: ~/mattermost/focalboard + docker: + - image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2 + resource_class: xlarge + steps: + - run: | + cd ../ + git clone --depth=10 --no-single-branch https://github.com/mattermost/focalboard.git + cd focalboard + git checkout $CIRCLE_BRANCH || git checkout main + echo $(git rev-parse HEAD) + - persist_to_workspace: + root: ~/mattermost + paths: + - focalboard check-deps: parameters: @@ -133,7 +156,6 @@ jobs: cd mattermost-server make app-layers if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi - check-go-mod-tidy: docker: - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 @@ -146,7 +168,6 @@ jobs: cd mattermost-server make modules-tidy if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; exit 1; fi - check-store-layers: docker: - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 @@ -159,7 +180,6 @@ jobs: cd mattermost-server make store-layers if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi - check-mocks: docker: - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 @@ -172,7 +192,6 @@ jobs: cd mattermost-server make mocks if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi - check-email-templates: docker: - image: cimg/go:1.17-node @@ -186,7 +205,6 @@ jobs: sudo npm install -g mjml@4.9.0 make build-templates if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi - check-gen-serialized: docker: - image: cimg/go:1.17 @@ -199,7 +217,6 @@ jobs: cd mattermost-server make gen-serialized if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi - check-migrations: docker: - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 @@ -212,7 +229,6 @@ jobs: cd mattermost-server make migrations-extract if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi - # Dedicated job for golangci-lint it does not need anything only the server code for that # and to make more clear when the job fails check-golangci-lint: @@ -240,7 +256,6 @@ jobs: make config-reset make plugin-checker make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' - # Dedicate job for sentry it does not need anything only the server code for that # and to make more clear when the job fails sentry: @@ -254,7 +269,6 @@ jobs: command: | sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD` sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD` - build-api-spec: docker: - image: circleci/node:lts @@ -274,7 +288,7 @@ jobs: paths: - mattermost-api-reference - build: + build-mattermost-server: docker: - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 resource_class: xlarge @@ -304,6 +318,38 @@ jobs: - mattermost-server - mattermost-webapp + build-focalboard: + docker: + - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 + resource_class: xlarge + working_directory: ~/mattermost + steps: + - attach_workspace: + at: ~/mattermost/ + - run: + command: | + cd focalboard + make server-linux + configure-multirepo: + docker: + - image: mattermost/mattermost-build-server:20220415_golang-1.18.1 + working_directory: ~/mattermost + steps: + - attach_workspace: + at: ~/mattermost/ + - run: + command: | + cd mattermost-server + make setup-go-work + cd ../focalboard + EXCLUDE_ENTERPRISE=TRUE make templates-archive + - persist_to_workspace: + root: ~/mattermost + paths: + - mattermost-server + - mattermost-webapp + - focalboard + test: executor: name: ubuntu @@ -410,13 +456,11 @@ jobs: -w /mattermost/mattermost-server \ mattermost/mattermost-build-server:20220415_golang-1.18.1 \ bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli" - echo "Generating dump" docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql echo "Removing databases created for db comparison" docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' - echo "Generating diff" diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) no_output_timeout: 1h @@ -430,7 +474,6 @@ jobs: echo "Importing mysql dump from version 6.0.0" docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" - docker run -it --net circleci_mm-test \ --env-file="dotenv/test-schema-validation.env" \ --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ @@ -439,14 +482,11 @@ jobs: -w /mattermost/mattermost-server \ mattermost/mattermost-build-server:20220415_golang-1.18.1 \ bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli" - echo "Generating dump" docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql - echo "Removing databases created for db comparison" docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' - echo "Generating diff" diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) no_output_timeout: 1h @@ -495,19 +535,28 @@ jobs: - run: command: | export TAG="${CIRCLE_SHA1:0:7}" - cd mattermost-server - export DOCKER_CLI_EXPERIMENTAL=enabled echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} build - workflows: version: 2 untagged-build: jobs: - - setup: + - setup-mattermost-server: + filters: + branches: + ignore: + - master + - /^release-.*/ + - setup-mattermost-webapp: + filters: + branches: + ignore: + - master + - /^release-.*/ + - setup-focalboard: filters: branches: ignore: @@ -524,7 +573,8 @@ workflows: # - setup - check-go-mod-tidy: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-golangci-lint: filters: branches: @@ -533,29 +583,46 @@ workflows: - /^release-.*/ - check-app-layers: requires: - - setup + - configure-multirepo - check-store-layers: requires: - - setup + - configure-multirepo - check-mocks: requires: - - setup + - configure-multirepo - check-email-templates: requires: - - setup + - configure-multirepo - check-gen-serialized: requires: - - setup + - configure-multirepo - check-mattermost-vet: requires: - - setup + - configure-multirepo - check-migrations: requires: - - setup + - configure-multirepo - build-api-spec: requires: - - setup - - build: + - configure-multirepo + - configure-multirepo: + requires: + - setup-mattermost-server + - setup-mattermost-webapp + - setup-focalboard + - build-mattermost-server: + requires: + - check-go-mod-tidy + - check-app-layers + - check-store-layers + - check-mocks + - check-email-templates + - check-gen-serialized + - check-mattermost-vet + - check-golangci-lint + - check-migrations + - build-api-spec + - build-focalboard: requires: - check-go-mod-tidy - check-app-layers @@ -570,11 +637,13 @@ workflows: - upload-s3-sha: context: mattermost-ci-pr-builds-s3 requires: - - build + - build-mattermost-server + - build-focalboard - upload-s3: context: mattermost-ci-pr-builds-s3 requires: - - build + - build-mattermost-server + - build-focalboard - build-docker: context: matterbuild-docker requires: @@ -621,13 +690,21 @@ workflows: - check-gen-serialized release-build: jobs: - - setup: + - setup-mattermost-server: filters: branches: only: - master - /^release-.*/ - cloud + - setup-mattermost-webapp: + filters: + branches: + only: + - master + - /^release-.*/ + - cloud + # Disabling check-deps since the new version is not on bintray and all builds will fail # we've are on top of this, and it's going to be temporary. #- check-deps: @@ -646,28 +723,36 @@ workflows: - cloud - check-go-mod-tidy: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-app-layers: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-store-layers: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - build-api-spec: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-mocks: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-email-templates: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-gen-serialized: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - check-mattermost-vet: requires: - - setup + - setup-mattermost-server + - setup-mattermost-webapp - sentry: context: matterbuild-sentry requires: @@ -680,7 +765,7 @@ workflows: - check-mattermost-vet - check-golangci-lint - build-api-spec - - build: + - build-mattermost-server: requires: - check-go-mod-tidy - check-app-layers