[MM-55017] Add API method to get users for Admin Reporting (#25499)

* Add store method to get reporting data

* Some store changes

* Added app layer

* Added API call, some miscellaneous fixes

* Fix lint

* Fix serialized check

* Add API docs

* Fix user store tests leaking users

* Fix test

* PR feedback

* Add filtering for role/team/activated user, filter out bot users

* Fix mock

* Fix test

* Oops

* Switch to using struct filter

* More PR feedback

* Fix gen

* Fix test

* Fix API docs

* Fix test

* Fix possible SQL injection, some query optimization

* Fix migrations

* Oops

* Add role to API

* Fix check

* Add Client4 API call for load testing

* Fix test

* Update server/channels/store/storetest/user_store.go

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

* PR feedback

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>
Этот коммит содержится в:
Devin Binnie
2023-12-08 10:30:08 -05:00
коммит произвёл GitHub
родитель 7afc14de36
Коммит 109f4643c6
26 изменённых файлов: 2399 добавлений и 1 удалений

Просмотреть файл

@@ -11892,6 +11892,24 @@ func (s *OpenTracingLayerUserStore) GetUnreadCountForChannel(userID string, chan
return result, err
}
func (s *OpenTracingLayerUserStore) GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.GetUserReport")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.UserStore.GetUserReport(filter)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.GetUsersBatchForIndexing")

Просмотреть файл

@@ -13577,6 +13577,27 @@ func (s *RetryLayerUserStore) GetUnreadCountForChannel(userID string, channelID
}
func (s *RetryLayerUserStore) GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error) {
tries := 0
for {
result, err := s.UserStore.GetUserReport(filter)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
tries := 0

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"fmt"
"sort"
"strings"
"time"
"unicode/utf8"
sq "github.com/mattermost/squirrel"
@@ -2266,3 +2267,102 @@ func (us SqlUserStore) RefreshPostStatsForUsers() error {
return nil
}
func (us SqlUserStore) GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error) {
isPostgres := us.DriverName() == model.DatabaseDriverPostgres
selectColumns := []string{"u.Id", "u.LastLogin", "MAX(s.LastActivityAt) AS LastStatusAt"}
for _, column := range model.UserReportSortColumns {
selectColumns = append(selectColumns, "u."+column)
}
if isPostgres {
selectColumns = append(selectColumns,
"MAX(ps.LastPostDate) AS LastPostDate",
"COUNT(ps.Day) AS DaysActive",
"SUM(ps.NumPosts) AS TotalPosts",
)
} else {
selectColumns = append(selectColumns,
"MAX(p.CreateAt) AS LastPostDate",
"COUNT(DATE(FROM_UNIXTIME(p.CreateAt / 1000))) AS DaysActive",
"COUNT(p.Id) AS TotalPosts",
)
}
sortColumnValue := filter.SortColumn
if filter.SortDesc {
sortColumnValue += " DESC"
}
query := us.getQueryBuilder().
Select(selectColumns...).
From("Users u").
LeftJoin("Status s ON s.UserId = u.Id").
Where(sq.Or{
sq.Gt{filter.SortColumn: filter.LastSortColumnValue},
sq.And{
sq.Eq{filter.SortColumn: filter.LastSortColumnValue},
sq.Gt{"u.Id": filter.LastUserId},
},
}).
Where(sq.Expr("u.Id NOT IN (SELECT UserId FROM Bots)")).
GroupBy("u.Id").
OrderBy(sortColumnValue, "u.Id")
if filter.PageSize > 0 {
query = query.Limit(uint64(filter.PageSize))
}
if isPostgres {
query = query.LeftJoin("PostStats ps ON ps.UserId = u.Id")
if filter.StartAt > 0 {
startDate := time.UnixMilli(filter.StartAt)
query = query.Where(sq.Or{
sq.Expr("ps.UserId IS NULL"),
sq.GtOrEq{"ps.Day": startDate.Format("2006-01-02")},
})
}
if filter.EndAt > 0 {
endDate := time.UnixMilli(filter.EndAt)
query = query.Where(sq.Or{
sq.Expr("ps.UserId IS NULL"),
sq.Lt{"ps.Day": endDate.Format("2006-01-02")},
})
}
} else {
query = query.LeftJoin("Posts p on p.UserId = u.Id")
if filter.StartAt > 0 {
query = query.Where(sq.Or{
sq.Expr("p.UserId IS NULL"),
sq.GtOrEq{"p.CreateAt": filter.StartAt},
})
}
if filter.EndAt > 0 {
query = query.Where(sq.Or{
sq.Expr("p.UserId IS NULL"),
sq.Lt{"p.CreateAt": filter.EndAt},
})
}
}
query = applyRoleFilter(query, filter.Role, isPostgres)
if filter.HasNoTeam {
query = query.Where(sq.Expr("u.Id NOT IN (SELECT UserId FROM TeamMembers WHERE DeleteAt = 0)"))
} else if filter.Team != "" {
query = query.Join("TeamMembers tm ON (tm.UserId = u.Id AND tm.DeleteAt = 0)").
Where(sq.Eq{"tm.TeamId": filter.Team})
}
if filter.HideActive {
query = query.Where(sq.Gt{"u.DeleteAt": 0})
}
if filter.HideInactive {
query = query.Where(sq.Eq{"u.DeleteAt": 0})
}
userResults := []*model.UserReportQuery{}
err := us.GetReplicaX().SelectBuilder(&userResults, query)
if err != nil {
return nil, errors.Wrap(err, "failed to get users for reporting")
}
return userResults, nil
}

Просмотреть файл

@@ -493,6 +493,7 @@ type UserStore interface {
GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error)
InsertUsers(users []*model.User) error
RefreshPostStatsForUsers() error
GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error)
}
type BotStore interface {

Просмотреть файл

@@ -1153,6 +1153,32 @@ func (_m *UserStore) GetUnreadCountForChannel(userID string, channelID string) (
return r0, r1
}
// GetUserReport provides a mock function with given fields: filter
func (_m *UserStore) GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error) {
ret := _m.Called(filter)
var r0 []*model.UserReportQuery
var r1 error
if rf, ok := ret.Get(0).(func(*model.UserReportOptions) ([]*model.UserReportQuery, error)); ok {
return rf(filter)
}
if rf, ok := ret.Get(0).(func(*model.UserReportOptions) []*model.UserReportQuery); ok {
r0 = rf(filter)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.UserReportQuery)
}
}
if rf, ok := ret.Get(1).(func(*model.UserReportOptions) error); ok {
r1 = rf(filter)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetUsersBatchForIndexing provides a mock function with given fields: startTime, startFileID, limit
func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
ret := _m.Called(startTime, startFileID, limit)

Просмотреть файл

@@ -96,6 +96,7 @@ func TestUserStore(t *testing.T, rctx request.CTX, ss store.Store, s SqlStore) {
t.Run("GetKnownUsers", func(t *testing.T) { testGetKnownUsers(t, rctx, ss) })
t.Run("GetUsersWithInvalidEmails", func(t *testing.T) { testGetUsersWithInvalidEmails(t, rctx, ss) })
t.Run("UpdateLastLogin", func(t *testing.T) { testUpdateLastLogin(t, rctx, ss) })
t.Run("GetUserReport", func(t *testing.T) { testGetUserReport(t, rctx, ss) })
}
func testUserStoreSave(t *testing.T, rctx request.CTX, ss store.Store) {
@@ -4904,6 +4905,7 @@ func testUserStoreGetUsersBatchForIndexing(t *testing.T, rctx request.CTX, ss st
CreateAt: model.GetMillis(),
})
require.NoError(t, err)
defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
time.Sleep(time.Millisecond)
@@ -4913,6 +4915,7 @@ func testUserStoreGetUsersBatchForIndexing(t *testing.T, rctx request.CTX, ss st
CreateAt: model.GetMillis(),
})
require.NoError(t, err)
defer func() { require.NoError(t, ss.User().PermanentDelete(u2.Id)) }()
_, nErr = ss.Team().SaveMember(&model.TeamMember{
UserId: u2.Id,
TeamId: t1.Id,
@@ -4939,6 +4942,7 @@ func testUserStoreGetUsersBatchForIndexing(t *testing.T, rctx request.CTX, ss st
CreateAt: model.GetMillis(),
})
require.NoError(t, err)
defer func() { require.NoError(t, ss.User().PermanentDelete(u3.Id)) }()
_, nErr = ss.Team().SaveMember(&model.TeamMember{
UserId: u3.Id,
TeamId: t1.Id,
@@ -6182,3 +6186,297 @@ func testUpdateLastLogin(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
require.Equal(t, int64(1234567890), user.LastLogin)
}
func testGetUserReport(t *testing.T, rctx request.CTX, ss store.Store) {
now := time.Now()
u1 := &model.User{Username: "u1" + model.NewId(), DeleteAt: 0}
u1.Email = MakeEmail()
u1, err := ss.User().Save(u1)
require.NoError(t, err)
defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
for i := 0; i < 5; i++ {
p := model.Post{UserId: u1.Id, ChannelId: model.NewId(), Message: NewTestId(), CreateAt: now.AddDate(0, 0, -i).UnixMilli()}
_, err = ss.Post().Save(&p)
require.NoError(t, err)
}
u2 := &model.User{Username: "u2" + model.NewId(), Roles: "system", DeleteAt: 0}
u2.Email = MakeEmail()
u2, err = ss.User().Save(u2)
require.NoError(t, err)
defer func() { require.NoError(t, ss.User().PermanentDelete(u2.Id)) }()
for i := 0; i < 5; i++ {
p := model.Post{UserId: u2.Id, ChannelId: model.NewId(), Message: NewTestId(), CreateAt: now.AddDate(0, 0, -i).UnixMilli()}
_, err = ss.Post().Save(&p)
require.NoError(t, err)
}
u3 := &model.User{Username: "u3" + model.NewId(), Roles: "guest", DeleteAt: now.UnixMilli()}
u3.Email = MakeEmail()
u3, err = ss.User().Save(u3)
require.NoError(t, err)
team, err := ss.Team().Save(&model.Team{
DisplayName: "DisplayName",
Name: NewTestId(),
Email: MakeEmail(),
Type: model.TeamOpen,
})
require.NoError(t, err)
_, err = ss.Team().SaveMember(&model.TeamMember{UserId: u3.Id, TeamId: team.Id}, 1)
require.NoError(t, err)
defer func() {
require.NoError(t, ss.Team().RemoveMember(rctx, team.Id, u3.Id))
require.NoError(t, ss.Team().PermanentDelete(team.Id))
}()
defer func() { require.NoError(t, ss.User().PermanentDelete(u3.Id)) }()
for i := 0; i < 5; i++ {
p := model.Post{UserId: u3.Id, ChannelId: model.NewId(), Message: NewTestId(), CreateAt: now.AddDate(0, 0, -i).UnixMilli()}
_, err = ss.Post().Save(&p)
require.NoError(t, err)
}
err = ss.User().RefreshPostStatsForUsers()
require.NoError(t, err)
t.Run("should return info for all the users", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
},
})
require.NoError(t, err)
require.NotNil(t, userReport)
require.Len(t, userReport, 3)
require.NotNil(t, userReport[0])
require.Equal(t, u1.Username, userReport[0].Username)
require.NotNil(t, userReport[1])
require.Equal(t, u2.Username, userReport[1].Username)
require.NotNil(t, userReport[2])
require.Equal(t, u3.Username, userReport[2].Username)
})
t.Run("should return in the correct order", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
SortDesc: true,
PageSize: 50,
},
})
require.NoError(t, err)
require.NotNil(t, userReport)
require.Equal(t, 3, len(userReport))
require.NotNil(t, userReport[0])
require.Equal(t, u3.Username, userReport[0].Username)
require.NotNil(t, userReport[1])
require.Equal(t, u2.Username, userReport[1].Username)
require.NotNil(t, userReport[2])
require.Equal(t, u1.Username, userReport[2].Username)
})
t.Run("should fail on invalid sort column", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "FakeColumn",
SortDesc: true,
PageSize: 50,
},
})
require.Error(t, err)
require.Nil(t, userReport)
})
t.Run("should only return amount of users in page", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 2,
},
})
require.NoError(t, err)
require.NotNil(t, userReport)
require.Equal(t, 2, len(userReport))
require.NotNil(t, userReport[0])
require.Equal(t, u1.Username, userReport[0].Username)
require.NotNil(t, userReport[1])
require.Equal(t, u2.Username, userReport[1].Username)
})
t.Run("should return correct paging", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
LastSortColumnValue: u2.Username,
LastUserId: u2.Id,
},
})
require.NoError(t, err)
require.NotNil(t, userReport)
require.Equal(t, 1, len(userReport))
require.NotNil(t, userReport[0])
require.Equal(t, u3.Username, userReport[0].Username)
})
t.Run("should return accurate post stats for various date ranges", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
},
})
require.NoError(t, err)
require.Len(t, userReport, 3)
require.Equal(t, 5, *userReport[0].TotalPosts)
require.Equal(t, 5, *userReport[0].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[0].LastPostDate)
require.Equal(t, 5, *userReport[1].TotalPosts)
require.Equal(t, 5, *userReport[1].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[1].LastPostDate)
require.Equal(t, 5, *userReport[2].TotalPosts)
require.Equal(t, 5, *userReport[2].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[2].LastPostDate)
userReport, err = ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
},
StartAt: now.AddDate(0, 0, -2).UnixMilli(),
})
require.NoError(t, err)
require.Len(t, userReport, 3)
require.Equal(t, 3, *userReport[0].TotalPosts)
require.Equal(t, 3, *userReport[0].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[0].LastPostDate)
require.Equal(t, 3, *userReport[1].TotalPosts)
require.Equal(t, 3, *userReport[1].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[1].LastPostDate)
require.Equal(t, 3, *userReport[2].TotalPosts)
require.Equal(t, 3, *userReport[2].DaysActive)
require.Equal(t, now.UnixMilli(), *userReport[2].LastPostDate)
userReport, err = ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
},
EndAt: now.AddDate(0, 0, -2).UnixMilli(),
})
require.NoError(t, err)
require.Len(t, userReport, 3)
require.Equal(t, 2, *userReport[0].TotalPosts)
require.Equal(t, 2, *userReport[0].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[0].LastPostDate)
require.Equal(t, 2, *userReport[1].TotalPosts)
require.Equal(t, 2, *userReport[1].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[1].LastPostDate)
require.Equal(t, 2, *userReport[2].TotalPosts)
require.Equal(t, 2, *userReport[2].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[2].LastPostDate)
userReport, err = ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
},
StartAt: now.AddDate(0, 0, -3).UnixMilli(),
EndAt: now.AddDate(0, 0, -2).UnixMilli(),
})
require.NoError(t, err)
require.Len(t, userReport, 3)
require.Equal(t, 1, *userReport[0].TotalPosts)
require.Equal(t, 1, *userReport[0].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[0].LastPostDate)
require.Equal(t, 1, *userReport[1].TotalPosts)
require.Equal(t, 1, *userReport[1].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[1].LastPostDate)
require.Equal(t, 1, *userReport[2].TotalPosts)
require.Equal(t, 1, *userReport[2].DaysActive)
require.Equal(t, now.AddDate(0, 0, -3).UnixMilli(), *userReport[2].LastPostDate)
})
t.Run("should filter on roles", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
Role: "system",
},
})
require.NoError(t, err)
require.Len(t, userReport, 1)
require.Equal(t, u2.Id, userReport[0].Id)
require.Equal(t, u2.Roles, "system")
})
t.Run("should filter on teams", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
HasNoTeam: true,
},
})
require.NoError(t, err)
require.Len(t, userReport, 2)
require.Equal(t, u1.Id, userReport[0].Id)
require.Equal(t, u2.Id, userReport[1].Id)
userReport, err = ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
Team: team.Id,
},
})
require.NoError(t, err)
require.Len(t, userReport, 1)
require.Equal(t, u3.Id, userReport[0].Id)
})
t.Run("should filter on activation", func(t *testing.T) {
userReport, err := ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
HideInactive: true,
},
})
require.NoError(t, err)
require.Len(t, userReport, 2)
require.Equal(t, u1.Id, userReport[0].Id)
require.Equal(t, u2.Id, userReport[1].Id)
userReport, err = ss.User().GetUserReport(&model.UserReportOptions{
UserReportOptionsWithoutDateRange: model.UserReportOptionsWithoutDateRange{
SortColumn: "Username",
PageSize: 50,
HideActive: true,
},
})
require.NoError(t, err)
require.Len(t, userReport, 1)
require.Equal(t, u3.Id, userReport[0].Id)
})
}

Просмотреть файл

@@ -10709,6 +10709,22 @@ func (s *TimerLayerUserStore) GetUnreadCountForChannel(userID string, channelID
return result, err
}
func (s *TimerLayerUserStore) GetUserReport(filter *model.UserReportOptions) ([]*model.UserReportQuery, error) {
start := time.Now()
result, err := s.UserStore.GetUserReport(filter)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("UserStore.GetUserReport", success, elapsed)
}
return result, err
}
func (s *TimerLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
start := time.Now()