MM-45899: Insights: least active channels (#20796)

* Add api endpoints, app layers for top inactive channels with dummy store calls

* Add store functions for top inactive channels

* Add model, store, app tests.

* Add client function and api tests

* Add participants information to TopInactiveChannel

* Translation fix

* Style fix while writing response

* Return channelmember IDs instead of profiles, query in batch avoiding inside the loop

* Make the following changes

 - move DeleteAt to subqueries, to avoid select, group by
 - Remove TeamId from response
 - Count bots and webhook posts

* SQL query lint fix, store test fix to include bot messages

* make app-layers

* Fix empty participant lists being sent as [""]

* Track channel joins, to distinguish 0 activity channels vs new channels

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Shivashis Padhi
2022-08-24 23:14:56 +05:30
коммит произвёл GitHub
родитель 8fd1762c3b
Коммит 6adbcc5d05
17 изменённых файлов: 1041 добавлений и 0 удалений

Просмотреть файл

@@ -1803,6 +1803,42 @@ func (s *OpenTracingLayerChannelStore) GetTopChannelsForUserSince(userID string,
return result, err
}
func (s *OpenTracingLayerChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GetTopInactiveChannelsForTeamSince")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.ChannelStore.GetTopInactiveChannelsForTeamSince(teamID, userID, since, offset, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GetTopInactiveChannelsForUserSince")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.ChannelStore.GetTopInactiveChannelsForUserSince(teamID, userID, since, offset, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerChannelStore) GroupSyncedChannelCount() (int64, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GroupSyncedChannelCount")

Просмотреть файл

@@ -2039,6 +2039,48 @@ func (s *RetryLayerChannelStore) GetTopChannelsForUserSince(userID string, teamI
}
func (s *RetryLayerChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
tries := 0
for {
result, err := s.ChannelStore.GetTopInactiveChannelsForTeamSince(teamID, userID, since, offset, limit)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
tries := 0
for {
result, err := s.ChannelStore.GetTopInactiveChannelsForUserSince(teamID, userID, since, offset, limit)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerChannelStore) GroupSyncedChannelCount() (int64, error) {
tries := 0

Просмотреть файл

@@ -4319,6 +4319,198 @@ func (s SqlChannelStore) GetTopChannelsForUserSince(userID string, teamID string
return model.GetTopChannelListWithPagination(channels, limit), nil
}
// GetTopInactiveChannelsForTeamSince returns the filtered post counts of the following Channels sets:
// a) those that are private channels in the given user's membership graph on the given team, and
// b) those that are public channels in the given team.
func (s SqlChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
channels := make([]*model.TopInactiveChannel, 0)
var args []any
query := `
SELECT
ID,
Type,
DisplayName,
Name,
MessageCount,
LastActivityAt
FROM
((SELECT
Posts.ChannelId AS ID,
'O' AS Type,
PublicChannels.DisplayName AS DisplayName,
PublicChannels.Name AS Name,
count(Posts.Id) AS MessageCount,
max(Posts.CreateAt) AS LastActivityAt
FROM
Posts
LEFT JOIN PublicChannels on Posts.ChannelId = PublicChannels.Id
WHERE
Posts.DeleteAt = 0
AND Posts.CreateAt > ?
AND (Posts.Type = '' OR Posts.Type = 'system_join_channel')
AND PublicChannels.TeamId = ?
AND PublicChannels.DeleteAt = 0
GROUP BY
Posts.ChannelId,
PublicChannels.DisplayName,
PublicChannels.Name,
PublicChannels.TeamId)
UNION ALL
(SELECT
Posts.ChannelId AS ID,
Channels.Type AS Type,
Channels.DisplayName AS DisplayName,
Channels.Name AS Name,
count(Posts.Id) AS MessageCount,
max(Posts.CreateAt) AS LastActivityAt
FROM
Posts
LEFT JOIN Channels on Posts.ChannelId = Channels.Id
LEFT JOIN ChannelMembers on Posts.ChannelId = ChannelMembers.ChannelId
WHERE
Posts.DeleteAt = 0
AND Posts.CreateAt > ?
AND (Posts.Type = '' OR Posts.Type = 'system_join_channel')
AND Channels.TeamId = ?
AND Channels.Type = 'P'
AND Channels.DeleteAt = 0
AND ChannelMembers.UserId = ?
GROUP BY
Posts.ChannelId,
Channels.Type,
Channels.DisplayName,
Channels.Name)) AS A
ORDER BY
MessageCount ASC,
Name ASC
LIMIT ?
OFFSET ?`
args = append(args, since, teamID, since, teamID, userID, limit+1, offset)
if err := s.GetReplicaX().Select(&channels, query, args...); err != nil {
return nil, errors.Wrap(err, "failed to get top Channels")
}
channels, err := postProcessTopInactiveChannels(s, channels)
if err != nil {
return nil, err
}
return model.GetTopInactiveChannelListWithPagination(channels, limit), nil
}
// GetTopInactiveChannelsForUserSince returns the filtered post counts of channels with with posts created by the user
// after the given timestamp within the given team (or across the workspace if no team is given). Excludes DM and GM channels.
func (s SqlChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
channels := make([]*model.TopInactiveChannel, 0)
var args []any
var query string
query = `
SELECT
Posts.ChannelId AS ID,
Channels.Type AS Type,
Channels.DisplayName AS DisplayName,
Channels.Name AS Name,
count(Posts.Id) AS MessageCount,
max(Posts.CreateAt) AS LastActivityAt
FROM
Posts
LEFT JOIN Channels on Posts.ChannelId = Channels.Id
LEFT JOIN ChannelMembers on Posts.ChannelId = ChannelMembers.ChannelId
WHERE
Posts.DeleteAt = 0
AND Posts.CreateAt > ?
AND (Posts.Type = '' OR Posts.Type = 'system_join_channel')
AND Channels.DeleteAt = 0
AND (Channels.Type = 'O' OR Channels.Type = 'P')
AND ChannelMembers.UserId = ? `
args = []any{since, userID}
if teamID != "" {
query += `
AND Channels.TeamID = ?`
args = append(args, teamID)
}
query += `
Group By
Posts.ChannelId,
Channels.Type,
Channels.DisplayName,
Channels.Name
ORDER BY
MessageCount ASC,
Name ASC
LIMIT ?
OFFSET ?`
args = append(args, limit+1, offset)
if err := s.GetReplicaX().Select(&channels, query, args...); err != nil {
return nil, errors.Wrap(err, "failed to get top Inactive Channels")
}
channels, err := postProcessTopInactiveChannels(s, channels)
if err != nil {
return nil, err
}
return model.GetTopInactiveChannelListWithPagination(channels, limit), nil
}
func postProcessTopInactiveChannels(s SqlChannelStore, channels []*model.TopInactiveChannel) ([]*model.TopInactiveChannel, error) {
// query channel members for Ids
var conditionalAggrSelector string
if s.DriverName() == model.DatabaseDriverMysql {
conditionalAggrSelector = "GROUP_CONCAT(UserId SEPARATOR ',') as UserIds"
} else if s.DriverName() == model.DatabaseDriverPostgres {
conditionalAggrSelector = "string_agg(UserId, ',') as UserIds"
}
var channelIds []string
for _, channel := range channels {
channelIds = append(channelIds, channel.ID)
}
q := s.getQueryBuilder().Select("ChannelId", conditionalAggrSelector).From("ChannelMembers").
Where(sq.Eq{
"ChannelId": channelIds,
}).GroupBy("ChannelId")
channelsUserIdsMap := make(map[string]string, len(channels))
type ChannelUserIdsResult struct {
ChannelId string
UserIds string
}
channelsUserIdsResultList := make([]ChannelUserIdsResult, len(channels))
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to stringify squirrel query")
}
if err := s.GetReplicaX().Select(&channelsUserIdsResultList, sql, args...); err != nil {
return nil, errors.Wrap(err, "failed to get top Inactive Channels users")
}
for _, channelUserIds := range channelsUserIdsResultList {
channelsUserIdsMap[channelUserIds.ChannelId] = channelUserIds.UserIds
}
for index, channel := range channels {
userIds := channelsUserIdsMap[channel.ID]
userIdsSlice := strings.Split(userIds, ",")
channels[index].Participants = userIdsSlice
// handle channels with 0 participants
if len(userIdsSlice) == 1 && userIdsSlice[0] == "" {
channels[index].Participants = make([]string, 0)
}
}
return channels, nil
}
func (s SqlChannelStore) PostCountsByDuration(channelIDs []string, sinceUnixMillis int64, userID *string, duration model.PostCountGrouping, atLocation *time.Location) ([]*model.DurationPostCount, error) {
var unixSelect string
var propsQuery string

Просмотреть файл

@@ -298,6 +298,10 @@ type ChannelStore interface {
GetTopChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopChannelList, error)
GetTopChannelsForUserSince(userID string, teamID string, since int64, offset int, limit int) (*model.TopChannelList, error)
PostCountsByDuration(channelIDs []string, sinceUnixMillis int64, userID *string, duration model.PostCountGrouping, groupingLocation *time.Location) ([]*model.DurationPostCount, error)
// Insights - inactive channels
GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error)
GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error)
}
type ChannelMemberHistoryStore interface {

Просмотреть файл

@@ -149,6 +149,7 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlStore) {
t.Run("SetShared", func(t *testing.T) { testSetShared(t, ss) })
t.Run("GetTeamForChannel", func(t *testing.T) { testGetTeamForChannel(t, ss) })
t.Run("PostCountsByDuration", func(t *testing.T) { testChannelPostCountsByDuration(t, ss) })
t.Run("GetTopInactiveChannels", func(t *testing.T) { testGetTopInactiveChannels(t, ss) })
}
func testChannelStoreSave(t *testing.T, ss store.Store) {
@@ -7961,3 +7962,162 @@ func testChannelPostCountsByDuration(t *testing.T, ss store.Store) {
require.Equal(t, channel.Id, dpc[0].ChannelID)
require.Equal(t, 1, dpc[0].PostCount)
}
func testGetTopInactiveChannels(t *testing.T, ss store.Store) {
team, err := ss.Team().Save(&model.Team{
Name: model.NewId(),
DisplayName: "DisplayName",
Email: MakeEmail(),
Type: model.TeamOpen,
})
require.NoError(t, err)
defer func() { ss.Team().PermanentDelete(team.Id) }()
channelPublic0 := &model.Channel{
TeamId: team.Id,
DisplayName: "test_share_flag asdf",
Name: "test_share_flag_public0",
Type: model.ChannelTypeOpen,
}
channelSaved0, err := ss.Channel().Save(channelPublic0, 999)
require.NoError(t, err)
defer func() { ss.Channel().PermanentDelete(channelSaved0.Id) }()
channelPublic1 := &model.Channel{
TeamId: team.Id,
DisplayName: "test_share_flag",
Name: "test_share_flag",
Type: model.ChannelTypeOpen,
}
channelSaved1, err := ss.Channel().Save(channelPublic1, 999)
require.NoError(t, err)
defer func() { ss.Channel().PermanentDelete(channelSaved1.Id) }()
// create private channel
c3 := model.Channel{}
c3.TeamId = team.Id
c3.DisplayName = "Channel3" + model.NewId()
c3.Name = NewTestId()
c3.Type = model.ChannelTypePrivate
channelPrivate, nErr := ss.Channel().Save(&c3, -1)
require.NoError(t, nErr)
// create dm channel
u1 := model.User{}
u1.Email = MakeEmail()
u1.Nickname = model.NewId()
_, err = ss.User().Save(&u1)
require.NoError(t, err)
u2 := model.User{}
u2.Email = MakeEmail()
u2.Nickname = model.NewId()
_, err = ss.User().Save(&u2)
require.NoError(t, err)
uBot := model.User{Id: model.NewId()}
_, nErr = ss.Channel().CreateDirectChannel(&u1, &u2)
require.NoError(t, nErr)
// add u1, u2 to channels
cm1 := &model.ChannelMember{ChannelId: channelPrivate.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()}
_, err = ss.Channel().SaveMember(cm1)
require.NoError(t, err)
cm1Public := &model.ChannelMember{ChannelId: channelPublic1.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()}
_, err = ss.Channel().SaveMember(cm1Public)
require.NoError(t, err)
cm2 := &model.ChannelMember{ChannelId: channelPublic0.Id, UserId: u2.Id, NotifyProps: model.GetDefaultChannelNotifyProps()}
_, err = ss.Channel().SaveMember(cm2)
require.NoError(t, err)
cmBot := &model.ChannelMember{ChannelId: channelPublic0.Id, UserId: uBot.Id, NotifyProps: model.GetDefaultChannelNotifyProps()}
_, err = ss.Channel().SaveMember(cmBot)
require.NoError(t, err)
_, err = ss.Post().Save(&model.Post{
UserId: u1.Id,
ChannelId: channelPrivate.Id,
Message: "test",
})
require.NoError(t, err)
_, err = ss.Post().Save(&model.Post{
UserId: u1.Id,
ChannelId: channelPrivate.Id,
Message: "test1",
})
require.NoError(t, err)
// create posts in channel public 0
postToCheckLastUpdateAt, err := ss.Post().Save(&model.Post{
UserId: u2.Id,
ChannelId: channelSaved0.Id,
Message: "test",
})
require.NoError(t, err)
_, err = ss.Post().Save(&model.Post{
UserId: model.NewId(),
ChannelId: channelPublic1.Id,
Message: "test",
Props: model.StringInterface{
"from_bot": true,
},
})
require.NoError(t, err)
// create posts in channel public 1
for i := 0; i < 3; i++ {
_, err = ss.Post().Save(&model.Post{
UserId: model.NewId(),
ChannelId: channelPublic1.Id,
Message: "test",
})
require.NoError(t, err)
}
// for u1
t.Run("top inactive channels for team - u1 ", func(t *testing.T) {
topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForTeamSince(team.Id, u1.Id, 0, 0, 10)
require.NoError(t, err)
require.Len(t, topInactiveChannels.Items, 3)
require.Equal(t, topInactiveChannels.Items[0].ID, channelSaved0.Id)
require.Equal(t, topInactiveChannels.Items[0].LastActivityAt, postToCheckLastUpdateAt.CreateAt)
require.Equal(t, topInactiveChannels.Items[1].ID, channelPrivate.Id)
require.Equal(t, topInactiveChannels.Items[2].ID, channelPublic1.Id)
// test bot posts are counted
require.Equal(t, topInactiveChannels.Items[2].MessageCount, int64(4))
// participants
require.Equal(t, topInactiveChannels.Items[1].Participants[0], u1.Id)
require.Equal(t, topInactiveChannels.Items[2].Participants[0], u1.Id)
})
t.Run("top inactive channels for user - u1 ", func(t *testing.T) {
topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForUserSince(team.Id, u1.Id, 0, 0, 10)
require.NoError(t, err)
require.Len(t, topInactiveChannels.Items, 2)
require.Equal(t, topInactiveChannels.Items[0].ID, channelPrivate.Id)
require.Equal(t, topInactiveChannels.Items[1].ID, channelPublic1.Id)
})
// for u2
t.Run("top inactive channels for team - u2 ", func(t *testing.T) {
topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForTeamSince(team.Id, u2.Id, 0, 0, 10)
require.NoError(t, err)
require.Len(t, topInactiveChannels.Items, 2)
require.Equal(t, topInactiveChannels.Items[0].ID, channelSaved0.Id)
require.Equal(t, topInactiveChannels.Items[0].LastActivityAt, postToCheckLastUpdateAt.CreateAt)
require.Equal(t, topInactiveChannels.Items[1].ID, channelPublic1.Id)
})
t.Run("top inactive channels for user - u2 ", func(t *testing.T) {
topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForUserSince(team.Id, u2.Id, 0, 0, 10)
require.NoError(t, err)
require.Len(t, topInactiveChannels.Items, 1)
require.Equal(t, topInactiveChannels.Items[0].ID, channelPublic0.Id)
})
}

Просмотреть файл

@@ -1579,6 +1579,52 @@ func (_m *ChannelStore) GetTopChannelsForUserSince(userID string, teamID string,
return r0, r1
}
// GetTopInactiveChannelsForTeamSince provides a mock function with given fields: teamID, userID, since, offset, limit
func (_m *ChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
ret := _m.Called(teamID, userID, since, offset, limit)
var r0 *model.TopInactiveChannelList
if rf, ok := ret.Get(0).(func(string, string, int64, int, int) *model.TopInactiveChannelList); ok {
r0 = rf(teamID, userID, since, offset, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.TopInactiveChannelList)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, int64, int, int) error); ok {
r1 = rf(teamID, userID, since, offset, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetTopInactiveChannelsForUserSince provides a mock function with given fields: teamID, userID, since, offset, limit
func (_m *ChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
ret := _m.Called(teamID, userID, since, offset, limit)
var r0 *model.TopInactiveChannelList
if rf, ok := ret.Get(0).(func(string, string, int64, int, int) *model.TopInactiveChannelList); ok {
r0 = rf(teamID, userID, since, offset, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.TopInactiveChannelList)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, string, int64, int, int) error); ok {
r1 = rf(teamID, userID, since, offset, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GroupSyncedChannelCount provides a mock function with given fields:
func (_m *ChannelStore) GroupSyncedChannelCount() (int64, error) {
ret := _m.Called()

Просмотреть файл

@@ -1653,6 +1653,38 @@ func (s *TimerLayerChannelStore) GetTopChannelsForUserSince(userID string, teamI
return result, err
}
func (s *TimerLayerChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
start := time.Now()
result, err := s.ChannelStore.GetTopInactiveChannelsForTeamSince(teamID, userID, since, offset, limit)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("ChannelStore.GetTopInactiveChannelsForTeamSince", success, elapsed)
}
return result, err
}
func (s *TimerLayerChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userID string, since int64, offset int, limit int) (*model.TopInactiveChannelList, error) {
start := time.Now()
result, err := s.ChannelStore.GetTopInactiveChannelsForUserSince(teamID, userID, since, offset, limit)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("ChannelStore.GetTopInactiveChannelsForUserSince", success, elapsed)
}
return result, err
}
func (s *TimerLayerChannelStore) GroupSyncedChannelCount() (int64, error) {
start := time.Now()