[MM-55014][MM-55015] Add last login timestamp for users, add materialized view and refresh job to keep track of post stats for Postgres (#25152)

* [MM-55014][MM-55015] Add last login timestamp for users, add materialized view and refresh job for Postgres

* Check fixes

* Fix type issue

* Add verification that lastlogin was updated

* PR feedback

* Morge'd

* Morge'd again

* Merge'd

* Update admin setting strings

* WIP

* PR feedback

* Oops

* Fix i18n

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/store/opentracinglayer/opentracinglayer.go

@@ -11921,6 +11921,24 @@ func (s *OpenTracingLayerUserStore) PromoteGuestToUser(userID string) error {
 	return err
 }
 
+func (s *OpenTracingLayerUserStore) RefreshPostStatsForUsers() error {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.RefreshPostStatsForUsers")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	err := s.UserStore.RefreshPostStatsForUsers()
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return err
+}
+
 func (s *OpenTracingLayerUserStore) ResetAuthDataToEmailForUsers(service string, userIDs []string, includeDeleted bool, dryRun bool) (int, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.ResetAuthDataToEmailForUsers")
@@ -12155,6 +12173,24 @@ func (s *OpenTracingLayerUserStore) UpdateFailedPasswordAttempts(userID string,
 	return err
 }
 
+func (s *OpenTracingLayerUserStore) UpdateLastLogin(userID string, lastLogin int64) error {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.UpdateLastLogin")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	err := s.UserStore.UpdateLastLogin(userID, lastLogin)
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return err
+}
+
 func (s *OpenTracingLayerUserStore) UpdateLastPictureUpdate(userID string) error {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.UpdateLastPictureUpdate")

server/channels/store/retrylayer/retrylayer.go

@@ -13585,6 +13585,27 @@ func (s *RetryLayerUserStore) PromoteGuestToUser(userID string) error {
 
 }
 
+func (s *RetryLayerUserStore) RefreshPostStatsForUsers() error {
+
+	tries := 0
+	for {
+		err := s.UserStore.RefreshPostStatsForUsers()
+		if err == nil {
+			return nil
+		}
+		if !isRepeatableError(err) {
+			return err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerUserStore) ResetAuthDataToEmailForUsers(service string, userIDs []string, includeDeleted bool, dryRun bool) (int, error) {
 
 	tries := 0
@@ -13858,6 +13879,27 @@ func (s *RetryLayerUserStore) UpdateFailedPasswordAttempts(userID string, attemp
 
 }
 
+func (s *RetryLayerUserStore) UpdateLastLogin(userID string, lastLogin int64) error {
+
+	tries := 0
+	for {
+		err := s.UserStore.UpdateLastLogin(userID, lastLogin)
+		if err == nil {
+			return nil
+		}
+		if !isRepeatableError(err) {
+			return err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerUserStore) UpdateLastPictureUpdate(userID string) error {
 
 	tries := 0

server/channels/store/sqlstore/user_store.go

@@ -54,7 +54,7 @@ func newSqlUserStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) s
 	// note: we are providing field names explicitly here to maintain order of columns (needed when using raw queries)
 	us.usersQuery = us.getQueryBuilder().
 		Select("u.Id", "u.CreateAt", "u.UpdateAt", "u.DeleteAt", "u.Username", "u.Password", "u.AuthData", "u.AuthService", "u.Email", "u.EmailVerified", "u.Nickname", "u.FirstName", "u.LastName", "u.Position", "u.Roles", "u.AllowMarketing", "u.Props", "u.NotifyProps", "u.LastPasswordUpdate", "u.LastPictureUpdate", "u.FailedAttempts", "u.Locale", "u.Timezone", "u.MfaActive", "u.MfaSecret",
-			"b.UserId IS NOT NULL AS IsBot", "COALESCE(b.Description, '') AS BotDescription", "COALESCE(b.LastIconUpdate, 0) AS BotLastIconUpdate", "u.RemoteId").
+			"b.UserId IS NOT NULL AS IsBot", "COALESCE(b.Description, '') AS BotDescription", "COALESCE(b.LastIconUpdate, 0) AS BotLastIconUpdate", "u.RemoteId", "u.LastLogin").
 		From("Users u").
 		LeftJoin("Bots b ON ( b.UserId = u.Id )")
 
@@ -193,6 +193,7 @@ func (us SqlUserStore) Update(user *model.User, trustedUpdateData bool) (*model.
 	user.FailedAttempts = oldUser.FailedAttempts
 	user.MfaSecret = oldUser.MfaSecret
 	user.MfaActive = oldUser.MfaActive
+	user.LastLogin = oldUser.LastLogin
 
 	if !trustedUpdateData {
 		user.Roles = oldUser.Roles
@@ -222,7 +223,7 @@ func (us SqlUserStore) Update(user *model.User, trustedUpdateData bool) (*model.
 				AllowMarketing=:AllowMarketing, Props=:Props, NotifyProps=:NotifyProps,
 				LastPasswordUpdate=:LastPasswordUpdate, LastPictureUpdate=:LastPictureUpdate,
 				FailedAttempts=:FailedAttempts,Locale=:Locale, Timezone=:Timezone, MfaActive=:MfaActive,
-				MfaSecret=:MfaSecret, RemoteId=:RemoteId
+				MfaSecret=:MfaSecret, RemoteId=:RemoteId, LastLogin=:LastLogin
 			WHERE Id=:Id`
 
 	user.Props = wrapBinaryParamStringMap(us.IsBinaryParamEnabled(), user.Props)
@@ -355,6 +356,27 @@ func (us SqlUserStore) UpdateAuthData(userId string, service string, authData *s
 	return userId, nil
 }
 
+func (us SqlUserStore) UpdateLastLogin(userId string, lastLogin int64) error {
+	updateAt := model.GetMillis()
+
+	updateQuery := us.getQueryBuilder().
+		Update("Users").
+		Set("LastLogin", lastLogin).
+		Set("UpdateAt", updateAt).
+		Where(sq.Eq{"Id": userId})
+
+	queryString, args, err := updateQuery.ToSql()
+	if err != nil {
+		return errors.Wrap(err, "update_last_login_tosql")
+	}
+
+	if _, err := us.GetMasterX().Exec(queryString, args...); err != nil {
+		return errors.Wrapf(err, "failed to update User with userId=%s", userId)
+	}
+
+	return nil
+}
+
 // ResetAuthDataToEmailForUsers resets the AuthData of users whose AuthService
 // is |service| to their Email. If userIDs is non-empty, only the users whose
 // IDs are in userIDs will be affected. If dryRun is true, only the number
@@ -449,7 +471,7 @@ func (us SqlUserStore) Get(ctx context.Context, id string) (*model.User, error)
 		&user.Nickname, &user.FirstName, &user.LastName, &user.Position, &user.Roles,
 		&user.AllowMarketing, &props, &notifyProps, &user.LastPasswordUpdate, &user.LastPictureUpdate,
 		&user.FailedAttempts, &user.Locale, &timezone, &user.MfaActive, &user.MfaSecret,
-		&user.IsBot, &user.BotDescription, &user.BotLastIconUpdate, &user.RemoteId)
+		&user.IsBot, &user.BotDescription, &user.BotLastIconUpdate, &user.RemoteId, &user.LastLogin)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, store.NewErrNotFound("User", id)
@@ -851,7 +873,7 @@ func (us SqlUserStore) GetAllProfilesInChannel(ctx context.Context, channelID st
 	for rows.Next() {
 		var user model.User
 		var props, notifyProps, timezone []byte
-		if err = rows.Scan(&user.Id, &user.CreateAt, &user.UpdateAt, &user.DeleteAt, &user.Username, &user.Password, &user.AuthData, &user.AuthService, &user.Email, &user.EmailVerified, &user.Nickname, &user.FirstName, &user.LastName, &user.Position, &user.Roles, &user.AllowMarketing, &props, &notifyProps, &user.LastPasswordUpdate, &user.LastPictureUpdate, &user.FailedAttempts, &user.Locale, &timezone, &user.MfaActive, &user.MfaSecret, &user.IsBot, &user.BotDescription, &user.BotLastIconUpdate, &user.RemoteId); err != nil {
+		if err = rows.Scan(&user.Id, &user.CreateAt, &user.UpdateAt, &user.DeleteAt, &user.Username, &user.Password, &user.AuthData, &user.AuthService, &user.Email, &user.EmailVerified, &user.Nickname, &user.FirstName, &user.LastName, &user.Position, &user.Roles, &user.AllowMarketing, &props, &notifyProps, &user.LastPasswordUpdate, &user.LastPictureUpdate, &user.FailedAttempts, &user.Locale, &timezone, &user.MfaActive, &user.MfaSecret, &user.IsBot, &user.BotDescription, &user.BotLastIconUpdate, &user.RemoteId, &user.LastLogin); err != nil {
 			return nil, errors.Wrap(err, "failed to scan values from rows into User entity")
 		}
 		if err = json.Unmarshal(props, &user.Props); err != nil {
@@ -2231,3 +2253,15 @@ func (us SqlUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrict
 
 	return users, nil
 }
+
+func (us SqlUserStore) RefreshPostStatsForUsers() error {
+	if us.DriverName() == model.DatabaseDriverPostgres {
+		if _, err := us.GetReplicaX().Exec("REFRESH MATERIALIZED VIEW poststats"); err != nil {
+			return errors.Wrap(err, "users_refresh_post_stats_exec")
+		}
+	} else {
+		mlog.Debug("Skipped running refresh post stats, only available on Postgres")
+	}
+
+	return nil
+}

server/channels/store/store.go

@@ -419,6 +419,7 @@ type UserStore interface {
 	UpdatePassword(userID, newPassword string) error
 	UpdateUpdateAt(userID string) (int64, error)
 	UpdateAuthData(userID string, service string, authData *string, email string, resetMfa bool) (string, error)
+	UpdateLastLogin(userID string, lastLogin int64) error
 	ResetAuthDataToEmailForUsers(service string, userIDs []string, includeDeleted bool, dryRun bool) (int, error)
 	UpdateMfaSecret(userID, secret string) error
 	UpdateMfaActive(userID string, active bool) error
@@ -488,6 +489,7 @@ type UserStore interface {
 	IsEmpty(excludeBots bool) (bool, error)
 	GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error)
 	InsertUsers(users []*model.User) error
+	RefreshPostStatsForUsers() error
 }
 
 type BotStore interface {

server/channels/store/storetest/mocks/UserStore.go

@@ -1308,6 +1308,20 @@ func (_m *UserStore) PromoteGuestToUser(userID string) error {
 	return r0
 }
 
+// RefreshPostStatsForUsers provides a mock function with given fields:
+func (_m *UserStore) RefreshPostStatsForUsers() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // ResetAuthDataToEmailForUsers provides a mock function with given fields: service, userIDs, includeDeleted, dryRun
 func (_m *UserStore) ResetAuthDataToEmailForUsers(service string, userIDs []string, includeDeleted bool, dryRun bool) (int, error) {
 	ret := _m.Called(service, userIDs, includeDeleted, dryRun)
@@ -1618,6 +1632,20 @@ func (_m *UserStore) UpdateFailedPasswordAttempts(userID string, attempts int) e
 	return r0
 }
 
+// UpdateLastLogin provides a mock function with given fields: userID, lastLogin
+func (_m *UserStore) UpdateLastLogin(userID string, lastLogin int64) error {
+	ret := _m.Called(userID, lastLogin)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, int64) error); ok {
+		r0 = rf(userID, lastLogin)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // UpdateLastPictureUpdate provides a mock function with given fields: userID
 func (_m *UserStore) UpdateLastPictureUpdate(userID string) error {
 	ret := _m.Called(userID)

server/channels/store/storetest/user_store.go

@@ -95,6 +95,7 @@ func TestUserStore(t *testing.T, ss store.Store, s SqlStore) {
 	t.Run("ResetLastPictureUpdate", func(t *testing.T) { testUserStoreResetLastPictureUpdate(t, ss) })
 	t.Run("GetKnownUsers", func(t *testing.T) { testGetKnownUsers(t, ss) })
 	t.Run("GetUsersWithInvalidEmails", func(t *testing.T) { testGetUsersWithInvalidEmails(t, ss) })
+	t.Run("UpdateLastLogin", func(t *testing.T) { testUpdateLastLogin(t, ss) })
 }
 
 func testUserStoreSave(t *testing.T, ss store.Store) {
@@ -6169,3 +6170,18 @@ func testGetUsersWithInvalidEmails(t *testing.T, ss store.Store) {
 	require.NoError(t, err)
 	assert.Len(t, users, 1)
 }
+
+func testUpdateLastLogin(t *testing.T, ss store.Store) {
+	u1 := model.User{}
+	u1.Email = MakeEmail()
+	_, err := ss.User().Save(&u1)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
+
+	err = ss.User().UpdateLastLogin(u1.Id, 1234567890)
+	require.NoError(t, err)
+
+	user, err := ss.User().Get(context.Background(), u1.Id)
+	require.NoError(t, err)
+	require.Equal(t, int64(1234567890), user.LastLogin)
+}

server/channels/store/timerlayer/timerlayer.go

@@ -10744,6 +10744,22 @@ func (s *TimerLayerUserStore) PromoteGuestToUser(userID string) error {
 	return err
 }
 
+func (s *TimerLayerUserStore) RefreshPostStatsForUsers() error {
+	start := time.Now()
+
+	err := s.UserStore.RefreshPostStatsForUsers()
+
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("UserStore.RefreshPostStatsForUsers", success, elapsed)
+	}
+	return err
+}
+
 func (s *TimerLayerUserStore) ResetAuthDataToEmailForUsers(service string, userIDs []string, includeDeleted bool, dryRun bool) (int, error) {
 	start := time.Now()
 
@@ -10952,6 +10968,22 @@ func (s *TimerLayerUserStore) UpdateFailedPasswordAttempts(userID string, attemp
 	return err
 }
 
+func (s *TimerLayerUserStore) UpdateLastLogin(userID string, lastLogin int64) error {
+	start := time.Now()
+
+	err := s.UserStore.UpdateLastLogin(userID, lastLogin)
+
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("UserStore.UpdateLastLogin", success, elapsed)
+	}
+	return err
+}
+
 func (s *TimerLayerUserStore) UpdateLastPictureUpdate(userID string) error {
 	start := time.Now()