Growth spike guest accounts (#19437)

* tools updates

* Revert "tools updates"

This reverts commit 6293297b55803c5a263e200ebd80192899666ae9.

* new endpoint to get users that should potentially be guests

* checking authservice to ensure they were an email signup

* adding tests for new endpoint

* fixing translation issue

* permissions for new endpoint

* fixing tests

* fixing when domain array is empty

* fixing when domain array is empty

* removing bots from request

Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.local>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MBP.ht.home>
Co-authored-by: mkraft <martinkraft@gmail.com>

store/opentracinglayer/opentracinglayer.go

@@ -10502,6 +10502,24 @@ func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, en
 	return result, err
 }
 
+func (s *OpenTracingLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.GetUsersWithInvalidEmails")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return result, err
+}
+
 func (s *OpenTracingLayerUserStore) InferSystemInstallDate() (int64, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.InferSystemInstallDate")

store/retrylayer/retrylayer.go

@@ -11977,6 +11977,27 @@ func (s *RetryLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime
 
 }
 
+func (s *RetryLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+
+	tries := 0
+	for {
+		result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerUserStore) InferSystemInstallDate() (int64, error) {
 
 	tries := 0

store/sqlstore/user_store.go

@@ -2076,3 +2076,36 @@ func (us SqlUserStore) IsEmpty(excludeBots bool) (bool, error) {
 	}
 	return !hasRows, nil
 }
+
+func (us SqlUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	domainArray := strings.Split(restrictedDomains, ",")
+	query := us.usersQuery.
+		LeftJoin("Bots ON u.Id = Bots.UserId").
+		Where("Bots.UserId IS NULL").
+		Where("u.Roles != 'system_guest'").
+		Where("u.DeleteAt = 0").
+		Where("(u.AuthService = '' OR u.AuthService IS NULL)")
+
+	for _, d := range domainArray {
+		if d != "" {
+			query = query.Where("u.Email NOT LIKE LOWER(?)", wildcardSearchTerm(d))
+		}
+	}
+	query = query.Offset(uint64(page * perPage)).Limit(uint64(perPage))
+
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "users_get_many_tosql")
+	}
+
+	var users []*model.User
+	if _, err := us.GetReplica().Select(&users, queryString, args...); err != nil {
+		return nil, errors.Wrap(err, "users_get_many_select")
+	}
+
+	for _, u := range users {
+		u.Sanitize(map[string]bool{})
+	}
+
+	return users, nil
+}

store/store.go

@@ -438,6 +438,7 @@ type UserStore interface {
 	AutocompleteUsersInChannel(teamID, channelID, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, error)
 	GetKnownUsers(userID string) ([]string, error)
 	IsEmpty(excludeBots bool) (bool, error)
+	GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error)
 	InsertUsers(users []*model.User) error
 }
 

store/storetest/mocks/UserStore.go

@@ -1002,6 +1002,29 @@ func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, li
 	return r0, r1
 }
 
+// GetUsersWithInvalidEmails provides a mock function with given fields: page, perPage, restrictedDomains
+func (_m *UserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	ret := _m.Called(page, perPage, restrictedDomains)
+
+	var r0 []*model.User
+	if rf, ok := ret.Get(0).(func(int, int, string) []*model.User); ok {
+		r0 = rf(page, perPage, restrictedDomains)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.User)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int, int, string) error); ok {
+		r1 = rf(page, perPage, restrictedDomains)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // InferSystemInstallDate provides a mock function with given fields:
 func (_m *UserStore) InferSystemInstallDate() (int64, error) {
 	ret := _m.Called()

store/storetest/user_store.go

@@ -91,6 +91,7 @@ func TestUserStore(t *testing.T, ss store.Store, s SqlStore) {
 	t.Run("DeactivateGuests", func(t *testing.T) { testDeactivateGuests(t, ss) })
 	t.Run("ResetLastPictureUpdate", func(t *testing.T) { testUserStoreResetLastPictureUpdate(t, ss) })
 	t.Run("GetKnownUsers", func(t *testing.T) { testGetKnownUsers(t, ss) })
+	t.Run("GetUsersWithInvalidEmails", func(t *testing.T) { testGetUsersWithInvalidEmails(t, ss) })
 }
 
 func testUserStoreSave(t *testing.T, ss store.Store) {
@@ -4656,6 +4657,7 @@ func testUserStoreGetTeamGroupUsers(t *testing.T, ss store.Store) {
 		require.NoError(t, userErr)
 		require.NotNil(t, user)
 		testUsers = append(testUsers, user)
+		defer func() { require.NoError(t, ss.User().PermanentDelete(user.Id)) }()
 	}
 	require.Len(t, testUsers, 3, "testUsers length doesn't meet required length")
 	userGroupA, userGroupB, userNoGroup := testUsers[0], testUsers[1], testUsers[2]
@@ -4776,6 +4778,7 @@ func testUserStoreGetChannelGroupUsers(t *testing.T, ss store.Store) {
 		require.NoError(t, userErr)
 		require.NotNil(t, user)
 		testUsers = append(testUsers, user)
+		defer func() { require.NoError(t, ss.User().PermanentDelete(user.Id)) }()
 	}
 	require.Len(t, testUsers, 3, "testUsers length doesn't meet required length")
 	userGroupA, userGroupB, userNoGroup := testUsers[0], testUsers[1], testUsers[2]
@@ -5778,3 +5781,17 @@ func testIsEmpty(t *testing.T, ss store.Store) {
 	require.NoError(t, err)
 	require.True(t, ok)
 }
+
+func testGetUsersWithInvalidEmails(t *testing.T, ss store.Store) {
+	u1, err := ss.User().Save(&model.User{
+		Email:    "ben@invalid.mattermost.com",
+		Username: "u1" + model.NewId(),
+	})
+
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
+
+	users, err := ss.User().GetUsersWithInvalidEmails(0, 50, "localhost,simulator.amazonses.com")
+	require.NoError(t, err)
+	assert.Len(t, users, 1)
+}

store/timerlayer/timerlayer.go

@@ -9461,6 +9461,22 @@ func (s *TimerLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime
 	return result, err
 }
 
+func (s *TimerLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	start := timemodule.Now()
+
+	result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("UserStore.GetUsersWithInvalidEmails", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerUserStore) InferSystemInstallDate() (int64, error) {
 	start := timemodule.Now()