Growth spike guest accounts (#19437)

* tools updates

* Revert "tools updates"

This reverts commit 6293297b55803c5a263e200ebd80192899666ae9.

* new endpoint to get users that should potentially be guests

* checking authservice to ensure they were an email signup

* adding tests for new endpoint

* fixing translation issue

* permissions for new endpoint

* fixing tests

* fixing when domain array is empty

* fixing when domain array is empty

* removing bots from request

Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.local>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MBP.ht.home>
Co-authored-by: mkraft <martinkraft@gmail.com>

api4/user.go

@@ -92,6 +92,8 @@ func (api *API) InitUser() {
 	api.BaseRoutes.User.Handle("/uploads", api.APISessionRequired(getUploadsForUser)).Methods("GET")
 	api.BaseRoutes.User.Handle("/channel_members", api.APISessionRequired(getChannelMembersForUser)).Methods("GET")
 
+	api.BaseRoutes.Users.Handle("/invalid_emails", api.APISessionRequired(getUsersWithInvalidEmails)).Methods("GET")
+
 	api.BaseRoutes.UserThreads.Handle("", api.APISessionRequired(getThreadsForUser)).Methods("GET")
 	api.BaseRoutes.UserThreads.Handle("/read", api.APISessionRequired(updateReadStateAllThreadsByUser)).Methods("PUT")
 
@@ -3171,3 +3173,24 @@ func updateReadStateAllThreadsByUser(c *Context, w http.ResponseWriter, r *http.
 	ReturnStatusOK(w)
 	auditRec.Success()
 }
+
+func getUsersWithInvalidEmails(c *Context, w http.ResponseWriter, r *http.Request) {
+	if *c.App.Config().TeamSettings.EnableOpenServer {
+		c.Err = model.NewAppError("GetUsersWithInvalidEmails", "api.users.invalid_emails.enable_open_server.app_error", nil, "", http.StatusBadRequest)
+		return
+	}
+
+	if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionSysconsoleReadUserManagementUsers) {
+		c.SetPermissionError(model.PermissionSysconsoleReadUserManagementUsers)
+		return
+	}
+
+	users, err := c.App.GetUsersWithInvalidEmails(c.Params.Page, c.Params.PerPage)
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	b, _ := json.Marshal(users)
+	w.Write(b)
+}

api4/user_test.go

@@ -6662,6 +6662,53 @@ func TestSetProfileImageWithProviderAttributes(t *testing.T) {
 	})
 }
 
+func TestGetUsersWithInvalidEmails(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+	client := th.SystemAdminClient
+
+	user := model.User{
+		Email:    "ben@invalid.mattermost.com",
+		Nickname: "Ben Cooke",
+		Password: "hello1",
+		Username: GenerateTestUsername(),
+		Roles:    model.SystemAdminRoleId + " " + model.SystemUserRoleId,
+	}
+
+	_, resp, err := client.CreateUser(&user)
+	require.NoError(t, err)
+	CheckCreatedStatus(t, resp)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.TeamSettings.EnableOpenServer = false
+		*cfg.TeamSettings.RestrictCreationToDomains = "localhost,simulator.amazonses.com"
+	})
+
+	users, _, err := client.GetUsersWithInvalidEmails(0, 50)
+	require.NoError(t, err)
+	assert.Len(t, users, 1)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.TeamSettings.EnableOpenServer = true
+	})
+
+	_, resp, err = client.GetUsersWithInvalidEmails(0, 50)
+	require.Error(t, err)
+	CheckBadRequestStatus(t, resp)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.TeamSettings.EnableOpenServer = false
+		*cfg.TeamSettings.RestrictCreationToDomains = "localhost,simulator.amazonses.com,invalid.mattermost.com"
+	})
+
+	users, _, err = client.GetUsersWithInvalidEmails(0, 50)
+	require.NoError(t, err)
+	assert.Len(t, users, 0)
+
+	_, resp, err = th.Client.GetUsersWithInvalidEmails(0, 50)
+	require.Error(t, err)
+	CheckForbiddenStatus(t, resp)
+}
 func TestUserUpdateEvents(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
@@ -6710,5 +6757,4 @@ func TestUserUpdateEvents(t *testing.T) {
 			require.Empty(t, eventUser.NotifyProps, "user event for non-source users should be sanitized")
 		})
 	})
-
 }

app/app_iface.go

@@ -785,6 +785,7 @@ type AppIface interface {
 	GetUsersNotInTeamEtag(teamID string, restrictionsHash string) string
 	GetUsersNotInTeamPage(teamID string, groupConstrained bool, page int, perPage int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError)
 	GetUsersPage(options *model.UserGetOptions, asAdmin bool) ([]*model.User, *model.AppError)
+	GetUsersWithInvalidEmails(page int, perPage int) ([]*model.User, *model.AppError)
 	GetUsersWithoutTeam(options *model.UserGetOptions) ([]*model.User, *model.AppError)
 	GetUsersWithoutTeamPage(options *model.UserGetOptions, asAdmin bool) ([]*model.User, *model.AppError)
 	GetVerifyEmailToken(token string) (*model.Token, *model.AppError)

app/opentracing/opentracing_layer.go

@@ -10246,6 +10246,28 @@ func (a *OpenTracingAppLayer) GetUsersPage(options *model.UserGetOptions, asAdmi
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) GetUsersWithInvalidEmails(page int, perPage int) ([]*model.User, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetUsersWithInvalidEmails")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.GetUsersWithInvalidEmails(page, perPage)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) GetUsersWithoutTeam(options *model.UserGetOptions) ([]*model.User, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetUsersWithoutTeam")

app/user.go

@@ -2440,6 +2440,15 @@ func (a *App) UpdateThreadReadForUser(currentSessionId, userID, teamID, threadID
 	return thread, nil
 }
 
+func (a *App) GetUsersWithInvalidEmails(page int, perPage int) ([]*model.User, *model.AppError) {
+	users, err := a.Srv().Store.User().GetUsersWithInvalidEmails(page, perPage, *a.Config().TeamSettings.RestrictCreationToDomains)
+	if err != nil {
+		return nil, model.NewAppError("GetUsersPage", "app.user.get_profiles.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return users, nil
+}
+
 func getProfileImagePath(userID string) string {
 	return filepath.Join("users", userID, "profile.png")
 }

i18n/en.json

@@ -4271,6 +4271,10 @@
     "id": "api.user.view_archived_channels.get_users_in_channel.app_error",
     "translation": "Cannot retrieve users for an archived channel"
   },
+  {
+    "id": "api.users.invalid_emails.enable_open_server.app_error",
+    "translation": " "
+  },
   {
     "id": "api.web_socket.connect.upgrade.app_error",
     "translation": "Failed to upgrade websocket connection."

model/client4.go

@@ -7848,3 +7848,20 @@ func (c *Client4) GetAncillaryPermissions(subsectionPermissions []string) ([]str
 	json.NewDecoder(r.Body).Decode(&returnedPermissions)
 	return returnedPermissions, BuildResponse(r), nil
 }
+
+func (c *Client4) GetUsersWithInvalidEmails(page, perPage int) ([]*User, *Response, error) {
+	query := fmt.Sprintf("/invalid_emails?page=%v&per_page=%v", page, perPage)
+	r, err := c.DoAPIGet(c.usersRoute()+query, "")
+	if err != nil {
+		return nil, BuildResponse(r), err
+	}
+	defer closeBody(r)
+	var list []*User
+	if r.StatusCode == http.StatusNotModified {
+		return list, BuildResponse(r), nil
+	}
+	if jsonErr := json.NewDecoder(r.Body).Decode(&list); jsonErr != nil {
+		return nil, nil, NewAppError("GetUsers", "api.unmarshal_error", nil, jsonErr.Error(), http.StatusInternalServerError)
+	}
+	return list, BuildResponse(r), nil
+}

store/opentracinglayer/opentracinglayer.go

@@ -10502,6 +10502,24 @@ func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, en
 	return result, err
 }
 
+func (s *OpenTracingLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.GetUsersWithInvalidEmails")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return result, err
+}
+
 func (s *OpenTracingLayerUserStore) InferSystemInstallDate() (int64, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.InferSystemInstallDate")

store/retrylayer/retrylayer.go

@@ -11977,6 +11977,27 @@ func (s *RetryLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime
 
 }
 
+func (s *RetryLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+
+	tries := 0
+	for {
+		result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerUserStore) InferSystemInstallDate() (int64, error) {
 
 	tries := 0

store/sqlstore/user_store.go

@@ -2076,3 +2076,36 @@ func (us SqlUserStore) IsEmpty(excludeBots bool) (bool, error) {
 	}
 	return !hasRows, nil
 }
+
+func (us SqlUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	domainArray := strings.Split(restrictedDomains, ",")
+	query := us.usersQuery.
+		LeftJoin("Bots ON u.Id = Bots.UserId").
+		Where("Bots.UserId IS NULL").
+		Where("u.Roles != 'system_guest'").
+		Where("u.DeleteAt = 0").
+		Where("(u.AuthService = '' OR u.AuthService IS NULL)")
+
+	for _, d := range domainArray {
+		if d != "" {
+			query = query.Where("u.Email NOT LIKE LOWER(?)", wildcardSearchTerm(d))
+		}
+	}
+	query = query.Offset(uint64(page * perPage)).Limit(uint64(perPage))
+
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "users_get_many_tosql")
+	}
+
+	var users []*model.User
+	if _, err := us.GetReplica().Select(&users, queryString, args...); err != nil {
+		return nil, errors.Wrap(err, "users_get_many_select")
+	}
+
+	for _, u := range users {
+		u.Sanitize(map[string]bool{})
+	}
+
+	return users, nil
+}

store/store.go

@@ -438,6 +438,7 @@ type UserStore interface {
 	AutocompleteUsersInChannel(teamID, channelID, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, error)
 	GetKnownUsers(userID string) ([]string, error)
 	IsEmpty(excludeBots bool) (bool, error)
+	GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error)
 	InsertUsers(users []*model.User) error
 }
 

store/storetest/mocks/UserStore.go

@@ -1002,6 +1002,29 @@ func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, li
 	return r0, r1
 }
 
+// GetUsersWithInvalidEmails provides a mock function with given fields: page, perPage, restrictedDomains
+func (_m *UserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	ret := _m.Called(page, perPage, restrictedDomains)
+
+	var r0 []*model.User
+	if rf, ok := ret.Get(0).(func(int, int, string) []*model.User); ok {
+		r0 = rf(page, perPage, restrictedDomains)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.User)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int, int, string) error); ok {
+		r1 = rf(page, perPage, restrictedDomains)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // InferSystemInstallDate provides a mock function with given fields:
 func (_m *UserStore) InferSystemInstallDate() (int64, error) {
 	ret := _m.Called()

store/storetest/user_store.go

@@ -91,6 +91,7 @@ func TestUserStore(t *testing.T, ss store.Store, s SqlStore) {
 	t.Run("DeactivateGuests", func(t *testing.T) { testDeactivateGuests(t, ss) })
 	t.Run("ResetLastPictureUpdate", func(t *testing.T) { testUserStoreResetLastPictureUpdate(t, ss) })
 	t.Run("GetKnownUsers", func(t *testing.T) { testGetKnownUsers(t, ss) })
+	t.Run("GetUsersWithInvalidEmails", func(t *testing.T) { testGetUsersWithInvalidEmails(t, ss) })
 }
 
 func testUserStoreSave(t *testing.T, ss store.Store) {
@@ -4656,6 +4657,7 @@ func testUserStoreGetTeamGroupUsers(t *testing.T, ss store.Store) {
 		require.NoError(t, userErr)
 		require.NotNil(t, user)
 		testUsers = append(testUsers, user)
+		defer func() { require.NoError(t, ss.User().PermanentDelete(user.Id)) }()
 	}
 	require.Len(t, testUsers, 3, "testUsers length doesn't meet required length")
 	userGroupA, userGroupB, userNoGroup := testUsers[0], testUsers[1], testUsers[2]
@@ -4776,6 +4778,7 @@ func testUserStoreGetChannelGroupUsers(t *testing.T, ss store.Store) {
 		require.NoError(t, userErr)
 		require.NotNil(t, user)
 		testUsers = append(testUsers, user)
+		defer func() { require.NoError(t, ss.User().PermanentDelete(user.Id)) }()
 	}
 	require.Len(t, testUsers, 3, "testUsers length doesn't meet required length")
 	userGroupA, userGroupB, userNoGroup := testUsers[0], testUsers[1], testUsers[2]
@@ -5778,3 +5781,17 @@ func testIsEmpty(t *testing.T, ss store.Store) {
 	require.NoError(t, err)
 	require.True(t, ok)
 }
+
+func testGetUsersWithInvalidEmails(t *testing.T, ss store.Store) {
+	u1, err := ss.User().Save(&model.User{
+		Email:    "ben@invalid.mattermost.com",
+		Username: "u1" + model.NewId(),
+	})
+
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
+
+	users, err := ss.User().GetUsersWithInvalidEmails(0, 50, "localhost,simulator.amazonses.com")
+	require.NoError(t, err)
+	assert.Len(t, users, 1)
+}

store/timerlayer/timerlayer.go

@@ -9461,6 +9461,22 @@ func (s *TimerLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime
 	return result, err
 }
 
+func (s *TimerLayerUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrictedDomains string) ([]*model.User, error) {
+	start := timemodule.Now()
+
+	result, err := s.UserStore.GetUsersWithInvalidEmails(page, perPage, restrictedDomains)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("UserStore.GetUsersWithInvalidEmails", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerUserStore) InferSystemInstallDate() (int64, error) {
 	start := timemodule.Now()