[MM-55726] Create batch report worker, add batch report job for exporting users to CSV (#25832)

* Split out migration logic and create generic BatchWorker * WIP * WIP * POC batch reporting * Oops * Job hookup * Working export to file * PR feedback * Merge'd * Fix error handling * Add API to start report, translations, couple fixes * Add DMs to send reports to users * Merge'd * Update types * A bit of cleanup * Some fixes * Add missing API doc * PR feedback * Fix generated * Fix bug with post creation * PR feedback * Add some tests * PR feedback * Fix lint * Some test changes * Fix tests * Add comment to explain why we forcibly stop * Rework of some tests * Batch report test * Restrict batch exports to Pro and Enterprise licenses * Fix erroneous comment --------- Co-authored-by: Mattermost Build <build@mattermost.com>
2024-01-19 15:22:17 -05:00
--- a/server/public/model/job.go
+++ b/server/public/model/job.go
@@ -38,6 +38,7 @@ const (
 	JobTypeCleanupDesktopTokens         = "cleanup_desktop_tokens"
 	JobTypeDeleteEmptyDraftsMigration   = "delete_empty_drafts_migration"
 	JobTypeRefreshPostStats             = "refresh_post_stats"
+	JobTypeExportUsersToCSV             = "export_users_to_csv"

 	JobStatusPending         = "pending"
 	JobStatusInProgress      = "in_progress"
--- a/server/public/model/post.go
+++ b/server/public/model/post.go
@@ -52,6 +52,7 @@ const (
 	PostTypeMe                     = "me"
 	PostCustomTypePrefix           = "custom_"
 	PostTypeReminder               = "reminder"
+	PostTypeAdminReport            = "system_admin_report"

 	PostFileidsMaxRunes   = 300
 	PostFilenamesMaxRunes = 4000
@@ -450,7 +451,8 @@ func (o *Post) IsValid(maxPostSize int) *AppError {
 		PostTypeReminder,
 		PostTypeMe,
 		PostTypeWrangler,
-		PostTypeGMConvertedToChannel:
+		PostTypeGMConvertedToChannel,
+		PostTypeAdminReport:
 	default:
 		if !strings.HasPrefix(o.Type, PostCustomTypePrefix) {
 			return NewAppError("Post.IsValid", "model.post.is_valid.type.app_error", nil, "id="+o.Type, http.StatusBadRequest)
--- a/server/public/model/report.go
+++ b/server/public/model/report.go
@@ -5,6 +5,7 @@ package model

 import (
 	"net/http"
+	"strconv"
 	"time"

 	pUtils "github.com/mattermost/mattermost/server/public/utils"
@@ -19,9 +20,15 @@ const (
 )

 var (
+	ReportExportFormats = []string{"csv"}
+
 	UserReportSortColumns = []string{"CreateAt", "Username", "FirstName", "LastName", "Nickname", "Email", "Roles"}
 )

+type ReportableObject interface {
+	ToReport() []string
+}
+
 type ReportingBaseOptions struct {
 	SortDesc        bool
 	Direction       string // Accepts only "prev" or "next"
@@ -34,20 +41,26 @@ type ReportingBaseOptions struct {
 	EndAt           int64
 }

-func (options *ReportingBaseOptions) PopulateDateRange(now time.Time) {
+func GetReportDateRange(dateRange string, now time.Time) (int64, int64) {
 	startAt := int64(0)
 	endAt := int64(0)

-	if options.DateRange == ReportDurationLast30Days {
+	if dateRange == ReportDurationLast30Days {
 		startAt = now.AddDate(0, 0, -30).UnixMilli()
-	} else if options.DateRange == ReportDurationPreviousMonth {
+	} else if dateRange == ReportDurationPreviousMonth {
 		startOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.Local)
 		startAt = startOfMonth.AddDate(0, -1, 0).UnixMilli()
 		endAt = startOfMonth.UnixMilli()
-	} else if options.DateRange == ReportDurationLast6Months {
+	} else if dateRange == ReportDurationLast6Months {
 		startAt = now.AddDate(0, -6, -0).UnixMilli()
 	}

+	return startAt, endAt
+}
+
+func (options *ReportingBaseOptions) PopulateDateRange(now time.Time) {
+	startAt, endAt := GetReportDateRange(options.DateRange, now)
+
 	options.StartAt = startAt
 	options.EndAt = endAt
 }
@@ -70,6 +83,43 @@ type UserReport struct {
 	UserPostStats
 }

+func (u *UserReport) ToReport() []string {
+	lastStatusAt := ""
+	if u.LastStatusAt != nil {
+		lastStatusAt = time.UnixMilli(*u.LastStatusAt).String()
+	}
+	lastPostDate := ""
+	if u.LastPostDate != nil {
+		lastPostDate = time.UnixMilli(*u.LastPostDate).String()
+	}
+	daysActive := ""
+	if u.DaysActive != nil {
+		daysActive = strconv.Itoa(*u.DaysActive)
+	}
+	totalPosts := ""
+	if u.TotalPosts != nil {
+		totalPosts = strconv.Itoa(*u.TotalPosts)
+	}
+	lastLogin := ""
+	if u.LastLogin > 0 {
+		lastLogin = time.UnixMilli(u.LastLogin).String()
+	}
+
+	return []string{
+		u.Id,
+		u.Username,
+		u.Email,
+		time.UnixMilli(u.CreateAt).String(),
+		u.User.GetDisplayName(ShowNicknameFullName),
+		u.Roles,
+		lastLogin,
+		lastStatusAt,
+		lastPostDate,
+		daysActive,
+		totalPosts,
+	}
+}
+
 type UserReportOptions struct {
 	ReportingBaseOptions
 	Role         string
@@ -99,3 +149,13 @@ func (u *UserReportQuery) ToReport() *UserReport {
 		UserPostStats: u.UserPostStats,
 	}
 }
+
+func IsValidReportExportFormat(format string) bool {
+	for _, fmt := range ReportExportFormats {
+		if format == fmt {
+			return true
+		}
+	}
+
+	return false
+}
--- a/server/public/model/user.go
+++ b/server/public/model/user.go
@@ -1015,7 +1015,6 @@ func (u *User) EmailDomain() string {
 }

 type UserPostStats struct {
-	LastLogin    int64  `json:"last_login_at,omitempty"`
 	LastStatusAt *int64 `json:"last_status_at,omitempty"`
 	LastPostDate *int64 `json:"last_post_date,omitempty"`
 	DaysActive   *int   `json:"days_active,omitempty"`
--- a/server/public/model/user_serial_gen.go
+++ b/server/public/model/user_serial_gen.go
@@ -875,12 +875,6 @@ func (z *UserPostStats) DecodeMsg(dc *msgp.Reader) (err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "LastLogin":
-			z.LastLogin, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "LastLogin")
-				return
-			}
 		case "LastStatusAt":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -966,19 +960,9 @@ func (z *UserPostStats) DecodeMsg(dc *msgp.Reader) (err error) {

 // EncodeMsg implements msgp.Encodable
 func (z *UserPostStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
-	// write "LastLogin"
-	err = en.Append(0x85, 0xa9, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x6f, 0x67, 0x69, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.LastLogin)
-	if err != nil {
-		err = msgp.WrapError(err, "LastLogin")
-		return
-	}
+	// map header, size 4
 	// write "LastStatusAt"
-	err = en.Append(0xac, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x41, 0x74)
+	err = en.Append(0x84, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x41, 0x74)
 	if err != nil {
 		return
 	}
@@ -1051,12 +1035,9 @@ func (z *UserPostStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *UserPostStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
-	// string "LastLogin"
-	o = append(o, 0x85, 0xa9, 0x4c, 0x61, 0x73, 0x74, 0x4c, 0x6f, 0x67, 0x69, 0x6e)
-	o = msgp.AppendInt64(o, z.LastLogin)
+	// map header, size 4
 	// string "LastStatusAt"
-	o = append(o, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x41, 0x74)
+	o = append(o, 0x84, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x41, 0x74)
 	if z.LastStatusAt == nil {
 		o = msgp.AppendNil(o)
 	} else {
@@ -1104,12 +1085,6 @@ func (z *UserPostStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "LastLogin":
-			z.LastLogin, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "LastLogin")
-				return
-			}
 		case "LastStatusAt":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -1192,7 +1167,7 @@ func (z *UserPostStats) UnmarshalMsg(bts []byte) (o []byte, err error) {

 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *UserPostStats) Msgsize() (s int) {
-	s = 1 + 10 + msgp.Int64Size + 13
+	s = 1 + 13
 	if z.LastStatusAt == nil {
 		s += msgp.NilSize
 	} else {