Audit logging -- convert audit logs to use the new schema (#20526)

* Audit logging - new schema added, old schema removed.

* fix linter error by running goimports

* Address review comments

* Address review comments

* Example usage of new audit logging API for the updateUserAuth call

* fixed unit test on auditing updating user record

* Changed the `TestUpdateConfigDiffInAuditRecord` testcase---it failed, because this PR changes how the `meta` field is serialized into the audit log records.

* fix linter error

* use string constants for record keys

* new audit api calls for api4/bot

* `Auditable` interface implementations for model classes

* New audit calls for channel api

* New audit calls for channel_local

* renamed receivers for required style reasons

* New audit calls for api4/command

* renamed receiver

* New audit calls for api4/command_local

* renamed receiver

* fix unit test to reflect changes in the Auditable implementation of the user class

* new audit calls for compliance

* new audit calls for configs

* remove auditRec.addMeta from updateConfig and patchConfig

* new audit calls for config_local

* new audit calls

* new audit calls for ldap, license apis

* new audit calls

* new audit calls

* new audit calls

* new audit calls

* new audit calls

* new audit calls

* new audit calls

* new audit calls

* fix linter error

* fixed linter error

* fixed "user update" test

* Don't include all of config when audit logging config changes. Also fix unit test on TestUpdateConfigDiffInAuditRecord

* address review comments

* Added Auditable() method for UserPatch

* Fix duplicative method declaration from merge

* Fix styling and API changes issues introduced with merge

* Fix broken test

Co-authored-by: Daniel Schalla <daniel@schalla.me>

model/bot.go

@@ -33,6 +33,20 @@ type Bot struct {
 	DeleteAt       int64  `json:"delete_at"`
 }
 
+func (b *Bot) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"user_id":          b.UserId,
+		"username":         b.Username,
+		"display_name":     b.DisplayName,
+		"description":      b.Description,
+		"owner_id":         b.OwnerId,
+		"last_icon_update": b.LastIconUpdate,
+		"create_at":        b.CreateAt,
+		"update_at":        b.UpdateAt,
+		"delete_at":        b.DeleteAt,
+	}
+}
+
 // BotPatch is a description of what fields to update on an existing bot.
 type BotPatch struct {
 	Username    *string `json:"username"`

model/channel.go

@@ -61,6 +61,27 @@ type Channel struct {
 	LastRootPostAt    int64          `json:"last_root_post_at"`
 }
 
+func (o *Channel) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"create_at":            o.CreateAt,
+		"creator_id":           o.CreatorId,
+		"delete_at":            o.DeleteAt,
+		"extra_group_at":       o.ExtraUpdateAt,
+		"group_constrained":    o.GroupConstrained,
+		"id":                   o.Id,
+		"last_post_at":         o.LastPostAt,
+		"last_root_post_at":    o.LastRootPostAt,
+		"policy_id":            o.PolicyID,
+		"props":                o.Props,
+		"scheme_id":            o.SchemeId,
+		"shared":               o.Shared,
+		"team_id":              o.TeamId,
+		"total_msg_count_root": o.TotalMsgCountRoot,
+		"type":                 o.Type,
+		"update_at":            o.UpdateAt,
+	}
+}
+
 type ChannelWithTeamData struct {
 	Channel
 	TeamDisplayName string `json:"team_display_name"`
@@ -81,6 +102,14 @@ type ChannelPatch struct {
 	GroupConstrained *bool   `json:"group_constrained"`
 }
 
+func (c *ChannelPatch) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"header":            c.Header,
+		"group_constrained": c.GroupConstrained,
+		"purpose":           c.Purpose,
+	}
+}
+
 type ChannelForExport struct {
 	Channel
 	TeamName   string

model/channel_member.go

@@ -60,6 +60,25 @@ type ChannelMember struct {
 	ExplicitRoles    string    `json:"explicit_roles"`
 }
 
+func (o *ChannelMember) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"channel_id":         o.ChannelId,
+		"user_id":            o.UserId,
+		"roles":              o.Roles,
+		"last_viewed_at":     o.LastViewedAt,
+		"msg_count":          o.MsgCount,
+		"mention_count":      o.MentionCount,
+		"mention_count_root": o.MentionCountRoot,
+		"msg_count_root":     o.MsgCountRoot,
+		"notify_props":       o.NotifyProps,
+		"last_update_at":     o.LastUpdateAt,
+		"scheme_guest":       o.SchemeGuest,
+		"scheme_user":        o.SchemeUser,
+		"scheme_admin":       o.SchemeAdmin,
+		"explicit_roles":     o.ExplicitRoles,
+	}
+}
+
 // The following are some GraphQL methods necessary to return the
 // data in float64 type. The spec doesn't support 64 bit integers,
 // so we have to pass the data in float64. The _ at the end is

model/command.go

@@ -41,6 +41,26 @@ type Command struct {
 	AutocompleteIconData string `db:"-" json:"autocomplete_icon_data,omitempty"`
 }
 
+func (o *Command) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":                 o.Id,
+		"create_at":          o.CreateAt,
+		"update_at":          o.UpdateAt,
+		"delete_at":          o.DeleteAt,
+		"creator_id":         o.CreatorId,
+		"team_id":            o.TeamId,
+		"trigger":            o.Trigger,
+		"username":           o.Username,
+		"icon_url":           o.IconURL,
+		"auto_complete":      o.AutoComplete,
+		"auto_complete_desc": o.AutoCompleteDesc,
+		"auto_complete_hint": o.AutoCompleteHint,
+		"display_name":       o.DisplayName,
+		"description":        o.Description,
+		"url":                o.URL,
+	}
+}
+
 func (o *Command) IsValid() *AppError {
 	if !IsValidId(o.Id) {
 		return NewAppError("Command.IsValid", "model.command.is_valid.id.app_error", nil, "", http.StatusBadRequest)

model/command_args.go

@@ -24,6 +24,19 @@ type CommandArgs struct {
 	Session Session `json:"-"`
 }
 
+func (o *CommandArgs) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"user_id":    o.UserId,
+		"channel_id": o.ChannelId,
+		"team_id":    o.TeamId,
+		"root_id":    o.RootId,
+		"parent_id":  o.ParentId,
+		"trigger_id": o.TriggerId,
+		"command":    o.Command,
+		"site_url":   o.SiteURL,
+	}
+}
+
 // AddUserMention adds or overrides an entry in UserMentions with name username
 // and identifier userId
 func (o *CommandArgs) AddUserMention(username, userId string) {

model/compliance.go

@@ -33,6 +33,22 @@ type Compliance struct {
 	Emails   string `json:"emails"`
 }
 
+func (c *Compliance) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":        c.Id,
+		"create_at": c.CreateAt,
+		"user_id":   c.UserId,
+		"status":    c.Status,
+		"count":     c.Count,
+		"desc":      c.Desc,
+		"type":      c.Type,
+		"start_at":  c.StartAt,
+		"end_at":    c.EndAt,
+		"keywords":  c.Keywords,
+		"emails":    c.Emails,
+	}
+}
+
 type Compliances []Compliance
 
 // ComplianceExportCursor is used for paginated iteration of posts

model/config.go

@@ -3129,6 +3129,12 @@ type Config struct {
 	ExportSettings            ExportSettings
 }
 
+func (o *Config) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		// TODO
+	}
+}
+
 func (o *Config) Clone() *Config {
 	buf, err := json.Marshal(o)
 	if err != nil {

model/data_retention_policy.go

@@ -24,12 +24,28 @@ type RetentionPolicyWithTeamAndChannelIDs struct {
 	ChannelIDs []string `json:"channel_ids"`
 }
 
+func (o *RetentionPolicyWithTeamAndChannelIDs) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"retention_policy": o.RetentionPolicy,
+		"team_ids":         o.TeamIDs,
+		"channel_ids":      o.ChannelIDs,
+	}
+}
+
 type RetentionPolicyWithTeamAndChannelCounts struct {
 	RetentionPolicy
 	ChannelCount int64 `json:"channel_count"`
 	TeamCount    int64 `json:"team_count"`
 }
 
+func (o *RetentionPolicyWithTeamAndChannelCounts) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"retention_policy": o.RetentionPolicy,
+		"channel_count":    o.ChannelCount,
+		"team_count":       o.TeamCount,
+	}
+}
+
 type RetentionPolicyChannel struct {
 	PolicyID  string `db:"PolicyId"`
 	ChannelID string `db:"ChannelId"`

model/emoji.go

@@ -25,6 +25,17 @@ type Emoji struct {
 	Name      string `json:"name"`
 }
 
+func (emoji *Emoji) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":         emoji.Id,
+		"create_at":  emoji.CreateAt,
+		"update_at":  emoji.UpdateAt,
+		"delete_at":  emoji.CreateAt,
+		"creator_id": emoji.CreatorId,
+		"name":       emoji.Name,
+	}
+}
+
 func inSystemEmoji(emojiName string) bool {
 	_, ok := SystemEmojis[emojiName]
 	return ok

model/group.go

@@ -45,6 +45,20 @@ type Group struct {
 	AllowReference bool        `json:"allow_reference"`
 }
 
+func (group *Group) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":              group.Id,
+		"source":          group.Source,
+		"remote_id":       group.RemoteId,
+		"create_at":       group.CreateAt,
+		"update_at":       group.UpdateAt,
+		"delete_at":       group.DeleteAt,
+		"has_syncables":   group.HasSyncables,
+		"member_count":    group.MemberCount,
+		"allow_reference": group.AllowReference,
+	}
+}
+
 type GroupWithUserIds struct {
 	Group
 	UserIds []string `json:"user_ids"`

model/group_syncable.go

@@ -42,6 +42,24 @@ type GroupSyncable struct {
 	TeamID             string `db:"-" json:"-"`
 }
 
+func (syncable *GroupSyncable) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"group_id":             syncable.GroupId,
+		"syncable_id":          syncable.SyncableId,
+		"auto_add":             syncable.AutoAdd,
+		"scheme_admin":         syncable.SchemeAdmin,
+		"create_at":            syncable.CreateAt,
+		"delete_at":            syncable.DeleteAt,
+		"update_at":            syncable.UpdateAt,
+		"type":                 syncable.Type,
+		"channel_display_name": syncable.ChannelDisplayName,
+		"team_display_name":    syncable.TeamDisplayName,
+		"team_type":            syncable.TeamType,
+		"channel_type":         syncable.ChannelType,
+		"team_id":              syncable.TeamID,
+	}
+}
+
 func (syncable *GroupSyncable) IsValid() *AppError {
 	if !IsValidId(syncable.GroupId) {
 		return NewAppError("GroupSyncable.SyncableIsValid", "model.group_syncable.group_id.app_error", nil, "", http.StatusBadRequest)

model/incoming_webhook.go

@@ -30,6 +30,23 @@ type IncomingWebhook struct {
 	ChannelLocked bool   `json:"channel_locked"`
 }
 
+func (o *IncomingWebhook) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":             o.Id,
+		"create_at":      o.CreateAt,
+		"update_at":      o.UpdateAt,
+		"delete_at":      o.DeleteAt,
+		"user_id":        o.UserId,
+		"channel_id":     o.ChannelId,
+		"team_id":        o.TeamId,
+		"display_name":   o.DisplayName,
+		"description":    o.Description,
+		"username":       o.Username,
+		"icon_url:":      o.IconURL,
+		"channel_locked": o.ChannelLocked,
+	}
+}
+
 type IncomingWebhookRequest struct {
 	Text        string             `json:"text"`
 	Username    string             `json:"username"`

model/job.go

@@ -71,6 +71,20 @@ type Job struct {
 	Data           StringMap `json:"data"`
 }
 
+func (j *Job) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":               j.Id,
+		"type":             j.Type,
+		"priority":         j.Priority,
+		"create_at":        j.CreateAt,
+		"start_at":         j.StartAt,
+		"last_activity_at": j.LastActivityAt,
+		"status":           j.Status,
+		"progress":         j.Progress,
+		"data":             j.Data, // TODO do we want this here
+	}
+}
+
 func (j *Job) IsValid() *AppError {
 	if !IsValidId(j.Id) {
 		return NewAppError("Job.IsValid", "model.job.is_valid.id.app_error", nil, "id="+j.Id, http.StatusBadRequest)

model/oauth.go

@@ -32,6 +32,22 @@ type OAuthApp struct {
 	MattermostAppID string      `json:"mattermost_app_id"`
 }
 
+func (a *OAuthApp) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":                a.Id,
+		"creator_id":        a.CreatorId,
+		"create_at":         a.CreateAt,
+		"update_at":         a.UpdateAt,
+		"name":              a.Name,
+		"description":       a.Description,
+		"icon_url":          a.IconURL,
+		"callback_urls:":    a.CallbackUrls,
+		"homepage":          a.Homepage,
+		"is_trusted":        a.IsTrusted,
+		"mattermost_app_id": a.MattermostAppID,
+	}
+}
+
 // IsValid validates the app and returns an error if it isn't configured
 // correctly.
 func (a *OAuthApp) IsValid() *AppError {

model/outgoing_webhook.go

@@ -30,6 +30,26 @@ type OutgoingWebhook struct {
 	IconURL      string      `json:"icon_url"`
 }
 
+func (o *OutgoingWebhook) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":            o.Id,
+		"create_at":     o.CreateAt,
+		"update_at":     o.UpdateAt,
+		"delete_at":     o.DeleteAt,
+		"creator_id":    o.CreatorId,
+		"channel_id":    o.ChannelId,
+		"team_id":       o.TeamId,
+		"trigger_words": o.TriggerWords,
+		"trigger_when":  o.TriggerWhen,
+		"callback_urls": o.CallbackURLs,
+		"display_name":  o.DisplayName,
+		"description":   o.Description,
+		"content_type":  o.ContentType,
+		"username":      o.Username,
+		"icon_url":      o.IconURL,
+	}
+}
+
 type OutgoingWebhookPayload struct {
 	Token       string `json:"token"`
 	TeamId      string `json:"team_id"`

model/post.go

@@ -112,6 +112,30 @@ type Post struct {
 	Metadata     *PostMetadata `json:"metadata,omitempty"`
 }
 
+func (o *Post) Auditable() map[string]interface{} {
+	return map[string]interface{}{ // TODO check this
+		"id":              o.Id,
+		"create_at":       o.CreateAt,
+		"update_at":       o.UpdateAt,
+		"edit_at":         o.EditAt,
+		"delete_at":       o.DeleteAt,
+		"is_pinned":       o.IsPinned,
+		"user_id":         o.UserId,
+		"channel_id":      o.ChannelId,
+		"root_id":         o.RootId,
+		"original_id":     o.OriginalId,
+		"type":            o.Type,
+		"props":           o.GetProps(),
+		"file_ids":        o.FileIds,
+		"pending_post_id": o.PendingPostId,
+		"remote_id":       o.RemoteId,
+		"reply_count":     o.ReplyCount,
+		"last_reply_at":   o.LastReplyAt,
+		"is_following":    o.IsFollowing,
+		"metadata":        o.Metadata,
+	}
+}
+
 type PostEphemeral struct {
 	UserID string `json:"user_id"`
 	Post   *Post  `json:"post"`

model/role.go

@@ -414,6 +414,21 @@ type Role struct {
 	BuiltIn       bool     `json:"built_in"`
 }
 
+func (r *Role) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":             r.Id,
+		"name":           r.Name,
+		"display_name":   r.DisplayName,
+		"description":    r.Description,
+		"create_at":      r.CreateAt,
+		"update_at":      r.UpdateAt,
+		"delete_at":      r.DeleteAt,
+		"permissions":    r.Permissions,
+		"scheme_managed": r.SchemeManaged,
+		"built_in":       r.BuiltIn,
+	}
+}
+
 type RolePatch struct {
 	Permissions *[]string `json:"permissions"`
 }

model/scheme.go

@@ -39,6 +39,29 @@ type Scheme struct {
 	DefaultRunMemberRole      string `json:"default_run_member_role"`
 }
 
+func (scheme *Scheme) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":                           scheme.Id,
+		"name":                         scheme.Name,
+		"display_name":                 scheme.DisplayName,
+		"description":                  scheme.Description,
+		"create_at":                    scheme.CreateAt,
+		"update_at":                    scheme.UpdateAt,
+		"delete_at":                    scheme.DeleteAt,
+		"scope":                        scheme.Scope,
+		"default_team_admin_role":      scheme.DefaultTeamAdminRole,
+		"default_team_user_role":       scheme.DefaultTeamUserRole,
+		"default_channel_admin_role":   scheme.DefaultChannelAdminRole,
+		"default_channel_user_role":    scheme.DefaultChannelUserRole,
+		"default_team_guest_role":      scheme.DefaultTeamGuestRole,
+		"default_channel_guest_role":   scheme.DefaultChannelGuestRole,
+		"default_playbook_admin_role":  scheme.DefaultPlaybookAdminRole,
+		"default_playbook_member_role": scheme.DefaultPlaybookMemberRole,
+		"default_run_admin_role":       scheme.DefaultRunAdminRole,
+		"default_run_member_role":      scheme.DefaultRunMemberRole,
+	}
+}
+
 type SchemePatch struct {
 	Name        *string `json:"name"`
 	DisplayName *string `json:"display_name"`
@@ -93,6 +116,10 @@ type SchemeRoles struct {
 	SchemeGuest bool `json:"scheme_guest"`
 }
 
+func (s *SchemeRoles) Auditable() map[string]interface{} {
+	return map[string]interface{}{}
+}
+
 func (scheme *Scheme) IsValid() bool {
 	if !IsValidId(scheme.Id) {
 		return false

model/session.go

@@ -58,6 +58,22 @@ type Session struct {
 	Local          bool          `json:"local" db:"-"`
 }
 
+func (s *Session) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":               s.Id,
+		"create_at":        s.CreateAt,
+		"expires_at":       s.ExpiresAt,
+		"last_activity_at": s.LastActivityAt,
+		"user_id":          s.UserId,
+		"device_id":        s.DeviceId,
+		"roles":            s.Roles,
+		"is_oauth":         s.IsOAuth,
+		"expired_notify":   s.ExpiredNotify,
+		"local":            s.Local,
+		// TODO: props and members?
+	}
+}
+
 // Returns true if the session is unrestricted, which should grant it
 // with all permissions. This is used for local mode sessions
 func (s *Session) IsUnrestricted() bool {

model/team.go

@@ -44,6 +44,22 @@ type Team struct {
 	CloudLimitsArchived bool    `json:"cloud_limits_archived"`
 }
 
+func (o *Team) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":                    o.Id,
+		"create_at":             o.CreateAt,
+		"update_at":             o.UpdateAt,
+		"delete_at":             o.DeleteAt,
+		"type":                  o.Type,
+		"invite_id":             o.InviteId,
+		"allow_open_invite":     o.AllowOpenInvite,
+		"scheme_id":             o.SchemeId,
+		"group_constrained":     o.GroupConstrained,
+		"policy_id":             o.PolicyID,
+		"cloud_limits_archived": o.CloudLimitsArchived,
+	}
+}
+
 type TeamPatch struct {
 	DisplayName         *string `json:"display_name"`
 	Description         *string `json:"description"`

model/team_member.go

@@ -27,6 +27,19 @@ type TeamMember struct {
 	ExplicitRoles string `json:"explicit_roles"`
 }
 
+func (o *TeamMember) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"team_id":        o.TeamId,
+		"user_id":        o.UserId,
+		"roles":          o.Roles,
+		"delete_at":      o.DeleteAt,
+		"scheme_guest":   o.SchemeGuest,
+		"scheme_user":    o.SchemeUser,
+		"scheme_admin":   o.SchemeAdmin,
+		"explicit_roles": o.ExplicitRoles,
+	}
+}
+
 //msgp:ignore TeamUnread
 type TeamUnread struct {
 	TeamId             string `json:"team_id"`

model/user.go

@@ -105,6 +105,38 @@ type User struct {
 	DisableWelcomeEmail    bool      `json:"disable_welcome_email"`
 }
 
+func (u *User) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"id":                         u.Id,
+		"create_at":                  u.CreateAt,
+		"update_at":                  u.UpdateAt,
+		"delete_at":                  u.DeleteAt,
+		"username":                   u.Username,
+		"auth_service":               u.AuthService,
+		"email":                      u.Email,
+		"email_verified":             u.EmailVerified,
+		"position":                   u.Position,
+		"roles":                      u.Roles,
+		"allow_marketing":            u.AllowMarketing,
+		"props":                      u.Props,
+		"notify_props":               u.NotifyProps,
+		"last_password_update":       u.LastPasswordUpdate,
+		"last_picture_update":        u.LastPictureUpdate,
+		"failed_attempts":            u.FailedAttempts,
+		"locale":                     u.Locale,
+		"timezone":                   u.Timezone,
+		"mfa_active":                 u.MfaActive,
+		"remote_id":                  u.RemoteId,
+		"last_activity_at":           u.LastActivityAt,
+		"is_bot":                     u.IsBot,
+		"bot_description":            u.BotDescription,
+		"bot_last_icon_update":       u.BotLastIconUpdate,
+		"terms_of_service_id":        u.TermsOfServiceId,
+		"terms_of_service_create_at": u.TermsOfServiceCreateAt,
+		"disable_welcome_email":      u.DisableWelcomeEmail,
+	}
+}
+
 //msgp UserMap
 
 // UserMap is a map from a userId to a user object.
@@ -133,6 +165,22 @@ type UserPatch struct {
 	RemoteId    *string   `json:"remote_id"`
 }
 
+func (u *UserPatch) Auditable() map[string]interface{} {
+	return map[string]interface{}{
+		"username":     u.Username,
+		"nickname":     u.Nickname,
+		"first_name":   u.FirstName,
+		"last_name":    u.LastName,
+		"position":     u.Position,
+		"email":        u.Email,
+		"props":        u.Props,
+		"notify_props": u.NotifyProps,
+		"locale":       u.Locale,
+		"timezone":     u.Timezone,
+		"remote_id":    u.RemoteId,
+	}
+}
+
 //msgp:ignore UserAuth
 type UserAuth struct {
 	Password    string  `json:"password,omitempty"` // DEPRECATED: It is not used.
@@ -835,13 +883,6 @@ func (u *User) ToPatch() *UserPatch {
 	}
 }
 
-func (u *User) Auditable() map[string]interface{} {
-	return map[string]interface{}{
-		"id":       u.Id,
-		"username": u.Username,
-	}
-}
-
 func (u *UserPatch) SetField(fieldName string, fieldValue string) {
 	switch fieldName {
 	case "FirstName":