mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-60115, MM-62436, MM-62493: Compliance export downloads from local and s3; e2e tests (#29806)

Просмотр исходного кода 
* add ZipReader method to filestore and file/s3 backends

to merge

to merge

* add WriteStreamResponse as an alternative to WriteFileResponse

* add generated layers

* enable download link; download job endpoint now streams export dir zips

* fix MM-62493

* re-enable e2e tests--we have download links, folks

* Add tests for ZipReader in filestore and s3store

* remove unnecessary error return on ZipReader

* little cleanup

* improve tests; some refactoring: s.Nil(err) -> s.NoError(err)

* blank commit

* backwards compatability for pre-10.5 job downloads

* compress file response; better errors; better comments; PR comments

* update generated app layers

* improve/widen tests; improve comments; simplify localstore ZipReader

* regenerate layers

* follow GoDoc conventions

* update generated layers

* remove unnecessary comment

* in jobs/job-id/download, clean exportDir before sending to ZipReader

* better comments; add an error return on ZipReader

* improve file permissions

* adjust tests for new error returns

* linting

* i18n
Этот коммит содержится в:
Christopher Poile
2025-01-26 22:58:07 -05:00
коммит произвёл GitHub
родитель 396ee06dcb
Коммит 737bed311c
21 изменённых файлов: 753 добавлений и 110 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

18
server/enterprise/message_export/actiance_export/actiance_export.go
Просмотреть файл

@@ -302,8 +302,11 @@ func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.File
var attachmentReader io.ReadCloser
attachmentReader, err = fileAttachmentBackend.Reader(fileInfo.Path)
if err != nil {
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessage+" - "+fileInfo.Path)
rctx.Logger().Warn(shared.MissingFileMessage, mlog.String("filename", fileInfo.Path))
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringBackendRead+" - "+fileInfo.Path)
rctx.Logger().Warn(shared.MissingFileMessageDuringBackendRead,
mlog.String("filename", fileInfo.Path),
mlog.Err(err),
)
continue
}
@@ -316,14 +319,21 @@ func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.File
return err
}
// CopyBuffer works with dirty buffers, no need to clear it.
if _, err = io.CopyBuffer(zipWriter, attachmentReader, buf); err != nil {
return err
}
return nil
}(); err != nil {
return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
// s3 only errors _here_ if the object key wasn't found. So to handle that: if there is a read
// error (even for local), let's add a warning instead of failing the export.
// Failing the export would fail the entire export run, and every future run would also fail on
// this non-existent file -- not good.
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringCopy+" - "+fileInfo.Path)
rctx.Logger().Warn(shared.MissingFileMessageDuringCopy,
mlog.String("filename", fileInfo.Path),
mlog.Err(err),
)
}
}

37
server/enterprise/message_export/actiance_export/actiance_export_test.go
Просмотреть файл

@@ -891,7 +891,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
expectedFiles: 3,
},
{
name: "joins and leaves after last post, one batch",
name: "joins and leaves after last post, one batch, and post from bot",
jobEndTime: 500,
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
@@ -899,7 +899,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
{JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
{JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
{JoinTime: 450, ChannelId: "channel-id", UserId: "test4", UserEmail: "test4@email", Username: "test4name", LeaveTime: model.NewPointer(int64(460))},
{JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
{JoinTime: 10, ChannelId: "channel-id", UserId: "test-bot", UserEmail: "test-bot@email", Username: "test-botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
},
},
activity: []string{"channel-id"},
@@ -965,6 +965,24 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
PostFileIds: []string{},
PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
},
{
PostId: model.NewPointer("post-id5"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(20)),
PostUpdateAt: model.NewPointer(int64(20)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test-bot@email"),
UserId: model.NewPointer("test-bot"),
Username: model.NewPointer("test-botname"),
IsBot: true,
ChannelType: &chanTypeDirect,
PostFileIds: []string{},
},
{
PostId: model.NewPointer("post-id4"),
PostRootId: model.NewPointer("post-root-id"),
@@ -1010,10 +1028,10 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" <CorporateEmailID>test2@email</CorporateEmailID>\n",
" </ParticipantEntered>\n",
" <ParticipantEntered>\n",
" <LoginName>test_bot@email</LoginName>\n",
" <LoginName>test-bot@email</LoginName>\n",
" <UserType>bot</UserType>\n",
" <DateTimeUTC>10</DateTimeUTC>\n",
" <CorporateEmailID>test_bot@email</CorporateEmailID>\n",
" <CorporateEmailID>test-bot@email</CorporateEmailID>\n",
" </ParticipantEntered>\n",
" <ParticipantEntered>\n",
" <LoginName>test3@email</LoginName>\n",
@@ -1060,6 +1078,13 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" <Content>delete message</Content>\n",
" </Message>\n",
" <Message>\n",
" <MessageId>post-id5</MessageId>\n",
" <LoginName>test-bot@email</LoginName>\n",
" <UserType>bot</UserType>\n",
" <DateTimeUTC>20</DateTimeUTC>\n",
" <Content>message</Content>\n",
" </Message>\n",
" <Message>\n",
" <MessageId>post-id4</MessageId>\n",
" <LoginName>test@test.com</LoginName>\n",
" <UserType>user</UserType>\n",
@@ -1067,10 +1092,10 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" <Content>message</Content>\n",
" </Message>\n",
" <ParticipantLeft>\n",
" <LoginName>test_bot@email</LoginName>\n",
" <LoginName>test-bot@email</LoginName>\n",
" <UserType>bot</UserType>\n",
" <DateTimeUTC>20</DateTimeUTC>\n",
" <CorporateEmailID>test_bot@email</CorporateEmailID>\n",
" <CorporateEmailID>test-bot@email</CorporateEmailID>\n",
" </ParticipantLeft>\n",
" <ParticipantLeft>\n",
" <LoginName>test2@email</LoginName>\n",

23
server/enterprise/message_export/csv_export/csv_export.go
Просмотреть файл

@@ -150,10 +150,14 @@ func CsvExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults
for _, attachment := range attachments {
var r io.ReadCloser
r, nErr := p.FileAttachmentBackend.Reader(attachment.Path)
if nErr != nil {
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessage+" - Post: "+*post.PostId+" - "+attachment.Path)
rctx.Logger().Warn(shared.MissingFileMessage, mlog.String("post_id", *post.PostId), mlog.String("filename", attachment.Path))
r, err = p.FileAttachmentBackend.Reader(attachment.Path)
if err != nil {
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringBackendRead+" - Post: "+*post.PostId+" - "+attachment.Path)
rctx.Logger().Warn(shared.MissingFileMessageDuringBackendRead,
mlog.String("post_id", *post.PostId),
mlog.String("filename", attachment.Path),
mlog.Err(err),
)
continue
}
@@ -173,7 +177,16 @@ func CsvExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults
return nil
}(); err != nil {
return results, fmt.Errorf("unable to copy the attachment into the zip file: %w", err)
// s3 only errors _here_ if the object key wasn't found. So to handle that: if there is a read
// error (even for local), let's add a warning instead of failing the export.
// Failing the export would fail the entire export run, and every future run would also fail on
// this non-existent file -- not good.
missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringCopy+" - Post: "+*post.PostId+" - "+attachment.Path)
rctx.Logger().Warn(shared.MissingFileMessageDuringCopy,
mlog.String("post_id", *post.PostId),
mlog.String("filename", attachment.Path),
mlog.Err(err),
)
}
}
}

2
server/enterprise/message_export/csv_export/csv_export_test.go
Просмотреть файл

@@ -906,7 +906,7 @@ func TestWriteExportWarnings(t *testing.T) {
warnings := string(data)
expectedWarnings := fmt.Sprintf("Warning:%[1]s - Post: post-id-1 - test1\nWarning:%[1]s - Post: post-id-3 - test2\n",
shared.MissingFileMessage)
shared.MissingFileMessageDuringBackendRead)
assert.Equal(t, expectedWarnings, warnings)
}

3
server/enterprise/message_export/shared/shared.go
Просмотреть файл

@@ -21,7 +21,8 @@ import (
)
const (
MissingFileMessage = "File missing for post; cannot copy file to archive"
MissingFileMessageDuringBackendRead = "File backend read: File missing for post; cannot copy file to archive"
MissingFileMessageDuringCopy = "Copy buffer: File missing for post; cannot copy file to archive"
EstimatedPostCount = 10_000_000

2
server/enterprise/message_export/worker.go
Просмотреть файл

@@ -251,7 +251,7 @@ func (w *MessageExportWorker) finishExport(rctx request.CTX, logger *mlog.Logger
// we've exported everything up to the current time
logger.Debug("FormatExport complete")
job.Data[shared.JobDataIsDownloadable] = "false"
job.Data[shared.JobDataIsDownloadable] = "true"
if totalWarningCount > 0 {
w.setJobWarning(logger, job)