This change includes the following:
 - Adds the job ID as a log field for bulk exports
 - Adds checkpoint logging for posts and direct posts so that
   progress can be tracked.
Этот коммит содержится в:
Gabe Jackson
2022-12-25 21:43:30 -05:00
коммит произвёл GitHub
родитель 7a9354cd09
Коммит 99b232e080
2 изменённых файлов: 19 добавлений и 1 удалений

Просмотреть файл

@@ -398,8 +398,15 @@ func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *imports.UserNot
func (a *App) exportAllPosts(ctx request.CTX, writer io.Writer, withAttachments bool) ([]imports.AttachmentImportData, *model.AppError) {
var attachments []imports.AttachmentImportData
afterId := strings.Repeat("0", 26)
var postProcessCount uint64
logCheckpoint := time.Now()
for {
if time.Since(logCheckpoint) > 5*time.Minute {
ctx.Logger().Debug(fmt.Sprintf("Bulk Export: processed %d posts", postProcessCount))
logCheckpoint = time.Now()
}
posts, nErr := a.Srv().Store().Post().GetParentsForExportAfter(1000, afterId)
if nErr != nil {
return nil, model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
@@ -411,6 +418,7 @@ func (a *App) exportAllPosts(ctx request.CTX, writer io.Writer, withAttachments
for _, post := range posts {
afterId = post.Id
postProcessCount++
// Skip deleted.
if post.DeleteAt != 0 {
@@ -677,7 +685,15 @@ func (a *App) buildFavoritedByList(channelID string) ([]string, *model.AppError)
func (a *App) exportAllDirectPosts(ctx request.CTX, writer io.Writer, withAttachments bool) ([]imports.AttachmentImportData, *model.AppError) {
var attachments []imports.AttachmentImportData
afterId := strings.Repeat("0", 26)
var postProcessCount uint64
logCheckpoint := time.Now()
for {
if time.Since(logCheckpoint) > 5*time.Minute {
ctx.Logger().Debug(fmt.Sprintf("Bulk Export: processed %d direct posts", postProcessCount))
logCheckpoint = time.Now()
}
posts, err := a.Srv().Store().Post().GetDirectPostParentsForExportAfter(1000, afterId)
if err != nil {
return nil, model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
@@ -689,6 +705,7 @@ func (a *App) exportAllDirectPosts(ctx request.CTX, writer io.Writer, withAttach
for _, post := range posts {
afterId = post.Id
postProcessCount++
// Skip deleted.
if post.DeleteAt != 0 {