MM-43733: Set concurrency limits via configuration (#20413)

```release-note
We create a new config option MaxImageDecoderConcurrency which
indicates how many images can be decoded concurrently at once.

The default is -1 which means number of CPUs present.

This affects the total memory consumption of the server. The
maximum memory of a single image is dictated by MaxImageResolution * 24 bytes.
Therefore, a good rule of thumb to follow is that MaxImageResolution
* MaxImageDecoderConcurrency * 24 should be less then the allocated memory for
image decoding.
```

https://mattermost.atlassian.net/browse/MM-43733
Этот коммит содержится в:
Agniva De Sarker
2022-06-11 00:24:46 +05:30
коммит произвёл GitHub
родитель 4b8cb4e272
Коммит 7c1b8cd937
4 изменённых файлов: 56 добавлений и 38 удалений

Просмотреть файл

@@ -197,8 +197,12 @@ func NewChannels(s *Server, services map[ServiceKey]interface{}) (*Channels, err
} }
var imgErr error var imgErr error
decoderConcurrency := int(*ch.cfgSvc.Config().FileSettings.MaxImageDecoderConcurrency)
if decoderConcurrency == -1 {
decoderConcurrency = runtime.NumCPU()
}
ch.imgDecoder, imgErr = imaging.NewDecoder(imaging.DecoderOptions{ ch.imgDecoder, imgErr = imaging.NewDecoder(imaging.DecoderOptions{
ConcurrencyLevel: runtime.NumCPU(), ConcurrencyLevel: decoderConcurrency,
}) })
if imgErr != nil { if imgErr != nil {
return nil, errors.Wrap(imgErr, "failed to create image decoder") return nil, errors.Wrap(imgErr, "failed to create image decoder")

Просмотреть файл

@@ -8095,6 +8095,10 @@
"id": "model.config.is_valid.group_unread_channels.app_error", "id": "model.config.is_valid.group_unread_channels.app_error",
"translation": "Invalid group unread channels for service settings. Must be 'disabled', 'default_on', or 'default_off'." "translation": "Invalid group unread channels for service settings. Must be 'disabled', 'default_on', or 'default_off'."
}, },
{
"id": "model.config.is_valid.image_decoder_concurrency.app_error",
"translation": "Invalid decoder concurrency {{.Value}}. Should be a positive number or -1."
},
{ {
"id": "model.config.is_valid.image_proxy_type.app_error", "id": "model.config.is_valid.image_proxy_type.app_error",
"translation": "Invalid image proxy type. Must be 'local' or 'atmos/camo'." "translation": "Invalid image proxy type. Must be 'local' or 'atmos/camo'."

Просмотреть файл

@@ -1406,6 +1406,7 @@ type FileSettings struct {
EnableMobileDownload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"` EnableMobileDownload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"`
MaxFileSize *int64 `access:"environment_file_storage,cloud_restrictable"` MaxFileSize *int64 `access:"environment_file_storage,cloud_restrictable"`
MaxImageResolution *int64 `access:"environment_file_storage,cloud_restrictable"` MaxImageResolution *int64 `access:"environment_file_storage,cloud_restrictable"`
MaxImageDecoderConcurrency *int64 `access:"environment_file_storage,cloud_restrictable"`
DriverName *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` DriverName *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
Directory *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` Directory *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
EnablePublicLink *bool `access:"site_public_links,cloud_restrictable"` EnablePublicLink *bool `access:"site_public_links,cloud_restrictable"`
@@ -1446,6 +1447,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
s.MaxImageResolution = NewInt64(7680 * 4320) // 8K, ~33MPX s.MaxImageResolution = NewInt64(7680 * 4320) // 8K, ~33MPX
} }
if s.MaxImageDecoderConcurrency == nil {
s.MaxImageDecoderConcurrency = NewInt64(-1) // Default to NumCPU
}
if s.DriverName == nil { if s.DriverName == nil {
s.DriverName = NewString(ImageDriverLocal) s.DriverName = NewString(ImageDriverLocal)
} }
@@ -3395,6 +3400,10 @@ func (s *FileSettings) isValid() *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.directory.app_error", nil, "", http.StatusBadRequest) return NewAppError("Config.IsValid", "model.config.is_valid.directory.app_error", nil, "", http.StatusBadRequest)
} }
if *s.MaxImageDecoderConcurrency < -1 || *s.MaxImageDecoderConcurrency == 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.image_decoder_concurrency.app_error", map[string]interface{}{"Value": *s.MaxImageDecoderConcurrency}, "", http.StatusBadRequest)
}
return nil return nil
} }

Просмотреть файл

@@ -537,6 +537,7 @@ func (ts *TelemetryService) trackConfig() {
"amazon_s3_trace": *cfg.FileSettings.AmazonS3Trace, "amazon_s3_trace": *cfg.FileSettings.AmazonS3Trace,
"max_file_size": *cfg.FileSettings.MaxFileSize, "max_file_size": *cfg.FileSettings.MaxFileSize,
"max_image_resolution": *cfg.FileSettings.MaxImageResolution, "max_image_resolution": *cfg.FileSettings.MaxImageResolution,
"max_image_decoder_concurrency": *cfg.FileSettings.MaxImageDecoderConcurrency,
"enable_file_attachments": *cfg.FileSettings.EnableFileAttachments, "enable_file_attachments": *cfg.FileSettings.EnableFileAttachments,
"enable_mobile_upload": *cfg.FileSettings.EnableMobileUpload, "enable_mobile_upload": *cfg.FileSettings.EnableMobileUpload,
"enable_mobile_download": *cfg.FileSettings.EnableMobileDownload, "enable_mobile_download": *cfg.FileSettings.EnableMobileDownload,