mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-43733: Set concurrency limits via configuration (#20413)

Просмотр исходного кода 
```release-note
We create a new config option MaxImageDecoderConcurrency which
indicates how many images can be decoded concurrently at once.

The default is -1 which means number of CPUs present.

This affects the total memory consumption of the server. The
maximum memory of a single image is dictated by MaxImageResolution * 24 bytes.
Therefore, a good rule of thumb to follow is that MaxImageResolution
* MaxImageDecoderConcurrency * 24 should be less then the allocated memory for
image decoding.
```

https://mattermost.atlassian.net/browse/MM-43733
Этот коммит содержится в:
Agniva De Sarker
2022-06-11 00:24:46 +05:30
коммит произвёл GitHub
родитель 4b8cb4e272
Коммит 7c1b8cd937
4 изменённых файлов: 56 добавлений и 38 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
app/channels.go
Просмотреть файл

@@ -197,8 +197,12 @@ func NewChannels(s *Server, services map[ServiceKey]interface{}) (*Channels, err
} }
var imgErr error var imgErr error
decoderConcurrency := int(*ch.cfgSvc.Config().FileSettings.MaxImageDecoderConcurrency)
if decoderConcurrency == -1 {
decoderConcurrency = runtime.NumCPU()
}
ch.imgDecoder, imgErr = imaging.NewDecoder(imaging.DecoderOptions{ ch.imgDecoder, imgErr = imaging.NewDecoder(imaging.DecoderOptions{
ConcurrencyLevel: runtime.NumCPU(), ConcurrencyLevel: decoderConcurrency,
}) })
if imgErr != nil { if imgErr != nil {
return nil, errors.Wrap(imgErr, "failed to create image decoder") return nil, errors.Wrap(imgErr, "failed to create image decoder")

4
i18n/en.json
Просмотреть файл

@@ -8095,6 +8095,10 @@
"id": "model.config.is_valid.group_unread_channels.app_error", "id": "model.config.is_valid.group_unread_channels.app_error",
"translation": "Invalid group unread channels for service settings. Must be 'disabled', 'default_on', or 'default_off'." "translation": "Invalid group unread channels for service settings. Must be 'disabled', 'default_on', or 'default_off'."
}, },
{
"id": "model.config.is_valid.image_decoder_concurrency.app_error",
"translation": "Invalid decoder concurrency {{.Value}}. Should be a positive number or -1."
},
{ {
"id": "model.config.is_valid.image_proxy_type.app_error", "id": "model.config.is_valid.image_proxy_type.app_error",
"translation": "Invalid image proxy type. Must be 'local' or 'atmos/camo'." "translation": "Invalid image proxy type. Must be 'local' or 'atmos/camo'."

53
model/config.go
Просмотреть файл

@@ -1401,28 +1401,29 @@ func (s *PasswordSettings) SetDefaults() {
} }
type FileSettings struct { type FileSettings struct {
EnableFileAttachments *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"` EnableFileAttachments *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"`
EnableMobileUpload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"` EnableMobileUpload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"`
EnableMobileDownload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"` EnableMobileDownload *bool `access:"site_file_sharing_and_downloads,cloud_restrictable"`
MaxFileSize *int64 `access:"environment_file_storage,cloud_restrictable"` MaxFileSize *int64 `access:"environment_file_storage,cloud_restrictable"`
MaxImageResolution *int64 `access:"environment_file_storage,cloud_restrictable"` MaxImageResolution *int64 `access:"environment_file_storage,cloud_restrictable"`
DriverName *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` MaxImageDecoderConcurrency *int64 `access:"environment_file_storage,cloud_restrictable"`
Directory *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` DriverName *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
EnablePublicLink *bool `access:"site_public_links,cloud_restrictable"` Directory *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
ExtractContent *bool `access:"environment_file_storage,write_restrictable"` EnablePublicLink *bool `access:"site_public_links,cloud_restrictable"`
ArchiveRecursion *bool `access:"environment_file_storage,write_restrictable"` ExtractContent *bool `access:"environment_file_storage,write_restrictable"`
PublicLinkSalt *string `access:"site_public_links,cloud_restrictable"` // telemetry: none ArchiveRecursion *bool `access:"environment_file_storage,write_restrictable"`
InitialFont *string `access:"environment_file_storage,cloud_restrictable"` // telemetry: none PublicLinkSalt *string `access:"site_public_links,cloud_restrictable"` // telemetry: none
AmazonS3AccessKeyId *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none InitialFont *string `access:"environment_file_storage,cloud_restrictable"` // telemetry: none
AmazonS3SecretAccessKey *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3AccessKeyId *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3Bucket *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3SecretAccessKey *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3PathPrefix *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3Bucket *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3Region *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3PathPrefix *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3Endpoint *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3Region *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3SSL *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"` AmazonS3Endpoint *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
AmazonS3SignV2 *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"` AmazonS3SSL *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
AmazonS3SSE *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"` AmazonS3SignV2 *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
AmazonS3Trace *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"` AmazonS3SSE *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
AmazonS3Trace *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
} }
func (s *FileSettings) SetDefaults(isUpdate bool) { func (s *FileSettings) SetDefaults(isUpdate bool) {
@@ -1446,6 +1447,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
s.MaxImageResolution = NewInt64(7680 * 4320) // 8K, ~33MPX s.MaxImageResolution = NewInt64(7680 * 4320) // 8K, ~33MPX
} }
if s.MaxImageDecoderConcurrency == nil {
s.MaxImageDecoderConcurrency = NewInt64(-1) // Default to NumCPU
}
if s.DriverName == nil { if s.DriverName == nil {
s.DriverName = NewString(ImageDriverLocal) s.DriverName = NewString(ImageDriverLocal)
} }
@@ -3395,6 +3400,10 @@ func (s *FileSettings) isValid() *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.directory.app_error", nil, "", http.StatusBadRequest) return NewAppError("Config.IsValid", "model.config.is_valid.directory.app_error", nil, "", http.StatusBadRequest)
} }
if *s.MaxImageDecoderConcurrency < -1 || *s.MaxImageDecoderConcurrency == 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.image_decoder_concurrency.app_error", map[string]interface{}{"Value": *s.MaxImageDecoderConcurrency}, "", http.StatusBadRequest)
}
return nil return nil
} }

31
services/telemetry/telemetry.go
Просмотреть файл

@@ -525,21 +525,22 @@ func (ts *TelemetryService) trackConfig() {
}) })
ts.SendTelemetry(TrackConfigFile, map[string]interface{}{ ts.SendTelemetry(TrackConfigFile, map[string]interface{}{
"enable_public_links": cfg.FileSettings.EnablePublicLink, "enable_public_links": cfg.FileSettings.EnablePublicLink,
"driver_name": *cfg.FileSettings.DriverName, "driver_name": *cfg.FileSettings.DriverName,
"isdefault_directory": isDefault(*cfg.FileSettings.Directory, model.FileSettingsDefaultDirectory), "isdefault_directory": isDefault(*cfg.FileSettings.Directory, model.FileSettingsDefaultDirectory),
"isabsolute_directory": filepath.IsAbs(*cfg.FileSettings.Directory), "isabsolute_directory": filepath.IsAbs(*cfg.FileSettings.Directory),
"extract_content": *cfg.FileSettings.ExtractContent, "extract_content": *cfg.FileSettings.ExtractContent,
"archive_recursion": *cfg.FileSettings.ArchiveRecursion, "archive_recursion": *cfg.FileSettings.ArchiveRecursion,
"amazon_s3_ssl": *cfg.FileSettings.AmazonS3SSL, "amazon_s3_ssl": *cfg.FileSettings.AmazonS3SSL,
"amazon_s3_sse": *cfg.FileSettings.AmazonS3SSE, "amazon_s3_sse": *cfg.FileSettings.AmazonS3SSE,
"amazon_s3_signv2": *cfg.FileSettings.AmazonS3SignV2, "amazon_s3_signv2": *cfg.FileSettings.AmazonS3SignV2,
"amazon_s3_trace": *cfg.FileSettings.AmazonS3Trace, "amazon_s3_trace": *cfg.FileSettings.AmazonS3Trace,
"max_file_size": *cfg.FileSettings.MaxFileSize, "max_file_size": *cfg.FileSettings.MaxFileSize,
"max_image_resolution": *cfg.FileSettings.MaxImageResolution, "max_image_resolution": *cfg.FileSettings.MaxImageResolution,
"enable_file_attachments": *cfg.FileSettings.EnableFileAttachments, "max_image_decoder_concurrency": *cfg.FileSettings.MaxImageDecoderConcurrency,
"enable_mobile_upload": *cfg.FileSettings.EnableMobileUpload, "enable_file_attachments": *cfg.FileSettings.EnableFileAttachments,
"enable_mobile_download": *cfg.FileSettings.EnableMobileDownload, "enable_mobile_upload": *cfg.FileSettings.EnableMobileUpload,
"enable_mobile_download": *cfg.FileSettings.EnableMobileDownload,
}) })
ts.SendTelemetry(TrackConfigEmail, map[string]interface{}{ ts.SendTelemetry(TrackConfigEmail, map[string]interface{}{