mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-13207 Add customizable timeout for link metadata and improve caching of errors (#10188)

Просмотр исходного кода 
* MM-13207 Add customizable timeout for link metadata and improve caching of errors

* Rename LinkMetadataTimeout to LinkMetadataTimeoutMilliseconds

* Add diagnostics for LinkMetadataTimeoutMilliseconds
Этот коммит содержится в:
Harrison Healey
2019-01-31 09:40:23 -05:00
коммит произвёл GitHub
родитель 2ca222033c
Коммит 7c677b6196
5 изменённых файлов: 105 добавлений и 18 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
app/diagnostics.go
Просмотреть файл

@@ -496,6 +496,7 @@ func (a *App) trackConfig() {
"client_side_cert_enable": *cfg.ExperimentalSettings.ClientSideCertEnable, "client_side_cert_enable": *cfg.ExperimentalSettings.ClientSideCertEnable,
"isdefault_client_side_cert_check": isDefault(*cfg.ExperimentalSettings.ClientSideCertCheck, model.CLIENT_SIDE_CERT_CHECK_PRIMARY_AUTH), "isdefault_client_side_cert_check": isDefault(*cfg.ExperimentalSettings.ClientSideCertCheck, model.CLIENT_SIDE_CERT_CHECK_PRIMARY_AUTH),
"enable_post_metadata": *cfg.ExperimentalSettings.EnablePostMetadata, "enable_post_metadata": *cfg.ExperimentalSettings.EnablePostMetadata,
"link_metadata_timeout_milliseconds": *cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds,
}) })
a.SendDiagnostic(TRACK_CONFIG_ANALYTICS, map[string]interface{}{ a.SendDiagnostic(TRACK_CONFIG_ANALYTICS, map[string]interface{}{

13
app/post_metadata.go
Просмотреть файл

@@ -9,6 +9,7 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
"time"
"github.com/dyatlov/go-opengraph/opengraph" "github.com/dyatlov/go-opengraph/opengraph"
"github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/mlog"
@@ -346,17 +347,19 @@ func (a *App) getLinkMetadata(requestURL string, timestamp int64, isNewPost bool
request.Header.Add("Accept", "text/html, image/*") request.Header.Add("Accept", "text/html, image/*")
res, err := a.HTTPService.MakeClient(false).Do(request) client := a.HTTPService.MakeClient(false)
if err != nil { client.Timeout = time.Duration(*a.Config().ExperimentalSettings.LinkMetadataTimeoutMilliseconds) * time.Millisecond
return nil, nil, err
}
res, err := client.Do(request)
if err == nil {
defer res.Body.Close() defer res.Body.Close()
// Parse the data // Parse the data
og, image, err = a.parseLinkMetadata(requestURL, res.Body, res.Header.Get("Content-Type")) og, image, err = a.parseLinkMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))
}
// Write back to cache and database // Write back to cache and database, even if there was an error and the results are nil
cacheLinkMetadata(requestURL, timestamp, og, image) cacheLinkMetadata(requestURL, timestamp, og, image)
a.saveLinkMetadataToDatabase(requestURL, timestamp, og, image) a.saveLinkMetadataToDatabase(requestURL, timestamp, og, image)

77
app/post_metadata_test.go
Просмотреть файл

@@ -11,6 +11,7 @@ import (
"io" "io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
@@ -1038,6 +1039,15 @@ func TestGetLinkMetadata(t *testing.T) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.Write([]byte("true")) w.Write([]byte("true"))
} else if strings.HasPrefix(r.URL.Path, "/timeout") {
w.Header().Set("Content-Type", "text/html")
w.Write([]byte("<html>"))
select {
case <-time.After(60 * time.Second):
case <-r.Context().Done():
}
w.Write([]byte("</html>"))
} else { } else {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
} }
@@ -1274,7 +1284,7 @@ func TestGetLinkMetadata(t *testing.T) {
assert.Exactly(t, img, fromDatabase) assert.Exactly(t, img, fromDatabase)
}) })
t.Run("should cache error results", func(t *testing.T) { t.Run("should cache general errors", func(t *testing.T) {
th := setup() th := setup()
defer th.TearDown() defer th.TearDown()
@@ -1304,6 +1314,71 @@ func TestGetLinkMetadata(t *testing.T) {
assert.Nil(t, imageFromDatabase) assert.Nil(t, imageFromDatabase)
}) })
t.Run("should cache invalid URL errors", func(t *testing.T) {
th := setup()
defer th.TearDown()
requestURL := "http://notarealdomainthatactuallyexists.ca/?name=" + t.Name()
timestamp := int64(1547510400000)
_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
require.False(t, ok, "data should not exist in in-memory cache")
_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
require.False(t, ok, "data should not exist in database")
og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
assert.Nil(t, og)
assert.Nil(t, img)
assert.IsType(t, &url.Error{}, err)
ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
assert.True(t, ok)
assert.Nil(t, ogFromCache)
assert.Nil(t, imgFromCache)
ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
assert.True(t, ok)
assert.Nil(t, ogFromDatabase)
assert.Nil(t, imageFromDatabase)
})
t.Run("should cache timeout errors", func(t *testing.T) {
th := setup()
defer th.TearDown()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds = 100
})
requestURL := server.URL + "/timeout?name=" + t.Name()
timestamp := int64(1547510400000)
_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
require.False(t, ok, "data should not exist in in-memory cache")
_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
require.False(t, ok, "data should not exist in database")
og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
assert.Nil(t, og)
assert.Nil(t, img)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Client.Timeout")
ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
assert.True(t, ok)
assert.Nil(t, ogFromCache)
assert.Nil(t, imgFromCache)
ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
assert.True(t, ok)
assert.Nil(t, ogFromDatabase)
assert.Nil(t, imageFromDatabase)
})
t.Run("should cache database results in memory", func(t *testing.T) { t.Run("should cache database results in memory", func(t *testing.T) {
th := setup() th := setup()
defer th.TearDown() defer th.TearDown()

3
config/default.json
Просмотреть файл

@@ -358,7 +358,8 @@
"ExperimentalSettings": { "ExperimentalSettings": {
"ClientSideCertEnable": false, "ClientSideCertEnable": false,
"ClientSideCertCheck": "secondary", "ClientSideCertCheck": "secondary",
"EnablePostMetadata": false "EnablePostMetadata": false,
"LinkMetadataTimeoutMilliseconds": 5000
}, },
"AnalyticsSettings": { "AnalyticsSettings": {
"MaxUsersForStatistics": 2500 "MaxUsersForStatistics": 2500

7
model/config.go
Просмотреть файл

@@ -139,6 +139,8 @@ const (
NATIVEAPP_SETTINGS_DEFAULT_ANDROID_APP_DOWNLOAD_LINK = "https://about.mattermost.com/mattermost-android-app/" NATIVEAPP_SETTINGS_DEFAULT_ANDROID_APP_DOWNLOAD_LINK = "https://about.mattermost.com/mattermost-android-app/"
NATIVEAPP_SETTINGS_DEFAULT_IOS_APP_DOWNLOAD_LINK = "https://about.mattermost.com/mattermost-ios-app/" NATIVEAPP_SETTINGS_DEFAULT_IOS_APP_DOWNLOAD_LINK = "https://about.mattermost.com/mattermost-ios-app/"
EXPERIMENTAL_SETTINGS_DEFAULT_LINK_METADATA_TIMEOUT_MILLISECONDS = 5000
ANALYTICS_SETTINGS_DEFAULT_MAX_USERS_FOR_STATISTICS = 2500 ANALYTICS_SETTINGS_DEFAULT_MAX_USERS_FOR_STATISTICS = 2500
ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_COLOR = "#f2a93b" ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_COLOR = "#f2a93b"
@@ -695,6 +697,7 @@ type ExperimentalSettings struct {
ClientSideCertEnable *bool ClientSideCertEnable *bool
ClientSideCertCheck *string ClientSideCertCheck *string
EnablePostMetadata *bool EnablePostMetadata *bool
LinkMetadataTimeoutMilliseconds *int64
} }
func (s *ExperimentalSettings) SetDefaults() { func (s *ExperimentalSettings) SetDefaults() {
@@ -709,6 +712,10 @@ func (s *ExperimentalSettings) SetDefaults() {
if s.EnablePostMetadata == nil { if s.EnablePostMetadata == nil {
s.EnablePostMetadata = NewBool(false) s.EnablePostMetadata = NewBool(false)
} }
if s.LinkMetadataTimeoutMilliseconds == nil {
s.LinkMetadataTimeoutMilliseconds = NewInt64(EXPERIMENTAL_SETTINGS_DEFAULT_LINK_METADATA_TIMEOUT_MILLISECONDS)
}
} }
type AnalyticsSettings struct { type AnalyticsSettings struct {