PLT-3383: image proxy support (#7991)

* image proxy support * go vet fix, remove mistakenly added coverage file * fix test compile error * add validation to config settings and documentation to model functions * add message_source field to post
2018-01-22 15:32:50 -06:00
--- a/model/config.go
+++ b/model/config.go
@@ -214,6 +214,9 @@ type ServiceSettings struct {
 	EnablePreviewFeatures                             *bool
 	EnableTutorial                                    *bool
 	ExperimentalEnableDefaultChannelLeaveJoinMessages *bool
+	ImageProxyType                                    *string
+	ImageProxyURL                                     *string
+	ImageProxyOptions                                 *string
 }

 func (s *ServiceSettings) SetDefaults() {
@@ -250,7 +253,7 @@ func (s *ServiceSettings) SetDefaults() {
 	}

 	if s.AllowedUntrustedInternalConnections == nil {
-		s.AllowedUntrustedInternalConnections = new(string)
+		s.AllowedUntrustedInternalConnections = NewString("")
 	}

 	if s.EnableMultifactorAuthentication == nil {
@@ -418,6 +421,18 @@ func (s *ServiceSettings) SetDefaults() {
 	if s.ExperimentalEnableDefaultChannelLeaveJoinMessages == nil {
 		s.ExperimentalEnableDefaultChannelLeaveJoinMessages = NewBool(true)
 	}
+
+	if s.ImageProxyType == nil {
+		s.ImageProxyType = NewString("")
+	}
+
+	if s.ImageProxyURL == nil {
+		s.ImageProxyURL = NewString("")
+	}
+
+	if s.ImageProxyOptions == nil {
+		s.ImageProxyOptions = NewString("")
+	}
 }

 type ClusterSettings struct {
@@ -2050,6 +2065,16 @@ func (ss *ServiceSettings) isValid() *AppError {
 		return NewAppError("Config.IsValid", "model.config.is_valid.listen_address.app_error", nil, "", http.StatusBadRequest)
 	}

+	switch *ss.ImageProxyType {
+	case "", "willnorris/imageproxy":
+	case "atmos/camo":
+		if *ss.ImageProxyOptions == "" {
+			return NewAppError("Config.IsValid", "model.config.is_valid.atmos_camo_image_proxy_options.app_error", nil, "", http.StatusBadRequest)
+		}
+	default:
+		return NewAppError("Config.IsValid", "model.config.is_valid.image_proxy_type.app_error", nil, "", http.StatusBadRequest)
+	}
+
 	return nil
 }

--- a/model/post.go
+++ b/model/post.go
@@ -8,8 +8,11 @@ import (
 	"io"
 	"net/http"
 	"regexp"
+	"sort"
 	"strings"
 	"unicode/utf8"
+
+	"github.com/mattermost/mattermost-server/utils/markdown"
 )

 const (
@@ -43,18 +46,25 @@ const (
 )

 type Post struct {
-	Id            string          `json:"id"`
-	CreateAt      int64           `json:"create_at"`
-	UpdateAt      int64           `json:"update_at"`
-	EditAt        int64           `json:"edit_at"`
-	DeleteAt      int64           `json:"delete_at"`
-	IsPinned      bool            `json:"is_pinned"`
-	UserId        string          `json:"user_id"`
-	ChannelId     string          `json:"channel_id"`
-	RootId        string          `json:"root_id"`
-	ParentId      string          `json:"parent_id"`
-	OriginalId    string          `json:"original_id"`
-	Message       string          `json:"message"`
+	Id         string `json:"id"`
+	CreateAt   int64  `json:"create_at"`
+	UpdateAt   int64  `json:"update_at"`
+	EditAt     int64  `json:"edit_at"`
+	DeleteAt   int64  `json:"delete_at"`
+	IsPinned   bool   `json:"is_pinned"`
+	UserId     string `json:"user_id"`
+	ChannelId  string `json:"channel_id"`
+	RootId     string `json:"root_id"`
+	ParentId   string `json:"parent_id"`
+	OriginalId string `json:"original_id"`
+
+	Message string `json:"message"`
+
+	// MessageSource will contain the message as submitted by the user if Message has been modified
+	// by Mattermost for presentation (e.g if an image proxy is being used). It should be used to
+	// populate edit boxes if present.
+	MessageSource string `json:"message_source,omitempty" db:"-"`
+
 	Type          string          `json:"type"`
 	Props         StringInterface `json:"props"`
 	Hashtags      string          `json:"hashtags"`
@@ -72,6 +82,14 @@ type PostPatch struct {
 	HasReactions *bool            `json:"has_reactions"`
 }

+func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch {
+	copy := *o
+	if copy.Message != nil {
+		*copy.Message = RewriteImageURLs(*o.Message, f)
+	}
+	return &copy
+}
+
 type PostForIndexing struct {
 	Post
 	TeamId         string `json:"team_id"`
@@ -392,3 +410,87 @@ func (o *Post) GenerateActionIds() {
 		}
 	}
 }
+
+var markdownDestinationEscaper = strings.NewReplacer(
+	`\`, `\\`,
+	`<`, `\<`,
+	`>`, `\>`,
+	`(`, `\(`,
+	`)`, `\)`,
+)
+
+// WithRewrittenImageURLs returns a new shallow copy of the post where the message has been
+// rewritten via RewriteImageURLs.
+func (o *Post) WithRewrittenImageURLs(f func(string) string) *Post {
+	copy := *o
+	copy.Message = RewriteImageURLs(o.Message, f)
+	if copy.MessageSource == "" && copy.Message != o.Message {
+		copy.MessageSource = o.Message
+	}
+	return &copy
+}
+
+// RewriteImageURLs takes a message and returns a copy that has all of the image URLs replaced
+// according to the function f. For each image URL, f will be invoked, and the resulting markdown
+// will contain the URL returned by that invocation instead.
+//
+// Image URLs are destination URLs used in inline images or reference definitions that are used
+// anywhere in the input markdown as an image.
+func RewriteImageURLs(message string, f func(string) string) string {
+	if !strings.Contains(message, "![") {
+		return message
+	}
+
+	var ranges []markdown.Range
+
+	markdown.Inspect(message, func(blockOrInline interface{}) bool {
+		switch v := blockOrInline.(type) {
+		case *markdown.ReferenceImage:
+			ranges = append(ranges, v.ReferenceDefinition.RawDestination)
+		case *markdown.InlineImage:
+			ranges = append(ranges, v.RawDestination)
+		default:
+			return true
+		}
+		return true
+	})
+
+	if ranges == nil {
+		return message
+	}
+
+	sort.Slice(ranges, func(i, j int) bool {
+		return ranges[i].Position < ranges[j].Position
+	})
+
+	copyRanges := make([]markdown.Range, 0, len(ranges))
+	urls := make([]string, 0, len(ranges))
+	resultLength := len(message)
+
+	start := 0
+	for i, r := range ranges {
+		switch {
+		case i == 0:
+		case r.Position != ranges[i-1].Position:
+			start = ranges[i-1].End
+		default:
+			continue
+		}
+		original := message[r.Position:r.End]
+		replacement := markdownDestinationEscaper.Replace(f(markdown.Unescape(original)))
+		resultLength += len(replacement) - len(original)
+		copyRanges = append(copyRanges, markdown.Range{Position: start, End: r.Position})
+		urls = append(urls, replacement)
+	}
+
+	result := make([]byte, resultLength)
+
+	offset := 0
+	for i, r := range copyRanges {
+		offset += copy(result[offset:], message[r.Position:r.End])
+		offset += copy(result[offset:], urls[i])
+	}
+	copy(result[offset:], message[ranges[len(ranges)-1].End:])
+
+	return string(result)
+}
--- a/model/post_list.go
+++ b/model/post_list.go
@@ -21,6 +21,15 @@ func NewPostList() *PostList {
 	}
 }

+func (o *PostList) WithRewrittenImageURLs(f func(string) string) *PostList {
+	copy := *o
+	copy.Posts = make(map[string]*Post)
+	for id, post := range o.Posts {
+		copy.Posts[id] = post.WithRewrittenImageURLs(f)
+	}
+	return &copy
+}
+
 func (o *PostList) StripActionIntegrations() {
 	posts := o.Posts
 	o.Posts = make(map[string]*Post)
--- a/model/post_test.go
+++ b/model/post_test.go
@@ -4,6 +4,7 @@
 package model

 import (
+	"io/ioutil"
 	"strings"
 	"testing"

@@ -173,3 +174,129 @@ func TestPostSanitizeProps(t *testing.T) {
 		t.Fatal("should not be nil")
 	}
 }
+
+var markdownSample, markdownSampleWithRewrittenImageURLs string
+
+func init() {
+	bytes, err := ioutil.ReadFile("testdata/markdown-sample.md")
+	if err != nil {
+		panic(err)
+	}
+	markdownSample = string(bytes)
+
+	bytes, err = ioutil.ReadFile("testdata/markdown-sample-with-rewritten-image-urls.md")
+	if err != nil {
+		panic(err)
+	}
+	markdownSampleWithRewrittenImageURLs = string(bytes)
+}
+
+func TestRewriteImageURLs(t *testing.T) {
+	for name, tc := range map[string]struct {
+		Markdown string
+		Expected string
+	}{
+		"Empty": {
+			Markdown: ``,
+			Expected: ``,
+		},
+		"NoImages": {
+			Markdown: `foo`,
+			Expected: `foo`,
+		},
+		"Link": {
+			Markdown: `[foo](/url)`,
+			Expected: `[foo](/url)`,
+		},
+		"Image": {
+			Markdown: `![foo](/url)`,
+			Expected: `![foo](rewritten:/url)`,
+		},
+		"SpacedURL": {
+			Markdown: `![foo]( /url )`,
+			Expected: `![foo]( rewritten:/url )`,
+		},
+		"Title": {
+			Markdown: `![foo](/url "title")`,
+			Expected: `![foo](rewritten:/url "title")`,
+		},
+		"Parentheses": {
+			Markdown: `![foo](/url(1) "title")`,
+			Expected: `![foo](rewritten:/url\(1\) "title")`,
+		},
+		"AngleBrackets": {
+			Markdown: `![foo](</url\<1\>\\> "title")`,
+			Expected: `![foo](<rewritten:/url\<1\>\\> "title")`,
+		},
+		"MultipleLines": {
+			Markdown: `![foo](
+				</url\<1\>\\>
+				"title"
+			)`,
+			Expected: `![foo](
+				<rewritten:/url\<1\>\\>
+				"title"
+			)`,
+		},
+		"ReferenceLink": {
+			Markdown: `[foo]: </url\<1\>\\> "title"
+		 		[foo]`,
+			Expected: `[foo]: </url\<1\>\\> "title"
+		 		[foo]`,
+		},
+		"ReferenceImage": {
+			Markdown: `[foo]: </url\<1\>\\> "title"
+		 		![foo]`,
+			Expected: `[foo]: <rewritten:/url\<1\>\\> "title"
+		 		![foo]`,
+		},
+		"MultipleReferenceImages": {
+			Markdown: `[foo]: </url1> "title"
+				[bar]: </url2>
+				[baz]: /url3 "title"
+				[qux]: /url4
+				![foo]![qux]`,
+			Expected: `[foo]: <rewritten:/url1> "title"
+				[bar]: </url2>
+				[baz]: /url3 "title"
+				[qux]: rewritten:/url4
+				![foo]![qux]`,
+		},
+		"DuplicateReferences": {
+			Markdown: `[foo]: </url1> "title"
+				[foo]: </url2>
+				[foo]: /url3 "title"
+				[foo]: /url4
+				![foo]![foo]![foo]`,
+			Expected: `[foo]: <rewritten:/url1> "title"
+				[foo]: </url2>
+				[foo]: /url3 "title"
+				[foo]: /url4
+				![foo]![foo]![foo]`,
+		},
+		"TrailingURL": {
+			Markdown: "![foo]\n\n[foo]: /url",
+			Expected: "![foo]\n\n[foo]: rewritten:/url",
+		},
+		"Sample": {
+			Markdown: markdownSample,
+			Expected: markdownSampleWithRewrittenImageURLs,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, tc.Expected, RewriteImageURLs(tc.Markdown, func(url string) string {
+				return "rewritten:" + url
+			}))
+		})
+	}
+}
+
+var rewriteImageURLsSink string
+
+func BenchmarkRewriteImageURLs(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		rewriteImageURLsSink = RewriteImageURLs(markdownSample, func(url string) string {
+			return "rewritten:" + url
+		})
+	}
+}
--- a/model/testdata/markdown-sample-with-rewritten-image-urls.md
+++ b/model/testdata/markdown-sample-with-rewritten-image-urls.md
@@ -0,0 +1,245 @@
+---
+__Advertisement :)__
+
+- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image
+  resize in browser.
+- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly
+  i18n with plurals support and easy syntax.
+
+You will like those projects!
+
+---
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Typographic replacements
+
+Enable typographer option to see result.
+
+(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+
+test.. test... test..... test?..... test!....
+
+!!!!!! ???? ,,  -- ---
+
+"Smartypants, double quotes" and 'single quotes'
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
+ Create a list by starting a line with `+`, `-`, or `*`
+ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
+ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/nodeca/pica (enable linkify to see)
+
+
+## Images
+
+![Minion](rewritten:https://octodex.github.com/images/minion.png)
+![Stormtroopocat](rewritten:https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat")
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: rewritten:https://octodex.github.com/images/dojocat.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum:
+>
+> Shortcuts (emoticons): :-) :-( 8-) ;)
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup)
+
+- 19^th^
+- H~2~O
+
+
+### [\<ins>](https://github.com/markdown-it/markdown-it-ins)
+
++Inserted text++
+
+
+### [\<mark>](https://github.com/markdown-it/markdown-it-mark)
+
+==Marked text==
+
+
+### [Footnotes](https://github.com/markdown-it/markdown-it-footnote)
+
+Footnote 1 link[^first].
+
+Footnote 2 link[^second].
+
+Inline footnote^[Text of inline footnote] definition.
+
+Duplicated footnote reference[^second].
+
+[^first]: Footnote **can have markup**
+
+    and multiple paragraphs.
+
+[^second]: Footnote text.
+
+
+### [Definition lists](https://github.com/markdown-it/markdown-it-deflist)
+
+Term 1
+
+:   Definition 1
+with lazy continuation.
+
+Term 2 with *inline markup*
+
+:   Definition 2
+
+        { some code, part of Definition 2 }
+
+    Third paragraph of definition 2.
+
+_Compact style:_
+
+Term 1
+  ~ Definition 1
+
+Term 2
+  ~ Definition 2a
+  ~ Definition 2b
+
+
+### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr)
+
+This is HTML abbreviation example.
+
+It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on.
+
+*[HTML]: Hyper Text Markup Language
+
+### [Custom containers](https://github.com/markdown-it/markdown-it-container)
+
+::: warning
+*here be dragons*
+:::
--- a/model/testdata/markdown-sample.md
+++ b/model/testdata/markdown-sample.md
@@ -0,0 +1,245 @@
+---
+__Advertisement :)__
+
+- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image
+  resize in browser.
+- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly
+  i18n with plurals support and easy syntax.
+
+You will like those projects!
+
+---
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Typographic replacements
+
+Enable typographer option to see result.
+
+(c) (C) (r) (R) (tm) (TM) (p) (P) +-
+
+test.. test... test..... test?..... test!....
+
+!!!!!! ???? ,,  -- ---
+
+"Smartypants, double quotes" and 'single quotes'
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
+ Create a list by starting a line with `+`, `-`, or `*`
+ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
+ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/nodeca/pica (enable linkify to see)
+
+
+## Images
+
+![Minion](https://octodex.github.com/images/minion.png)
+![Stormtroopocat](https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat")
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: https://octodex.github.com/images/dojocat.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum:
+>
+> Shortcuts (emoticons): :-) :-( 8-) ;)
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup)
+
+- 19^th^
+- H~2~O
+
+
+### [\<ins>](https://github.com/markdown-it/markdown-it-ins)
+
++Inserted text++
+
+
+### [\<mark>](https://github.com/markdown-it/markdown-it-mark)
+
+==Marked text==
+
+
+### [Footnotes](https://github.com/markdown-it/markdown-it-footnote)
+
+Footnote 1 link[^first].
+
+Footnote 2 link[^second].
+
+Inline footnote^[Text of inline footnote] definition.
+
+Duplicated footnote reference[^second].
+
+[^first]: Footnote **can have markup**
+
+    and multiple paragraphs.
+
+[^second]: Footnote text.
+
+
+### [Definition lists](https://github.com/markdown-it/markdown-it-deflist)
+
+Term 1
+
+:   Definition 1
+with lazy continuation.
+
+Term 2 with *inline markup*
+
+:   Definition 2
+
+        { some code, part of Definition 2 }
+
+    Third paragraph of definition 2.
+
+_Compact style:_
+
+Term 1
+  ~ Definition 1
+
+Term 2
+  ~ Definition 2a
+  ~ Definition 2b
+
+
+### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr)
+
+This is HTML abbreviation example.
+
+It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on.
+
+*[HTML]: Hyper Text Markup Language
+
+### [Custom containers](https://github.com/markdown-it/markdown-it-container)
+
+::: warning
+*here be dragons*
+:::