PLT-495 Improves slack markup conversion (#4914)

* improves slack markup conversion

* bold
* strikethrough
* blockquotes

* handles blockquotes in slack_import

* removes unnecessary formatting

* fixes various format problems

app/slackimport.go

@@ -564,12 +564,63 @@ func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]Sla
 }
 
 func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost {
-	// Convert URLs in Slack's format to Markdown format.
+	regexReplaceAllString := []struct {
-	regex := regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`)
+		regex *regexp.Regexp
+		rpl   string
+	}{
+		// URL
+		{
+			regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`),
+			"[$2]($1)",
+		},
+		// bold
+		{
+			regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
+			"$1**$2**",
+		},
+		// strikethrough
+		{
+			regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
+			"$1~~$2~~",
+		},
+		// single paragraph blockquote
+		// Slack converts > character to &gt;
+		{
+			regexp.MustCompile(`(?sm)^&gt;`),
+			">",
+		},
+	}
+
+	regexReplaceAllStringFunc := []struct {
+		regex *regexp.Regexp
+		fn    func(string) string
+	}{
+		// multiple paragraphs blockquotes
+		{
+			regexp.MustCompile(`(?sm)^>&gt;&gt;(.+)$`),
+			func(src string) string {
+				// remove >>> prefix, might have leading \n
+				prefixRegexp := regexp.MustCompile(`^([\n])?>&gt;&gt;(.*)`)
+				src = prefixRegexp.ReplaceAllString(src, "$1$2")
+				// append > to start of line
+				appendRegexp := regexp.MustCompile(`(?m)^`)
+				return appendRegexp.ReplaceAllString(src, ">$0")
+			},
+		},
+	}
 
 	for channelName, channelPosts := range posts {
 		for postIdx, post := range channelPosts {
-			posts[channelName][postIdx].Text = regex.ReplaceAllString(post.Text, "[$2]($1)")
+			result := post.Text
+
+			for _, rule := range regexReplaceAllString {
+				result = rule.regex.ReplaceAllString(result, rule.rpl)
+			}
+
+			for _, rule := range regexReplaceAllStringFunc {
+				result = rule.regex.ReplaceAllStringFunc(result, rule.fn)
+			}
+			posts[channelName][postIdx].Text = result
 		}
 	}
 

app/slackimport_test.go

@@ -190,7 +190,7 @@ func TestSlackSanitiseChannelProperties(t *testing.T) {
 
 	c1s := SlackSanitiseChannelProperties(c1)
 	if c1.DisplayName != c1s.DisplayName || c1.Name != c1s.Name || c1.Purpose != c1s.Purpose || c1.Header != c1s.Header {
-		t.Fatalf("Unexpected alterations to the channel properties.")
+		t.Fatal("Unexpected alterations to the channel properties.")
 	}
 
 	c2 := model.Channel{
@@ -227,14 +227,103 @@ func TestSlackConvertPostsMarkup(t *testing.T) {
 		{
 			Text: "This message contains a mailto link to <mailto:me@example.com|me@example.com> in it.",
 		},
+		{
+			Text: "This message contains a *bold* word.",
+		},
+		{
+			Text: "This is not a * bold * word.",
+		},
+		{
+			Text: `There is *no bold word
+in this*.`,
+		},
+		{
+			Text: "*This* is not a*bold* word.*This* is a bold word, *and* this; *and* this too.",
+		},
+		{
+			Text: "This message contains a ~strikethrough~ word.",
+		},
+		{
+			Text: "This is not a ~ strikethrough ~ word.",
+		},
+		{
+			Text: `There is ~no strikethrough word
+in this~.`,
+		},
+		{
+			Text: "~This~ is not a~strikethrough~ word.~This~ is a strikethrough word, ~and~ this; ~and~ this too.",
+		},
+		{
+			Text: `This message contains multiple paragraphs blockquotes
+&gt;&gt;&gt;first
+second
+third`,
+		},
+		{
+			Text: `This message contains single paragraph blockquotes
+&gt;something
+&gt;another thing`,
+		},
+		{
+			Text: "This message has no > block quote",
+		},
 	}
 
-	output := SlackConvertPostsMarkup(input)
+	expectedOutput := make(map[string][]SlackPost)
-
+	expectedOutput["test"] = []SlackPost{
-	if output["test"][0].Text != "This message contains a link to [Google](https://google.com)." {
+		{
-		t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
+			Text: "This message contains a link to [Google](https://google.com).",
+		},
+		{
+			Text: "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it.",
+		},
+		{
+			Text: "This message contains a **bold** word.",
+		},
+		{
+			Text: "This is not a * bold * word.",
+		},
+		{
+			Text: `There is *no bold word
+in this*.`,
+		},
+		{
+			Text: "**This** is not a*bold* word.**This** is a bold word, **and** this; **and** this too.",
+		},
+		{
+			Text: "This message contains a ~~strikethrough~~ word.",
+		},
+		{
+			Text: "This is not a ~ strikethrough ~ word.",
+		},
+		{
+			Text: `There is ~no strikethrough word
+in this~.`,
+		},
+		{
+			Text: "~~This~~ is not a~strikethrough~ word.~~This~~ is a strikethrough word, ~~and~~ this; ~~and~~ this too.",
+		},
+		{
+			Text: `This message contains multiple paragraphs blockquotes
+>first
+>second
+>third`,
+		},
+		{
+			Text: `This message contains single paragraph blockquotes
+>something
+>another thing`,
+		},
+		{
+			Text: "This message has no > block quote",
+		},
 	}
-	if output["test"][1].Text != "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it." {
+
-		t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
+	actualOutput := SlackConvertPostsMarkup(input)
+
+	for i := range actualOutput["test"] {
+		if actualOutput["test"][i].Text != expectedOutput["test"][i].Text {
+			t.Errorf("Unexpected message after markup translation: %v", actualOutput["test"][i].Text)
+		}
 	}
 }