PLT-2283 Improved trimming of punctuation from hashtags and search terms (#3178)

* Improved trimming of punctuation from hashtags and search terms

* Separated punctuation regexes used for hashtags and for search terms

model/search_params.go

@@ -4,9 +4,13 @@
 package model
 
 import (
+	"regexp"
 	"strings"
 )
 
+var searchTermPuncStart = regexp.MustCompile(`^[^\pL\d\s#"]+`)
+var searchTermPuncEnd = regexp.MustCompile(`[^\pL\d\s*"]+$`)
+
 type SearchParams struct {
 	Terms      string
 	IsHashtag  bool
@@ -91,8 +95,8 @@ func parseSearchFlags(input []string) ([]string, [][2]string) {
 
 		if !isFlag {
 			// trim off surrounding punctuation (note that we leave trailing asterisks to allow wildcards)
-			word = puncStart.ReplaceAllString(word, "")
+			word = searchTermPuncStart.ReplaceAllString(word, "")
-			word = puncEndWildcard.ReplaceAllString(word, "")
+			word = searchTermPuncEnd.ReplaceAllString(word, "")
 
 			// and remove extra pound #s
 			word = hashtagStart.ReplaceAllString(word, "#")

model/utils.go

@@ -315,10 +315,9 @@ func Etag(parts ...interface{}) string {
 }
 
 var validHashtag = regexp.MustCompile(`^(#[A-Za-zäöüÄÖÜß]+[A-Za-z0-9äöüÄÖÜß_\-]*[A-Za-z0-9äöüÄÖÜß])$`)
-var puncStart = regexp.MustCompile(`^[.,()&$!\?\[\]{}':;\\<>\-+=%^*|]+`)
+var puncStart = regexp.MustCompile(`^[^\pL\d\s#]+`)
 var hashtagStart = regexp.MustCompile(`^#{2,}`)
-var puncEnd = regexp.MustCompile(`[.,()&$#!\?\[\]{}':;\\<>\-+=%^*|]+$`)
+var puncEnd = regexp.MustCompile(`[^\pL\d\s]+$`)
-var puncEndWildcard = regexp.MustCompile(`[.,()&$#!\?\[\]{}':;\\<>\-+=%^|]+$`)
 
 func ParseHashtags(text string) (string, string) {
 	words := strings.Fields(text)

model/utils_test.go

@@ -82,7 +82,7 @@ func TestEtag(t *testing.T) {
 	}
 }
 
-var hashtags map[string]string = map[string]string{
+var hashtags = map[string]string{
 	"#test":           "#test",
 	"test":            "",
 	"#test123":        "#test123",
@@ -101,6 +101,7 @@ var hashtags map[string]string = map[string]string{
 	"<#less_than<":    "#less_than",
 	">#greater_than>": "#greater_than",
 	"-#minus-":        "#minus",
+	"_#under_":        "#under",
 	"+#plus+":         "#plus",
 	"=#equals=":       "#equals",
 	"%#pct%":          "#pct",
@@ -111,12 +112,21 @@ var hashtags map[string]string = map[string]string{
 	"|#pipe|":         "#pipe",
 	":#colon:":        "#colon",
 	";#semi;":         "#semi",
+	"#Mötley;":        "#Mötley",
+	".#period.":       "#period",
+	"¿#upside¿":       "#upside",
+	"\"#quote\"":      "#quote",
+	"/#slash/":        "#slash",
+	"\\#backslash\\":  "#backslash",
+	"#a":              "",
+	"#1":              "",
+	"foo#bar":         "",
 }
 
 func TestParseHashtags(t *testing.T) {
 	for input, output := range hashtags {
 		if o, _ := ParseHashtags(input); o != output {
-			t.Fatal("expected=" + output + " actual=" + o)
+			t.Fatal("failed to parse hashtags from input=" + input + " expected=" + output + " actual=" + o)
 		}
 	}
 }