Upgrading server dependancies (#6431)

vendor/github.com/mssola/user_agent/bot.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2014 Miquel Sabaté Solà <mikisabate@gmail.com>
+// Copyright (C) 2014-2017 Miquel Sabaté Solà <mikisabate@gmail.com>
 // This file is licensed under the MIT license.
 // See the LICENSE file.
 
@@ -9,6 +9,8 @@ import (
 	"strings"
 )
 
+var botFromSiteRegexp = regexp.MustCompile("http://.+\\.\\w+")
+
 // Get the name of the bot from the website that may be in the given comment. If
 // there is no website in the comment, then an empty string is returned.
 func getFromSite(comment []string) string {
@@ -23,8 +25,7 @@ func getFromSite(comment []string) string {
 	}
 
 	// Pick the site.
-	re := regexp.MustCompile("http://.+\\.\\w+")
-	results := re.FindStringSubmatch(comment[idx])
+	results := botFromSiteRegexp.FindStringSubmatch(comment[idx])
 	if len(results) == 1 {
 		// If it's a simple comment, just return the name of the site.
 		if idx == 0 {
@@ -74,6 +75,8 @@ func (p *UserAgent) fixOther(sections []section) {
 	}
 }
 
+var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
+
 // Check if we're dealing with a bot or with some weird browser. If that is the
 // case, the receiver will be modified accordingly.
 func (p *UserAgent) checkBot(sections []section) {
@@ -82,9 +85,8 @@ func (p *UserAgent) checkBot(sections []section) {
 	if len(sections) == 1 && sections[0].name != "Mozilla" {
 		p.mozilla = ""
 
-		// Check whether the name has some suspicious "bot" in his name.
-		reg, _ := regexp.Compile("(?i)bot")
-		if reg.Match([]byte(sections[0].name)) {
+		// Check whether the name has some suspicious "bot" or "crawler" in his name.
+		if botRegex.Match([]byte(sections[0].name)) {
 			p.setSimple(sections[0].name, "", true)
 			return
 		}