Adding default robots.txt (#10016)

web/static.go

@@ -19,6 +19,8 @@ import (
 	"github.com/mattermost/mattermost-server/utils/fileutils"
 )
 
+var robotsTxt = []byte("User-agent: *\nDisallow: /\n")
+
 func (w *Web) InitStatic() {
 	if *w.ConfigService.Config().ServiceSettings.WebserverMode != "disabled" {
 		utils.UpdateAssetsSubpathFromConfig(w.ConfigService.Config())
@@ -40,6 +42,7 @@ func (w *Web) InitStatic() {
 
 		w.MainRouter.PathPrefix("/static/plugins/").Handler(pluginHandler)
 		w.MainRouter.PathPrefix("/static/").Handler(staticHandler)
+		w.MainRouter.Handle("/robots.txt", http.HandlerFunc(robotsHandler))
 		w.MainRouter.Handle("/{anything:.*}", w.NewStaticHandler(root)).Methods("GET")
 
 		// When a subpath is defined, it's necessary to handle redirects without a
@@ -84,3 +87,11 @@ func staticFilesHandler(handler http.Handler) http.Handler {
 		handler.ServeHTTP(w, r)
 	})
 }
+
+func robotsHandler(w http.ResponseWriter, r *http.Request) {
+	if strings.HasSuffix(r.URL.Path, "/") {
+		http.NotFound(w, r)
+		return
+	}
+	w.Write(robotsTxt)
+}