Merge pull request #205 from mattermost/mm-1351

fixes mm-1351 adds local server storage

.gitignore

@@ -47,3 +47,7 @@ web/sass-files/sass/.sass-cache/
 *config.codekit
 *.sass-cache
 *styles.css
+
+# Default local file storage
+data/*
+api/data/*

Dockerfile

@@ -95,5 +95,8 @@ RUN cd /go/src/github.com/mattermost/platform/web/react; npm install
 RUN chmod +x /go/src/github.com/mattermost/platform/docker/docker-entry.sh
 ENTRYPOINT /go/src/github.com/mattermost/platform/docker/docker-entry.sh
 
+# Create default storage directory
+RUN mkdir /mattermost/
+
 # Ports
 EXPOSE 80

Makefile

@@ -112,6 +112,8 @@ clean:
 	rm -f web/static/js/bundle*.js
 	rm -f web/static/css/styles.css
 
+	rm -rf data/*
+	rm -rf api/data/*
 	rm -rf logs/*
 
 

api/file.go

@@ -18,8 +18,10 @@ import (
 	_ "image/gif"
 	"image/jpeg"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -27,7 +29,7 @@ import (
 )
 
 func InitFile(r *mux.Router) {
-	l4g.Debug("Initializing post api routes")
+	l4g.Debug("Initializing file api routes")
 
 	sr := r.PathPrefix("/files").Subrouter()
 	sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
@@ -36,8 +38,8 @@ func InitFile(r *mux.Router) {
 }
 
 func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
-	if !utils.IsS3Configured() {
+	if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
+		c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
@@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var auth aws.Auth
-	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
-	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
-
-	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
-
 	m := r.MultipartForm
 
 	props := m.Value
@@ -94,28 +89,25 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 		buf := bytes.NewBuffer(nil)
 		io.Copy(buf, file)
 
-		ext := filepath.Ext(files[i].Filename)
+		filename := filepath.Base(files[i].Filename)
 
 		uid := model.NewId()
 
-		path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename
+		path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename
 
-		if model.IsFileExtImage(ext) {
+		if err := writeFile(buf.Bytes(), path); err != nil {
-			options := s3.Options{}
+			c.Err = err
-			err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
-			imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
-			imageDataList = append(imageDataList, buf.Bytes())
-		} else {
-			options := s3.Options{}
-			err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
-		}
-
-		if err != nil {
-			c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
 			return
 		}
 
-		fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
+		if model.IsFileExtImage(filepath.Ext(files[i].Filename)) {
+			imageNameList = append(imageNameList, uid+"/"+filename)
+			imageDataList = append(imageDataList, buf.Bytes())
+		}
+
+		encName := utils.UrlEncode(filename)
+
+		fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName
 		resStruct.Filenames = append(resStruct.Filenames, fileUrl)
 	}
 
@@ -127,13 +119,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {
 
 	go func() {
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
-		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
-
-		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
-
 		dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"
 
 		for i, filename := range filenames {
@@ -169,11 +154,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
 						return
 					}
 
-					// Upload thumbnail to S3
+					if err := writeFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil {
-					options := s3.Options{}
+						l4g.Error("Unable to upload thumbnail channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
-					err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
-					if err != nil {
-						l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
 						return
 					}
 				}()
@@ -188,19 +170,15 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
 					}
 
 					buf := new(bytes.Buffer)
-					err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
 
-					//err = png.Encode(buf, preview)
+					err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
 					if err != nil {
 						l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
 						return
 					}
 
-					// Upload preview to S3
+					if err := writeFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil {
-					options := s3.Options{}
+						l4g.Error("Unable to upload preview channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
-					err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
-					if err != nil {
-						l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
 						return
 					}
 				}()
@@ -215,8 +193,8 @@ type ImageGetResult struct {
 }
 
 func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
-	if !utils.IsS3Configured() {
+	if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
+		c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
@@ -247,13 +225,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
 
-	var auth aws.Auth
-	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
-	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
-
-	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
-
 	path := ""
 	if len(teamId) == 26 {
 		path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
@@ -262,7 +233,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	fileData := make(chan []byte)
-	asyncGetFile(bucket, path, fileData)
+	asyncGetFile(path, fileData)
 
 	if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
 		if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
@@ -283,26 +254,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
 	f := <-fileData
 
 	if f == nil {
-		var f2 []byte
+		c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path)
-		tries := 0
-		for {
-			time.Sleep(3000 * time.Millisecond)
-			tries++
-
-			asyncGetFile(bucket, path, fileData)
-			f2 = <-fileData
-
-			if f2 != nil {
-				w.Header().Set("Cache-Control", "max-age=2592000, public")
-				w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
-				w.Write(f2)
-				return
-			} else if tries >= 2 {
-				break
-			}
-		}
-
-		c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
 		c.Err.StatusCode = http.StatusNotFound
 		return
 	}
@@ -312,10 +264,11 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Write(f)
 }
 
-func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
+func asyncGetFile(path string, fileData chan []byte) {
 	go func() {
-		data, getErr := bucket.Get(path)
+		data, getErr := readFile(path)
 		if getErr != nil {
+			l4g.Error(getErr)
 			fileData <- nil
 		} else {
 			fileData <- data
@@ -329,8 +282,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
 		c.Err.StatusCode = http.StatusForbidden
 	}
 
-	if !utils.IsS3Configured() {
+	if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
+		c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
@@ -344,15 +297,14 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1)
-	if len(matches) == 0 || len(matches[0]) < 5 {
+	if len(matches) == 0 || len(matches[0]) < 4 {
 		c.SetInvalidParam("getPublicLink", "filename")
 		return
 	}
 
-	getType := matches[0][1]
+	channelId := matches[0][1]
-	channelId := matches[0][2]
+	userId := matches[0][2]
-	userId := matches[0][3]
+	filename = matches[0][3]
-	filename = matches[0][4]
 
 	cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
 
@@ -363,7 +315,7 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
 	data := model.MapToJson(newProps)
 	hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt))
 
-	url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)
+	url := fmt.Sprintf("%s/api/v1/files/get/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)
 
 	if !c.HasPermissionsToChannel(cchan, "getPublicLink") {
 		return
@@ -374,3 +326,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	w.Write([]byte(model.MapToJson(rData)))
 }
+
+func writeFile(f []byte, path string) *model.AppError {
+
+	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
+		var auth aws.Auth
+		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+
+		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+
+		ext := filepath.Ext(path)
+
+		var err error
+		if model.IsFileExtImage(ext) {
+			options := s3.Options{}
+			err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
+
+		} else {
+			options := s3.Options{}
+			err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
+		}
+
+		if err != nil {
+			return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
+		}
+	} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
+		if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil {
+			return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error())
+		}
+
+		if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil {
+			return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error())
+		}
+	} else {
+		return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
+	}
+
+	return nil
+}
+
+func readFile(path string) ([]byte, *model.AppError) {
+
+	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
+		var auth aws.Auth
+		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+
+		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+
+		// try to get the file from S3 with some basic retry logic
+		tries := 0
+		for {
+			tries++
+
+			f, err := bucket.Get(path)
+
+			if f != nil {
+				return f, nil
+			} else if tries >= 3 {
+				return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
+			}
+			time.Sleep(3000 * time.Millisecond)
+		}
+	} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
+		if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
+			return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
+		} else {
+			return f, nil
+		}
+	} else {
+		return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
+	}
+}

api/file_test.go

@@ -38,7 +38,7 @@ func TestUploadFile(t *testing.T) {
 
 	body := &bytes.Buffer{}
 	writer := multipart.NewWriter(body)
-	part, err := writer.CreateFormFile("files", "test.png")
+	part, err := writer.CreateFormFile("files", "../test.png")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -68,12 +68,17 @@ func TestUploadFile(t *testing.T) {
 	}
 
 	resp, appErr := Client.UploadFile("/files/upload", body.Bytes(), writer.FormDataContentType())
-	if utils.IsS3Configured() {
+	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
 		if appErr != nil {
 			t.Fatal(appErr)
 		}
 
-		filenames := resp.Data.(*model.FileUploadResponse).Filenames
+		filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+		filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+		if strings.Contains(filename, "../") {
+			t.Fatal("relative path should have been sanitized out")
+		}
+		fileId := strings.Split(filename, ".")[0]
 
 		var auth aws.Auth
 		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
@@ -82,12 +87,10 @@ func TestUploadFile(t *testing.T) {
 		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
 		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
 
-		fileId := strings.Split(filenames[0], ".")[0]
-
 		// wait a bit for files to ready
 		time.Sleep(5 * time.Second)
 
-		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0])
+		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -97,13 +100,38 @@ func TestUploadFile(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png")
+		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
 		if err != nil {
 			t.Fatal(err)
 		}
+	} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
+		filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+		filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+		if strings.Contains(filename, "../") {
+			t.Fatal("relative path should have been sanitized out")
+		}
+		fileId := strings.Split(filename, ".")[0]
+
+		// wait a bit for files to ready
+		time.Sleep(5 * time.Second)
+
+		path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
+		if err := os.Remove(path); err != nil {
+			t.Fatal("Couldn't remove file at " + path)
+		}
+
+		path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
+		if err := os.Remove(path); err != nil {
+			t.Fatal("Couldn't remove file at " + path)
+		}
+
+		path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
+		if err := os.Remove(path); err != nil {
+			t.Fatal("Couldn't remove file at " + path)
+		}
 	} else {
 		if appErr == nil {
-			t.Fatal("S3 not configured, should have failed")
+			t.Fatal("S3 and local storage not configured, should have failed")
 		}
 	}
 }
@@ -123,7 +151,7 @@ func TestGetFile(t *testing.T) {
 	channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
 	channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel)
 
-	if utils.IsS3Configured() {
+	if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
 
 		body := &bytes.Buffer{}
 		writer := multipart.NewWriter(body)
@@ -169,8 +197,8 @@ func TestGetFile(t *testing.T) {
 		// wait a bit for files to ready
 		time.Sleep(5 * time.Second)
 
-		if _, downErr := Client.GetFile(filenames[0], true); downErr != nil {
+		if _, downErr := Client.GetFile(filenames[0], false); downErr != nil {
-			t.Fatal("file get failed")
+			t.Fatal(downErr)
 		}
 
 		team2 := &model.Team{DisplayName: "Name", Name: "z-z-" + model.NewId() + "a", Email: "test@nowhere.com", Type: model.TEAM_OPEN}
@@ -189,35 +217,35 @@ func TestGetFile(t *testing.T) {
 
 		Client.LoginByEmail(team2.Name, user2.Email, "pwd")
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr != nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr != nil {
 			t.Fatal(downErr)
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), false); downErr == nil {
 			t.Fatal("Should have errored - missing team id")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", false); downErr == nil {
 			t.Fatal("Should have errored - bad team id")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", false); downErr == nil {
 			t.Fatal("Should have errored - bad team id")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, false); downErr == nil {
 			t.Fatal("Should have errored - missing hash")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, false); downErr == nil {
 			t.Fatal("Should have errored - bad hash")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil {
 			t.Fatal("Should have errored - missing data")
 		}
 
-		if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil {
+		if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil {
 			t.Fatal("Should have errored - bad data")
 		}
 
@@ -225,28 +253,51 @@ func TestGetFile(t *testing.T) {
 			t.Fatal("Should have errored - user not logged in and link not public")
 		}
 
-		var auth aws.Auth
+		if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			var auth aws.Auth
-		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+			auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
 
-		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+			s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+			bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
 
-		fileId := strings.Split(filenames[0], ".")[0]
+			filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+			filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+			fileId := strings.Split(filename, ".")[0]
 
-		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0])
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
-		if err != nil {
+			if err != nil {
-			t.Fatal(err)
+				t.Fatal(err)
-		}
+			}
 
-		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
-		if err != nil {
+			if err != nil {
-			t.Fatal(err)
+				t.Fatal(err)
-		}
+			}
 
-		err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png")
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
-		if err != nil {
+			if err != nil {
-			t.Fatal(err)
+				t.Fatal(err)
+			}
+		} else {
+			filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+			filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+			fileId := strings.Split(filename, ".")[0]
+
+			path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
+
+			path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
+
+			path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
 		}
 	} else {
 		if _, downErr := Client.GetFile("/files/get/yxebdmbz5pgupx7q6ez88rw11a/n3btzxu9hbnapqk36iwaxkjxhc/junk.jpg", false); downErr.StatusCode != http.StatusNotImplemented {
@@ -274,7 +325,7 @@ func TestGetPublicLink(t *testing.T) {
 	channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
 	channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel)
 
-	if utils.IsS3Configured() {
+	if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
 
 		body := &bytes.Buffer{}
 		writer := multipart.NewWriter(body)
@@ -350,26 +401,52 @@ func TestGetPublicLink(t *testing.T) {
 			t.Fatal("should have errored, user not member of channel")
 		}
 
-		// perform clean-up on s3
+		if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		var auth aws.Auth
+			// perform clean-up on s3
-		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			var auth aws.Auth
-		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+			auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
 
-		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+			s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+			bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
 
-		fileId := strings.Split(filenames[0], ".")[0]
+			filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+			filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+			fileId := strings.Split(filename, ".")[0]
 
-		if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + filenames[0]); err != nil {
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
-			t.Fatal(err)
+			if err != nil {
-		}
+				t.Fatal(err)
+			}
 
-		if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_thumb.jpg"); err != nil {
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
-			t.Fatal(err)
+			if err != nil {
-		}
+				t.Fatal(err)
+			}
 
-		if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_preview.png"); err != nil {
+			err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
-			t.Fatal(err)
+			if err != nil {
+				t.Fatal(err)
+			}
+		} else {
+			filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
+			filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
+			fileId := strings.Split(filename, ".")[0]
+
+			path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
+
+			path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
+
+			path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
 		}
 	} else {
 		data := make(map[string]string)

api/post.go

@@ -160,6 +160,39 @@ func CreatePost(c *Context, post *model.Post, doUpdateLastViewed bool) (*model.P
 
 	post.UserId = c.Session.UserId
 
+	if len(post.Filenames) > 0 {
+		doRemove := false
+		for i := len(post.Filenames) - 1; i >= 0; i-- {
+			path := post.Filenames[i]
+
+			doRemove = false
+			if model.UrlRegex.MatchString(path) {
+				continue
+			} else if model.PartialUrlRegex.MatchString(path) {
+				matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1)
+				if len(matches) == 0 || len(matches[0]) < 4 {
+					doRemove = true
+				}
+
+				channelId := matches[0][1]
+				if channelId != post.ChannelId {
+					doRemove = true
+				}
+
+				userId := matches[0][2]
+				if userId != post.UserId {
+					doRemove = true
+				}
+			} else {
+				doRemove = true
+			}
+			if doRemove {
+				l4g.Error("Bad filename discarded, filename=%v", path)
+				post.Filenames = append(post.Filenames[:i], post.Filenames[i+1:]...)
+			}
+		}
+	}
+
 	var rpost *model.Post
 	if result := <-Srv.Store.Post().Save(post); result.Err != nil {
 		return nil, result.Err

api/post_test.go

@@ -37,7 +37,7 @@ func TestCreatePost(t *testing.T) {
 	channel2 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
 	channel2 = Client.Must(Client.CreateChannel(channel2)).Data.(*model.Channel)
 
-	filenames := []string{"/api/v1/files/get/12345678901234567890123456/12345678901234567890123456/test.png", "/api/v1/files/get/" + channel1.Id + "/" + user1.Id + "/test.png"}
+	filenames := []string{"/12345678901234567890123456/12345678901234567890123456/12345678901234567890123456/test.png", "/" + channel1.Id + "/" + user1.Id + "/test.png", "www.mattermost.com/fake/url", "junk"}
 
 	post1 := &model.Post{ChannelId: channel1.Id, Message: "#hashtag a" + model.NewId() + "a", Filenames: filenames}
 	rpost1, err := Client.CreatePost(post1)

api/user.go

@@ -7,8 +7,6 @@ import (
 	"bytes"
 	l4g "code.google.com/p/log4go"
 	"fmt"
-	"github.com/goamz/goamz/aws"
-	"github.com/goamz/goamz/s3"
 	"github.com/gorilla/mux"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/store"
@@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError
 	buf := new(bytes.Buffer)
 
 	if imgErr := png.Encode(buf, img); imgErr != nil {
-		return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error())
+		return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error())
 	} else {
 		return buf.Bytes(), nil
 	}
@@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 		var img []byte
-		var err *model.AppError
 
-		if !utils.IsS3Configured() {
+		if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-			img, err = createProfileImage(result.Data.(*model.User).Username, id)
+			var err *model.AppError
-			if err != nil {
+			if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
 				c.Err = err
 				return
 			}
 		} else {
-			var auth aws.Auth
-			auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
-			auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
-
-			s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-			bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
-
 			path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png"
 
-			if data, getErr := bucket.Get(path); getErr != nil {
+			if data, err := readFile(path); err != nil {
-				img, err = createProfileImage(result.Data.(*model.User).Username, id)
+
-				if err != nil {
+				if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
 					c.Err = err
 					return
 				}
 
-				options := s3.Options{}
+				if err := writeFile(img, path); err != nil {
-				if err := bucket.Put(path, img, "image", s3.Private, options); err != nil {
+					c.Err = err
-					c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error())
 					return
 				}
 
@@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
 }
 
 func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
-	if !utils.IsS3Configured() {
+	if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "")
+		c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
@@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var auth aws.Auth
-	auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
-	auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
-
-	s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-	bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
-
 	m := r.MultipartForm
 
 	imageArray, ok := m.File["image"]
@@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png"
 
-	options := s3.Options{}
+	if err := writeFile(buf.Bytes(), path); err != nil {
-	if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil {
 		c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "")
 		return
 	}

api/user_test.go

@@ -356,6 +356,24 @@ func TestUserCreateImage(t *testing.T) {
 
 	Client.DoGet("/users/"+user.Id+"/image", "", "")
 
+	if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
+		var auth aws.Auth
+		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+
+		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+
+		if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"
+		if err := os.Remove(path); err != nil {
+			t.Fatal("Couldn't remove file at " + path)
+		}
+	}
+
 }
 
 func TestUserUploadProfileImage(t *testing.T) {
@@ -368,7 +386,7 @@ func TestUserUploadProfileImage(t *testing.T) {
 	user = Client.Must(Client.CreateUser(user, "")).Data.(*model.User)
 	store.Must(Srv.Store.User().VerifyEmail(user.Id))
 
-	if utils.IsS3Configured() {
+	if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
 
 		body := &bytes.Buffer{}
 		writer := multipart.NewWriter(body)
@@ -436,15 +454,22 @@ func TestUserUploadProfileImage(t *testing.T) {
 
 		Client.DoGet("/users/"+user.Id+"/image", "", "")
 
-		var auth aws.Auth
+		if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
-		auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			var auth aws.Auth
-		auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
+			auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
+			auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
 
-		s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
+			s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
-		bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
+			bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
 
-		if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
+			if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
-			t.Fatal(err)
+				t.Fatal(err)
+			}
+		} else {
+			path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"
+			if err := os.Remove(path); err != nil {
+				t.Fatal("Couldn't remove file at " + path)
+			}
 		}
 	} else {
 		body := &bytes.Buffer{}

config/config.json

@@ -19,7 +19,9 @@
         "InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
         "PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
         "ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
-        "AnalyticsUrl": ""
+        "AnalyticsUrl": "",
+        "UseLocalStorage": true,
+        "StorageDirectory": "./data/"
     },
     "SqlSettings": {
         "DriverName": "mysql",

config/config_docker.json

@@ -19,7 +19,9 @@
         "InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
         "PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
         "ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
-        "AnalyticsUrl": ""
+        "AnalyticsUrl": "",
+        "UseLocalStorage": true,
+        "StorageDirectory": "/mattermost/data/"
     },
     "SqlSettings": {
         "DriverName": "mysql",

model/client.go

@@ -550,7 +550,7 @@ func (c *Client) GetFile(url string, isFullUrl bool) (*Result, *AppError) {
 	if isFullUrl {
 		rq, _ = http.NewRequest("GET", url, nil)
 	} else {
-		rq, _ = http.NewRequest("GET", c.Url+url, nil)
+		rq, _ = http.NewRequest("GET", c.Url+"/files/get"+url, nil)
 	}
 
 	if len(c.AuthToken) > 0 {

model/utils.go

@@ -319,6 +319,6 @@ func ClearMentionTags(post string) string {
 }
 
 var UrlRegex = regexp.MustCompile(`^((?:[a-z]+:\/\/)?(?:(?:[a-z0-9\-]+\.)+(?:[a-z]{2}|aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|local|internal))(:[0-9]{1,5})?(?:\/[a-z0-9_\-\.~]+)*(\/([a-z0-9_\-\.]*)(?:\?[a-z0-9+_~\-\.%=&]*)?)?(?:#[a-zA-Z0-9!$&'()*+.=-_~:@/?]*)?)(?:\s+|$)$`)
-var PartialUrlRegex = regexp.MustCompile(`/api/v1/files/(get|get_image)/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/(([A-Za-z0-9]+/)?.+\.[A-Za-z0-9]{3,})`)
+var PartialUrlRegex = regexp.MustCompile(`/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/((?:[A-Za-z0-9]{26})?.+\.[A-Za-z0-9]{3,})`)
 
 var SplitRunes = map[rune]bool{',': true, ' ': true, '.': true, '!': true, '?': true, ':': true, ';': true, '\n': true, '<': true, '>': true, '(': true, ')': true, '{': true, '}': true, '[': true, ']': true, '+': true, '/': true, '\\': true}

utils/config.go

@@ -18,16 +18,18 @@ const (
 )
 
 type ServiceSettings struct {
-	SiteName       string
+	SiteName         string
-	Mode           string
+	Mode             string
-	AllowTesting   bool
+	AllowTesting     bool
-	UseSSL         bool
+	UseSSL           bool
-	Port           string
+	Port             string
-	Version        string
+	Version          string
-	InviteSalt     string
+	InviteSalt       string
-	PublicLinkSalt string
+	PublicLinkSalt   string
-	ResetSalt      string
+	ResetSalt        string
-	AnalyticsUrl   string
+	AnalyticsUrl     string
+	UseLocalStorage  bool
+	StorageDirectory string
 }
 
 type SqlSettings struct {

utils/urlencode.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package utils
+
+import (
+	"net/url"
+	"strings"
+)
+
+func UrlEncode(str string) string {
+	strs := strings.Split(str, " ")
+
+	for i, s := range strs {
+		strs[i] = url.QueryEscape(s)
+	}
+
+	return strings.Join(strs, "%20")
+}

web/react/components/view_image.jsx

@@ -155,7 +155,7 @@ module.exports = React.createClass({
                 var imgClass = "hidden";
                 if (this.state.loaded[id] && this.state.imgId == id) imgClass = "";
 
-                img[info['path']] = <a key={info['path']} className={imgClass} href={this.props.filenames[id]} target="_blank"><img ref="image" src={preview_filename}/></a>;
+                img[info['path']] = <a key={info['path']} className={imgClass} href={info.path+"."+info.ext} target="_blank"><img ref="image" src={preview_filename}/></a>;
             }
         }