mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Merge pull request #205 from mattermost/mm-1351

Просмотр исходного кода 
fixes mm-1351 adds local server storage
Этот коммит содержится в:
Corey Hulen
2015-07-21 15:46:12 -08:00
родитель 06bac01e88 237920e314
Коммит 269185fb0e
16 изменённых файлов: 372 добавлений и 195 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
.gitignore поставляемый
Просмотреть файл

@@ -47,3 +47,7 @@ web/sass-files/sass/.sass-cache/
*config.codekit
*.sass-cache
*styles.css
# Default local file storage
data/*
api/data/*

11
Dockerfile
Просмотреть файл

@@ -40,7 +40,7 @@ WORKDIR /go
#
# Install SQL
#
#
ENV MYSQL_ROOT_PASSWORD=mostest
ENV MYSQL_USER=mmuser
@@ -60,7 +60,7 @@ RUN echo "deb http://repo.mysql.com/apt/debian/ wheezy mysql-${MYSQL_MAJOR}" > /
RUN apt-get update \
&& export DEBIAN_FRONTEND=noninteractive \
&& apt-get -y install mysql-server \
&& apt-get -y install mysql-server \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql
@@ -88,12 +88,15 @@ ADD . /go/src/github.com/mattermost/platform
ADD ./docker/main.cf /etc/postfix/
RUN go get github.com/tools/godep
RUN cd /go/src/github.com/mattermost/platform; godep restore
RUN cd /go/src/github.com/mattermost/platform; godep restore
RUN go install github.com/mattermost/platform
RUN cd /go/src/github.com/mattermost/platform/web/react; npm install
RUN cd /go/src/github.com/mattermost/platform/web/react; npm install
RUN chmod +x /go/src/github.com/mattermost/platform/docker/docker-entry.sh
ENTRYPOINT /go/src/github.com/mattermost/platform/docker/docker-entry.sh
# Create default storage directory
RUN mkdir /mattermost/
# Ports
EXPOSE 80

2
Makefile
Просмотреть файл

@@ -112,6 +112,8 @@ clean:
rm -f web/static/js/bundle*.js
rm -f web/static/css/styles.css
rm -rf data/*
rm -rf api/data/*
rm -rf logs/*

195
api/file.go
Просмотреть файл

@@ -18,8 +18,10 @@ import (
_ "image/gif"
"image/jpeg"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
@@ -27,7 +29,7 @@ import (
)
func InitFile(r *mux.Router) {
l4g.Debug("Initializing post api routes")
l4g.Debug("Initializing file api routes")
sr := r.PathPrefix("/files").Subrouter()
sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
@@ -36,8 +38,8 @@ func InitFile(r *mux.Router) {
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
props := m.Value
@@ -94,28 +89,25 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
ext := filepath.Ext(files[i].Filename)
filename := filepath.Base(files[i].Filename)
uid := model.NewId()
path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename
path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
imageDataList = append(imageDataList, buf.Bytes())
} else {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
}
if err != nil {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = err
return
}
fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
if model.IsFileExtImage(filepath.Ext(files[i].Filename)) {
imageNameList = append(imageNameList, uid+"/"+filename)
imageDataList = append(imageDataList, buf.Bytes())
}
encName := utils.UrlEncode(filename)
fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName
resStruct.Filenames = append(resStruct.Filenames, fileUrl)
}
@@ -127,13 +119,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {
go func() {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"
for i, filename := range filenames {
@@ -169,11 +154,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
return
}
// Upload thumbnail to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
if err != nil {
l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
if err := writeFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil {
l4g.Error("Unable to upload thumbnail channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
}
}()
@@ -188,19 +170,15 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
}
buf := new(bytes.Buffer)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
//err = png.Encode(buf, preview)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
if err != nil {
l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
}
// Upload preview to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
if err != nil {
l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
if err := writeFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil {
l4g.Error("Unable to upload preview channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
}
}()
@@ -215,8 +193,8 @@ type ImageGetResult struct {
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -247,13 +225,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := ""
if len(teamId) == 26 {
path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
@@ -262,7 +233,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
fileData := make(chan []byte)
asyncGetFile(bucket, path, fileData)
asyncGetFile(path, fileData)
if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
@@ -283,26 +254,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
f := <-fileData
if f == nil {
var f2 []byte
tries := 0
for {
time.Sleep(3000 * time.Millisecond)
tries++
asyncGetFile(bucket, path, fileData)
f2 = <-fileData
if f2 != nil {
w.Header().Set("Cache-Control", "max-age=2592000, public")
w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
w.Write(f2)
return
} else if tries >= 2 {
break
}
}
c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path)
c.Err.StatusCode = http.StatusNotFound
return
}
@@ -312,10 +264,11 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write(f)
}
func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
func asyncGetFile(path string, fileData chan []byte) {
go func() {
data, getErr := bucket.Get(path)
data, getErr := readFile(path)
if getErr != nil {
l4g.Error(getErr)
fileData <- nil
} else {
fileData <- data
@@ -329,8 +282,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
c.Err.StatusCode = http.StatusForbidden
}
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -344,15 +297,14 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
}
matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1)
if len(matches) == 0 || len(matches[0]) < 5 {
if len(matches) == 0 || len(matches[0]) < 4 {
c.SetInvalidParam("getPublicLink", "filename")
return
}
getType := matches[0][1]
channelId := matches[0][2]
userId := matches[0][3]
filename = matches[0][4]
channelId := matches[0][1]
userId := matches[0][2]
filename = matches[0][3]
cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
@@ -363,7 +315,7 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
data := model.MapToJson(newProps)
hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt))
url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)
url := fmt.Sprintf("%s/api/v1/files/get/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId)
if !c.HasPermissionsToChannel(cchan, "getPublicLink") {
return
@@ -374,3 +326,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(model.MapToJson(rData)))
}
func writeFile(f []byte, path string) *model.AppError {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
ext := filepath.Ext(path)
var err error
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
} else {
options := s3.Options{}
err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
}
if err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil {
return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error())
}
if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error())
}
} else {
return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
return nil
}
func readFile(path string) ([]byte, *model.AppError) {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
// try to get the file from S3 with some basic retry logic
tries := 0
for {
tries++
f, err := bucket.Get(path)
if f != nil {
return f, nil
} else if tries >= 3 {
return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
}
time.Sleep(3000 * time.Millisecond)
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
} else {
return f, nil
}
} else {
return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
}

181
api/file_test.go
Просмотреть файл

@@ -38,7 +38,7 @@ func TestUploadFile(t *testing.T) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("files", "test.png")
part, err := writer.CreateFormFile("files", "../test.png")
if err != nil {
t.Fatal(err)
}
@@ -68,12 +68,17 @@ func TestUploadFile(t *testing.T) {
}
resp, appErr := Client.UploadFile("/files/upload", body.Bytes(), writer.FormDataContentType())
if utils.IsS3Configured() {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
if appErr != nil {
t.Fatal(appErr)
}
filenames := resp.Data.(*model.FileUploadResponse).Filenames
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
if strings.Contains(filename, "../") {
t.Fatal("relative path should have been sanitized out")
}
fileId := strings.Split(filename, ".")[0]
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
@@ -82,12 +87,10 @@ func TestUploadFile(t *testing.T) {
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
fileId := strings.Split(filenames[0], ".")[0]
// wait a bit for files to ready
time.Sleep(5 * time.Second)
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0])
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
if err != nil {
t.Fatal(err)
}
@@ -97,13 +100,38 @@ func TestUploadFile(t *testing.T) {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png")
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
if err != nil {
t.Fatal(err)
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
if strings.Contains(filename, "../") {
t.Fatal("relative path should have been sanitized out")
}
fileId := strings.Split(filename, ".")[0]
// wait a bit for files to ready
time.Sleep(5 * time.Second)
path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
} else {
if appErr == nil {
t.Fatal("S3 not configured, should have failed")
t.Fatal("S3 and local storage not configured, should have failed")
}
}
}
@@ -123,7 +151,7 @@ func TestGetFile(t *testing.T) {
channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel)
if utils.IsS3Configured() {
if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
@@ -169,8 +197,8 @@ func TestGetFile(t *testing.T) {
// wait a bit for files to ready
time.Sleep(5 * time.Second)
if _, downErr := Client.GetFile(filenames[0], true); downErr != nil {
t.Fatal("file get failed")
if _, downErr := Client.GetFile(filenames[0], false); downErr != nil {
t.Fatal(downErr)
}
team2 := &model.Team{DisplayName: "Name", Name: "z-z-" + model.NewId() + "a", Email: "test@nowhere.com", Type: model.TEAM_OPEN}
@@ -189,35 +217,35 @@ func TestGetFile(t *testing.T) {
Client.LoginByEmail(team2.Name, user2.Email, "pwd")
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr != nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr != nil {
t.Fatal(downErr)
}
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash), false); downErr == nil {
t.Fatal("Should have errored - missing team id")
}
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=junk", false); downErr == nil {
t.Fatal("Should have errored - bad team id")
}
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h="+url.QueryEscape(hash)+"&t=12345678901234567890123456", false); downErr == nil {
t.Fatal("Should have errored - bad team id")
}
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&t="+team.Id, false); downErr == nil {
t.Fatal("Should have errored - missing hash")
}
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d="+url.QueryEscape(data)+"&h=junk&t="+team.Id, false); downErr == nil {
t.Fatal("Should have errored - bad hash")
}
if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil {
t.Fatal("Should have errored - missing data")
}
if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, true); downErr == nil {
if _, downErr := Client.GetFile(filenames[0]+"?d=junk&h="+url.QueryEscape(hash)+"&t="+team.Id, false); downErr == nil {
t.Fatal("Should have errored - bad data")
}
@@ -225,28 +253,51 @@ func TestGetFile(t *testing.T) {
t.Fatal("Should have errored - user not logged in and link not public")
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
fileId := strings.Split(filenames[0], ".")[0]
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
fileId := strings.Split(filename, ".")[0]
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filenames[0])
if err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
if err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
if err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
if err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.png")
if err != nil {
t.Fatal(err)
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
if err != nil {
t.Fatal(err)
}
} else {
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
fileId := strings.Split(filename, ".")[0]
path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
}
} else {
if _, downErr := Client.GetFile("/files/get/yxebdmbz5pgupx7q6ez88rw11a/n3btzxu9hbnapqk36iwaxkjxhc/junk.jpg", false); downErr.StatusCode != http.StatusNotImplemented {
@@ -274,7 +325,7 @@ func TestGetPublicLink(t *testing.T) {
channel1 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
channel1 = Client.Must(Client.CreateChannel(channel1)).Data.(*model.Channel)
if utils.IsS3Configured() {
if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
@@ -350,26 +401,52 @@ func TestGetPublicLink(t *testing.T) {
t.Fatal("should have errored, user not member of channel")
}
// perform clean-up on s3
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
// perform clean-up on s3
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
fileId := strings.Split(filenames[0], ".")[0]
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
fileId := strings.Split(filename, ".")[0]
if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + filenames[0]); err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename)
if err != nil {
t.Fatal(err)
}
if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_thumb.jpg"); err != nil {
t.Fatal(err)
}
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg")
if err != nil {
t.Fatal(err)
}
if err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + rpost1.Data.(*model.Post).UserId + "/" + fileId + "_preview.png"); err != nil {
t.Fatal(err)
err = bucket.Del("teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg")
if err != nil {
t.Fatal(err)
}
} else {
filenames := strings.Split(resp.Data.(*model.FileUploadResponse).Filenames[0], "/")
filename := filenames[len(filenames)-2] + "/" + filenames[len(filenames)-1]
fileId := strings.Split(filename, ".")[0]
path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + filename
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_thumb.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
path = utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + team.Id + "/channels/" + channel1.Id + "/users/" + user1.Id + "/" + fileId + "_preview.jpg"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
}
} else {
data := make(map[string]string)

33
api/post.go
Просмотреть файл

@@ -160,6 +160,39 @@ func CreatePost(c *Context, post *model.Post, doUpdateLastViewed bool) (*model.P
post.UserId = c.Session.UserId
if len(post.Filenames) > 0 {
doRemove := false
for i := len(post.Filenames) - 1; i >= 0; i-- {
path := post.Filenames[i]
doRemove = false
if model.UrlRegex.MatchString(path) {
continue
} else if model.PartialUrlRegex.MatchString(path) {
matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1)
if len(matches) == 0 || len(matches[0]) < 4 {
doRemove = true
}
channelId := matches[0][1]
if channelId != post.ChannelId {
doRemove = true
}
userId := matches[0][2]
if userId != post.UserId {
doRemove = true
}
} else {
doRemove = true
}
if doRemove {
l4g.Error("Bad filename discarded, filename=%v", path)
post.Filenames = append(post.Filenames[:i], post.Filenames[i+1:]...)
}
}
}
var rpost *model.Post
if result := <-Srv.Store.Post().Save(post); result.Err != nil {
return nil, result.Err

2
api/post_test.go
Просмотреть файл

@@ -37,7 +37,7 @@ func TestCreatePost(t *testing.T) {
channel2 := &model.Channel{DisplayName: "Test API Name", Name: "a" + model.NewId() + "a", Type: model.CHANNEL_OPEN, TeamId: team.Id}
channel2 = Client.Must(Client.CreateChannel(channel2)).Data.(*model.Channel)
filenames := []string{"/api/v1/files/get/12345678901234567890123456/12345678901234567890123456/test.png", "/api/v1/files/get/" + channel1.Id + "/" + user1.Id + "/test.png"}
filenames := []string{"/12345678901234567890123456/12345678901234567890123456/12345678901234567890123456/test.png", "/" + channel1.Id + "/" + user1.Id + "/test.png", "www.mattermost.com/fake/url", "junk"}
post1 := &model.Post{ChannelId: channel1.Id, Message: "#hashtag a" + model.NewId() + "a", Filenames: filenames}
rpost1, err := Client.CreatePost(post1)

43
api/user.go
Просмотреть файл

@@ -7,8 +7,6 @@ import (
"bytes"
l4g "code.google.com/p/log4go"
"fmt"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/gorilla/mux"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
@@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError
buf := new(bytes.Buffer)
if imgErr := png.Encode(buf, img); imgErr != nil {
return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error())
return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error())
} else {
return buf.Bytes(), nil
}
@@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
} else {
var img []byte
var err *model.AppError
if !utils.IsS3Configured() {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var err *model.AppError
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
} else {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png"
if data, getErr := bucket.Get(path); getErr != nil {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if data, err := readFile(path); err != nil {
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
options := s3.Options{}
if err := bucket.Put(path, img, "image", s3.Private, options); err != nil {
c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error())
if err := writeFile(img, path); err != nil {
c.Err = err
return
}
@@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
}
func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
imageArray, ok := m.File["image"]
@@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png"
options := s3.Options{}
if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil {
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "")
return
}

41
api/user_test.go
Просмотреть файл

@@ -356,6 +356,24 @@ func TestUserCreateImage(t *testing.T) {
Client.DoGet("/users/"+user.Id+"/image", "", "")
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
t.Fatal(err)
}
} else {
path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
}
}
func TestUserUploadProfileImage(t *testing.T) {
@@ -368,7 +386,7 @@ func TestUserUploadProfileImage(t *testing.T) {
user = Client.Must(Client.CreateUser(user, "")).Data.(*model.User)
store.Must(Srv.Store.User().VerifyEmail(user.Id))
if utils.IsS3Configured() {
if utils.IsS3Configured() || utils.Cfg.ServiceSettings.UseLocalStorage {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
@@ -436,15 +454,22 @@ func TestUserUploadProfileImage(t *testing.T) {
Client.DoGet("/users/"+user.Id+"/image", "", "")
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
t.Fatal(err)
if err := bucket.Del("teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"); err != nil {
t.Fatal(err)
}
} else {
path := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + user.TeamId + "/users/" + user.Id + "/profile.png"
if err := os.Remove(path); err != nil {
t.Fatal("Couldn't remove file at " + path)
}
}
} else {
body := &bytes.Buffer{}

4
config/config.json
Просмотреть файл

@@ -19,7 +19,9 @@
"InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
"PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
"ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
"AnalyticsUrl": ""
"AnalyticsUrl": "",
"UseLocalStorage": true,
"StorageDirectory": "./data/"
},
"SqlSettings": {
"DriverName": "mysql",

4
config/config_docker.json
Просмотреть файл

@@ -19,7 +19,9 @@
"InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
"PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
"ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
"AnalyticsUrl": ""
"AnalyticsUrl": "",
"UseLocalStorage": true,
"StorageDirectory": "/mattermost/data/"
},
"SqlSettings": {
"DriverName": "mysql",

2
model/client.go
Просмотреть файл

@@ -550,7 +550,7 @@ func (c *Client) GetFile(url string, isFullUrl bool) (*Result, *AppError) {
if isFullUrl {
rq, _ = http.NewRequest("GET", url, nil)
} else {
rq, _ = http.NewRequest("GET", c.Url+url, nil)
rq, _ = http.NewRequest("GET", c.Url+"/files/get"+url, nil)
}
if len(c.AuthToken) > 0 {

2
model/utils.go
Просмотреть файл

@@ -319,6 +319,6 @@ func ClearMentionTags(post string) string {
}
var UrlRegex = regexp.MustCompile(`^((?:[a-z]+:\/\/)?(?:(?:[a-z0-9\-]+\.)+(?:[a-z]{2}|aero|arpa|biz|com|coop|edu|gov|info|int|jobs|mil|museum|name|nato|net|org|pro|travel|local|internal))(:[0-9]{1,5})?(?:\/[a-z0-9_\-\.~]+)*(\/([a-z0-9_\-\.]*)(?:\?[a-z0-9+_~\-\.%=&amp;]*)?)?(?:#[a-zA-Z0-9!$&'()*+.=-_~:@/?]*)?)(?:\s+|$)$`)
var PartialUrlRegex = regexp.MustCompile(`/api/v1/files/(get|get_image)/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/(([A-Za-z0-9]+/)?.+\.[A-Za-z0-9]{3,})`)
var PartialUrlRegex = regexp.MustCompile(`/([A-Za-z0-9]{26})/([A-Za-z0-9]{26})/((?:[A-Za-z0-9]{26})?.+\.[A-Za-z0-9]{3,})`)
var SplitRunes = map[rune]bool{',': true, ' ': true, '.': true, '!': true, '?': true, ':': true, ';': true, '\n': true, '<': true, '>': true, '(': true, ')': true, '{': true, '}': true, '[': true, ']': true, '+': true, '/': true, '\\': true}

22
utils/config.go
Просмотреть файл

@@ -18,16 +18,18 @@ const (
)
type ServiceSettings struct {
SiteName string
Mode string
AllowTesting bool
UseSSL bool
Port string
Version string
InviteSalt string
PublicLinkSalt string
ResetSalt string
AnalyticsUrl string
SiteName string
Mode string
AllowTesting bool
UseSSL bool
Port string
Version string
InviteSalt string
PublicLinkSalt string
ResetSalt string
AnalyticsUrl string
UseLocalStorage bool
StorageDirectory string
}
type SqlSettings struct {

19
utils/urlencode.go Обычный файл
Просмотреть файл

@@ -0,0 +1,19 @@
// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"net/url"
"strings"
)
func UrlEncode(str string) string {
strs := strings.Split(str, " ")
for i, s := range strs {
strs[i] = url.QueryEscape(s)
}
return strings.Join(strs, "%20")
}

2
web/react/components/view_image.jsx
Просмотреть файл

@@ -155,7 +155,7 @@ module.exports = React.createClass({
var imgClass = "hidden";
if (this.state.loaded[id] && this.state.imgId == id) imgClass = "";
img[info['path']] = <a key={info['path']} className={imgClass} href={this.props.filenames[id]} target="_blank"><img ref="image" src={preview_filename}/></a>;
img[info['path']] = <a key={info['path']} className={imgClass} href={info.path+"."+info.ext} target="_blank"><img ref="image" src={preview_filename}/></a>;
}
}