mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

initial implementation of local server storage for files

Просмотр исходного кода
Этот коммит содержится в:
JoramWilander
2015-07-16 08:54:09 -04:00
родитель 06bac01e88
Коммит ada84835ee
5 изменённых файлов: 127 добавлений и 115 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

169
api/file.go
Просмотреть файл

@@ -18,8 +18,10 @@ import (
_ "image/gif"
"image/jpeg"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
@@ -27,7 +29,7 @@ import (
)
func InitFile(r *mux.Router) {
l4g.Debug("Initializing post api routes")
l4g.Debug("Initializing file api routes")
sr := r.PathPrefix("/files").Subrouter()
sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
@@ -36,8 +38,8 @@ func InitFile(r *mux.Router) {
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
props := m.Value
@@ -94,25 +89,18 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
ext := filepath.Ext(files[i].Filename)
uid := model.NewId()
path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
imageDataList = append(imageDataList, buf.Bytes())
} else {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = err
return
}
if err != nil {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
return
if model.IsFileExtImage(filepath.Ext(files[i].Filename)) {
imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
imageDataList = append(imageDataList, buf.Bytes())
}
fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
@@ -127,13 +115,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {
go func() {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"
for i, filename := range filenames {
@@ -169,9 +150,7 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
return
}
// Upload thumbnail to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
err = writeFile(buf.Bytes(), dest+name+"_thumb.jpg")
if err != nil {
l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
@@ -188,17 +167,14 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
}
buf := new(bytes.Buffer)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
//err = png.Encode(buf, preview)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
if err != nil {
l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
}
// Upload preview to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
err = writeFile(buf.Bytes(), dest+name+"_preview.jpg")
if err != nil {
l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
@@ -215,8 +191,8 @@ type ImageGetResult struct {
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -247,13 +223,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := ""
if len(teamId) == 26 {
path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
@@ -262,7 +231,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
fileData := make(chan []byte)
asyncGetFile(bucket, path, fileData)
asyncGetFile(path, fileData)
if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
@@ -283,26 +252,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
f := <-fileData
if f == nil {
var f2 []byte
tries := 0
for {
time.Sleep(3000 * time.Millisecond)
tries++
asyncGetFile(bucket, path, fileData)
f2 = <-fileData
if f2 != nil {
w.Header().Set("Cache-Control", "max-age=2592000, public")
w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
w.Write(f2)
return
} else if tries >= 2 {
break
}
}
c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path)
c.Err.StatusCode = http.StatusNotFound
return
}
@@ -312,9 +262,9 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write(f)
}
func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
func asyncGetFile(path string, fileData chan []byte) {
go func() {
data, getErr := bucket.Get(path)
data, getErr := readFile(path)
if getErr != nil {
fileData <- nil
} else {
@@ -329,8 +279,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
c.Err.StatusCode = http.StatusForbidden
}
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -374,3 +324,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(model.MapToJson(rData)))
}
func writeFile(f []byte, path string) *model.AppError {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
ext := filepath.Ext(path)
var err error
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
} else {
options := s3.Options{}
err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
}
if err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil {
return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error())
}
if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error())
}
} else {
return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
return nil
}
func readFile(path string) ([]byte, *model.AppError) {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
// try to get the file from S3 with some basic retry logic
tries := 0
for {
tries++
f, err := bucket.Get(path)
if f != nil {
return f, nil
} else if tries >= 3 {
return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
}
time.Sleep(3000 * time.Millisecond)
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
} else {
return f, nil
}
} else {
return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
}

43
api/user.go
Просмотреть файл

@@ -7,8 +7,6 @@ import (
"bytes"
l4g "code.google.com/p/log4go"
"fmt"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/gorilla/mux"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
@@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError
buf := new(bytes.Buffer)
if imgErr := png.Encode(buf, img); imgErr != nil {
return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error())
return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error())
} else {
return buf.Bytes(), nil
}
@@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
} else {
var img []byte
var err *model.AppError
if !utils.IsS3Configured() {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var err *model.AppError
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
} else {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png"
if data, getErr := bucket.Get(path); getErr != nil {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if data, err := readFile(path); err != nil {
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
options := s3.Options{}
if err := bucket.Put(path, img, "image", s3.Private, options); err != nil {
c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error())
if err := writeFile(img, path); err != nil {
c.Err = err
return
}
@@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
}
func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
imageArray, ok := m.File["image"]
@@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png"
options := s3.Options{}
if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil {
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "")
return
}

4
config/config.json
Просмотреть файл

@@ -19,7 +19,9 @@
"InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
"PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
"ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
"AnalyticsUrl": ""
"AnalyticsUrl": "",
"UseLocalStorage": true,
"StorageDirectory": "/mattermost/"
},
"SqlSettings": {
"DriverName": "mysql",

4
config/config_docker.json
Просмотреть файл

@@ -19,7 +19,9 @@
"InviteSalt": "gxHVDcKUyP2y1eiyW8S8na1UYQAfq6J6",
"PublicLinkSalt": "TO3pTyXIZzwHiwyZgGql7lM7DG3zeId4",
"ResetSalt": "IPxFzSfnDFsNsRafZxz8NaYqFKhf9y2t",
"AnalyticsUrl": ""
"AnalyticsUrl": "",
"UseLocalStorage": "",
"StorageDirectory": "/mattermost/"
},
"SqlSettings": {
"DriverName": "mysql",

22
utils/config.go
Просмотреть файл

@@ -18,16 +18,18 @@ const (
)
type ServiceSettings struct {
SiteName string
Mode string
AllowTesting bool
UseSSL bool
Port string
Version string
InviteSalt string
PublicLinkSalt string
ResetSalt string
AnalyticsUrl string
SiteName string
Mode string
AllowTesting bool
UseSSL bool
Port string
Version string
InviteSalt string
PublicLinkSalt string
ResetSalt string
AnalyticsUrl string
UseLocalStorage bool
StorageDirectory string
}
type SqlSettings struct {