mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

initial implementation of local server storage for files

Просмотр исходного кода
Этот коммит содержится в:
JoramWilander
2015-07-16 08:54:09 -04:00
родитель 06bac01e88
Коммит ada84835ee
5 изменённых файлов: 127 добавлений и 115 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

169
api/file.go
Просмотреть файл

@@ -18,8 +18,10 @@ import (
_ "image/gif"
"image/jpeg"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
@@ -27,7 +29,7 @@ import (
)
func InitFile(r *mux.Router) {
l4g.Debug("Initializing post api routes")
l4g.Debug("Initializing file api routes")
sr := r.PathPrefix("/files").Subrouter()
sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
@@ -36,8 +38,8 @@ func InitFile(r *mux.Router) {
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
props := m.Value
@@ -94,25 +89,18 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
ext := filepath.Ext(files[i].Filename)
uid := model.NewId()
path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options)
imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
imageDataList = append(imageDataList, buf.Bytes())
} else {
options := s3.Options{}
err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options)
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = err
return
}
if err != nil {
c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error())
return
if model.IsFileExtImage(filepath.Ext(files[i].Filename)) {
imageNameList = append(imageNameList, uid+"/"+files[i].Filename)
imageDataList = append(imageDataList, buf.Bytes())
}
fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename)
@@ -127,13 +115,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {
go func() {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"
for i, filename := range filenames {
@@ -169,9 +150,7 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
return
}
// Upload thumbnail to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
err = writeFile(buf.Bytes(), dest+name+"_thumb.jpg")
if err != nil {
l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
@@ -188,17 +167,14 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch
}
buf := new(bytes.Buffer)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
//err = png.Encode(buf, preview)
err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
if err != nil {
l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
}
// Upload preview to S3
options := s3.Options{}
err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options)
err = writeFile(buf.Bytes(), dest+name+"_preview.jpg")
if err != nil {
l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err)
return
@@ -215,8 +191,8 @@ type ImageGetResult struct {
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -247,13 +223,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId)
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := ""
if len(teamId) == 26 {
path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
@@ -262,7 +231,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
fileData := make(chan []byte)
asyncGetFile(bucket, path, fileData)
asyncGetFile(path, fileData)
if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 {
if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) {
@@ -283,26 +252,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
f := <-fileData
if f == nil {
var f2 []byte
tries := 0
for {
time.Sleep(3000 * time.Millisecond)
tries++
asyncGetFile(bucket, path, fileData)
f2 = <-fileData
if f2 != nil {
w.Header().Set("Cache-Control", "max-age=2592000, public")
w.Header().Set("Content-Length", strconv.Itoa(len(f2)))
w.Write(f2)
return
} else if tries >= 2 {
break
}
}
c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path)
c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path)
c.Err.StatusCode = http.StatusNotFound
return
}
@@ -312,9 +262,9 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write(f)
}
func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) {
func asyncGetFile(path string, fileData chan []byte) {
go func() {
data, getErr := bucket.Get(path)
data, getErr := readFile(path)
if getErr != nil {
fileData <- nil
} else {
@@ -329,8 +279,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
c.Err.StatusCode = http.StatusForbidden
}
if !utils.IsS3Configured() {
c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -374,3 +324,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(model.MapToJson(rData)))
}
func writeFile(f []byte, path string) *model.AppError {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
ext := filepath.Ext(path)
var err error
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
} else {
options := s3.Options{}
err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
}
if err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error())
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil {
return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error())
}
if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil {
return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error())
}
} else {
return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
return nil
}
func readFile(path string) ([]byte, *model.AppError) {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
// try to get the file from S3 with some basic retry logic
tries := 0
for {
tries++
f, err := bucket.Get(path)
if f != nil {
return f, nil
} else if tries >= 3 {
return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error())
}
time.Sleep(3000 * time.Millisecond)
}
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error())
} else {
return f, nil
}
} else {
return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
}

43
api/user.go
Просмотреть файл

@@ -7,8 +7,6 @@ import (
"bytes"
l4g "code.google.com/p/log4go"
"fmt"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/gorilla/mux"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
@@ -598,7 +596,7 @@ func createProfileImage(username string, userId string) ([]byte, *model.AppError
buf := new(bytes.Buffer)
if imgErr := png.Encode(buf, img); imgErr != nil {
return nil, model.NewAppError("getProfileImage", "Could not encode default profile image", imgErr.Error())
return nil, model.NewAppError("createProfileImage", "Could not encode default profile image", imgErr.Error())
} else {
return buf.Bytes(), nil
}
@@ -613,34 +611,25 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
} else {
var img []byte
var err *model.AppError
if !utils.IsS3Configured() {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
var err *model.AppError
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
} else {
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
path := "teams/" + c.Session.TeamId + "/users/" + id + "/profile.png"
if data, getErr := bucket.Get(path); getErr != nil {
img, err = createProfileImage(result.Data.(*model.User).Username, id)
if err != nil {
if data, err := readFile(path); err != nil {
if img, err = createProfileImage(result.Data.(*model.User).Username, id); err != nil {
c.Err = err
return
}
options := s3.Options{}
if err := bucket.Put(path, img, "image", s3.Private, options); err != nil {
c.Err = model.NewAppError("getImage", "Couldn't upload default profile image", err.Error())
if err := writeFile(img, path); err != nil {
c.Err = err
return
}
@@ -660,8 +649,8 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
}
func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
if !utils.IsS3Configured() {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload image. Amazon S3 not configured. ", "")
if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
c.Err = model.NewAppError("uploadProfileImage", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
@@ -671,13 +660,6 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
var auth aws.Auth
auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId
auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region])
bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket)
m := r.MultipartForm
imageArray, ok := m.File["image"]
@@ -721,8 +703,7 @@ func uploadProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
path := "teams/" + c.Session.TeamId + "/users/" + c.Session.UserId + "/profile.png"
options := s3.Options{}
if err := bucket.Put(path, buf.Bytes(), "image", s3.Private, options); err != nil {
if err := writeFile(buf.Bytes(), path); err != nil {
c.Err = model.NewAppError("uploadProfileImage", "Couldn't upload profile image", "")
return
}