mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

PLT-???? Prepare file upload infrastructure for Data Retention. (#7266)

Просмотр исходного кода 
* Prepare file upload infrastructure for Data Retention.

This commit prepares the file upload infrastructure for the data
retention feature that is under construction. Changes are:

* Move file management code to utils to allow access to it from jobs.

* From now on, store all file uploads in a top level folder which is the
  date of the day on which they were uploaded.

This commit is based on Harrison Healey's branch, but updated to work
with the latest master.

* Use NewAppError
Этот коммит содержится в:
George Goldberg
2017-08-25 15:38:13 +01:00
коммит произвёл Harrison Healey
родитель 99acf61068
Коммит 50fc6e1e9e
20 изменённых файлов: 682 добавлений и 303 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
app/emoji.go
Просмотреть файл

@@ -101,7 +101,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := gif.EncodeAll(newbuf, resized_gif); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest)
}
if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -113,13 +113,13 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := png.Encode(newbuf, resized_image); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest)
}
if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
}
} else {
if err := WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -159,7 +159,7 @@ func GetEmojiImage(emojiId string) (imageByte []byte, imageType string, err *mod
} else {
var img []byte
if data, err := ReadFile(getEmojiImagePath(emojiId)); err != nil {
if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil {
return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound)
} else {
img = data
@@ -219,7 +219,7 @@ func imageToPaletted(img image.Image) *image.Paletted {
}
func deleteEmojiImage(id string) {
if err := MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
l4g.Error("Failed to rename image when deleting emoji %v", id)
}
}

257
app/file.go
Просмотреть файл

@@ -14,23 +14,21 @@ import (
_ "image/gif"
"image/jpeg"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
l4g "github.com/alecthomas/log4go"
"github.com/disintegration/imaging"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
s3 "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
const (
@@ -57,222 +55,8 @@ const (
IMAGE_THUMBNAIL_PIXEL_WIDTH = 120
IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100
IMAGE_PREVIEW_PIXEL_WIDTH = 1024
TEST_FILE_PATH = "/testfile"
)
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) {
var creds *credentials.Credentials
if signV2 {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4)
}
return s3.NewWithCredentials(endpoint, creds, secure, region)
}
func TestFileConnection() *model.AppError {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
region := utils.Cfg.FileSettings.AmazonS3Region
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error())
}
exists, err := s3Clnt.BucketExists(bucket)
if err != nil {
return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error())
}
if !exists {
l4g.Warn("Bucket specified does not exist. Attempting to create...")
err := s3Clnt.MakeBucket(bucket, region)
if err != nil {
l4g.Error("Unable to create bucket.")
return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
}
}
l4g.Info("Connection to S3 or minio is good. Bucket exists.")
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
f := []byte("testingwrite")
if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil {
return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
}
os.Remove(utils.Cfg.FileSettings.Directory + TEST_FILE_PATH)
l4g.Info("Able to write files to local storage.")
} else {
return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "")
}
return nil
}
func ReadFile(path string) ([]byte, *model.AppError) {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
region := utils.Cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
minioObject, err := s3Clnt.GetObject(bucket, path)
defer minioObject.Close()
if err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
}
if f, err := ioutil.ReadAll(minioObject); err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
} else {
return f, nil
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error())
} else {
return f, nil
}
} else {
return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented)
}
}
func MoveFile(oldPath, newPath string) *model.AppError {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
region := utils.Cfg.FileSettings.AmazonS3Region
encrypt := false
if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance {
encrypt = true
}
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
source := s3.NewSourceInfo(bucket, oldPath, nil)
destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt))
if err != nil {
return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
if err = s3Clnt.CopyObject(destination, source); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
}
if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
}
if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
}
} else {
return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "")
}
return nil
}
func WriteFile(f []byte, path string) *model.AppError {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
region := utils.Cfg.FileSettings.AmazonS3Region
encrypt := false
if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance {
encrypt = true
}
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
ext := filepath.Ext(path)
metaData := S3Metadata(encrypt, "binary/octet-stream")
if model.IsFileExtImage(ext) {
metaData = S3Metadata(encrypt, model.GetImageMimeType(ext))
}
_, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil)
if err != nil {
return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
return err
}
} else {
return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "")
}
return nil
}
func writeFileLocally(f []byte, path string) *model.AppError {
if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
directory, _ := filepath.Abs(filepath.Dir(path))
return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error())
}
if err := ioutil.WriteFile(path, f, 0644); err != nil {
return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error())
}
return nil
}
func openFileWriteStream(path string) (io.Writer, *model.AppError) {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.s3.app_error", nil, "")
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+path), 0774); err != nil {
return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.creating_dir.app_error", nil, err.Error())
}
if fileHandle, err := os.Create(utils.Cfg.FileSettings.Directory + path); err != nil {
return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.local_server.app_error", nil, err.Error())
} else {
fileHandle.Chmod(0644)
return fileHandle, nil
}
}
return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "")
}
func closeFileWriteStream(file io.Writer) {
file.(*os.File).Close()
}
func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo {
// Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension}
split := strings.SplitN(filename, "/", 5)
@@ -295,7 +79,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model
// Open the file and populate the fields of the FileInfo
var info *model.FileInfo
if data, err := ReadFile(path); err != nil {
if data, err := utils.ReadFile(path); err != nil {
l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err)
return nil
} else {
@@ -337,7 +121,7 @@ func FindTeamIdForFilename(post *model.Post, filename string) string {
} else {
for _, team := range teams {
path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name)
if _, err := ReadFile(path); err == nil {
if _, err := utils.ReadFile(path); err == nil {
// Found the team that this file was posted from
return team.Id
}
@@ -484,7 +268,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []*
io.Copy(buf, file)
data := buf.Bytes()
info, err := DoUploadFile(teamId, channelId, userId, fileHeader.Filename, data)
info, err := DoUploadFile(time.Now(), teamId, channelId, userId, fileHeader.Filename, data)
if err != nil {
return nil, err
}
@@ -507,7 +291,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []*
return resStruct, nil
}
func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
func DoUploadFile(now time.Time, teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
@@ -519,7 +303,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st
info.Id = model.NewId()
info.CreatorId = userId
pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
@@ -535,7 +319,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := WriteFile(data, info.Path); err != nil {
if err := utils.WriteFile(data, info.Path); err != nil {
return nil, err
}
@@ -652,7 +436,7 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he
return
}
if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
@@ -674,7 +458,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) {
return
}
if err := WriteFile(buf.Bytes(), previewPath); err != nil {
if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
@@ -687,20 +471,3 @@ func GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) {
return result.Data.(*model.FileInfo), nil
}
}
func S3Metadata(encrypt bool, contentType string) map[string][]string {
metaData := make(map[string][]string)
if contentType != "" {
metaData["Content-Type"] = []string{"contentType"}
}
if encrypt {
metaData["x-amz-server-side-encryption"] = []string{"AES256"}
}
return metaData
}
func CopyMetadata(encrypt bool) map[string]string {
metaData := make(map[string]string)
metaData["x-amz-server-side-encryption"] = "AES256"
return metaData
}

55
app/file_test.go
Просмотреть файл

@@ -4,9 +4,12 @@
package app
import (
"fmt"
"testing"
"time"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
func TestGeneratePublicLinkHash(t *testing.T) {
@@ -31,3 +34,55 @@ func TestGeneratePublicLinkHash(t *testing.T) {
t.Fatal("hashes for the same file with different salts should not be equal")
}
}
func TestDoUploadFile(t *testing.T) {
Setup()
teamId := model.NewId()
channelId := model.NewId()
userId := model.NewId()
filename := "test"
data := []byte("abcd")
info1, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
if err != nil {
t.Fatal(err)
} else {
defer func() {
<-Srv.Store.FileInfo().PermanentDelete(info1.Id)
utils.RemoveFile(info1.Path)
}()
}
if info1.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info1.Id, filename) {
t.Fatal("stored file at incorrect path", info1.Path)
}
info2, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
if err != nil {
t.Fatal(err)
} else {
defer func() {
<-Srv.Store.FileInfo().PermanentDelete(info2.Id)
utils.RemoveFile(info2.Path)
}()
}
if info2.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info2.Id, filename) {
t.Fatal("stored file at incorrect path", info2.Path)
}
info3, err := DoUploadFile(time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
if err != nil {
t.Fatal(err)
} else {
defer func() {
<-Srv.Store.FileInfo().PermanentDelete(info3.Id)
utils.RemoveFile(info3.Path)
}()
}
if info3.Path != fmt.Sprintf("20080305/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info3.Id, filename) {
t.Fatal("stored file at incorrect path", info3.Path)
}
}

6
app/import.go
Просмотреть файл

@@ -12,9 +12,11 @@ import (
"regexp"
"strings"
"sync"
"time"
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
@@ -1486,12 +1488,12 @@ func OldImportChannel(channel *model.Channel) *model.Channel {
}
}
func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
func OldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
data := buf.Bytes()
fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data)
fileInfo, err := DoUploadFile(timestamp, teamId, channelId, userId, fileName, data)
if err != nil {
return nil, err
}

3
app/slackimport.go
Просмотреть файл

@@ -401,7 +401,8 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
}
defer openFile.Close()
uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp))
uploadedFile, err := OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
if err != nil {
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
return nil, false

6
app/user.go
Просмотреть файл

@@ -758,7 +758,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
} else {
path := "users/" + user.Id + "/profile.png"
if data, err := ReadFile(path); err != nil {
if data, err := utils.ReadFile(path); err != nil {
readFailed = true
if img, err = CreateProfileImage(user.Username, user.Id); err != nil {
@@ -766,7 +766,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
}
if user.LastPictureUpdate == 0 {
if err := WriteFile(img, path); err != nil {
if err := utils.WriteFile(img, path); err != nil {
return nil, false, err
}
}
@@ -819,7 +819,7 @@ func SetProfileImage(userId string, imageData *multipart.FileHeader) *model.AppE
path := "users/" + userId + "/profile.png"
if err := WriteFile(buf.Bytes(), path); err != nil {
if err := utils.WriteFile(buf.Bytes(), path); err != nil {
return model.NewLocAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "")
}