PLT-3105 Files table migration (#4068)

* Implemented initial changes for files table

* Removed *_benchmark_test.go files

* Re-implemented GetPublicFile and added support for old path

* Localization for files table

* Moved file system code into utils package

* Finished server-side changes and added initial upgrade script

* Added getPostFiles api

* Re-add Extension and HasPreviewImage fields to FileInfo

* Removed unused translation

* Fixed merge conflicts left over after permissions changes

* Forced FileInfo.extension to be lower case

* Changed FileUploadResponse to contain the FileInfos instead of FileIds

* Fixed permissions on getFile* calls

* Fixed notifications for file uploads

* Added initial version of client code for files changes

* Permanently added FileIds field to Post object and removed Post.HasFiles

* Updated PostStore.Update to be usable in more circumstances

* Re-added Filenames field and switched file migration to be entirely lazy-loaded

* Increased max listener count for FileStore

* Removed unused fileInfoCache

* Moved file system code back into api

* Removed duplicate test case

* Fixed unit test running on ports other than 8065

* Renamed HasPermissionToPostContext to HasPermissionToChannelByPostContext

* Refactored handleImages to make it more easily understandable

* Renamed getPostFiles to getFileInfosForPost

* Re-added pre-FileIds posts to analytics

* Changed files to be saved as their ids as opposed to id/filename.ext

* Renamed FileInfo.UserId to FileInfo.CreatorId

* Fixed detection of language in CodePreview

* Fixed switching between threads in the RHS not loading new files

* Add serverside protection against a rare bug where the client sends the same file twice for a single post

* Refactored the important parts of uploadFile api call into a function that can be called without a web context

api/post.go

@@ -49,6 +49,7 @@ func InitPost() {
 	BaseRoutes.NeedPost.Handle("/delete", ApiUserRequired(deletePost)).Methods("POST")
 	BaseRoutes.NeedPost.Handle("/before/{offset:[0-9]+}/{num_posts:[0-9]+}", ApiUserRequired(getPostsBefore)).Methods("GET")
 	BaseRoutes.NeedPost.Handle("/after/{offset:[0-9]+}/{num_posts:[0-9]+}", ApiUserRequired(getPostsAfter)).Methods("GET")
+	BaseRoutes.NeedPost.Handle("/get_file_infos", ApiUserRequired(getFileInfosForPost)).Methods("GET")
 }
 
 func createPost(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -135,48 +136,26 @@ func CreatePost(c *Context, post *model.Post, triggerWebhooks bool) (*model.Post
 
 	post.Hashtags, _ = model.ParseHashtags(post.Message)
 
-	if len(post.Filenames) > 0 {
-		doRemove := false
-		for i := len(post.Filenames) - 1; i >= 0; i-- {
-			path := post.Filenames[i]
-
-			doRemove = false
-			if model.UrlRegex.MatchString(path) {
-				continue
-			} else if model.PartialUrlRegex.MatchString(path) {
-				matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1)
-				if len(matches) == 0 || len(matches[0]) < 4 {
-					doRemove = true
-				}
-
-				channelId := matches[0][1]
-				if channelId != post.ChannelId {
-					doRemove = true
-				}
-
-				userId := matches[0][2]
-				if userId != post.UserId {
-					doRemove = true
-				}
-			} else {
-				doRemove = true
-			}
-			if doRemove {
-				l4g.Error(utils.T("api.post.create_post.bad_filename.error"), path)
-				post.Filenames = append(post.Filenames[:i], post.Filenames[i+1:]...)
-			}
-		}
-	}
-
 	var rpost *model.Post
 	if result := <-Srv.Store.Post().Save(post); result.Err != nil {
 		return nil, result.Err
 	} else {
 		rpost = result.Data.(*model.Post)
-
-		go handlePostEvents(c, rpost, triggerWebhooks)
 	}
 
+	if len(post.FileIds) > 0 {
+		// There's a rare bug where the client sends up duplicate FileIds so protect against that
+		post.FileIds = utils.RemoveDuplicatesFromStringArray(post.FileIds)
+
+		for _, fileId := range post.FileIds {
+			if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
+				l4g.Error(utils.T("api.post.create_post.attach_files.error"), post.Id, post.FileIds, c.Session.UserId, result.Err)
+			}
+		}
+	}
+
+	go handlePostEvents(c, rpost, triggerWebhooks)
+
 	return rpost, nil
 }
 
@@ -566,6 +545,7 @@ func sendNotifications(c *Context, post *model.Post, team *model.Team, channel *
 	pchan := Srv.Store.User().GetProfiles(c.TeamId)
 	dpchan := Srv.Store.User().GetDirectProfiles(c.Session.UserId)
 	mchan := Srv.Store.Channel().GetMembers(post.ChannelId)
+	fchan := Srv.Store.FileInfo().GetForPost(post.Id)
 
 	var profileMap map[string]*model.User
 	if result := <-pchan; result.Err != nil {
@@ -785,12 +765,18 @@ func sendNotifications(c *Context, post *model.Post, team *model.Team, channel *
 	message.Add("sender_name", senderName)
 	message.Add("team_id", team.Id)
 
-	if len(post.Filenames) != 0 {
+	if len(post.FileIds) != 0 {
 		message.Add("otherFile", "true")
 
-		for _, filename := range post.Filenames {
-			ext := filepath.Ext(filename)
-			if model.IsFileExtImage(ext) {
+		var infos []*model.FileInfo
+		if result := <-fchan; result.Err != nil {
+			l4g.Warn(utils.T("api.post.send_notifications.files.error"), post.Id, result.Err)
+		} else {
+			infos = result.Data.([]*model.FileInfo)
+		}
+
+		for _, info := range infos {
+			if info.IsImage() {
 				message.Add("image", "true")
 				break
 			}
@@ -915,22 +901,29 @@ func sendNotificationEmail(c *Context, post *model.Post, user *model.User, chann
 }
 
 func getMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string {
-	if len(strings.TrimSpace(post.Message)) != 0 || len(post.Filenames) == 0 {
+	if len(strings.TrimSpace(post.Message)) != 0 || len(post.FileIds) == 0 {
 		return post.Message
 	}
 
 	// extract the filenames from their paths and determine what type of files are attached
-	filenames := make([]string, len(post.Filenames))
+	var infos []*model.FileInfo
+	if result := <-Srv.Store.FileInfo().GetForPost(post.Id); result.Err != nil {
+		l4g.Warn(utils.T("api.post.get_message_for_notification.get_files.error"), post.Id, result.Err)
+	} else {
+		infos = result.Data.([]*model.FileInfo)
+	}
+
+	filenames := make([]string, len(infos))
 	onlyImages := true
-	for i, filename := range post.Filenames {
-		var err error
-		if filenames[i], err = url.QueryUnescape(filepath.Base(filename)); err != nil {
+	for i, info := range infos {
+		if escaped, err := url.QueryUnescape(filepath.Base(info.Name)); err != nil {
 			// this should never error since filepath was escaped using url.QueryEscape
-			filenames[i] = filepath.Base(filename)
+			filenames[i] = escaped
+		} else {
+			filenames[i] = info.Name
 		}
 
-		ext := filepath.Ext(filename)
-		onlyImages = onlyImages && model.IsFileExtImage(ext)
+		onlyImages = onlyImages && info.IsImage()
 	}
 
 	props := map[string]interface{}{"Filenames": strings.Join(filenames, ", ")}
@@ -1099,9 +1092,6 @@ func SendEphemeralPost(teamId, userId string, post *model.Post) {
 	if post.Props == nil {
 		post.Props = model.StringInterface{}
 	}
-	if post.Filenames == nil {
-		post.Filenames = []string{}
-	}
 
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil)
 	message.Add("post", post.ToJson())
@@ -1156,9 +1146,13 @@ func updatePost(c *Context, w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	hashtags, _ := model.ParseHashtags(post.Message)
+	newPost := &model.Post{}
+	*newPost = *oldPost
 
-	if result := <-Srv.Store.Post().Update(oldPost, post.Message, hashtags); result.Err != nil {
+	newPost.Message = post.Message
+	newPost.Hashtags, _ = model.ParseHashtags(post.Message)
+
+	if result := <-Srv.Store.Post().Update(newPost, oldPost); result.Err != nil {
 		c.Err = result.Err
 		return
 	} else {
@@ -1449,7 +1443,7 @@ func deletePost(c *Context, w http.ResponseWriter, r *http.Request) {
 		message.Add("post", post.ToJson())
 
 		go Publish(message)
-		go DeletePostFiles(c.TeamId, post)
+		go DeletePostFiles(post)
 		go DeleteFlaggedPost(c.Session.UserId, post)
 
 		result := make(map[string]string)
@@ -1465,17 +1459,13 @@ func DeleteFlaggedPost(userId string, post *model.Post) {
 	}
 }
 
-func DeletePostFiles(teamId string, post *model.Post) {
-	if len(post.Filenames) == 0 {
+func DeletePostFiles(post *model.Post) {
+	if len(post.FileIds) != 0 {
 		return
 	}
 
-	prefix := "teams/" + teamId + "/channels/" + post.ChannelId + "/users/" + post.UserId + "/"
-	for _, filename := range post.Filenames {
-		splitUrl := strings.Split(filename, "/")
-		oldPath := prefix + splitUrl[len(splitUrl)-2] + "/" + splitUrl[len(splitUrl)-1]
-		newPath := prefix + splitUrl[len(splitUrl)-2] + "/deleted_" + splitUrl[len(splitUrl)-1]
-		MoveFile(oldPath, newPath)
+	if result := <-Srv.Store.FileInfo().DeleteForPost(post.Id); result.Err != nil {
+		l4g.Warn(utils.T("api.post.delete_post_files.app_error.warn"), post.Id, result.Err)
 	}
 }
 
@@ -1583,3 +1573,59 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
 	w.Write([]byte(posts.ToJson()))
 }
+
+func getFileInfosForPost(c *Context, w http.ResponseWriter, r *http.Request) {
+	params := mux.Vars(r)
+
+	channelId := params["channel_id"]
+	if len(channelId) != 26 {
+		c.SetInvalidParam("getFileInfosForPost", "channelId")
+		return
+	}
+
+	postId := params["post_id"]
+	if len(postId) != 26 {
+		c.SetInvalidParam("getFileInfosForPost", "postId")
+		return
+	}
+
+	pchan := Srv.Store.Post().Get(postId)
+	fchan := Srv.Store.FileInfo().GetForPost(postId)
+
+	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) {
+		return
+	}
+
+	var infos []*model.FileInfo
+	if result := <-fchan; result.Err != nil {
+		c.Err = result.Err
+		return
+	} else {
+		infos = result.Data.([]*model.FileInfo)
+	}
+
+	if len(infos) == 0 {
+		// No FileInfos were returned so check if they need to be created for this post
+		var post *model.Post
+		if result := <-pchan; result.Err != nil {
+			c.Err = result.Err
+			return
+		} else {
+			post = result.Data.(*model.PostList).Posts[postId]
+		}
+
+		if len(post.Filenames) > 0 {
+			// The post has Filenames that need to be replaced with FileInfos
+			infos = migrateFilenamesToFileInfos(post)
+		}
+	}
+
+	etag := model.GetEtagForFileInfos(infos)
+
+	if HandleEtag(etag, w, r) {
+		return
+	} else {
+		w.Header().Set(model.HEADER_ETAG_SERVER, etag)
+		w.Write([]byte(model.FileInfosToJson(infos)))
+	}
+}