PLT-7666: Clean up files on disk/s3 in data retention. (#7503)

i18n/en.json

@@ -6575,6 +6575,18 @@
     "id": "utils.file.remove_directory.s3.app_error",
     "translation": "Encountered an error removing directory from S3."
   },
+  {
+    "id": "utils.file.list_directory.configured.app_error",
+    "translation": "File storage not configured properly. Please configure for either S3 or local server file storage."
+  },
+  {
+    "id": "utils.file.list_directory.local.app_error",
+    "translation": "Encountered an error listing directory from local server file storage."
+  },
+  {
+    "id": "utils.file.list_directory.s3.app_error",
+    "translation": "Encountered an error listing directory from S3."
+  },
   {
     "id": "utils.file.remove_file.configured.app_error",
     "translation": "File storage not configured properly. Please configure for either S3 or local server file storage."

utils/file.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	l4g "github.com/alecthomas/log4go"
 	s3 "github.com/minio/minio-go"
@@ -269,6 +270,51 @@ func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
 	return out
 }
 
+// Returns a list of all the directories within the path directory provided.
+func ListDirectory(path string) (*[]string, *model.AppError) {
+	var paths []string
+
+	if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+		endpoint := Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *Cfg.FileSettings.AmazonS3SSL
+		signV2 := *Cfg.FileSettings.AmazonS3SignV2
+		region := Cfg.FileSettings.AmazonS3Region
+
+		s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+		if err != nil {
+			return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		doneCh := make(chan struct{})
+
+		defer close(doneCh)
+
+		bucket := Cfg.FileSettings.AmazonS3Bucket
+		for object := range s3Clnt.ListObjects(bucket, path, false, doneCh) {
+			if object.Err != nil {
+				return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)
+			}
+			paths = append(paths, strings.Trim(object.Key, "/"))
+		}
+	} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+		if fileInfos, err := ioutil.ReadDir(Cfg.FileSettings.Directory + path); err != nil {
+			return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
+		} else {
+			for _, fileInfo := range fileInfos {
+				if fileInfo.IsDir() {
+					paths = append(paths, filepath.Join(path, fileInfo.Name()))
+				}
+			}
+		}
+	} else {
+		return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.configured.app_error", nil, "", http.StatusInternalServerError)
+	}
+
+	return &paths, nil
+}
+
 func RemoveDirectory(path string) *model.AppError {
 	if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
 		endpoint := Cfg.FileSettings.AmazonS3Endpoint

utils/file_test.go

@@ -125,6 +125,32 @@ func (s *FileTestSuite) TestRemoveFile() {
 	s.Nil(RemoveDirectory("tests2"))
 }
 
+func (s *FileTestSuite) TestListDirectory() {
+	b := []byte("test")
+	path1 := "19700101/" + model.NewId()
+	path2 := "19800101/" + model.NewId()
+
+	s.Nil(WriteFile(b, path1))
+	defer RemoveFile(path1)
+	s.Nil(WriteFile(b, path2))
+	defer RemoveFile(path2)
+
+	paths, err := ListDirectory("")
+	s.Nil(err)
+
+	found1 := false
+	found2 := false
+	for _, path := range *paths {
+		if path == "19700101" {
+			found1 = true
+		} else if path == "19800101" {
+			found2 = true
+		}
+	}
+	s.True(found1)
+	s.True(found2)
+}
+
 func (s *FileTestSuite) TestRemoveDirectory() {
 	b := []byte("test")