PLT-2115 adding compliance feature to enterprise

i18n/en.json

@@ -2771,6 +2771,10 @@
     "id": "store.sql_post.analytics_posts_count.app_error",
     "translation": "We couldn't get post counts"
   },
+  {
+    "id": "store.sql_post.compliance_export.app_error",
+    "translation": "We couldn't get posts for compliance export"
+  },
   {
     "id": "store.sql_post.analytics_posts_count_by_day.app_error",
     "translation": "We couldn't get post counts by day"

model/compliance_post.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"time"
+)
+
+type CompliancePost struct {
+
+	// From Team
+	TeamName        string
+	TeamDisplayName string
+
+	// From Channel
+	ChannelName        string
+	ChannelDisplayName string
+
+	// From User
+	UserUsername string
+	UserEmail    string
+	UserNickname string
+
+	// From Post
+	PostId         string
+	PostCreateAt   int64
+	PostUpdateAt   int64
+	PostDeleteAt   int64
+	PostRootId     string
+	PostParentId   string
+	PostOriginalId string
+	PostMessage    string
+	PostType       string
+	PostProps      string
+	PostHashtags   string
+	PostFilenames  string
+}
+
+func CompliancePostHeader() []string {
+	return []string{
+		"TeamName",
+		"TeamDisplayName",
+
+		"ChannelName",
+		"ChannelDisplayName",
+
+		"UserUsername",
+		"UserEmail",
+		"UserNickname",
+
+		"PostId",
+		"PostCreateAt",
+		"PostUpdateAt",
+		"PostDeleteAt",
+		"PostRootId",
+		"PostParentId",
+		"PostOriginalId",
+		"PostMessage",
+		"PostType",
+		"PostProps",
+		"PostHashtags",
+		"PostFilenames",
+	}
+}
+
+func (me *CompliancePost) Row() []string {
+	return []string{
+		me.TeamName,
+		me.TeamDisplayName,
+
+		me.ChannelName,
+		me.ChannelDisplayName,
+
+		me.UserUsername,
+		me.UserEmail,
+		me.UserNickname,
+
+		me.PostId,
+		time.Unix(0, me.PostCreateAt*1000).Format(time.RFC3339),
+		time.Unix(0, me.PostUpdateAt*1000).Format(time.RFC3339),
+		time.Unix(0, me.PostDeleteAt*1000).Format(time.RFC3339),
+		me.PostRootId,
+		me.PostParentId,
+		me.PostOriginalId,
+		me.PostMessage,
+		me.PostType,
+		me.PostProps,
+		me.PostHashtags,
+		me.PostFilenames,
+	}
+}

model/compliance_post_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"testing"
+)
+
+func TestCompliancePostHeader(t *testing.T) {
+	if CompliancePostHeader()[0] != "TeamName" {
+		t.Fatal()
+	}
+}
+
+func TestCompliancePost(t *testing.T) {
+	o := CompliancePost{TeamName: "test", PostFilenames: "files", PostCreateAt: GetMillis()}
+	r := o.Row()
+
+	if r[0] != "test" {
+		t.Fatal()
+	}
+
+	if r[len(r)-1] != "files" {
+		t.Fatal()
+	}
+}

store/sql_post_store.go

@@ -979,3 +979,59 @@ func (s SqlPostStore) AnalyticsPostCount(teamId string, mustHaveFile bool, mustH
 
 	return storeChannel
 }
+
+func (s SqlPostStore) ComplianceExport(startTime int64, endTime int64) StoreChannel {
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		query :=
+			`SELECT
+			    Teams.Name AS TeamName,
+			    Teams.DisplayName AS TeamDisplayName,
+			    Channels.Name AS ChannelName,
+			    Channels.DisplayName AS ChannelDisplayName,
+			    Users.Username AS UserUsername,
+			    Users.Email AS UserEmail,
+			    Users.Nickname AS UserNickname,
+			    Posts.Id AS PostId,
+			    Posts.CreateAt AS PostCreateAt,
+			    Posts.UpdateAt AS PostUpdateAt,
+			    Posts.DeleteAt AS PostDeleteAt,
+			    Posts.RootId AS PostRootId,
+			    Posts.ParentId AS PostParentId,
+			    Posts.OriginalId AS PostOriginalId,
+			    Posts.Message AS PostMessage,
+			    Posts.Type AS PostType,
+			    Posts.Props AS PostProps,
+			    Posts.Hashtags AS PostHashtags,
+			    Posts.Filenames AS PostFilenames
+			FROM
+			    Teams,
+			    Channels,
+			    Users,
+			    Posts
+			WHERE
+			    Teams.Id = Channels.TeamId
+			        AND Posts.ChannelId = Channels.Id
+			        AND Posts.UserId = Users.Id
+			        AND Posts.CreateAt > :StartTime
+			        AND Posts.CreateAt <= :EndTime
+			ORDER BY Posts.CreateAt
+			LIMIT 50000`
+
+		var cposts []*model.CompliancePost
+
+		if _, err := s.GetReplica().Select(&cposts, query, map[string]interface{}{"StartTime": startTime, "EndTime": endTime}); err != nil {
+			result.Err = model.NewLocAppError("SqlPostStore.ComplianceExport", "store.sql_post.compliance_export.app_error", nil, err.Error())
+		} else {
+			result.Data = cposts
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+	}()
+
+	return storeChannel
+}

store/sql_post_store_test.go

@@ -895,3 +895,79 @@ func TestPostCountsByDay(t *testing.T) {
 		}
 	}
 }
+
+func TestComplianceExport(t *testing.T) {
+	Setup()
+
+	time.Sleep(100 * time.Millisecond)
+
+	t1 := &model.Team{}
+	t1.DisplayName = "DisplayName"
+	t1.Name = "a" + model.NewId() + "b"
+	t1.Email = model.NewId() + "@nowhere.com"
+	t1.Type = model.TEAM_OPEN
+	t1 = Must(store.Team().Save(t1)).(*model.Team)
+
+	u1 := &model.User{}
+	u1.TeamId = t1.Id
+	u1.Email = model.NewId()
+	u1.Username = model.NewId()
+	u1 = Must(store.User().Save(u1)).(*model.User)
+
+	c1 := &model.Channel{}
+	c1.TeamId = t1.Id
+	c1.DisplayName = "Channel2"
+	c1.Name = "a" + model.NewId() + "b"
+	c1.Type = model.CHANNEL_OPEN
+	c1 = Must(store.Channel().Save(c1)).(*model.Channel)
+
+	o1 := &model.Post{}
+	o1.ChannelId = c1.Id
+	o1.UserId = u1.Id
+	o1.CreateAt = model.GetMillis()
+	o1.Message = "a" + model.NewId() + "b"
+	o1 = Must(store.Post().Save(o1)).(*model.Post)
+
+	o1a := &model.Post{}
+	o1a.ChannelId = c1.Id
+	o1a.UserId = u1.Id
+	o1a.CreateAt = o1.CreateAt + 10
+	o1a.Message = "a" + model.NewId() + "b"
+	o1a = Must(store.Post().Save(o1a)).(*model.Post)
+
+	o2 := &model.Post{}
+	o2.ChannelId = c1.Id
+	o2.UserId = u1.Id
+	o2.CreateAt = o1.CreateAt + 20
+	o2.Message = "a" + model.NewId() + "b"
+	o2 = Must(store.Post().Save(o2)).(*model.Post)
+
+	o2a := &model.Post{}
+	o2a.ChannelId = c1.Id
+	o2a.UserId = u1.Id
+	o2a.CreateAt = o1.CreateAt + 30
+	o2a.Message = "a" + model.NewId() + "b"
+	o2a = Must(store.Post().Save(o2a)).(*model.Post)
+
+	time.Sleep(100 * time.Millisecond)
+
+	if r1 := <-store.Post().ComplianceExport(o1.CreateAt-1, o2a.CreateAt+1); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+		t.Log(cposts)
+
+		if len(cposts) != 4 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o1.Id {
+			t.Fatal("Wrong sort")
+		}
+
+		if cposts[3].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+
+	}
+}

store/store.go

@@ -105,6 +105,7 @@ type PostStore interface {
 	AnalyticsUserCountsWithPostsByDay(teamId string) StoreChannel
 	AnalyticsPostCountsByDay(teamId string) StoreChannel
 	AnalyticsPostCount(teamId string, mustHaveFile bool, mustHaveHashtag bool) StoreChannel
+	ComplianceExport(startTime int64, endTime int64) StoreChannel
 }
 
 type UserStore interface {