PLT-6408 Framework for job server (#6404)

* Added initial job server

* Added job server to be ran as part of platform

* Added test job to the enterprise repo

* Fixed job server not loading license

* Renamed job package to jobs

* Fixed TE not being buildable

* Added JobStatus table to database

* Changed fields used by JobStatus

* Added APIs to query job status

* Added config change listener to server

* Added option to run job server from Makefile

* Added ability to enable/disable jobs from config

* Commented out placeholder for search indexing job

* Fixed govet

* Removed debug messages and fixed job api init message

model/client4.go

@@ -254,6 +254,10 @@ func (c *Client4) GetOpenGraphRoute() string {
 	return fmt.Sprintf("/opengraph")
 }
 
+func (c *Client4) GetJobsRoute() string {
+	return fmt.Sprintf("/jobs")
+}
+
 func (c *Client4) DoApiGet(url string, etag string) (*http.Response, *AppError) {
 	return c.DoApiRequest(http.MethodGet, c.ApiUrl+url, "", etag)
 }
@@ -2638,3 +2642,25 @@ func (c *Client4) OpenGraph(url string) (map[string]string, *Response) {
 		return MapFromJson(r.Body), BuildResponse(r)
 	}
 }
+
+// Jobs Section
+
+// GetJobStatus gets the status of a single job.
+func (c *Client4) GetJobStatus(id string) (*JobStatus, *Response) {
+	if r, err := c.DoApiGet(c.GetJobsRoute()+fmt.Sprintf("/%v/status", id), ""); err != nil {
+		return nil, &Response{StatusCode: r.StatusCode, Error: err}
+	} else {
+		defer closeBody(r)
+		return JobStatusFromJson(r.Body), BuildResponse(r)
+	}
+}
+
+// GetJobStatusesByType gets the status of all jobs of a given type, sorted with the job that most recently started first.
+func (c *Client4) GetJobStatusesByType(jobType string, page int, perPage int) ([]*JobStatus, *Response) {
+	if r, err := c.DoApiGet(c.GetJobsRoute()+fmt.Sprintf("/type/%v/statuses?page=%v&per_page=%v", jobType, page, perPage), ""); err != nil {
+		return nil, &Response{StatusCode: r.StatusCode, Error: err}
+	} else {
+		defer closeBody(r)
+		return JobStatusesFromJson(r.Body), BuildResponse(r)
+	}
+}

model/config.go

@@ -410,6 +410,10 @@ type ElasticSearchSettings struct {
 	Sniff           *bool
 }
 
+type DataRetentionSettings struct {
+	Enable *bool
+}
+
 type Config struct {
 	ServiceSettings       ServiceSettings
 	TeamSettings          TeamSettings
@@ -434,6 +438,7 @@ type Config struct {
 	AnalyticsSettings     AnalyticsSettings
 	WebrtcSettings        WebrtcSettings
 	ElasticSearchSettings ElasticSearchSettings
+	DataRetentionSettings DataRetentionSettings
 }
 
 func (o *Config) ToJson() string {
@@ -1257,6 +1262,11 @@ func (o *Config) SetDefaults() {
 		*o.ElasticSearchSettings.Sniff = true
 	}
 
+	if o.DataRetentionSettings.Enable == nil {
+		o.DataRetentionSettings.Enable = new(bool)
+		*o.DataRetentionSettings.Enable = false
+	}
+
 	o.defaultWebrtcSettings()
 }
 

model/job.go

@@ -1,110 +1,9 @@
-// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
 // See License.txt for license information.
 
 package model
 
-import (
-	"fmt"
-	"sync"
-	"time"
-)
-
-type TaskFunc func()
-
-type ScheduledTask struct {
-	Name      string        `json:"name"`
-	Interval  time.Duration `json:"interval"`
-	Recurring bool          `json:"recurring"`
-	function  TaskFunc
-	timer     *time.Timer
-}
-
-var taskMutex = sync.Mutex{}
-var tasks = make(map[string]*ScheduledTask)
-
-func addTask(task *ScheduledTask) {
-	taskMutex.Lock()
-	defer taskMutex.Unlock()
-	tasks[task.Name] = task
-}
-
-func removeTaskByName(name string) {
-	taskMutex.Lock()
-	defer taskMutex.Unlock()
-	delete(tasks, name)
-}
-
-func GetTaskByName(name string) *ScheduledTask {
-	taskMutex.Lock()
-	defer taskMutex.Unlock()
-	if task, ok := tasks[name]; ok {
-		return task
-	}
-	return nil
-}
-
-func GetAllTasks() *map[string]*ScheduledTask {
-	taskMutex.Lock()
-	defer taskMutex.Unlock()
-	return &tasks
-}
-
-func CreateTask(name string, function TaskFunc, timeToExecution time.Duration) *ScheduledTask {
-	task := &ScheduledTask{
-		Name:      name,
-		Interval:  timeToExecution,
-		Recurring: false,
-		function:  function,
-	}
-
-	taskRunner := func() {
-		go task.function()
-		removeTaskByName(task.Name)
-	}
-
-	task.timer = time.AfterFunc(timeToExecution, taskRunner)
-
-	addTask(task)
-
-	return task
-}
-
-func CreateRecurringTask(name string, function TaskFunc, interval time.Duration) *ScheduledTask {
-	task := &ScheduledTask{
-		Name:      name,
-		Interval:  interval,
-		Recurring: true,
-		function:  function,
-	}
-
-	taskRecurer := func() {
-		go task.function()
-		task.timer.Reset(task.Interval)
-	}
-
-	task.timer = time.AfterFunc(interval, taskRecurer)
-
-	addTask(task)
-
-	return task
-}
-
-func (task *ScheduledTask) Cancel() {
-	task.timer.Stop()
-	removeTaskByName(task.Name)
-}
-
-// Executes the task immediatly. A recurring task will be run regularally after interval.
-func (task *ScheduledTask) Execute() {
-	task.function()
-	task.timer.Reset(task.Interval)
-}
-
-func (task *ScheduledTask) String() string {
-	return fmt.Sprintf(
-		"%s\nInterval: %s\nRecurring: %t\n",
-		task.Name,
-		task.Interval.String(),
-		task.Recurring,
-	)
+type Job interface {
+	Run()
+	Stop()
 }

model/job_status.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"encoding/json"
+	"io"
+)
+
+const (
+	JOB_TYPE_DATA_RETENTION  = "data_retention"
+	JOB_TYPE_SEARCH_INDEXING = "search_indexing"
+)
+
+type JobStatus struct {
+	Id                 string                 `json:"id"`
+	Type               string                 `json:"type"`
+	StartAt            int64                  `json:"start_at"`
+	LastActivityAt     int64                  `json:"last_activity_at"`
+	LastRunStartedAt   int64                  `json:"last_run_started_at"`
+	LastRunCompletedAt int64                  `json:"last_run_completed_at"`
+	Status             string                 `json:"status"`
+	Data               map[string]interface{} `json:"data"`
+}
+
+func (js *JobStatus) ToJson() string {
+	if b, err := json.Marshal(js); err != nil {
+		return ""
+	} else {
+		return string(b)
+	}
+}
+
+func JobStatusFromJson(data io.Reader) *JobStatus {
+	var status JobStatus
+	if err := json.NewDecoder(data).Decode(&status); err == nil {
+		return &status
+	} else {
+		return nil
+	}
+}
+
+func JobStatusesToJson(statuses []*JobStatus) string {
+	if b, err := json.Marshal(statuses); err != nil {
+		return ""
+	} else {
+		return string(b)
+	}
+}
+
+func JobStatusesFromJson(data io.Reader) []*JobStatus {
+	var statuses []*JobStatus
+	if err := json.NewDecoder(data).Decode(&statuses); err == nil {
+		return statuses
+	} else {
+		return nil
+	}
+}

model/scheduled_task.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+type TaskFunc func()
+
+type ScheduledTask struct {
+	Name      string        `json:"name"`
+	Interval  time.Duration `json:"interval"`
+	Recurring bool          `json:"recurring"`
+	function  TaskFunc
+	timer     *time.Timer
+}
+
+var taskMutex = sync.Mutex{}
+var tasks = make(map[string]*ScheduledTask)
+
+func addTask(task *ScheduledTask) {
+	taskMutex.Lock()
+	defer taskMutex.Unlock()
+	tasks[task.Name] = task
+}
+
+func removeTaskByName(name string) {
+	taskMutex.Lock()
+	defer taskMutex.Unlock()
+	delete(tasks, name)
+}
+
+func GetTaskByName(name string) *ScheduledTask {
+	taskMutex.Lock()
+	defer taskMutex.Unlock()
+	if task, ok := tasks[name]; ok {
+		return task
+	}
+	return nil
+}
+
+func GetAllTasks() *map[string]*ScheduledTask {
+	taskMutex.Lock()
+	defer taskMutex.Unlock()
+	return &tasks
+}
+
+func CreateTask(name string, function TaskFunc, timeToExecution time.Duration) *ScheduledTask {
+	task := &ScheduledTask{
+		Name:      name,
+		Interval:  timeToExecution,
+		Recurring: false,
+		function:  function,
+	}
+
+	taskRunner := func() {
+		go task.function()
+		removeTaskByName(task.Name)
+	}
+
+	task.timer = time.AfterFunc(timeToExecution, taskRunner)
+
+	addTask(task)
+
+	return task
+}
+
+func CreateRecurringTask(name string, function TaskFunc, interval time.Duration) *ScheduledTask {
+	task := &ScheduledTask{
+		Name:      name,
+		Interval:  interval,
+		Recurring: true,
+		function:  function,
+	}
+
+	taskRecurer := func() {
+		go task.function()
+		task.timer.Reset(task.Interval)
+	}
+
+	task.timer = time.AfterFunc(interval, taskRecurer)
+
+	addTask(task)
+
+	return task
+}
+
+func (task *ScheduledTask) Cancel() {
+	task.timer.Stop()
+	removeTaskByName(task.Name)
+}
+
+// Executes the task immediatly. A recurring task will be run regularally after interval.
+func (task *ScheduledTask) Execute() {
+	task.function()
+	task.timer.Reset(task.Interval)
+}
+
+func (task *ScheduledTask) String() string {
+	return fmt.Sprintf(
+		"%s\nInterval: %s\nRecurring: %t\n",
+		task.Name,
+		task.Interval.String(),
+		task.Recurring,
+	)
+}