PLT-6896 per-paging for logs (#7903)

* PLT-6896 Read logs from last

* Getting rid of file.Stats

* remove deprecated value

* Make non-reassigned value constant

api4/params.go

@@ -11,9 +11,11 @@ import (
 )
 
 const (
-	PAGE_DEFAULT     = 0
+	PAGE_DEFAULT          = 0
-	PER_PAGE_DEFAULT = 60
+	PER_PAGE_DEFAULT      = 60
-	PER_PAGE_MAXIMUM = 200
+	PER_PAGE_MAXIMUM      = 200
+	LOGS_PER_PAGE_DEFAULT = 10000
+	LOGS_PER_PAGE_MAXIMUM = 10000
 )
 
 type ApiParams struct {
@@ -43,6 +45,7 @@ type ApiParams struct {
 	ActionId       string
 	Page           int
 	PerPage        int
+	LogsPerPage    int
 	Permanent      bool
 }
 
@@ -165,5 +168,13 @@ func ApiParamsFromRequest(r *http.Request) *ApiParams {
 		params.PerPage = val
 	}
 
+	if val, err := strconv.Atoi(r.URL.Query().Get("logs_per_page")); err != nil || val < 0 {
+		params.LogsPerPage = LOGS_PER_PAGE_DEFAULT
+	} else if val > LOGS_PER_PAGE_MAXIMUM {
+		params.LogsPerPage = LOGS_PER_PAGE_MAXIMUM
+	} else {
+		params.LogsPerPage = val
+	}
+
 	return params
 }

api4/system.go

@@ -187,7 +187,7 @@ func getLogs(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	lines, err := c.App.GetLogs(c.Params.Page, c.Params.PerPage)
+	lines, err := c.App.GetLogs(c.Params.Page, c.Params.LogsPerPage)
 	if err != nil {
 		c.Err = err
 		return

api4/system_test.go

@@ -308,18 +308,18 @@ func TestGetLogs(t *testing.T) {
 	logs, resp := th.SystemAdminClient.GetLogs(0, 10)
 	CheckNoError(t, resp)
 
-	// if len(logs) != 10 {
+	if len(logs) != 10 {
-	// 	t.Log(len(logs))
+		t.Log(len(logs))
-	// 	t.Fatal("wrong length")
+		t.Fatal("wrong length")
-	// }
+	}
 
 	logs, resp = th.SystemAdminClient.GetLogs(1, 10)
 	CheckNoError(t, resp)
 
-	// if len(logs) != 10 {
+	if len(logs) != 10 {
-	// 	t.Log(len(logs))
+		t.Log(len(logs))
-	// 	t.Fatal("wrong length")
+		t.Fatal("wrong length")
-	// }
+	}
 
 	logs, resp = th.SystemAdminClient.GetLogs(-1, -1)
 	CheckNoError(t, resp)

app/admin.go

@@ -4,7 +4,7 @@
 package app
 
 import (
-	"bufio"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -20,9 +20,6 @@ import (
 )
 
 func (a *App) GetLogs(page, perPage int) ([]string, *model.AppError) {
-
-	perPage = 10000
-
 	var lines []string
 	if a.Cluster != nil && *a.Config().ClusterSettings.Enable {
 		lines = append(lines, "-----------------------------------------------------------------------------------------------------------")
@@ -62,20 +59,48 @@ func (a *App) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
 
 		defer file.Close()
 
-		offsetCount := 0
+		var newLine = []byte{'\n'}
-		limitCount := 0
+		var lineCount int
-		scanner := bufio.NewScanner(file)
+		const searchPos = -1
-		for scanner.Scan() {
+		lineEndPos, err := file.Seek(0, io.SeekEnd)
-			if limitCount >= perPage {
+		if err != nil {
-				break
+			return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+		for {
+			pos, err := file.Seek(searchPos, io.SeekCurrent)
+			if err != nil {
+				return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, err.Error(), http.StatusInternalServerError)
 			}
 
-			if offsetCount >= page*perPage {
+			b := make([]byte, 1)
-				lines = append(lines, scanner.Text())
+			_, err = file.ReadAt(b, pos)
-				limitCount++
+			if err != nil {
-			} else {
+				return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, err.Error(), http.StatusInternalServerError)
-				offsetCount++
 			}
+
+			if b[0] == newLine[0] || pos == 0 {
+				lineCount++
+				if lineCount > page*perPage {
+					line := make([]byte, lineEndPos-pos)
+					_, err := file.ReadAt(line, pos)
+					if err != nil {
+						return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, err.Error(), http.StatusInternalServerError)
+					}
+					lines = append(lines, string(line))
+				}
+				if pos == 0 {
+					break
+				}
+				lineEndPos = pos
+			}
+
+			if len(lines) == perPage {
+				break
+			}
+		}
+
+		for i, j := 0, len(lines)-1; i < j; i, j = i+1, j-1 {
+			lines[i], lines[j] = lines[j], lines[i]
 		}
 	} else {
 		lines = append(lines, "")

model/client4.go

@@ -2649,7 +2649,7 @@ func (c *Client4) UploadBrandImage(data []byte) (bool, *Response) {
 
 // GetLogs page of logs as a string array.
 func (c *Client4) GetLogs(page, perPage int) ([]string, *Response) {
-	query := fmt.Sprintf("?page=%v&per_page=%v", page, perPage)
+	query := fmt.Sprintf("?page=%v&logs_per_page=%v", page, perPage)
 	if r, err := c.DoApiGet("/logs"+query, ""); err != nil {
 		return nil, BuildErrorResponse(r, err)
 	} else {