Includes mmctl into the mono-repo (#23091)

* Includes mmctl into the mono-repo

* Update to use the new public module paths

* Adds docs check to the mmctl CI

* Fix public utils import path

* Tidy up modules

* Fix linter

* Update CI tasks to use the new file structure

* Update CI references
Этот коммит содержится в:
Miguel de la Cruz
2023-06-05 12:42:55 +02:00
коммит произвёл GitHub
родитель 412109b02e
Коммит 951456c780
305 изменённых файлов: 47027 добавлений и 88 удалений

52
server/cmd/mmctl/printer/human/entry.go Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package human
import (
"fmt"
"strings"
"time"
"github.com/mattermost/mattermost-server/server/public/shared/mlog"
)
type LogEntry struct {
Time time.Time
Level string
Message string
Caller string
Fields []mlog.Field
}
// Provide default string representation. Used by SimpleWriter
func (f LogEntry) String() string {
var sb strings.Builder
if !f.Time.IsZero() {
sb.WriteString(f.Time.Format(time.RFC3339Nano))
sb.WriteRune(' ')
}
if f.Level != "" {
sb.WriteString(f.Level)
sb.WriteRune(' ')
}
if f.Caller != "" {
sb.WriteString(f.Caller)
sb.WriteRune(' ')
}
for _, field := range f.Fields {
sb.WriteString(field.Key)
sb.WriteRune('=')
sb.WriteString(fmt.Sprint(field.Interface))
sb.WriteRune(' ')
}
if f.Message != "" {
// If the message is multiple lines, start the whole message on a new line
if strings.ContainsRune(f.Message, '\n') {
sb.WriteRune('\n')
}
sb.WriteString(f.Message)
}
return sb.String()
}

Просмотреть файл

@@ -0,0 +1,77 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package human
import (
"fmt"
"io"
"time"
"github.com/sirupsen/logrus"
)
type LogrusWriter struct {
logger *logrus.Logger
}
func (w *LogrusWriter) Write(e LogEntry) {
if e.Level == "" {
fmt.Fprintln(w.logger.Out, e.Message)
return
}
lvl, err := logrus.ParseLevel(e.Level)
if err != nil {
fmt.Fprintln(w.logger.Out, err)
lvl = logrus.TraceLevel + 1 // will invoke Println
}
logger := w.logger.WithTime(e.Time)
if e.Caller != "" {
// logrus has a system of reporting the caller, but there's no easy way to override it
logger = logger.WithField("caller", e.Caller)
}
for _, field := range e.Fields {
logger = logger.WithField(field.Key, field.Interface)
}
switch lvl {
case logrus.PanicLevel:
// Prevent panic from causing us to exit
defer func() {
_ = recover()
}()
logger.Panic(e.Message)
case logrus.FatalLevel:
logger.Fatal(e.Message)
case logrus.ErrorLevel:
logger.Error(e.Message)
case logrus.WarnLevel:
logger.Warn(e.Message)
case logrus.InfoLevel:
logger.Info(e.Message)
case logrus.DebugLevel:
logger.Debug(e.Message)
case logrus.TraceLevel:
logger.Trace(e.Message)
default:
logger.Println(e.Message)
}
}
func NewLogrusWriter(output io.Writer) *LogrusWriter {
w := new(LogrusWriter)
w.logger = logrus.New()
w.logger.SetLevel(logrus.TraceLevel) // don't filter any logs
w.logger.ExitFunc = func(int) {} // prevent Fatal from causing us to exit
w.logger.SetReportCaller(false)
w.logger.SetOutput(output)
var tf logrus.TextFormatter
tf.FullTimestamp = true
tf.TimestampFormat = time.RFC3339Nano
w.logger.SetFormatter(&tf)
return w
}

180
server/cmd/mmctl/printer/human/parser.go Обычный файл
Просмотреть файл

@@ -0,0 +1,180 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package human
import (
"encoding/json"
"errors"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/mattermost/mattermost-server/server/public/shared/mlog"
)
func ParseLogMessage(msg string) LogEntry {
result, err := parseLogMessage(msg)
if err != nil {
// If failed to parse, just output a LogEntry where all fields are blank, but Message is the original string
var result2 LogEntry
result2.Message = msg
return result2
}
return result
}
func parseLogMessage(msg string) (result LogEntry, err error) {
// Note: This implementation uses a custom json decoding loop.
// The primary advantage of this versus decoding directly into a map is to
// preserve the order of the fields. This can be simplified if we end up
// having the formatter sort fields alphabetically (logrus does by default)
dec := json.NewDecoder(strings.NewReader(msg))
// look for an initial "{"
token, err := dec.Token()
if err != nil {
return result, err
}
d, ok := token.(json.Delim)
if !ok || d != '{' {
return result, fmt.Errorf("input is not a JSON object, found: %v", token)
}
// read all key-value pairs
for dec.More() {
key, err2 := dec.Token()
if err2 != nil {
return result, err2
}
skey, ok2 := key.(string)
if !ok2 {
return result, errors.New("key is not a value string")
}
if !dec.More() {
return result, errors.New("missing value pair")
}
switch skey {
case "ts":
var ts json.Number
if err2 := dec.Decode(&ts); err2 != nil {
return result, err2
}
timeVal, err2 := numberToTime(ts)
if err2 != nil {
return result, err2
}
result.Time = timeVal
case "level":
s, err2 := decodeAsString(dec)
if err2 != nil {
return result, err2
}
result.Level = s
case "msg":
s, err2 := decodeAsString(dec)
if err2 != nil {
return result, err2
}
result.Message = s
case "caller":
s, err2 := decodeAsString(dec)
if err2 != nil {
return result, err2
}
result.Caller = s
default:
var p interface{}
if err2 := dec.Decode(&p); err2 != nil {
return result, err2
}
var f mlog.Field
f.Key = skey
f.Interface = p
result.Fields = append(result.Fields, f)
}
}
// read the "}"
token, err = dec.Token()
if err != nil {
return result, err
}
d, ok = token.(json.Delim)
if !ok || d != '}' {
return result, fmt.Errorf("failed to read '}', read: %v", token)
}
// make sure nothing else trailing
if token, err := dec.Token(); err != io.EOF {
return result, err
} else if token != nil {
return result, errors.New("found trailing data")
}
return result, nil
}
// Translate a number into a time
func numberToTime(v json.Number) (time.Time, error) {
// Using floating point math to extract the nanoseconds leads to a time that doesn't exactly match the input
// Instead, parse out the components from the string representation
var t time.Time
// First make sure it is a number...
flt, err := v.Float64()
if err != nil {
return t, err
}
s := v.String()
if strings.ContainsAny(s, "eE") {
// input is in scientific notation. Convert to standard decimal notation
s = strconv.FormatFloat(flt, 'f', -1, 64)
}
// extract the seconds and nanoseconds separately
var nanos, sec int64
parts := strings.SplitN(s, ".", 2)
sec, err = strconv.ParseInt(parts[0], 10, 64)
if err != nil {
return t, err
}
if len(parts) == 2 {
nanosText := parts[1] + "000000000"
nanosText = nanosText[:9]
nanos, err = strconv.ParseInt(nanosText, 10, 64)
if err != nil {
return t, err
}
}
t = time.Unix(sec, nanos)
return t, nil
}
// Decodes a value from JSON, coercing it to a string value as necessary
func decodeAsString(dec *json.Decoder) (s string, err error) {
var v interface{}
if err = dec.Decode(&v); err != nil {
return s, err
}
var ok bool
if s, ok = v.(string); ok {
return s, err
}
s = fmt.Sprint(v)
return s, err
}

23
server/cmd/mmctl/printer/human/process.go Обычный файл
Просмотреть файл

@@ -0,0 +1,23 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package human
import (
"bufio"
"io"
)
type LogWriter interface {
Write(e LogEntry)
}
// Read JSON logs from input and write formatted logs to the output
func ProcessLogs(reader io.Reader, writer LogWriter) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
s := scanner.Text()
e := ParseLogMessage(s)
writer.Write(e)
}
}

Просмотреть файл

@@ -0,0 +1,23 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package human
import (
"fmt"
"io"
)
type SimpleWriter struct {
out io.Writer
}
func (w *SimpleWriter) Write(e LogEntry) {
fmt.Fprintln(w.out, e)
}
func NewSimpleWriter(out io.Writer) *SimpleWriter {
w := new(SimpleWriter)
w.out = out
return w
}

57
server/cmd/mmctl/printer/keys.go Обычный файл
Просмотреть файл

@@ -0,0 +1,57 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package printer
// These are the key that aliases
const (
ArrowLeft = rune(KeyCtrlB)
ArrowRight = rune(KeyCtrlF)
ArrowUp = rune(KeyCtrlP)
ArrowDown = rune(KeyCtrlN)
Space = ' '
Enter = '\r'
NewLine = '\n'
Backspace = rune(KeyCtrlH)
Backspace2 = rune(KeyDEL)
)
// Key is the ascii codes of a keys
type Key int16
// These are the control keys. Note that they overlap with other keys.
const (
KeyCtrlSpace Key = iota
KeyCtrlA // KeySOH
KeyCtrlB // KeySTX
KeyCtrlC // KeyETX
KeyCtrlD // KeyEOT
KeyCtrlE // KeyENQ
KeyCtrlF // KeyACK
KeyCtrlG // KeyBEL
KeyCtrlH // KeyBS
KeyCtrlI // KeyTAB
KeyCtrlJ // KeyLF
KeyCtrlK // KeyVT
KeyCtrlL // KeyFF
KeyCtrlM // KeyCR
KeyCtrlN // KeySO
KeyCtrlO // KeySI
KeyCtrlP // KeyDLE
KeyCtrlQ // KeyDC1
KeyCtrlR // KeyDC2
KeyCtrlS // KeyDC3
KeyCtrlT // KeyDC4
KeyCtrlU // KeyNAK
KeyCtrlV // KeySYN
KeyCtrlW // KeyETB
KeyCtrlX // KeyCAN
KeyCtrlY // KeyEM
KeyCtrlZ // KeySUB
KeyESC // KeyESC
KeyCtrlBackslash // KeyFS
KeyCtrlRightSq // KeyGS
KeyCtrlCarat // KeyRS
KeyCtrlUnderscore // KeyUS
KeyDEL = 0x7F
)

325
server/cmd/mmctl/printer/printer.go Обычный файл
Просмотреть файл

@@ -0,0 +1,325 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package printer
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
"text/template"
"github.com/fatih/color"
"github.com/spf13/cobra"
)
const (
FormatPlain = "plain"
FormatJSON = "json"
)
type Printer struct { //nolint
writer io.Writer
eWriter io.Writer
Format string
Single bool
NoNewline bool
templateFuncs template.FuncMap
pager bool
Quiet bool
Lines []interface{}
ErrorLines []interface{}
cmd *cobra.Command
serverAddr string
}
type printOpts struct {
format string
pagerPath string
single bool
usePager bool
shortStat bool
noNewline bool
}
var printer Printer
func init() {
printer.writer = os.Stdout
printer.eWriter = os.Stderr
printer.pager = true
printer.templateFuncs = make(template.FuncMap)
}
// SetFormat sets the format for the final output of the printer
func SetFormat(t string) {
printer.Format = t
}
func SetCommand(cmd *cobra.Command) {
printer.cmd = cmd
}
func SetServerAddres(addr string) {
printer.serverAddr = addr
}
func OverrideEnablePager(enable bool) {
printer.pager = enable
}
// SetFormat sets the format for the final output of the printer
func SetQuiet(q bool) {
printer.Quiet = q
}
// SetNoNewline prevents the addition of a newline at the end of the plain output.
// This may be useful when you want to handle newlines yourself.
func SetNoNewline(no bool) {
printer.NoNewline = no
}
func SetTemplateFunc(name string, f interface{}) {
printer.templateFuncs[name] = f
}
// SetSingle sets the single flag on the printer. If this flag is set, the
// printer will check the size of stored elements before printing, and
// if there is only one, it will be printed on its own instead of
// inside a list
func SetSingle(single bool) {
printer.Single = single
}
// PrintT prints an element. Depending on the format, the element can be
// formatted and printed as a structure or used to populate the
// template
func PrintT(templateString string, v interface{}) {
if printer.Quiet {
return
}
switch printer.Format {
case FormatPlain:
tpl := template.Must(template.New("").Funcs(printer.templateFuncs).Parse(templateString))
sb := &strings.Builder{}
if err := tpl.Execute(sb, v); err != nil {
PrintError("Can't print the message using the provided template: " + templateString)
return
}
printer.Lines = append(printer.Lines, sb.String())
case FormatJSON:
printer.Lines = append(printer.Lines, v)
}
}
func PrintPreparedT(tpl *template.Template, v interface{}) {
if printer.Quiet {
return
}
switch printer.Format {
case FormatPlain:
sb := &strings.Builder{}
if err := tpl.Execute(sb, v); err != nil {
PrintError("Can't print the message using the provided template: " + err.Error())
return
}
printer.Lines = append(printer.Lines, sb.String())
case FormatJSON:
printer.Lines = append(printer.Lines, v)
}
}
// Print an element. If the format requires a template, the element
// will be printed as a structure with field names using the print
// verb %+v
func Print(v interface{}) {
PrintT("{{printf \"%+v\" .}}", v)
}
// Flush writes the elements accumulated in the printer
func Flush() error {
if printer.Quiet {
return nil
}
opts := printOpts{
format: printer.Format,
single: printer.Single,
noNewline: printer.NoNewline,
}
cmd := printer.cmd
if cmd != nil {
shortStat, err := printer.cmd.Flags().GetBool("short-stat")
if err == nil && printer.cmd.Name() == "list" && printer.cmd.Parent().Name() != "auth" {
opts.shortStat = shortStat
}
}
b, err := printer.linesToBytes(opts)
if err != nil {
return err
}
lines := lineCount(b)
isTTY := checkInteractiveTerminal() == nil
var enablePager bool
termHeight, err := termHeight(os.Stdout)
if err == nil {
enablePager = isTTY && (termHeight < lines) // calculate if we should enable paging
}
pager := os.Getenv("PAGER")
if enablePager {
enablePager = pager != ""
}
opts.usePager = enablePager && printer.pager
opts.pagerPath = pager
err = printer.printBytes(b, opts)
if err != nil {
return err
}
// after all, print errors
printer.printErrors()
defer func() {
printer.Lines = []interface{}{}
printer.ErrorLines = []interface{}{}
}()
if cmd == nil || cmd.Name() != "list" || printer.cmd.Parent().Name() == "auth" {
return nil
}
// the command is a list command, we may want to
// take care of the stat flags
noStat, err := cmd.Flags().GetBool("no-stat")
if err != nil {
return err
}
// print stats
switch {
case noStat:
// do nothing
case !opts.shortStat:
// should not go to pager
if isTTY && !enablePager {
fmt.Fprintf(printer.eWriter, "\n") // add a one line space before statistical data
}
fallthrough
case len(printer.Lines) > 0:
entity := cmd.Parent().Name()
container := strings.TrimSuffix(printer.serverAddr, "api/v4")
if container != "" {
container = fmt.Sprintf(" on %s", container)
}
fmt.Fprintf(printer.eWriter, "There are %d %ss%s\n", len(printer.Lines), entity, container)
}
return nil
}
// Clean resets the printer's accumulated lines
func Clean() {
printer.Lines = []interface{}{}
printer.ErrorLines = []interface{}{}
}
// GetLines returns the printer's accumulated lines
func GetLines() []interface{} {
return printer.Lines
}
// GetErrorLines returns the printer's accumulated error lines
func GetErrorLines() []interface{} {
return printer.ErrorLines
}
// PrintError prints to the stderr.
func PrintError(msg string) {
printer.ErrorLines = append(printer.ErrorLines, msg)
}
// PrintWarning prints warning message to the error output, unlike Print and PrintError
// functions, PrintWarning writes the output immediately instead of waiting command to finish.
func PrintWarning(msg string) {
if printer.Quiet {
return
}
fmt.Fprintf(printer.eWriter, "%s\n", color.YellowString("WARNING: %s", msg))
}
func (p Printer) linesToBytes(opts printOpts) (b []byte, err error) {
if opts.shortStat {
return
}
newline := "\n"
if opts.noNewline {
newline = ""
}
switch opts.format {
case FormatPlain:
var buf bytes.Buffer
for i := range p.Lines {
fmt.Fprintf(&buf, "%s%s", p.Lines[i], newline)
}
b = buf.Bytes()
case FormatJSON:
switch {
case opts.single && len(p.Lines) == 0:
return
case opts.single && len(p.Lines) == 1:
b, err = json.MarshalIndent(p.Lines[0], "", " ")
default:
b, err = json.MarshalIndent(p.Lines, "", " ")
}
b = append(b, '\n')
}
return
}
func (p Printer) printBytes(b []byte, opts printOpts) error {
if !opts.usePager {
fmt.Fprintf(p.writer, "%s", b)
return nil
}
c := exec.Command(opts.pagerPath) // nolint:gosec
in, err := c.StdinPipe()
if err != nil {
return fmt.Errorf("could not create the stdin pipe: %w", err)
}
c.Stdout = p.writer
c.Stderr = p.eWriter
go func() {
defer in.Close()
_, _ = io.Copy(in, bytes.NewReader(b))
}()
if err := c.Start(); err != nil {
return fmt.Errorf("could not start the pager: %w", err)
}
return c.Wait()
}
func (p Printer) printErrors() {
for i := range printer.ErrorLines {
fmt.Fprintln(printer.eWriter, printer.ErrorLines[i])
}
}

161
server/cmd/mmctl/printer/printer_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,161 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package printer
import (
"bufio"
"bytes"
"testing"
"text/template"
"github.com/stretchr/testify/assert"
)
type mockWriter []byte
func (w *mockWriter) Write(b []byte) (n int, err error) {
*w = append(*w, b...)
return len(*w) - len(b), nil
}
func TestPrintT(t *testing.T) {
w := bufio.NewWriter(&bytes.Buffer{})
printer.writer = w
printer.Format = FormatPlain
ts := struct {
ID int
}{
ID: 123,
}
t.Run("should execute template", func(t *testing.T) {
tpl := `testing template {{.ID}}`
PrintT(tpl, ts)
assert.Len(t, GetLines(), 1)
assert.Equal(t, "testing template 123", printer.Lines[0])
_ = Flush()
})
t.Run("should fail to execute, no method or field", func(t *testing.T) {
Clean()
tpl := `testing template {{.Name}}`
PrintT(tpl, ts)
assert.Len(t, GetErrorLines(), 1)
assert.Equal(t, "Can't print the message using the provided template: "+tpl, printer.ErrorLines[0])
_ = Flush()
})
}
func TestPrintPreparedT(t *testing.T) {
w := bufio.NewWriter(&bytes.Buffer{})
printer.writer = w
printer.Format = FormatPlain
ts := struct {
ID int
}{
ID: 123,
}
t.Run("should execute template", func(t *testing.T) {
tpl := template.Must(template.New("").Parse(`testing template {{.ID}}`))
PrintPreparedT(tpl, ts)
assert.Len(t, GetLines(), 1)
assert.Equal(t, "testing template 123", printer.Lines[0])
_ = Flush()
})
t.Run("should fail to execute, no method or field", func(t *testing.T) {
Clean()
tpl := template.Must(template.New("").Parse(`testing template {{.Name}}`))
PrintPreparedT(tpl, ts)
assert.Len(t, GetErrorLines(), 1)
assert.Contains(t, printer.ErrorLines[0], "Can't print the message using the provided template")
_ = Flush()
})
}
func TestFlushJSON(t *testing.T) {
printer.Format = FormatJSON
t.Run("should print a line in JSON format", func(t *testing.T) {
mw := &mockWriter{}
printer.writer = mw
Clean()
Print("test string")
assert.Len(t, GetLines(), 1)
_ = Flush()
assert.Equal(t, "[\n \"test string\"\n]\n", string(*mw))
assert.Empty(t, GetLines(), 0)
})
t.Run("should print multi line in JSON format", func(t *testing.T) {
mw := &mockWriter{}
printer.writer = mw
Clean()
Print("test string-1")
Print("test string-2")
assert.Len(t, GetLines(), 2)
_ = Flush()
assert.Equal(t, "[\n \"test string-1\",\n \"test string-2\"\n]\n", string(*mw))
assert.Empty(t, GetLines(), 0)
})
}
func TestFlushPlain(t *testing.T) {
printer.Format = FormatPlain
t.Run("should print a line in plain format", func(t *testing.T) {
mw := &mockWriter{}
printer.writer = mw
Clean()
Print("test string")
assert.Len(t, GetLines(), 1)
_ = Flush()
assert.Equal(t, "test string\n", string(*mw))
assert.Empty(t, GetLines(), 0)
})
t.Run("should print multi line in plain format", func(t *testing.T) {
mw := &mockWriter{}
printer.writer = mw
Clean()
Print("test string-1")
Print("test string-2")
assert.Len(t, GetLines(), 2)
_ = Flush()
assert.Equal(t, "test string-1\ntest string-2\n", string(*mw))
assert.Empty(t, GetLines(), 0)
})
t.Run("should print multi line in plain format without a newline", func(t *testing.T) {
mw := &mockWriter{}
printer.writer = mw
printer.NoNewline = true
Clean()
Print("test string-1")
Print("test string-2")
assert.Len(t, GetLines(), 2)
_ = Flush()
assert.Equal(t, "test string-1test string-2", string(*mw))
assert.Empty(t, GetLines(), 0)
})
}

38
server/cmd/mmctl/printer/util.go Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package printer
import (
"bytes"
"errors"
"os"
"golang.org/x/term"
)
func checkInteractiveTerminal() error {
fileInfo, err := os.Stdout.Stat()
if err != nil {
return err
}
if (fileInfo.Mode() & os.ModeCharDevice) == 0 {
return errors.New("this is not an interactive shell")
}
return nil
}
func termHeight(file *os.File) (int, error) {
_, h, err := term.GetSize(int(file.Fd()))
if err != nil {
return -1, err
}
return h, nil
}
func lineCount(b []byte) int {
return bytes.Count(b, []byte{'\n'})
}