Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/channels/store/layer_generators/main.go

@@ -0,0 +1,326 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"text/template"
+)
+
+const (
+	OpenTracingParamsMarker = "@openTracingParams"
+	ErrorType               = "error"
+)
+
+func isError(typeName string) bool {
+	return strings.Contains(typeName, ErrorType)
+}
+
+func main() {
+	if err := buildTimerLayer(); err != nil {
+		log.Fatal(err)
+	}
+	if err := buildOpenTracingLayer(); err != nil {
+		log.Fatal(err)
+	}
+	if err := buildRetryLayer(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func buildRetryLayer() error {
+	code, err := generateLayer("RetryLayer", "retry_layer.go.tmpl")
+	if err != nil {
+		return err
+	}
+	formatedCode, err := format.Source(code)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(path.Join("retrylayer/retrylayer.go"), formatedCode, 0644)
+}
+
+func buildTimerLayer() error {
+	code, err := generateLayer("TimerLayer", "timer_layer.go.tmpl")
+	if err != nil {
+		return err
+	}
+	formatedCode, err := format.Source(code)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(path.Join("timerlayer", "timerlayer.go"), formatedCode, 0644)
+}
+
+func buildOpenTracingLayer() error {
+	code, err := generateLayer("OpenTracingLayer", "opentracing_layer.go.tmpl")
+	if err != nil {
+		return err
+	}
+	formatedCode, err := format.Source(code)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(path.Join("opentracinglayer", "opentracinglayer.go"), formatedCode, 0644)
+}
+
+type methodParam struct {
+	Name string
+	Type string
+}
+
+type methodData struct {
+	Params        []methodParam
+	Results       []string
+	ParamsToTrace map[string]bool
+}
+
+type subStore struct {
+	Methods map[string]methodData
+}
+
+type storeMetadata struct {
+	Name      string
+	SubStores map[string]subStore
+	Methods   map[string]methodData
+}
+
+func extractMethodMetadata(method *ast.Field, src []byte) methodData {
+	params := []methodParam{}
+	results := []string{}
+	paramsToTrace := map[string]bool{}
+	ast.Inspect(method.Type, func(expr ast.Node) bool {
+		switch e := expr.(type) {
+		case *ast.FuncType:
+			if method.Doc != nil {
+				for _, comment := range method.Doc.List {
+					s := comment.Text
+					if idx := strings.Index(s, OpenTracingParamsMarker); idx != -1 {
+						for _, p := range strings.Split(s[idx+len(OpenTracingParamsMarker):], ",") {
+							paramsToTrace[strings.TrimSpace(p)] = true
+						}
+					}
+				}
+			}
+			if e.Params != nil {
+				for _, param := range e.Params.List {
+					for _, paramName := range param.Names {
+						params = append(params, methodParam{Name: paramName.Name, Type: string(src[param.Type.Pos()-1 : param.Type.End()-1])})
+					}
+				}
+			}
+			if e.Results != nil {
+				for _, result := range e.Results.List {
+					results = append(results, string(src[result.Type.Pos()-1:result.Type.End()-1]))
+				}
+			}
+
+			for paramName := range paramsToTrace {
+				found := false
+				for _, param := range params {
+					if param.Name == paramName {
+						found = true
+						break
+					}
+				}
+				if !found {
+					log.Fatalf("Unable to find a parameter called '%s' (method '%s') that is mentioned in the '%s' comment. Maybe it was renamed?", paramName, method.Names[0].Name, OpenTracingParamsMarker)
+				}
+			}
+		}
+		return true
+	})
+	return methodData{Params: params, Results: results, ParamsToTrace: paramsToTrace}
+}
+
+func extractStoreMetadata() (*storeMetadata, error) {
+	// Create the AST by parsing src.
+	fset := token.NewFileSet() // positions are relative to fset
+
+	file, err := os.Open("store.go")
+	if err != nil {
+		return nil, fmt.Errorf("unable to open store/store.go file: %w", err)
+	}
+	src, err := io.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+	file.Close()
+	f, err := parser.ParseFile(fset, "", src, parser.AllErrors|parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	topLevelFunctions := map[string]bool{
+		"MarkSystemRanUnitTests":   false,
+		"Close":                    false,
+		"LockToMaster":             false,
+		"UnlockFromMaster":         false,
+		"DropAllTables":            false,
+		"TotalMasterDbConnections": true,
+		"TotalReadDbConnections":   true,
+		"SetContext":               true,
+		"TotalSearchDbConnections": true,
+		"GetCurrentSchemaVersion":  true,
+	}
+
+	metadata := storeMetadata{Methods: map[string]methodData{}, SubStores: map[string]subStore{}}
+
+	ast.Inspect(f, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.TypeSpec:
+			if x.Name.Name == "Store" {
+				for _, method := range x.Type.(*ast.InterfaceType).Methods.List {
+					methodName := method.Names[0].Name
+					if _, ok := topLevelFunctions[methodName]; ok {
+						metadata.Methods[methodName] = extractMethodMetadata(method, src)
+					}
+				}
+			} else if strings.HasSuffix(x.Name.Name, "Store") {
+				subStoreName := strings.TrimSuffix(x.Name.Name, "Store")
+				metadata.SubStores[subStoreName] = subStore{Methods: map[string]methodData{}}
+				for _, method := range x.Type.(*ast.InterfaceType).Methods.List {
+					methodName := method.Names[0].Name
+					metadata.SubStores[subStoreName].Methods[methodName] = extractMethodMetadata(method, src)
+				}
+			}
+		}
+		return true
+	})
+
+	return &metadata, nil
+}
+
+func generateLayer(name, templateFile string) ([]byte, error) {
+	out := bytes.NewBufferString("")
+	metadata, err := extractStoreMetadata()
+	if err != nil {
+		return nil, err
+	}
+	metadata.Name = name
+
+	myFuncs := template.FuncMap{
+		"joinResults": func(results []string) string {
+			return strings.Join(results, ", ")
+		},
+		"joinResultsForSignature": func(results []string) string {
+			if len(results) == 0 {
+				return ""
+			}
+			returns := []string{}
+			for _, result := range results {
+				switch result {
+				case "*PostReminderMetadata":
+					returns = append(returns, fmt.Sprintf("*store.%s", strings.TrimPrefix(result, "*")))
+				default:
+					returns = append(returns, result)
+				}
+			}
+
+			if len(returns) == 1 {
+				return strings.Join(returns, ", ")
+			}
+			return fmt.Sprintf("(%s)", strings.Join(returns, ", "))
+		},
+		"genResultsVars": func(results []string, withNilError bool) string {
+			vars := []string{}
+			for i, typeName := range results {
+				if isError(typeName) {
+					if withNilError {
+						vars = append(vars, "nil")
+					} else {
+						vars = append(vars, "err")
+					}
+				} else if i == 0 {
+					vars = append(vars, "result")
+				} else {
+					vars = append(vars, fmt.Sprintf("resultVar%d", i))
+				}
+			}
+			return strings.Join(vars, ", ")
+		},
+		"errorToBoolean": func(results []string) string {
+			for _, typeName := range results {
+				if isError(typeName) {
+					return "err == nil"
+				}
+			}
+			return "true"
+		},
+		"errorPresent": func(results []string) bool {
+			for _, typeName := range results {
+				if isError(typeName) {
+					return true
+				}
+			}
+			return false
+		},
+		"errorVar": func(results []string) string {
+			for _, typeName := range results {
+				if isError(typeName) {
+					return "err"
+				}
+			}
+			return ""
+		},
+		"joinParams": func(params []methodParam) string {
+			paramsNames := make([]string, 0, len(params))
+			for _, param := range params {
+				tParams := ""
+				if strings.HasPrefix(param.Type, "...") {
+					tParams = "..."
+				}
+				paramsNames = append(paramsNames, param.Name+tParams)
+			}
+			return strings.Join(paramsNames, ", ")
+		},
+		"joinParamsWithType": func(params []methodParam) string {
+			paramsWithType := []string{}
+			for _, param := range params {
+				switch param.Type {
+				case "ChannelSearchOpts", "UserGetByIdsOpts", "ThreadMembershipOpts":
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s store.%s", param.Name, param.Type))
+				case "*UserGetByIdsOpts", "*ChannelMemberGraphQLSearchOpts", "*SidebarCategorySearchOpts":
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s *store.%s", param.Name, strings.TrimPrefix(param.Type, "*")))
+				default:
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s %s", param.Name, param.Type))
+				}
+			}
+			return strings.Join(paramsWithType, ", ")
+		},
+		"joinParamsWithTypeOutsideStore": func(params []methodParam) string {
+			paramsWithType := []string{}
+			for _, param := range params {
+				switch param.Type {
+				case "ChannelSearchOpts", "UserGetByIdsOpts", "ThreadMembershipOpts":
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s store.%s", param.Name, param.Type))
+				case "*UserGetByIdsOpts", "*ChannelMemberGraphQLSearchOpts", "*SidebarCategorySearchOpts":
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s *store.%s", param.Name, strings.TrimPrefix(param.Type, "*")))
+				default:
+					paramsWithType = append(paramsWithType, fmt.Sprintf("%s %s", param.Name, param.Type))
+				}
+			}
+			return strings.Join(paramsWithType, ", ")
+		},
+	}
+
+	t := template.Must(template.New(templateFile).Funcs(myFuncs).ParseFiles("layer_generators/" + templateFile))
+	if err = t.Execute(out, metadata); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}

server/channels/store/layer_generators/opentracing_layer.go.tmpl

@@ -0,0 +1,84 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "make store-layers"
+// DO NOT EDIT
+
+package opentracinglayer
+
+import (
+	"context"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/tracing"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/opentracing/opentracing-go/ext"
+	spanlog "github.com/opentracing/opentracing-go/log"
+)
+
+type {{.Name}} struct {
+	store.Store
+{{range $index, $element := .SubStores}}	{{$index}}Store store.{{$index}}Store
+{{end}}
+}
+
+{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
+	return s.{{$index}}Store
+}
+
+{{end}}
+
+{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
+	store.{{$index}}Store
+	Root *{{$.Name}}
+}
+
+{{end}}
+
+{{range $substoreName, $substore := .SubStores}}
+{{range $index, $element := $substore.Methods}}
+func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "{{$substoreName}}Store.{{$index}}")
+	s.Root.Store.SetContext(newCtx)
+	defer func(){
+		s.Root.Store.SetContext(origCtx)
+	}()
+	{{range $paramName, $param := $element.Params}}
+		{{if index $element.ParamsToTrace $param.Name }}
+			span.SetTag("{{$param.Name}}", {{$param.Name}})
+		{{end}}
+	{{end}}
+	defer span.Finish()
+	{{- if $element.Results | len | eq 0}}
+	s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}
+		{{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+		{{- if $element.Results | errorPresent }}
+			if {{$element.Results | errorVar}} != nil {
+				span.LogFields(spanlog.Error({{$element.Results | errorVar}}))
+				ext.Error.Set(span, true)
+			}
+		{{end}}
+		return {{ genResultsVars $element.Results false -}}
+	{{end}}
+}
+{{end}}
+{{end}}
+
+{{range $index, $element := .Methods}}
+func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}}
+{{end}}
+
+func New(childStore store.Store, ctx context.Context) *{{.Name}} {
+	newStore := {{.Name}}{
+		Store: childStore,
+	}
+	{{range $substoreName, $substore := .SubStores}}
+	newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
+	return &newStore
+}

server/channels/store/layer_generators/retry_layer.go.tmpl

@@ -0,0 +1,103 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "make store-layers"
+// DO NOT EDIT
+
+package retrylayer
+
+import (
+	"context"
+	timepkg "time"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/mattermost/mattermost-server/v6/model"
+    "github.com/mattermost/mattermost-server/v6/server/channels/store"
+    "github.com/pkg/errors"
+	"github.com/go-sql-driver/mysql"
+)
+
+const mySQLDeadlockCode = uint16(1213)
+
+type {{.Name}} struct {
+	store.Store
+{{range $index, $element := .SubStores}}	{{$index}}Store store.{{$index}}Store
+{{end}}
+}
+
+{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
+	return s.{{$index}}Store
+}
+
+{{end}}
+
+{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
+	store.{{$index}}Store
+	Root *{{$.Name}}
+}
+
+{{end}}
+
+func isRepeatableError(err error) bool {
+	var pqErr *pq.Error
+	var mysqlErr *mysql.MySQLError
+	switch {
+	case errors.As(errors.Cause(err), &pqErr):
+		if pqErr.Code == "40001" || pqErr.Code == "40P01" {
+			return true
+		}
+	case errors.As(errors.Cause(err), &mysqlErr):
+		if mysqlErr.Number == mySQLDeadlockCode {
+			return true
+		}
+	}
+	return false
+}
+
+{{range $substoreName, $substore := .SubStores}}
+{{range $index, $element := $substore.Methods}}
+func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithTypeOutsideStore}}) {{$element.Results | joinResultsForSignature}} {
+    {{if $element.Results | len | eq 0}}
+        s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+    {{else}}
+        {{if $element.Results | errorPresent}}
+            tries := 0
+            for {
+                {{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+                if {{$element.Results | errorVar}} == nil {
+                    return {{genResultsVars $element.Results true }}
+                }
+                if !isRepeatableError({{$element.Results | errorVar}}) {
+                    return {{genResultsVars $element.Results false }}
+                }
+                tries++
+                if tries >= 3 {
+                    {{$element.Results | errorVar}} = errors.Wrap({{$element.Results | errorVar}}, "giving up after 3 consecutive repeatable transaction failures")
+                    return {{genResultsVars $element.Results false }}
+                }
+                timepkg.Sleep(100 * timepkg.Millisecond)
+            }
+        {{else}}
+            return s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+        {{end}}
+    {{end}}
+}
+{{end}}
+{{end}}
+
+{{range $index, $element := .Methods}}
+func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithTypeOutsideStore}}) {{$element.Results | joinResultsForSignature}} {
+	{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}}
+{{end}}
+
+func New(childStore store.Store) *{{.Name}} {
+	newStore := {{.Name}}{
+		Store: childStore,
+	}
+	{{range $substoreName, $substore := .SubStores}}
+	newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
+	return &newStore
+}

server/channels/store/layer_generators/timer_layer.go.tmpl

@@ -0,0 +1,79 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "make store-layers"
+// DO NOT EDIT
+
+package timerlayer
+
+import (
+	"context"
+    "time"
+
+    "github.com/mattermost/mattermost-server/v6/server/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+type {{.Name}} struct {
+	store.Store
+	Metrics einterfaces.MetricsInterface
+{{range $index, $element := .SubStores}}	{{$index}}Store store.{{$index}}Store
+{{end}}
+}
+
+{{range $index, $element := .SubStores}}func (s *{{$.Name}}) {{$index}}() store.{{$index}}Store {
+	return s.{{$index}}Store
+}
+
+{{end}}
+
+{{range $index, $element := .SubStores}}type {{$.Name}}{{$index}}Store struct {
+	store.{{$index}}Store
+	Root *{{$.Name}}
+}
+
+{{end}}
+
+{{range $substoreName, $substore := .SubStores}}
+{{range $index, $element := $substore.Methods}}
+func (s *{{$.Name}}{{$substoreName}}Store) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	start := time.Now()
+	{{if $element.Results | len | eq 0}}
+	s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}
+	{{genResultsVars $element.Results false }} := s.{{$substoreName}}Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if {{$element.Results | errorToBoolean}} {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("{{$substoreName}}Store.{{$index}}", success, elapsed)
+	{{ with (genResultsVars $element.Results false ) -}}
+	}
+	return {{ . }}
+	{{- else -}}
+	}
+	{{- end }}
+}
+{{end}}
+{{end}}
+
+{{range $index, $element := .Methods}}
+func (s *{{$.Name}}) {{$index}}({{$element.Params | joinParamsWithType}}) {{$element.Results | joinResultsForSignature}} {
+	{{if $element.Results | len | eq 0}}s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{else}}return s.Store.{{$index}}({{$element.Params | joinParams}})
+	{{end}}}
+{{end}}
+
+func New(childStore store.Store, metrics einterfaces.MetricsInterface) *{{.Name}} {
+	newStore := {{.Name}}{
+		Store: childStore,
+		Metrics: metrics,
+	}
+	{{range $substoreName, $substore := .SubStores}}
+	newStore.{{$substoreName}}Store = &{{$.Name}}{{$substoreName}}Store{{"{"}}{{$substoreName}}Store: childStore.{{$substoreName}}(), Root: &newStore}{{end}}
+	return &newStore
+}