Files
Agniva De Sarker 05cdb36886 MM-49353: Setup intermediate signal handlers for plugin shutdown (#28653)
There are several steps that a server runs through inside (*Server).Start
after ch.initPlugins() till the signal handler is reached which handles
the server shutdown procedure.

The issue arises when the server is shutdown after ch.initPlugins() completes
but before (*Server).Start finishes. In that case, the plugins are all started
but they won't be shut down cleanly.

To fix this edge-case, we set up an intermediate signal handler, which
attaches itself as soon as ch.initPlugins is finished, allowing us to run
the cleanup code in case the shutdown happens before (*Server).Start finishes.

And when we do reach the main signal handler, we don't need this intermediate
handler any more. So we just reset the handlers and use the main signal handler
which takes care of shutting down the whole server.

Note: This is still not 100% bug-proof because ch.initPlugins() will initialize
_all_ plugins, and the shutdown can happen just after one plugin is initialized.
To handle that case will require the need to set up signal handlers after every
plugin init which feels like overkill to me.

A sample flow diagram to visualize better:

Edge-case
server.Start()
|
ch.initPlugins()
|
<ctrl-c>
|
execute signal handler, os.Exit(1)

Happy-path
server.Start()
|
ch.initPlugins()
|
server.Start() finished
|
reset old signal handler
|
setup main signal handler
|
server runs on as usual until shutdown

https://mattermost.atlassian.net/browse/MM-49353

```release-note
NONE
```
2025-01-21 11:15:19 +05:30

152 строки
3.9 KiB
Go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package commands
import (
"bytes"
"net"
"os"
"os/signal"
"runtime/debug"
"runtime/pprof"
"syscall"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/api4"
"github.com/mattermost/mattermost/server/v8/channels/app"
"github.com/mattermost/mattermost/server/v8/channels/utils"
"github.com/mattermost/mattermost/server/v8/channels/web"
"github.com/mattermost/mattermost/server/v8/channels/wsapi"
"github.com/mattermost/mattermost/server/v8/config"
)
var serverCmd = &cobra.Command{
Use: "server",
Short: "Run the Mattermost server",
RunE: serverCmdF,
SilenceUsage: true,
}
func init() {
RootCmd.AddCommand(serverCmd)
RootCmd.RunE = serverCmdF
}
func serverCmdF(command *cobra.Command, args []string) error {
interruptChan := make(chan os.Signal, 1)
if err := utils.TranslationsPreInit(); err != nil {
return errors.Wrap(err, "unable to load Mattermost translation files")
}
customDefaults, err := loadCustomDefaults()
if err != nil {
mlog.Warn("Error loading custom configuration defaults: " + err.Error())
}
configStore, err := config.NewStoreFromDSN(getConfigDSN(command, config.GetEnvironment()), false, customDefaults, true)
if err != nil {
return errors.Wrap(err, "failed to load configuration")
}
defer configStore.Close()
return runServer(configStore, interruptChan)
}
func runServer(configStore *config.Store, interruptChan chan os.Signal) error {
// Setting the highest traceback level from the code.
// This is done to print goroutines from all threads (see golang.org/issue/13161)
// and also preserve a crash dump for later investigation.
debug.SetTraceback("crash")
options := []app.Option{
// The option order is important as app.Config option reads app.StartMetrics option.
app.StartMetrics,
app.ConfigStore(configStore),
app.RunEssentialJobs,
app.JoinCluster,
}
server, err := app.NewServer(options...)
if err != nil {
mlog.Error(err.Error())
return err
}
defer server.Shutdown()
// We add this after shutdown so that it can be called
// before server shutdown happens as it can close
// the advanced logger and prevent the mlog call from working properly.
defer func() {
// A panic pass-through layer which just logs it
// and sends it upwards.
if x := recover(); x != nil {
var buf bytes.Buffer
pprof.Lookup("goroutine").WriteTo(&buf, 2)
mlog.Error("A panic occurred",
mlog.Any("error", x),
mlog.String("stack", buf.String()))
panic(x)
}
}()
_, err = api4.Init(server)
if err != nil {
mlog.Error(err.Error())
return err
}
wsapi.Init(server)
web.New(server)
err = server.Start()
if err != nil {
mlog.Error(err.Error())
return err
}
notifyReady()
// Wiping off any signal handlers set before.
// This may come from intermediary signal handlers requiring to clean
// up resources before server.Start can finish.
signal.Reset(syscall.SIGINT, syscall.SIGTERM)
// wait for kill signal before attempting to gracefully shutdown
// the running service
signal.Notify(interruptChan, syscall.SIGINT, syscall.SIGTERM)
<-interruptChan
return nil
}
func notifyReady() {
// If the environment vars provide a systemd notification socket,
// notify systemd that the server is ready.
systemdSocket := os.Getenv("NOTIFY_SOCKET")
if systemdSocket != "" {
mlog.Info("Sending systemd READY notification.")
err := sendSystemdReadyNotification(systemdSocket)
if err != nil {
mlog.Error(err.Error())
}
}
}
func sendSystemdReadyNotification(socketPath string) error {
msg := "READY=1"
addr := &net.UnixAddr{
Name: socketPath,
Net: "unixgram",
}
conn, err := net.DialUnix(addr.Net, nil, addr)
if err != nil {
return err
}
defer conn.Close()
_, err = conn.Write([]byte(msg))
return err
}