MM-21019 - Fix race in (*Environment).SetPluginState() (#13610)

* MM-21019 - Fix race in (*Environment).SetPluginState()

- We change from passing pointers to registeredPlugin to passing
the struct by value.
- We also add a mutex to the supervisor struct to protect
from racy data access.

* move the immutability comment to the godoc of the Active method

* Changing mut to lock

Co-authored-by: mattermod <mattermod@users.noreply.github.com>

plugin/health_check.go

@@ -77,7 +77,7 @@ func (job *PluginHealthCheckJob) checkPlugin(id string) {
 	if !ok {
 		return
 	}
-	rp := p.(*registeredPlugin)
+	rp := p.(registeredPlugin)
 
 	sup := rp.supervisor
 	if sup == nil {
@@ -98,14 +98,16 @@ func (job *PluginHealthCheckJob) handleHealthCheckFail(id string, err error) {
 	if !ok {
 		return
 	}
-	p := rp.(*registeredPlugin)
+	p := rp.(registeredPlugin)
 
 	// Append current failure before checking for deactivate vs restart action
 	p.failTimeStamps = append(p.failTimeStamps, time.Now())
 	p.lastError = err
+	job.env.registeredPlugins.Store(id, p)
 
 	if shouldDeactivatePlugin(p) {
 		p.failTimeStamps = []time.Time{}
+		job.env.registeredPlugins.Store(id, p)
 		mlog.Debug("Deactivating plugin due to multiple crashes", mlog.String("id", id))
 		job.env.Deactivate(id)
 		job.env.SetPluginState(id, model.PluginStateFailedToStayRunning)
@@ -135,7 +137,7 @@ func (job *PluginHealthCheckJob) Cancel() {
 // shouldDeactivatePlugin determines if a plugin needs to be deactivated after certain criteria is met.
 //
 // The criteria is based on if the plugin has consistently failed during the configured number of restarts, within the configured time window.
-func shouldDeactivatePlugin(rp *registeredPlugin) bool {
+func shouldDeactivatePlugin(rp registeredPlugin) bool {
 	if len(rp.failTimeStamps) >= HEALTH_CHECK_RESTART_LIMIT {
 		index := len(rp.failTimeStamps) - HEALTH_CHECK_RESTART_LIMIT
 		t := rp.failTimeStamps[index]