MM-59932: Migrate remaining caches to Redis (#27880)

- We introduce 2 new APIs:
1. Scan: this allows incremental iteration
without blocking the Redis server and is the
recommended way to iterate over keys. With this,
we have entirely removed the need for Keys.
2. RemoveMulti: this allows deletion of multiple
keys in a single operation which optimizes
network round trips.

- While here, we make a small improvement to
GetStatusFromCache, where we remove the shallow
copy which wasn't necessary because we always
serialize the data from the cache.
- We do not use Redis for session cache because of
frequent requests to iterate the entire cache which leads
to a lot of `SCAN` calls.
- Avoid broadcasting status update messages for Redis case.
- Setting cache expiry for status cache
- Removing .Set method altogether to prevent
any chances of setting an item with no expiry.

https://mattermost.atlassian.net/browse/MM-59932

```release-note
NONE
```

server/channels/store/localcachelayer/layer.go

@@ -340,7 +340,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
 	}); err != nil {
 		return
 	}
-	if localCacheStore.profilesInChannelCache, err = cache.NewProvider().NewCache(&cache.CacheOptions{
+	if localCacheStore.profilesInChannelCache, err = cacheProvider.NewCache(&cache.CacheOptions{
 		Size:                   ProfilesInChannelCacheSize,
 		Name:                   "ProfilesInChannel",
 		DefaultExpiry:          ProfilesInChannelCacheSec * time.Second,
@@ -450,7 +450,10 @@ func (s LocalCacheStore) DropAllTables() {
 }
 
 func (s *LocalCacheStore) doInvalidateCacheCluster(cache cache.Cache, key string, props map[string]string) {
-	cache.Remove(key)
+	err := cache.Remove(key)
+	if err != nil {
+		s.logger.Warn("Error while removing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
+	}
 	if s.cluster != nil && s.cacheType == model.CacheTypeLRU {
 		msg := &model.ClusterMessage{
 			Event:    cache.GetInvalidateClusterEvent(),
@@ -464,20 +467,46 @@ func (s *LocalCacheStore) doInvalidateCacheCluster(cache cache.Cache, key string
 	}
 }
 
-func (s *LocalCacheStore) doStandardAddToCache(cache cache.Cache, key string, value any) {
-	cache.SetWithDefaultExpiry(key, value)
+func (s *LocalCacheStore) doMultiInvalidateCacheCluster(cache cache.Cache, keys []string, props map[string]string) {
+	err := cache.RemoveMulti(keys)
+	if err != nil {
+		s.logger.Warn("Error while removing cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
+	}
+	if s.cluster != nil && s.cacheType == model.CacheTypeLRU {
+		for _, key := range keys {
+			msg := &model.ClusterMessage{
+				Event:    cache.GetInvalidateClusterEvent(),
+				SendType: model.ClusterSendBestEffort,
+				Data:     []byte(key),
+			}
+			if props != nil {
+				msg.Props = props
+			}
+			s.cluster.SendClusterMessage(msg)
+		}
+	}
 }
 
-func (s *LocalCacheStore) doStandardReadCache(cache cache.Cache, key string, value any) error {
-	err := cache.Get(key, value)
+func (s *LocalCacheStore) doStandardAddToCache(cache cache.Cache, key string, value any) {
+	err := cache.SetWithDefaultExpiry(key, value)
+	if err != nil {
+		s.logger.Warn("Error while setting cache entry", mlog.Err(err), mlog.String("cache_name", cache.Name()))
+	}
+}
+
+func (s *LocalCacheStore) doStandardReadCache(c cache.Cache, key string, value any) error {
+	err := c.Get(key, value)
 	if err == nil {
 		if s.metrics != nil {
-			s.metrics.IncrementMemCacheHitCounter(cache.Name())
+			s.metrics.IncrementMemCacheHitCounter(c.Name())
 		}
 		return nil
 	}
+	if err != cache.ErrKeyNotFound {
+		s.logger.Warn("Error while reading from cache", mlog.Err(err), mlog.String("cache_name", c.Name()))
+	}
 	if s.metrics != nil {
-		s.metrics.IncrementMemCacheMissCounter(cache.Name())
+		s.metrics.IncrementMemCacheMissCounter(c.Name())
 	}
 	return err
 }