Support Elasticsearch v9 (for v10.11) (#35925)

* Support Elasticsearch v9 alongside v8

* Add CI workflow changes for Elasticsearch v8/v9 testing

* Support Elasticsearch v7 in addition to v8/v9, add v7 CI test job

Lowers the minimum supported ES version from 8 to 7 to avoid dropping
v7 support in a dot release. Adds a dedicated CI job to verify v7
compatibility alongside the existing v8 and v9 (default) jobs.

* Fix ES7 plugin install crash on cgroup v2 hosts

* Fix ES 7 container startup on cgroup v2 Linux (GitHub Actions)

ES 7 bundles JDK 11, which crashes with a NullPointerException in
CgroupV2Subsystem.getMountPoint() on modern Linux kernels that use
cgroup v2 (including GitHub Actions ubuntu-latest runners). The flag
was already set during the Dockerfile RUN step, but not at runtime.

Adding -XX:-UseContainerSupport to ES_JAVA_OPTS in docker-compose
fixes the crash. The flag is harmless on ES 8/9 which ship JDK 17+
where the cgroup v2 bug is fixed (it simply opts out of container-aware
JVM sizing).

* Capture docker compose logs in CI test artifact

* Use ES 7.17.29 for v7 CI test; remove cgroup v2 workarounds

ES 7.17.0 bundled JDK 17.0.1 which had a cgroup v2 bug
(CgroupV2Subsystem NPE) not fixable via -XX:-UseContainerSupport.
ES 7.17.29 bundles JDK 22 where the bug is long fixed.

Reverts the -XX:-UseContainerSupport workarounds added in the
previous two commits as they were based on a wrong diagnosis
and are no longer needed.

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

.github/workflows/server-ci-template.yml

@@ -264,6 +264,30 @@ jobs:
       datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&multiStatements=true&maxAllowedPacket=4194304
       drivername: mysql
       logsartifact: mysql-server-test-logs
+  test-elasticsearch-v8:
+    name: Elasticsearch v8 Compatibility
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/server-test-template.yml
+    secrets: inherit
+    with:
+      name: Elasticsearch v8 Compatibility
+      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
+      drivername: postgres
+      logsartifact: elasticsearch-v8-server-test-logs
+      elasticsearch-version: "8.9.0"
+      test-target: "test-server-elasticsearch"
+  test-elasticsearch-v7:
+    name: Elasticsearch v7 Compatibility
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/server-test-template.yml
+    secrets: inherit
+    with:
+      name: Elasticsearch v7 Compatibility
+      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
+      drivername: postgres
+      logsartifact: elasticsearch-v7-server-test-logs
+      elasticsearch-version: "7.17.29"
+      test-target: "test-server-elasticsearch"
   test-coverage:
     # Skip coverage generation for cherry-pick PRs into release branches.
     if: ${{ github.event_name != 'pull_request' || !startsWith(github.event.pull_request.base.ref, 'release-') }}

.github/workflows/server-test-template.yml

@@ -22,6 +22,14 @@ on:
         required: false
         type: boolean
         default: false
+      elasticsearch-version:
+        required: false
+        type: string
+        default: "9.0.0"
+      test-target:
+        required: false
+        type: string
+        default: "test-server"
       # -- Test sharding inputs (leave defaults for non-sharded callers) --
       shard-index:
         required: false
@@ -75,6 +83,8 @@ jobs:
           echo "${{ inputs.name }}" > server/test-name
           echo "${{ github.event.pull_request.number }}" > server/pr-number
       - name: Run docker compose
+        env:
+          ELASTICSEARCH_VERSION: ${{ inputs.elasticsearch-version }}
         run: |
           cd server/build
           docker compose --ansi never run --rm start_dependencies
@@ -143,11 +153,11 @@ jobs:
         env:
           BUILD_IMAGE: mattermost/mattermost-build-server:${{ steps.go.outputs.GO_VERSION }}
         run: |
-          if [[ ${{ github.ref_name }} == 'master' && ${{ inputs.fullyparallel }} != true ]]; then
+          if [[ ${{ github.ref_name }} == 'master' && ${{ inputs.fullyparallel }} != true && "${{ inputs.test-target }}" == "test-server" ]]; then
             export RACE_MODE="-race"
           fi
 
-          TEST_TARGET="test-server${RACE_MODE}"
+          TEST_TARGET="${{ inputs.test-target }}${RACE_MODE}"
           BUILD_NUMBER="${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}"
           DOCKER_CMD="make ${TEST_TARGET}"
 
@@ -186,8 +196,10 @@ jobs:
           disable_search: true
           files: server/cover.out
       - name: Stop docker compose
+        if: ${{ always() }}
         run: |
           cd server/build
+          docker compose --ansi never logs --no-color > ../../docker-compose.log 2>&1
           docker compose --ansi never stop
       - name: Archive logs
         if: ${{ always() }}
@@ -200,4 +212,5 @@ jobs:
             server/cover.out
             server/test-name
             server/pr-number
+            docker-compose.log
 

server/Makefile

@@ -1,4 +1,4 @@
-.PHONY: build package run stop run-client run-server run-node run-haserver stop-haserver stop-client stop-server restart restart-server restart-client restart-haserver start-docker update-docker clean-dist clean nuke check-style check-client-style check-server-style check-unit-tests test dist run-client-tests setup-run-client-tests cleanup-run-client-tests test-client build-linux build-osx build-windows package-prep package-linux package-osx package-windows internal-test-web-client vet run-server-for-web-client-tests diff-config prepackaged-plugins prepackaged-binaries test-server test-server-ee test-server-quick test-server-race test-mmctl-unit test-mmctl-e2e test-mmctl test-mmctl-coverage mmctl-build mmctl-docs new-migration migrations-extract test-public mocks-public
+.PHONY: build package run stop run-client run-server run-node run-haserver stop-haserver stop-client stop-server restart restart-server restart-client restart-haserver start-docker update-docker clean-dist clean nuke check-style check-client-style check-server-style check-unit-tests test dist run-client-tests setup-run-client-tests cleanup-run-client-tests test-client build-linux build-osx build-windows package-prep package-linux package-osx package-windows internal-test-web-client vet run-server-for-web-client-tests diff-config prepackaged-plugins prepackaged-binaries test-server test-server-ee test-server-elasticsearch test-server-quick test-server-race test-mmctl-unit test-mmctl-e2e test-mmctl test-mmctl-coverage mmctl-build mmctl-docs new-migration migrations-extract test-public mocks-public
 
 ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
@@ -476,6 +476,15 @@ test-server-ee: check-prereqs-enterprise start-docker gotestsum ## Runs EE tests
 	@echo Running only EE tests
 	$(GOBIN)/gotestsum --packages="$(EE_PACKAGES)" -- $(GOFLAGS) -timeout=20m
 
+ES_PACKAGES=$(shell $(GO) list ./enterprise/elasticsearch/...)
+
+test-server-elasticsearch: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-server-elasticsearch: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-server-elasticsearch: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-server-elasticsearch: check-prereqs-enterprise start-docker gotestsum ## Runs Elasticsearch tests.
+	@echo Running only Elasticsearch tests
+	$(GOBIN)/gotestsum --rerun-fails=3 --packages="$(ES_PACKAGES)" -- $(GOFLAGS) -timeout=20m
+
 test-server-quick: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
 test-server-quick: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
 test-server-quick: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)

server/build/Dockerfile.elasticsearch

@@ -0,0 +1,4 @@
+ARG ELASTICSEARCH_VERSION=9.0.0
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch analysis-icu analysis-nori analysis-kuromoji analysis-smartcn

server/build/docker-compose.common.yml

@@ -87,7 +87,11 @@ services:
       LDAP_DOMAIN: "mm.test.com"
       LDAP_ADMIN_PASSWORD: "mostest"
   elasticsearch:
-    image: "mattermostdevelopment/mattermost-elasticsearch:8.9.0"
+    build:
+      context: .
+      dockerfile: ./Dockerfile.elasticsearch
+      args:
+        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-9.0.0}
     networks:
       - mm-test
     environment:

server/enterprise/elasticsearch/elasticsearch/check_version_test.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	elastic "github.com/elastic/go-elasticsearch/v8"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newTestClient(t *testing.T, handler http.Handler) *elastic.TypedClient {
+	t.Helper()
+	ts := httptest.NewServer(handler)
+	t.Cleanup(ts.Close)
+
+	client, err := elastic.NewTypedClient(elastic.Config{
+		Addresses: []string{ts.URL},
+	})
+	require.NoError(t, err)
+	return client
+}
+
+func infoHandler(version string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("X-Elastic-Product", "Elasticsearch")
+		fmt.Fprintf(w, `{"cluster_name":"test","version":{"number":%q,"build_flavor":"default","build_hash":"abc","build_date":"2024-01-01","build_snapshot":false,"build_type":"docker","lucene_version":"9.0.0","minimum_wire_compatibility_version":"7.0.0","minimum_index_compatibility_version":"7.0.0"}}`, version)
+	}
+}
+
+func TestCheckVersion(t *testing.T) {
+	tests := []struct {
+		name        string
+		version     string
+		wantVersion string
+		wantMajor   int
+		wantErrID   string
+	}{
+		{
+			name:        "ES 8 is supported",
+			version:     "8.9.0",
+			wantVersion: "8.9.0",
+			wantMajor:   8,
+		},
+		{
+			name:        "ES 9 is supported",
+			version:     "9.0.0",
+			wantVersion: "9.0.0",
+			wantMajor:   9,
+		},
+		{
+			name:        "ES 7 is supported",
+			version:     "7.17.0",
+			wantVersion: "7.17.0",
+			wantMajor:   7,
+		},
+		{
+			name:      "ES 6 is too old",
+			version:   "6.8.0",
+			wantErrID: "ent.elasticsearch.min_version.app_error",
+		},
+		{
+			name:      "ES 10 is too new",
+			version:   "10.0.0",
+			wantErrID: "ent.elasticsearch.max_version.app_error",
+		},
+		{
+			name:      "invalid version string",
+			version:   "invalid",
+			wantErrID: "ent.elasticsearch.start.parse_server_version.app_error",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			client := newTestClient(t, infoHandler(tc.version))
+			version, major, appErr := checkVersion(client, nil)
+			if tc.wantErrID != "" {
+				require.NotNil(t, appErr)
+				assert.Equal(t, tc.wantErrID, appErr.Id)
+			} else {
+				require.Nil(t, appErr)
+				assert.Equal(t, tc.wantVersion, version)
+				assert.Equal(t, tc.wantMajor, major)
+			}
+		})
+	}
+}
+
+func TestCheckVersionConnectionError(t *testing.T) {
+	ts := httptest.NewServer(http.NotFoundHandler())
+	ts.Close() // close immediately to force connection error
+
+	client, err := elastic.NewTypedClient(elastic.Config{
+		Addresses:  []string{ts.URL},
+		MaxRetries: 0,
+	})
+	require.NoError(t, err)
+
+	_, _, appErr := checkVersion(client, nil)
+	require.NotNil(t, appErr)
+	assert.Equal(t, "ent.elasticsearch.start.get_server_version.app_error", appErr.Id)
+}

server/enterprise/elasticsearch/elasticsearch/elasticsearch.go

@@ -28,7 +28,8 @@ import (
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder"
 )
 
-const elasticsearchMaxVersion = 8
+const elasticsearchMinVersion = 7
+const elasticsearchMaxVersion = 9
 
 var (
 	purgeIndexListAllowedIndexes = []string{common.IndexBaseChannels}
@@ -106,7 +107,7 @@ func (es *ElasticsearchInterfaceImpl) Start() *model.AppError {
 		return appErr
 	}
 
-	version, major, appErr := checkMaxVersion(es.client, es.Platform.Config())
+	version, major, appErr := checkVersion(es.client, es.Platform.Config())
 	if appErr != nil {
 		return appErr
 	}
@@ -1245,7 +1246,7 @@ func (es *ElasticsearchInterfaceImpl) TestConfig(rctx request.CTX, cfg *model.Co
 		return appErr
 	}
 
-	_, _, appErr = checkMaxVersion(client, cfg)
+	_, _, appErr = checkVersion(client, cfg)
 	if appErr != nil {
 		return appErr
 	}
@@ -1830,19 +1831,22 @@ func (es *ElasticsearchInterfaceImpl) DeleteFilesBatch(rctx request.CTX, endTime
 	return nil
 }
 
-func checkMaxVersion(client *elastic.TypedClient, cfg *model.Config) (string, int, *model.AppError) {
+func checkVersion(client *elastic.TypedClient, cfg *model.Config) (string, int, *model.AppError) {
 	resp, err := client.API.Core.Info().Do(context.Background())
 	if err != nil {
-		return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.start.get_server_version.app_error", map[string]any{"Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusInternalServerError).Wrap(err)
+		return "", 0, model.NewAppError("Elasticsearch.checkVersion", "ent.elasticsearch.start.get_server_version.app_error", map[string]any{"Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusInternalServerError).Wrap(err)
 	}
 
 	major, _, _, esErr := common.GetVersionComponents(resp.Version.Int)
 	if esErr != nil {
-		return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.start.parse_server_version.app_error", map[string]any{"Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusInternalServerError).Wrap(err)
+		return "", 0, model.NewAppError("Elasticsearch.checkVersion", "ent.elasticsearch.start.parse_server_version.app_error", map[string]any{"Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusInternalServerError).Wrap(esErr)
 	}
 
+	if major < elasticsearchMinVersion {
+		return "", 0, model.NewAppError("Elasticsearch.checkVersion", "ent.elasticsearch.min_version.app_error", map[string]any{"Version": major, "MinVersion": elasticsearchMinVersion, "Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusBadRequest)
+	}
 	if major > elasticsearchMaxVersion {
-		return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.max_version.app_error", map[string]any{"Version": major, "MaxVersion": elasticsearchMaxVersion, "Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusBadRequest)
+		return "", 0, model.NewAppError("Elasticsearch.checkVersion", "ent.elasticsearch.max_version.app_error", map[string]any{"Version": major, "MaxVersion": elasticsearchMaxVersion, "Backend": model.ElasticsearchSettingsESBackend}, "", http.StatusBadRequest)
 	}
 	return resp.Version.Int, major, nil
 }

server/i18n/en.json

@@ -8228,6 +8228,10 @@
     "id": "ent.elasticsearch.max_version.app_error",
     "translation": "{{.Backend}} version {{.Version}} is higher than max supported version of {{.MaxVersion}}"
   },
+  {
+    "id": "ent.elasticsearch.min_version.app_error",
+    "translation": "{{.Backend}} version {{.Version}} is lower than min supported version of {{.MinVersion}}"
+  },
   {
     "id": "ent.elasticsearch.not_started.error",
     "translation": "{{.Backend}} is not started"