build: add targets for linux/darwin arm64 and include those in the package (#18491)

* build: add targets for linux/darwin arm64 and include those in the package

* update based on feedback

* store arm64 targets

.circleci/config.yml

@@ -267,6 +267,10 @@ jobs:
           path: ~/mattermost/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz
       - store_artifacts:
           path: ~/mattermost/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz
+      - store_artifacts:
+          path: ~/mattermost/mattermost-server/dist/mattermost-team-linux-arm64.tar.gz
+      - store_artifacts:
+          path: ~/mattermost/mattermost-server/dist/mattermost-team-osx-arm64.tar.gz
       - store_artifacts:
           path: ~/mattermost/mattermost-server/dist/mattermost-team-windows-amd64.zip
       - persist_to_workspace:

Makefile

@@ -95,8 +95,10 @@ PLATFORM_FILES="./cmd/mattermost"
 # Output paths
 DIST_ROOT=dist
 DIST_PATH=$(DIST_ROOT)/mattermost
-DIST_PATH_LIN=$(DIST_ROOT)/linux/mattermost
+DIST_PATH_LIN_AMD64=$(DIST_ROOT)/linux_amd64/mattermost
-DIST_PATH_OSX=$(DIST_ROOT)/osx/mattermost
+DIST_PATH_LIN_ARM64=$(DIST_ROOT)/linux_arm64/mattermost
+DIST_PATH_OSX_AMD64=$(DIST_ROOT)/osx_amd64/mattermost
+DIST_PATH_OSX_ARM64=$(DIST_ROOT)/osx_arm64/mattermost
 DIST_PATH_WIN=$(DIST_ROOT)/windows/mattermost
 
 # Tests

build/release.mk

@@ -8,6 +8,13 @@ else
 	mkdir -p $(GOBIN)/linux_amd64
 	env GOOS=linux GOARCH=amd64 $(GO) build -o $(GOBIN)/linux_amd64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
 endif
+	@echo Build Linux arm64
+ifeq ($(BUILDER_GOOS_GOARCH),"linux_arm64")
+	env GOOS=linux GOARCH=arm64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
+else
+	mkdir -p $(GOBIN)/linux_arm64
+	env GOOS=linux GOARCH=arm64 $(GO) build -o $(GOBIN)/linux_arm64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
+endif
 
 build-osx:
 	@echo Build OSX amd64
@@ -17,6 +24,13 @@ else
 	mkdir -p $(GOBIN)/darwin_amd64
 	env GOOS=darwin GOARCH=amd64 $(GO) build -o $(GOBIN)/darwin_amd64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
 endif
+	@echo Build OSX arm64
+ifeq ($(BUILDER_GOOS_GOARCH),"darwin_arm64")
+	env GOOS=darwin GOARCH=arm64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
+else
+	mkdir -p $(GOBIN)/darwin_arm64
+	env GOOS=darwin GOARCH=arm64 $(GO) build -o $(GOBIN)/darwin_arm64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./...
+endif
 
 build-windows:
 	@echo Build Windows amd64
@@ -28,25 +42,39 @@ else
 endif
 
 build-cmd-linux:
-	@echo Build Linux amd64
+	@echo Build CMD Linux amd64
 ifeq ($(BUILDER_GOOS_GOARCH),"linux_amd64")
 	env GOOS=linux GOARCH=amd64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
 else
 	mkdir -p $(GOBIN)/linux_amd64
 	env GOOS=linux GOARCH=amd64 $(GO) build -o $(GOBIN)/linux_amd64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
 endif
+	@echo Build CMD Linux arm64
+ifeq ($(BUILDER_GOOS_GOARCH),"linux_arm64")
+	env GOOS=linux GOARCH=arm64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
+else
+	mkdir -p $(GOBIN)/linux_arm64
+	env GOOS=linux GOARCH=arm64 $(GO) build -o $(GOBIN)/linux_arm64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
+endif
 
 build-cmd-osx:
-	@echo Build OSX amd64
+	@echo Build CMD OSX amd64
 ifeq ($(BUILDER_GOOS_GOARCH),"darwin_amd64")
 	env GOOS=darwin GOARCH=amd64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
 else
 	mkdir -p $(GOBIN)/darwin_amd64
 	env GOOS=darwin GOARCH=amd64 $(GO) build -o $(GOBIN)/darwin_amd64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
 endif
+	@echo Build CMD OSX arm64
+ifeq ($(BUILDER_GOOS_GOARCH),"darwin_arm64")
+	env GOOS=darwin GOARCH=arm64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
+else
+	mkdir -p $(GOBIN)/darwin_arm64
+	env GOOS=darwin GOARCH=arm64 $(GO) build -o $(GOBIN)/darwin_arm64 $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
+endif
 
 build-cmd-windows:
-	@echo Build Windows amd64
+	@echo Build CMD Windows amd64
 ifeq ($(BUILDER_GOOS_GOARCH),"windows_amd64")
 	env GOOS=windows GOARCH=amd64 $(GO) build -o $(GOBIN) $(GOFLAGS) -trimpath -ldflags '$(LDFLAGS)' ./cmd/...
 else
@@ -118,41 +146,75 @@ endif
 		done; \
 	done
 
-
+package-general:
-package-osx: package-prep
 	@# Create needed directories
-	mkdir -p $(DIST_PATH_OSX)/bin
+	mkdir -p $(DIST_PATH_GENERIC)/bin
-	mkdir -p $(DIST_PATH_OSX)/logs
+	mkdir -p $(DIST_PATH_GENERIC)/logs
-	mkdir -p $(DIST_PATH_OSX)/prepackaged_plugins
+	mkdir -p $(DIST_PATH_GENERIC)/prepackaged_plugins
 
 	@# Copy binary
-ifeq ($(BUILDER_GOOS_GOARCH),"darwin_amd64")
+ifeq ($(BUILDER_GOOS_GOARCH),"$(CURRENT_PACKAGE_ARCH)")
-	cp $(GOBIN)/mattermost $(DIST_PATH_OSX)/bin # from native bin dir, not cross-compiled
+	cp $(GOBIN)/$(MM_BIN_NAME) $(DIST_PATH_GENERIC)/bin # from native bin dir, not cross-compiled
 else
-	cp $(GOBIN)/darwin_amd64/mattermost $(DIST_PATH_OSX)/bin # from cross-compiled bin dir
+	cp $(GOBIN)/$(CURRENT_PACKAGE_ARCH)/$(MM_BIN_NAME) $(DIST_PATH_GENERIC)/bin # from cross-compiled bin dir
 endif
-	#Download MMCTL for OSX
+
-	scripts/download_mmctl_release.sh "Darwin" $(DIST_PATH_OSX)/bin
+ifeq ("darwin_arm64","$(CURRENT_PACKAGE_ARCH)")
+	echo "No plugins or mmctl yet for $(CURRENT_PACKAGE_ARCH) platform, skipping..."
+else ifeq ("linux_arm64","$(CURRENT_PACKAGE_ARCH)")
+	echo "No plugins or mmctl yet for $(CURRENT_PACKAGE_ARCH) platform, skipping..."
+else
+	#Download MMCTL for $(MMCTL_PLATFORM)
+	scripts/download_mmctl_release.sh $(MMCTL_PLATFORM) $(DIST_PATH_GENERIC)/bin
 	@# Prepackage plugins
 	@for plugin_package in $(PLUGIN_PACKAGES) ; do \
-		ARCH="osx-amd64"; \
+		ARCH=$(PLUGIN_ARCH); \
-		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz $(DIST_PATH_OSX)/prepackaged_plugins; \
+		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz $(DIST_PATH_GENERIC)/prepackaged_plugins; \
-		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_OSX)/prepackaged_plugins; \
+		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_GENERIC)/prepackaged_plugins; \
-		HAS_ARCH=`tar -tf $(DIST_PATH_OSX)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz | grep -oE "dist/plugin-.*"`; \
+		HAS_ARCH=`tar -tf $(DIST_PATH_GENERIC)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz | grep -oE "dist/plugin-.*"`; \
-		if [ "$$HAS_ARCH" != "dist/plugin-darwin-amd64" ]; then \
+		if [ "$$HAS_ARCH" != "dist/plugin-$(subst _,-,$(CURRENT_PACKAGE_ARCH))" ]; then \
-			echo "Contains $$HAS_ARCH in $$plugin_package-$$ARCH.tar.gz but needs dist/plugin-darwin-amd64"; \
+			echo "Contains $$HAS_ARCH in $$plugin_package-$$ARCH.tar.gz but needs dist/plugin-$(subst _,-,$(CURRENT_PACKAGE_ARCH))"; \
 			exit 1; \
 		fi; \
-		gpg --verify $(DIST_PATH_OSX)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_OSX)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz; \
+		gpg --verify $(DIST_PATH_GENERIC)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_GENERIC)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz; \
 		if [ $$? -ne 0 ]; then \
 			echo "Failed to verify $$plugin_package-$$ARCH.tar.gz|$$plugin_package-$$ARCH.tar.gz.sig"; \
 			exit 1; \
 		fi; \
 	done
+endif
+
+package-osx-amd64: package-prep
+	DIST_PATH_GENERIC=$(DIST_PATH_OSX_AMD64) CURRENT_PACKAGE_ARCH=darwin_amd64 PLUGIN_ARCH=osx-amd64 MMCTL_PLATFORM="Darwin" MM_BIN_NAME=mattermost $(MAKE) package-general
 	@# Package
-	tar -C $(DIST_PATH_OSX)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-osx-amd64.tar.gz mattermost ../mattermost
+	tar -C $(DIST_PATH_OSX_AMD64)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-osx-amd64.tar.gz mattermost ../mattermost
 	@# Cleanup
-	rm -rf $(DIST_PATH_OSX)
+	rm -rf $(DIST_ROOT)/osx_amd64
+
+package-osx-arm64: package-prep
+	DIST_PATH_GENERIC=$(DIST_PATH_OSX_ARM64) CURRENT_PACKAGE_ARCH=darwin_arm64 PLUGIN_ARCH=osx-arm64 MMCTL_PLATFORM="Darwin" MM_BIN_NAME=mattermost $(MAKE) package-general
+	@# Package
+	tar -C $(DIST_PATH_OSX_ARM64)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-osx-arm64.tar.gz mattermost ../mattermost
+	@# Cleanup
+	rm -rf $(DIST_ROOT)/osx_arm64
+
+package-osx: package-osx-amd64 package-osx-arm64
+
+package-linux-amd64: package-prep
+	DIST_PATH_GENERIC=$(DIST_PATH_LIN_AMD64) CURRENT_PACKAGE_ARCH=linux_amd64 PLUGIN_ARCH=linux-amd64 MMCTL_PLATFORM="Linux" MM_BIN_NAME=mattermost $(MAKE) package-general
+	@# Package
+	tar -C $(DIST_PATH_LIN_AMD64)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-linux-amd64.tar.gz mattermost ../mattermost
+	@# Cleanup
+	rm -rf $(DIST_ROOT)/linux_amd64
+
+package-linux-arm64: package-prep
+	DIST_PATH_GENERIC=$(DIST_PATH_LIN_ARM64) CURRENT_PACKAGE_ARCH=linux_arm64 PLUGIN_ARCH=linux-arm64 MMCTL_PLATFORM="Linux" MM_BIN_NAME=mattermost $(MAKE) package-general
+	@# Package
+	tar -C $(DIST_PATH_LIN_ARM64)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-linux-amd64.tar.gz mattermost ../mattermost
+	@# Cleanup
+	rm -rf $(DIST_ROOT)/linux_arm64
+
+package-linux: package-linux-amd64 package-linux-arm64
 
 package-windows: package-prep
 	@# Create needed directories
@@ -187,45 +249,8 @@ endif
 	@# Package
 	cd $(DIST_PATH_WIN)/.. && zip -9 -r -q -l ../mattermost-$(BUILD_TYPE_NAME)-windows-amd64.zip mattermost ../mattermost && cd ../..
 	@# Cleanup
-	rm -rf $(DIST_PATH_WIN)
+	rm -rf $(DIST_ROOT)/windows
 
-package-linux: package-prep
+package: package-osx package-linux package-windows
-	@# Create needed directories
-	mkdir -p $(DIST_PATH_LIN)/bin
-	mkdir -p $(DIST_PATH_LIN)/logs
-	mkdir -p $(DIST_PATH_LIN)/prepackaged_plugins
-
-	@# Copy binary
-ifeq ($(BUILDER_GOOS_GOARCH),"linux_amd64")
-	cp $(GOBIN)/mattermost $(DIST_PATH_LIN)/bin # from native bin dir, not cross-compiled
-else
-	cp $(GOBIN)/linux_amd64/mattermost $(DIST_PATH_LIN)/bin # from cross-compiled bin dir
-endif
-	#Download MMCTL for Linux
-	scripts/download_mmctl_release.sh "Linux" $(DIST_PATH_LIN)/bin
-	@# Prepackage plugins
-	@for plugin_package in $(PLUGIN_PACKAGES) ; do \
-		ARCH="linux-amd64"; \
-		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz $(DIST_PATH_LIN)/prepackaged_plugins; \
-		cp tmpprepackaged/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_LIN)/prepackaged_plugins; \
-		HAS_ARCH=`tar -tf $(DIST_PATH_LIN)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz | grep -oE "dist/plugin-.*"`; \
-		if [ "$$HAS_ARCH" != "dist/plugin-linux-amd64" ]; then \
-			echo "Contains $$HAS_ARCH in $$plugin_package-$$ARCH.tar.gz but needs dist/plugin-linux-amd64"; \
-			exit 1; \
-		fi; \
-		gpg --verify $(DIST_PATH_LIN)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz.sig $(DIST_PATH_LIN)/prepackaged_plugins/$$plugin_package-$$ARCH.tar.gz; \
-		if [ $$? -ne 0 ]; then \
-			echo "Failed to verify $$plugin_package-$$ARCH.tar.gz|$$plugin_package-$$ARCH.tar.gz.sig"; \
-			exit 1; \
-		fi; \
-	done
-	@# Package
-	tar -C $(DIST_PATH_LIN)/.. -czf $(DIST_PATH)-$(BUILD_TYPE_NAME)-linux-amd64.tar.gz mattermost ../mattermost
-	@# Preserve native package so dev machines will have an unzipped package available
-	mv $(DIST_PATH_LIN)/bin $(DIST_PATH)/bin
-	@# Cleanup
-	rm -rf $(DIST_PATH_LIN)
-
-package: package-osx package-windows package-linux
 	rm -rf tmpprepackaged
 	rm -rf $(DIST_PATH)