[MM-28303]: check if mmctl exists before checking on Github and reduce requests to Github (#15387)

Summary:
Every make call would ping Github to find out the mmctl version to download. Each check made 4 requests to Github. So every make execution resulted in 4 requests to Github. This leads to frequent rate-limit errors from Github.
In this PR we check for the mmctl version only if mmctl doesn't already exist. We also print a more helpful error message.
Reduce the number the number of requests to Github from 4 to 2.

Ticket Link:
https://mattermost.atlassian.net/browse/MM-28303

Makefile

@@ -124,9 +124,6 @@ else
 ALL_PACKAGES=$(TE_PACKAGES)
 endif
 
-# Decide what version of prebuilt binaries to download. This will use the release-* branch names or change to the latest.
-MMCTL_REL_TO_DOWNLOAD:=$(shell scripts/get_latest_release.sh 'mattermost/mmctl' 'release-')
-
 all: run ## Alias for 'run'.
 
 -include config.override.mk
@@ -198,24 +195,15 @@ prepackaged-plugins: ## Populate the prepackaged-plugins directory
 	done
 
 prepackaged-binaries: ## Populate the prepackaged-binaries to the bin directory
-ifeq ($(MMCTL_REL_TO_DOWNLOAD),)
-	@echo "An error has occured trying to get the latest mmctl release. Aborting. Perhaps api.github.com is down?"
-	@exit 1
-endif
-# Externally built binaries
 ifeq ($(shell test -f bin/mmctl && printf "yes"),yes)
-	@echo mmctl installed
+	@echo "mmctl already exists in bin/mmctl not downloading a new version."
-else ifeq ($(PLATFORM),Darwin)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="darwin_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && tar -xvf $$MMCTL_FILE -C bin && rm $$MMCTL_FILE
-else ifeq ($(PLATFORM),Linux)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="linux_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && tar -xvf $$MMCTL_FILE -C bin && rm $$MMCTL_FILE
-else ifeq ($(PLATFORM),Windows)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="windows_amd64.zip" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && unzip -o $$MMCTL_FILE -d bin && rm $$MMCTL_FILE
 else
-	@echo "mmctl error: can't detect OS"
+	@MMCTL_VERSION=$$(scripts/get_latest_release.sh mattermost/mmctl release-); if [ $$? -eq 0 ]; then \
+		scripts/download_mmctl_release.sh $$MMCTL_VERSION; \
+	else \
+		echo $$MMCTL_VERSION; \
+		exit 1; \
+	fi;
 endif
 
 golangci-lint: ## Run golangci-lint on codebase

build/release.mk

@@ -97,6 +97,9 @@ package:
 	mkdir -p $(DIST_PATH)/client
 	cp -RL $(BUILD_WEBAPP_DIR)/dist/* $(DIST_PATH)/client
 
+	$(eval MMCTL_REL_TO_DOWNLOAD := $(shell scripts/get_latest_release.sh 'mattermost/mmctl' 'release-'))
+	@echo "Using mmctl version $(MMCTL_REL_TO_DOWNLOAD)"
+
 	@# Help files
 ifeq ($(BUILD_ENTERPRISE_READY),true)
 	cp $(BUILD_ENTERPRISE_DIR)/ENTERPRISE-EDITION-LICENSE.txt $(DIST_PATH)

scripts/download_mmctl_release.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# $1 - version to download
+
+if [[ "$OS" = "Windows_NT" ]]
+then
+  PLATFORM="Windows"
+else
+  PLATFORM=$(uname)
+fi
+
+# strip whitespace
+RELEASE_TO_DOWNLOAD=$(echo "$1" | xargs echo)
+echo "Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$RELEASE_TO_DOWNLOAD";
+
+case "$PLATFORM" in
+
+Linux)
+  MMCTL_FILE="linux_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && tar -xvf "$MMCTL_FILE" -C bin && rm "$MMCTL_FILE";
+  ;;
+
+Darwin)
+  MMCTL_FILE="darwin_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && tar -xvf "$MMCTL_FILE" -C bin && rm "$MMCTL_FILE";
+  ;;
+
+Windows)
+  MMCTL_FILE="windows_amd64.zip" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && unzip -o "$MMCTL_FILE" -d bin && rm "$MMCTL_FILE";
+  ;;
+
+*)
+  echo "error downloading mmctl: can't detect OS";
+  ;;
+
+esac

scripts/get_latest_release.sh

@@ -14,39 +14,44 @@ BASIC_AUTH=""
 # much more strict for unauthenticated requests.
 if [[ ! -z "$GITHUB_USERNAME" && ! -z "$GITHUB_TOKEN" ]];
 then
-	BASIC_AUTH="--user $GITHUB_USERNAME:$GITHUB_TOKEN"
+  BASIC_AUTH="--user $GITHUB_USERNAME:$GITHUB_TOKEN"
 fi
 
-LATEST_REL=$(curl \
+LATEST_RELEASE=$(curl \
   --silent \
-	$BASIC_AUTH \
+  $BASIC_AUTH \
-  "https://api.github.com/repos/$REPO_TO_USE/releases/latest" \
+  "https://api.github.com/repos/$REPO_TO_USE/releases/latest")
-  | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
 
-DRAFT=$(curl \
+RELEASES=$(curl \
   --silent \
-	$BASIC_AUTH \
+  $BASIC_AUTH \
-  "https://api.github.com/repos/$REPO_TO_USE/releases/latest" \
+  "https://api.github.com/repos/$REPO_TO_USE/releases")
-  | grep '"draft":' | sed -E 's/.*: ([^,]+).*/\1/')
 
-PRERELEASE=$(curl \
+LATEST_REL=$(echo "$LATEST_RELEASE" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-  --silent \
+
-	$BASIC_AUTH \
+DRAFT=$(echo "$LATEST_RELEASE" | grep '"draft":' | sed -E 's/.*: ([^,]+).*/\1/')
-  "https://api.github.com/repos/$REPO_TO_USE/releases" \
+
-  | grep '"prerelease":' | sed -E 's/.*: ([^,]+).*/\1/')
+PRERELEASE=$(echo "$RELEASES" | grep '"prerelease":' | sed -E 's/.*: ([^,]+).*/\1/')
 
 # Check if this is a release branch
 THIS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 #THIS_BRANCH="release-5.27"# - Used to test release logic on a non release branch
 
 if [[ "$THIS_BRANCH" =~ $BRANCH_TO_USE || $DRAFT =~ "true" ]]; then
-    VERSION_REL=${THIS_BRANCH//$BRANCH_TO_USE/v}
+  VERSION_REL=${THIS_BRANCH//$BRANCH_TO_USE/v}
-    REL_TO_USE=$(curl --silent $BASIC_AUTH "https://api.github.com/repos/$REPO_TO_USE/releases" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed -n "/$VERSION_REL/p" | sort -rV | head -n 1)
+  REL_TO_USE=$(echo "$RELEASES" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed -n "/$VERSION_REL/p" | sort -rV | head -n 1)
 elif [[ "$THIS_BRANCH"  =~ "master" ]]; then
-    # Get the latest release even if its a pre-release
+  # Get the latest release even if its a pre-release
-    REL_TO_USE=$(curl --silent $BASIC_AUTH "https://api.github.com/repos/$REPO_TO_USE/releases" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sort -rV | head -n 1)
+  REL_TO_USE=$(echo "$RELEASES" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sort -rV | head -n 1)
 else
-    REL_TO_USE=$LATEST_REL
+  REL_TO_USE=$LATEST_REL
 fi
 
-echo "$REL_TO_USE"
+if [[ -z "$REL_TO_USE" ]]
+then
+  echo "An error has occured trying to get the latest mmctl release. Aborting. Perhaps api.github.com is down, or you are being rate-limited.";
+  echo "Set the GITHUB_USERNAME and GITHUB_TOKEN environment variables to the appropriate values to work around Github rate-limiting.";
+  exit 1;
+else
+  echo "$REL_TO_USE"
+fi