From ee76ad435b9ee10451bc6a880240d92ba73eb439 Mon Sep 17 00:00:00 2001
From: Ashish Bhate <bhate.ashish@gmail.com>
Date: Thu, 24 Sep 2020 14:54:57 +0530
Subject: [PATCH] [MM-28303]: check if mmctl exists before checking on Github
 and reduce requests to Github (#15387)

Summary:
Every make call would ping Github to find out the mmctl version to download. Each check made 4 requests to Github. So every make execution resulted in 4 requests to Github. This leads to frequent rate-limit errors from Github.
In this PR we check for the mmctl version only if mmctl doesn't already exist. We also print a more helpful error message.
Reduce the number the number of requests to Github from 4 to 2.

Ticket Link:
https://mattermost.atlassian.net/browse/MM-28303
---
 Makefile                          | 26 +++++-------------
 build/release.mk                  |  3 +++
 scripts/download_mmctl_release.sh | 33 +++++++++++++++++++++++
 scripts/get_latest_release.sh     | 45 +++++++++++++++++--------------
 4 files changed, 68 insertions(+), 39 deletions(-)
 create mode 100755 scripts/download_mmctl_release.sh

diff --git a/Makefile b/Makefile
index 82a2c95233..02eec43722 100644
--- a/Makefile
+++ b/Makefile
@@ -124,9 +124,6 @@ else
 ALL_PACKAGES=$(TE_PACKAGES)
 endif
 
-# Decide what version of prebuilt binaries to download. This will use the release-* branch names or change to the latest.
-MMCTL_REL_TO_DOWNLOAD:=$(shell scripts/get_latest_release.sh 'mattermost/mmctl' 'release-')
-
 all: run ## Alias for 'run'.
 
 -include config.override.mk
@@ -198,24 +195,15 @@ prepackaged-plugins: ## Populate the prepackaged-plugins directory
 	done
 
 prepackaged-binaries: ## Populate the prepackaged-binaries to the bin directory
-ifeq ($(MMCTL_REL_TO_DOWNLOAD),)
-	@echo "An error has occured trying to get the latest mmctl release. Aborting. Perhaps api.github.com is down?"
-	@exit 1
-endif
-# Externally built binaries
 ifeq ($(shell test -f bin/mmctl && printf "yes"),yes)
-	@echo mmctl installed
-else ifeq ($(PLATFORM),Darwin)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="darwin_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && tar -xvf $$MMCTL_FILE -C bin && rm $$MMCTL_FILE
-else ifeq ($(PLATFORM),Linux)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="linux_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && tar -xvf $$MMCTL_FILE -C bin && rm $$MMCTL_FILE
-else ifeq ($(PLATFORM),Windows)
-	@echo Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$(MMCTL_REL_TO_DOWNLOAD)
-	@MMCTL_FILE="windows_amd64.zip" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/$(MMCTL_REL_TO_DOWNLOAD)/$$MMCTL_FILE && unzip -o $$MMCTL_FILE -d bin && rm $$MMCTL_FILE
+	@echo "mmctl already exists in bin/mmctl not downloading a new version."
 else
-	@echo "mmctl error: can't detect OS"
+	@MMCTL_VERSION=$$(scripts/get_latest_release.sh mattermost/mmctl release-); if [ $$? -eq 0 ]; then \
+		scripts/download_mmctl_release.sh $$MMCTL_VERSION; \
+	else \
+		echo $$MMCTL_VERSION; \
+		exit 1; \
+	fi;
 endif
 
 golangci-lint: ## Run golangci-lint on codebase
diff --git a/build/release.mk b/build/release.mk
index 0c5478ac61..b9711ff41c 100644
--- a/build/release.mk
+++ b/build/release.mk
@@ -97,6 +97,9 @@ package:
 	mkdir -p $(DIST_PATH)/client
 	cp -RL $(BUILD_WEBAPP_DIR)/dist/* $(DIST_PATH)/client
 
+	$(eval MMCTL_REL_TO_DOWNLOAD := $(shell scripts/get_latest_release.sh 'mattermost/mmctl' 'release-'))
+	@echo "Using mmctl version $(MMCTL_REL_TO_DOWNLOAD)"
+
 	@# Help files
 ifeq ($(BUILD_ENTERPRISE_READY),true)
 	cp $(BUILD_ENTERPRISE_DIR)/ENTERPRISE-EDITION-LICENSE.txt $(DIST_PATH)
diff --git a/scripts/download_mmctl_release.sh b/scripts/download_mmctl_release.sh
new file mode 100755
index 0000000000..0734b8af49
--- /dev/null
+++ b/scripts/download_mmctl_release.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# $1 - version to download
+
+if [[ "$OS" = "Windows_NT" ]]
+then
+  PLATFORM="Windows"
+else
+  PLATFORM=$(uname)
+fi
+
+# strip whitespace
+RELEASE_TO_DOWNLOAD=$(echo "$1" | xargs echo)
+echo "Downloading prepackaged binary: https://github.com/mattermost/mmctl/releases/$RELEASE_TO_DOWNLOAD";
+
+case "$PLATFORM" in
+
+Linux)
+  MMCTL_FILE="linux_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && tar -xvf "$MMCTL_FILE" -C bin && rm "$MMCTL_FILE";
+  ;;
+
+Darwin)
+  MMCTL_FILE="darwin_amd64.tar" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && tar -xvf "$MMCTL_FILE" -C bin && rm "$MMCTL_FILE";
+  ;;
+
+Windows)
+  MMCTL_FILE="windows_amd64.zip" && curl -f -O -L https://github.com/mattermost/mmctl/releases/download/"$RELEASE_TO_DOWNLOAD"/"$MMCTL_FILE" && unzip -o "$MMCTL_FILE" -d bin && rm "$MMCTL_FILE";
+  ;;
+
+*)
+  echo "error downloading mmctl: can't detect OS";
+  ;;
+
+esac
diff --git a/scripts/get_latest_release.sh b/scripts/get_latest_release.sh
index b0e5c125ba..322e5fa325 100755
--- a/scripts/get_latest_release.sh
+++ b/scripts/get_latest_release.sh
@@ -14,39 +14,44 @@ BASIC_AUTH=""
 # much more strict for unauthenticated requests.
 if [[ ! -z "$GITHUB_USERNAME" && ! -z "$GITHUB_TOKEN" ]];
 then
-	BASIC_AUTH="--user $GITHUB_USERNAME:$GITHUB_TOKEN"
+  BASIC_AUTH="--user $GITHUB_USERNAME:$GITHUB_TOKEN"
 fi
 
-LATEST_REL=$(curl \
+LATEST_RELEASE=$(curl \
   --silent \
-	$BASIC_AUTH \
-  "https://api.github.com/repos/$REPO_TO_USE/releases/latest" \
-  | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+  $BASIC_AUTH \
+  "https://api.github.com/repos/$REPO_TO_USE/releases/latest")
 
-DRAFT=$(curl \
+RELEASES=$(curl \
   --silent \
-	$BASIC_AUTH \
-  "https://api.github.com/repos/$REPO_TO_USE/releases/latest" \
-  | grep '"draft":' | sed -E 's/.*: ([^,]+).*/\1/')
+  $BASIC_AUTH \
+  "https://api.github.com/repos/$REPO_TO_USE/releases")
 
-PRERELEASE=$(curl \
-  --silent \
-	$BASIC_AUTH \
-  "https://api.github.com/repos/$REPO_TO_USE/releases" \
-  | grep '"prerelease":' | sed -E 's/.*: ([^,]+).*/\1/')
+LATEST_REL=$(echo "$LATEST_RELEASE" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+
+DRAFT=$(echo "$LATEST_RELEASE" | grep '"draft":' | sed -E 's/.*: ([^,]+).*/\1/')
+
+PRERELEASE=$(echo "$RELEASES" | grep '"prerelease":' | sed -E 's/.*: ([^,]+).*/\1/')
 
 # Check if this is a release branch
 THIS_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 #THIS_BRANCH="release-5.27"# - Used to test release logic on a non release branch
 
 if [[ "$THIS_BRANCH" =~ $BRANCH_TO_USE || $DRAFT =~ "true" ]]; then
-    VERSION_REL=${THIS_BRANCH//$BRANCH_TO_USE/v}
-    REL_TO_USE=$(curl --silent $BASIC_AUTH "https://api.github.com/repos/$REPO_TO_USE/releases" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed -n "/$VERSION_REL/p" | sort -rV | head -n 1)
+  VERSION_REL=${THIS_BRANCH//$BRANCH_TO_USE/v}
+  REL_TO_USE=$(echo "$RELEASES" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed -n "/$VERSION_REL/p" | sort -rV | head -n 1)
 elif [[ "$THIS_BRANCH"  =~ "master" ]]; then
-    # Get the latest release even if its a pre-release
-    REL_TO_USE=$(curl --silent $BASIC_AUTH "https://api.github.com/repos/$REPO_TO_USE/releases" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sort -rV | head -n 1)
+  # Get the latest release even if its a pre-release
+  REL_TO_USE=$(echo "$RELEASES" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sort -rV | head -n 1)
 else
-    REL_TO_USE=$LATEST_REL
+  REL_TO_USE=$LATEST_REL
 fi
 
-echo "$REL_TO_USE"
+if [[ -z "$REL_TO_USE" ]]
+then
+  echo "An error has occured trying to get the latest mmctl release. Aborting. Perhaps api.github.com is down, or you are being rate-limited.";
+  echo "Set the GITHUB_USERNAME and GITHUB_TOKEN environment variables to the appropriate values to work around Github rate-limiting.";
+  exit 1;
+else
+  echo "$REL_TO_USE"
+fi