MM-13606 Remove consumeAndClose and clean up integration response handling (#10066)

* MM-13606 Remove consumeAndClose

* Allow overriding HTTPService's request timeout

* MM-13606 Clean up integration response handling

* Properly close httptest servers

* Address feedback

* Only call buf.Bytes when necessary

* Properly check for errors in doOutgoingWebhookRequest

* Add comment explaining ignored ioutil.ReadAll errors

app/webhook.go

@@ -20,6 +20,8 @@ import (
 const (
 	TRIGGERWORDS_EXACT_MATCH = 0
 	TRIGGERWORDS_STARTS_WITH = 1
+
+	MaxIntegrationResponseSize = 1024 * 1024 // Posts can be <100KB at most, so this is likely more than enough
 )
 
 func (a *App) handleWebhookEvents(post *model.Post, team *model.Team, channel *model.Channel, user *model.User) *model.AppError {
@@ -101,55 +103,70 @@ func (a *App) TriggerWebhook(payload *model.OutgoingWebhookPayload, hook *model.
 		contentType = "application/x-www-form-urlencoded"
 	}
 
-	for _, url := range hook.CallbackURLs {
-		a.Srv.Go(func(url string) func() {
-			return func() {
-				req, _ := http.NewRequest("POST", url, body)
-				req.Header.Set("Content-Type", contentType)
-				req.Header.Set("Accept", "application/json")
-				if resp, err := a.HTTPService.MakeClient(false).Do(req); err != nil {
-					mlog.Error(fmt.Sprintf("Event POST failed, err=%s", err.Error()))
-				} else {
-					defer consumeAndClose(resp)
+	for i := range hook.CallbackURLs {
+		// Get the callback URL by index to properly capture it for the go func
+		url := hook.CallbackURLs[i]
 
-					webhookResp := model.OutgoingWebhookResponseFromJson(resp.Body)
+		a.Srv.Go(func() {
+			webhookResp, err := a.doOutgoingWebhookRequest(url, body, contentType)
+			if err != nil {
+				mlog.Error(fmt.Sprintf("Event POST failed, err=%s", err.Error()))
+				return
+			}
 
-					if webhookResp != nil && (webhookResp.Text != nil || len(webhookResp.Attachments) > 0) {
-						postRootId := ""
-						if webhookResp.ResponseType == model.OUTGOING_HOOK_RESPONSE_TYPE_COMMENT {
-							postRootId = post.Id
-						}
-						if len(webhookResp.Props) == 0 {
-							webhookResp.Props = make(model.StringInterface)
-						}
-						webhookResp.Props["webhook_display_name"] = hook.DisplayName
+			if webhookResp != nil && (webhookResp.Text != nil || len(webhookResp.Attachments) > 0) {
+				postRootId := ""
+				if webhookResp.ResponseType == model.OUTGOING_HOOK_RESPONSE_TYPE_COMMENT {
+					postRootId = post.Id
+				}
+				if len(webhookResp.Props) == 0 {
+					webhookResp.Props = make(model.StringInterface)
+				}
+				webhookResp.Props["webhook_display_name"] = hook.DisplayName
 
-						text := ""
-						if webhookResp.Text != nil {
-							text = a.ProcessSlackText(*webhookResp.Text)
-						}
-						webhookResp.Attachments = a.ProcessSlackAttachments(webhookResp.Attachments)
-						// attachments is in here for slack compatibility
-						if len(webhookResp.Attachments) > 0 {
-							webhookResp.Props["attachments"] = webhookResp.Attachments
-						}
-						if a.Config().ServiceSettings.EnablePostUsernameOverride && hook.Username != "" && webhookResp.Username == "" {
-							webhookResp.Username = hook.Username
-						}
+				text := ""
+				if webhookResp.Text != nil {
+					text = a.ProcessSlackText(*webhookResp.Text)
+				}
+				webhookResp.Attachments = a.ProcessSlackAttachments(webhookResp.Attachments)
+				// attachments is in here for slack compatibility
+				if len(webhookResp.Attachments) > 0 {
+					webhookResp.Props["attachments"] = webhookResp.Attachments
+				}
+				if a.Config().ServiceSettings.EnablePostUsernameOverride && hook.Username != "" && webhookResp.Username == "" {
+					webhookResp.Username = hook.Username
+				}
 
-						if a.Config().ServiceSettings.EnablePostIconOverride && hook.IconURL != "" && webhookResp.IconURL == "" {
-							webhookResp.IconURL = hook.IconURL
-						}
-						if _, err := a.CreateWebhookPost(hook.CreatorId, channel, text, webhookResp.Username, webhookResp.IconURL, webhookResp.Props, webhookResp.Type, postRootId); err != nil {
-							mlog.Error(fmt.Sprintf("Failed to create response post, err=%v", err))
-						}
-					}
+				if a.Config().ServiceSettings.EnablePostIconOverride && hook.IconURL != "" && webhookResp.IconURL == "" {
+					webhookResp.IconURL = hook.IconURL
+				}
+				if _, err := a.CreateWebhookPost(hook.CreatorId, channel, text, webhookResp.Username, webhookResp.IconURL, webhookResp.Props, webhookResp.Type, postRootId); err != nil {
+					mlog.Error(fmt.Sprintf("Failed to create response post, err=%v", err))
 				}
 			}
-		}(url))
+		})
 	}
 }
 
+func (a *App) doOutgoingWebhookRequest(url string, body io.Reader, contentType string) (*model.OutgoingWebhookResponse, error) {
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := a.HTTPService.MakeClient(false).Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	return model.OutgoingWebhookResponseFromJson(io.LimitReader(resp.Body, MaxIntegrationResponseSize))
+}
+
 func SplitWebhookPost(post *model.Post, maxPostSize int) ([]*model.Post, *model.AppError) {
 	splits := make([]*model.Post, 0)
 	remainingText := post.Message