Сравнить коммиты
10 Коммитов
31b97d9ee1
...
release-10
| Автор | SHA1 | Дата | |
|---|---|---|---|
|
|
08e7ea3ccd | ||
|
|
c748a7a47d | ||
|
|
bfede3d018 | ||
|
|
638007314e | ||
|
|
cbfcecb37c | ||
|
|
9966d111d2 | ||
|
|
2ff29e375b | ||
|
|
acc19baca0 | ||
|
|
775c36f827 | ||
|
|
c5e77eab62 |
127
.gitlab-ci.yml
Обычный файл
127
.gitlab-ci.yml
Обычный файл
@@ -0,0 +1,127 @@
|
||||
---
|
||||
image: debian:bookworm
|
||||
stages:
|
||||
- build
|
||||
- publish
|
||||
|
||||
variables:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/usr/local/go/bin
|
||||
GO_VERSION: 1.25.8
|
||||
GO_HASHSUM: ceb5e041bbc3893846bd1614d76cb4681c91dadee579426cf21a63f2d7e03be6
|
||||
|
||||
build:
|
||||
stage: build
|
||||
before_script:
|
||||
- mkdir artifacts
|
||||
- apt update
|
||||
- apt install -qq -y build-essential libpng-dev libpng16-16 wget curl git
|
||||
- ulimit -n 8096
|
||||
- cd /tmp
|
||||
- wget -q "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz"
|
||||
- >
|
||||
echo \
|
||||
"${GO_HASHSUM} go${GO_VERSION}.linux-amd64.tar.gz" \
|
||||
> "go${GO_VERSION}.linux-amd64.tar.gz.sha256sum"
|
||||
- sha256sum -c "go${GO_VERSION}.linux-amd64.tar.gz.sha256sum"
|
||||
- tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
|
||||
- cd -
|
||||
script:
|
||||
- >
|
||||
sed -e "s/^\(BUILD_NUMBER\) ?= .*/\1 = $CI_JOB_ID/" \
|
||||
-e "s/^\(BUILD_HASH = \).*/\1$(git rev-parse HEAD)/" \
|
||||
-i server/Makefile
|
||||
- git apply limitless.patch
|
||||
- cd server
|
||||
- make validate-go-version
|
||||
- make setup-go-work
|
||||
- make build-linux-arm64
|
||||
- make build-linux-amd64
|
||||
- cd -
|
||||
- >
|
||||
mv server/bin/mostlymatter \
|
||||
"mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless)"
|
||||
- >
|
||||
mv server/bin/linux_arm64/mostlymatter \
|
||||
"mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless)"
|
||||
cache:
|
||||
key: "$CI_COMMIT_TAG"
|
||||
policy: push
|
||||
paths:
|
||||
- "mostlymatter-*"
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /limitless/'
|
||||
|
||||
# Release
|
||||
publish:
|
||||
stage: publish
|
||||
image: framasoft/upload-packages:latest
|
||||
needs:
|
||||
- build
|
||||
before_script:
|
||||
- mkdir -p ~/.minisign
|
||||
- chmod 700 ~/.minisign
|
||||
- >
|
||||
echo 'untrusted comment: minisign encrypted secret key' \
|
||||
> ~/.minisign/minisign.key
|
||||
- echo "$MINISIG_KEY" >> ~/.minisign/minisign.key
|
||||
- chmod 600 ~/.minisign/minisign.key
|
||||
- >
|
||||
echo "$MINISIG_PWD" |
|
||||
minisign -Sm "mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless)"
|
||||
- >
|
||||
sha512sum "mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless)" \
|
||||
> "mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless).sha512"
|
||||
- >
|
||||
echo "$MINISIG_PWD" |
|
||||
minisign -Sm "mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless)"
|
||||
- >
|
||||
sha512sum "mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless)" \
|
||||
> "mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless).sha512"
|
||||
script:
|
||||
- eval $(ssh-agent -s)
|
||||
- ssh-add <(echo "${DEPLOYEMENT_KEY}" | base64 --decode -i)
|
||||
- >
|
||||
echo "put mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless)" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
echo "put mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless).minisig" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
echo "put mostlymatter-amd64-$(basename "$CI_COMMIT_TAG" -limitless).sha512" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
echo "put mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless)" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
echo "put mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless).minisig" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
echo "put mostlymatter-arm64-$(basename "$CI_COMMIT_TAG" -limitless).sha512" |
|
||||
sftp -o "VerifyHostKeyDNS yes" \
|
||||
-o "StrictHostKeyChecking accept-new" \
|
||||
${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:public/
|
||||
- >
|
||||
cat <<EOF
|
||||
================================================================================================
|
||||
== mostlymatter-$(basename "$CI_COMMIT_TAG" -limitless) published on https://packages.framasoft.org/projects/mostlymatter/ ==
|
||||
================================================================================================
|
||||
EOF
|
||||
cache:
|
||||
key: "$CI_COMMIT_TAG"
|
||||
policy: pull
|
||||
paths:
|
||||
- "mostlymatter-amd64-$CI_COMMIT_TAG"
|
||||
- "mostlymatter-arm64-$CI_COMMIT_TAG"
|
||||
rules:
|
||||
- if: '$DEPLOYEMENT_HOST && $DEPLOYEMENT_USER && $DEPLOYEMENT_KEY && $MINISIG_KEY && $MINISIG_PWD && $CI_COMMIT_TAG =~ /limitless/'
|
||||
109
MOSTLYMATTER_HOW_TO.md
Обычный файл
109
MOSTLYMATTER_HOW_TO.md
Обычный файл
@@ -0,0 +1,109 @@
|
||||
# How to use Framasoft’s patch to compile Mostlymatter
|
||||
|
||||
## Setup the repository
|
||||
|
||||
```bash
|
||||
git clone https://framagit.org/framasoft/framateam/mostlymatter.git
|
||||
cd mostlymatter
|
||||
git remote add upstream https://github.com/mattermost/mattermost.git
|
||||
```
|
||||
|
||||
## New version
|
||||
|
||||
Refresh you local repository.
|
||||
```bash
|
||||
git fetch -p --all
|
||||
```
|
||||
|
||||
Set some env vars.
|
||||
```bash
|
||||
export NEW_VERSION=10.5.1
|
||||
```
|
||||
|
||||
As you will need to cherry-pick some commits (the main fork commit and a fix-patch commit), you will need to go on an old release branch.
|
||||
```bash
|
||||
export OLD_VERSION=10.5.0
|
||||
```
|
||||
|
||||
```bash
|
||||
export BASE_VERSION=$(echo "$NEW_VERSION" | sed -e "s/\.[^.]\+$//")
|
||||
git checkout "release-$OLD_VERSION"
|
||||
git log --graph --abbrev-commit --date=relative \
|
||||
--pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr by %an)%Creset' \
|
||||
--max-count=3
|
||||
```
|
||||
|
||||
Note the commits you’ll need (usually, the two first commits).
|
||||
|
||||
Go to the main branch of the version you want and reset the code to this version, then create a new branch with the version you want to compile from the main branch of this version.
|
||||
```bash
|
||||
git branch | grep -q "release-$BASE_VERSION\$" &&
|
||||
git checkout "release-$BASE_VERSION" ||
|
||||
git checkout -b "release-$BASE_VERSION" "upstream/release-$BASE_VERSION"
|
||||
|
||||
git reset --hard "v$NEW_VERSION"
|
||||
git checkout -b "release-$NEW_VERSION" "release-$BASE_VERSION"
|
||||
```
|
||||
|
||||
Cherry-pick the commits (use the oldest first!).
|
||||
|
||||
```bash
|
||||
for i in 4b3d29da 056a5f8c
|
||||
do
|
||||
git cherry-pick "$i"
|
||||
done
|
||||
```
|
||||
|
||||
If you compile a bugfix version (ex: `10.5.1`, using the commits of the `10.5.0` version), you should be just fine
|
||||
But if you compile a new version (ex: `10.6.0`), there is a lot of chances that you need to fix the `limitless.patch` file.
|
||||
|
||||
To test the patch:
|
||||
```bash
|
||||
git apply limitless.patch &&
|
||||
echo -e "\033[0;36mPatch applied successfully\033[0;36m" &&
|
||||
rm -rf server/cmd/mostlymatter &&
|
||||
git checkout -- server
|
||||
```
|
||||
|
||||
If the patch does not apply, fix it. The fix is usually those steps:
|
||||
|
||||
- remove the `server/.golangci.yml` part of the patch
|
||||
- manually apply this part (it’s mostly replacing `mattermost` by `mostlymatter` in this file)
|
||||
- `git apply limitless.patch`
|
||||
- `git add server`
|
||||
- `git diff --cached > limitless.patch`
|
||||
- `git restore --staged -- server`
|
||||
- `git checkout -- server`
|
||||
- `rm -rf server/cmd/mostlymatter`
|
||||
- `git add limitless.patch`
|
||||
- `git commit --amend`
|
||||
|
||||
Now, you can retest the patch:
|
||||
```bash
|
||||
git apply limitless.patch &&
|
||||
echo -e "\033[0;36mPatch applied successfully\033[0;36m" &&
|
||||
rm -rf server/cmd/mostlymatter &&
|
||||
git checkout -- server
|
||||
```
|
||||
|
||||
Tag the new version (`limitless` is needed in the tag name for the CI to run) and push to Gitlab:
|
||||
```bash
|
||||
git tag "v$NEW_VERSION-limitless" -m "v$NEW_VERSION-limitless"
|
||||
git push -u origin "release-$NEW_VERSION"
|
||||
```
|
||||
|
||||
The CI will compile mostlymatter. Note that it will not be available as an artifact.
|
||||
|
||||
You can use the step of [.gitlab-ci.yml](.gitlab-ci.yml) to compile Mostlymatter without using the CI.
|
||||
|
||||
## Publish
|
||||
|
||||
The secrets are set in GitLab CI variables :
|
||||
|
||||
- `MINISIG_KEY`
|
||||
- `MINISIG_PWD`
|
||||
- `DEPLOYEMENT_KEY` (ssh private key encoded with base64)
|
||||
- `DEPLOYEMENT_USER`
|
||||
- `DEPLOYEMENT_HOST`
|
||||
|
||||
The CI will publish the compiled mostlymatter binary through sftp.
|
||||
92
README.md
92
README.md
@@ -1,97 +1,33 @@
|
||||
# [](https://mattermost.com)
|
||||
 Mostlymatter is a fork of [Mattermost](https://mattermost.com) meant to remove the users and messages limits.
|
||||
|
||||
[Mattermost](https://mattermost.com) is an open core, self-hosted collaboration platform that offers chat, workflow automation, voice calling, screen sharing, and AI integration. This repo is the primary source for core development on the Mattermost platform; it's written in Go and React, runs as a single Linux binary, and relies on PostgreSQL. A new compiled version is released under an MIT license every month on the 16th.
|
||||
|
||||
[Deploy Mattermost on-premises](https://mattermost.com/deploy/?utm_source=github-mattermost-server-readme), or [try it for free in the cloud](https://mattermost.com/sign-up/?utm_source=github-mattermost-server-readme).
|
||||
Please go to <https://github.com/mattermost/mattermost/> for installation instructions.
|
||||
|
||||
<img width="1006" alt="mattermost user interface" src="https://user-images.githubusercontent.com/7205829/136107976-7a894c9e-290a-490d-8501-e5fdbfc3785a.png">
|
||||
## Differences between Mostlymatter and Mattermost
|
||||
|
||||
Learn more about the following use cases with Mattermost:
|
||||
Our fork is limited to the backend (the binary), we don’t modify the front-end and we don’t provide compiled version of the other tools of Mattermost (like `mmctl`).
|
||||
|
||||
- [DevSecOps](https://mattermost.com/solutions/use-cases/devops/?utm_source=github-mattermost-server-readme)
|
||||
- [Incident Resolution](https://mattermost.com/solutions/use-cases/incident-resolution/?utm_source=github-mattermost-server-readme)
|
||||
- [IT Service Desk](https://mattermost.com/solutions/use-cases/it-service-desk/?utm_source=github-mattermost-server-readme)
|
||||
We multiplied the limits in `server/channels/app/limits.go` by 1,000.
|
||||
|
||||
Other useful resources:
|
||||
The user limits should be 5,000,000 and 11,000,000.
|
||||
|
||||
- [Download and Install Mattermost](https://docs.mattermost.com/guides/deployment.html) - Install, setup, and configure your own Mattermost instance.
|
||||
- [Product documentation](https://docs.mattermost.com/) - Learn how to run a Mattermost instance and take advantage of all the features.
|
||||
- [Developer documentation](https://developers.mattermost.com/) - Contribute code to Mattermost or build an integration via APIs, Webhooks, slash commands, Apps, and plugins.
|
||||
We replaced the name `Mattermost` by `Mostlymatter` (and `mattermost` by `mostlymatter`) in the server strings (logging, help pages…) to avoid confusion between our the official build and our own but we kept the copyright comments.
|
||||
|
||||
Table of contents
|
||||
=================
|
||||
The modifications are contained in the file [`limitless.patch`](limitless.patch).
|
||||
|
||||
- [Install Mattermost](#install-mattermost)
|
||||
- [Native mobile and desktop apps](#native-mobile-and-desktop-apps)
|
||||
- [Get security bulletins](#get-security-bulletins)
|
||||
- [Get involved](#get-involved)
|
||||
- [Learn more](#learn-more)
|
||||
- [License](#license)
|
||||
- [Get the latest news](#get-the-latest-news)
|
||||
- [Contributing](#contributing)
|
||||
To apply our modifications and compile Mostlymatter, please have a look at [MOSTLYMATTER_HOW_TO.md](MOSTLYMATTER_HOW_TO.md).
|
||||
|
||||
## Install Mattermost
|
||||
## Get Mostlymatter binaries
|
||||
|
||||
- [Download and Install Mattermost Self-Hosted](https://docs.mattermost.com/guides/deployment.html) - Deploy a Mattermost Self-hosted instance in minutes via Docker, Ubuntu, or tar.
|
||||
- [Get started in the cloud](https://mattermost.com/sign-up/?utm_source=github-mattermost-server-readme) to try Mattermost today.
|
||||
- [Developer machine setup](https://developers.mattermost.com/contribute/server/developer-setup) - Follow this guide if you want to write code for Mattermost.
|
||||
Go to <https://packages.framasoft.org/projects/mostlymatter/> to get the binaries you want.
|
||||
|
||||
|
||||
Other install guides:
|
||||
|
||||
- [Deploy Mattermost on Docker](https://docs.mattermost.com/install/install-docker.html)
|
||||
- [Mattermost Omnibus](https://docs.mattermost.com/install/installing-mattermost-omnibus.html)
|
||||
- [Install Mattermost from Tar](https://docs.mattermost.com/install/install-tar.html)
|
||||
- [Ubuntu 20.04 LTS](https://docs.mattermost.com/install/installing-ubuntu-2004-LTS.html)
|
||||
- [Kubernetes](https://docs.mattermost.com/install/install-kubernetes.html)
|
||||
- [Helm](https://docs.mattermost.com/install/install-kubernetes.html#installing-the-operators-via-helm)
|
||||
- [Debian Buster](https://docs.mattermost.com/install/install-debian.html)
|
||||
- [RHEL 8](https://docs.mattermost.com/install/install-rhel-8.html)
|
||||
- [More server install guides](https://docs.mattermost.com/guides/deployment.html)
|
||||
|
||||
## Native mobile and desktop apps
|
||||
|
||||
In addition to the web interface, you can also download Mattermost clients for [Android](https://mattermost.com/pl/android-app/), [iOS](https://mattermost.com/pl/ios-app/), [Windows PC](https://docs.mattermost.com/install/desktop-app-install.html#windows-10-windows-8-1), [macOS](https://docs.mattermost.com/install/desktop-app-install.html#macos-10-9), and [Linux](https://docs.mattermost.com/install/desktop-app-install.html#linux).
|
||||
|
||||
[<img src="https://user-images.githubusercontent.com/30978331/272826427-6200c98f-7319-42c3-86d4-0b33ae99e01a.png" alt="Get Mattermost on Google Play" height="50px"/>](https://mattermost.com/pl/android-app/) [<img src="https://developer.apple.com/assets/elements/badges/download-on-the-app-store.svg" alt="Get Mattermost on the App Store" height="50px"/>](https://itunes.apple.com/us/app/mattermost/id1257222717?mt=8) [](https://docs.mattermost.com/install/desktop.html#windows-10-windows-8-1-windows-7) [](https://docs.mattermost.com/install/desktop.html#macos-10-9) [](https://docs.mattermost.com/install/desktop.html#linux)
|
||||
|
||||
## Get security bulletins
|
||||
|
||||
Receive notifications of critical security updates. The sophistication of online attackers is perpetually increasing. If you're deploying Mattermost it's highly recommended you subscribe to the Mattermost Security Bulletin mailing list for updates on critical security releases.
|
||||
|
||||
[Subscribe here](https://mattermost.com/security-updates/#sign-up)
|
||||
|
||||
## Get involved
|
||||
|
||||
- [Contribute to Mattermost](https://handbook.mattermost.com/contributors/contributors/ways-to-contribute)
|
||||
- [Find "Help Wanted" projects](https://github.com/mattermost/mattermost-server/issues?page=1&q=is%3Aissue+is%3Aopen+%22Help+Wanted%22&utf8=%E2%9C%93)
|
||||
- [Join Developer Discussion on a Mattermost server for contributors](https://community.mattermost.com/signup_user_complete/?id=f1924a8db44ff3bb41c96424cdc20676)
|
||||
- [Get Help With Mattermost](https://docs.mattermost.com/guides/get-help.html)
|
||||
|
||||
## Learn more
|
||||
|
||||
- [API options - webhooks, slash commands, drivers, and web service](https://api.mattermost.com/)
|
||||
- [See who's using Mattermost](https://mattermost.com/customers/)
|
||||
- [Browse over 700 Mattermost integrations](https://mattermost.com/marketplace/)
|
||||
Follow the instructions on top of the page to verify the binaries you downloaded.
|
||||
|
||||
## License
|
||||
|
||||
See the [LICENSE file](LICENSE.txt) for license rights and limitations.
|
||||
|
||||
## Get the latest news
|
||||
## License of Mostlymatter’s logo
|
||||
|
||||
- **X** - Follow [Mattermost on X, formerly Twitter](https://twitter.com/mattermost).
|
||||
- **Blog** - Get the latest updates from the [Mattermost blog](https://mattermost.com/blog/).
|
||||
- **Facebook** - Follow [Mattermost on Facebook](https://www.facebook.com/MattermostHQ).
|
||||
- **LinkedIn** - Follow [Mattermost on LinkedIn](https://www.linkedin.com/company/mattermost/).
|
||||
- **Email** - Subscribe to our [newsletter](https://mattermost.us11.list-manage.com/subscribe?u=6cdba22349ae374e188e7ab8e&id=2add1c8034) (1 or 2 per month).
|
||||
- **Mattermost** - Join the ~contributors channel on [the Mattermost Community Server](https://community.mattermost.com).
|
||||
- **IRC** - Join the #matterbridge channel on [Freenode](https://freenode.net/) (thanks to [matterircd](https://github.com/42wim/matterircd)).
|
||||
- **YouTube** - Subscribe to [Mattermost](https://www.youtube.com/@MattermostHQ).
|
||||
|
||||
## Contributing
|
||||
|
||||
[](https://gitpod.io/#https://github.com/mattermost/mattermost)
|
||||
|
||||
Please see [CONTRIBUTING.md](./CONTRIBUTING.md).
|
||||
[Join the Mattermost Contributors server](https://community.mattermost.com/signup_user_complete/?id=codoy5s743rq5mk18i7u5ksz7e) to join community discussions about contributions, development, and more.
|
||||
[CC-By-NC v4.0](https://creativecommons.org/licenses/by-nc/4.0/deed.en) [Geoffrey Dorne](https://geoffreydorne.com)
|
||||
|
||||
@@ -215,6 +215,7 @@
|
||||
"Directory": "./data/",
|
||||
"EnablePublicLink": false,
|
||||
"ExtractContent": true,
|
||||
"ExtractContentTimeout": 10,
|
||||
"ArchiveRecursion": false,
|
||||
"PublicLinkSalt": "",
|
||||
"InitialFont": "nunito-bold.ttf",
|
||||
|
||||
@@ -310,6 +310,7 @@ const defaultServerConfig: AdminConfig = {
|
||||
Directory: './data/',
|
||||
EnablePublicLink: false,
|
||||
ExtractContent: true,
|
||||
ExtractContentTimeout: 10,
|
||||
ArchiveRecursion: false,
|
||||
PublicLinkSalt: '',
|
||||
InitialFont: 'nunito-bold.ttf',
|
||||
|
||||
418
limitless.patch
Обычный файл
418
limitless.patch
Обычный файл
@@ -0,0 +1,418 @@
|
||||
diff --git a/server/.golangci.yml b/server/.golangci.yml
|
||||
index e7b25ff604..7eb81a08a2 100644
|
||||
--- a/server/.golangci.yml
|
||||
+++ b/server/.golangci.yml
|
||||
@@ -133,19 +133,19 @@ linters:
|
||||
channels/store/storetest/scheme_store.go|\
|
||||
channels/store/storetest/shared_channel_store.go|\
|
||||
channels/store/storetest/team_store.go|\
|
||||
channels/store/storetest/thread_store.go|\
|
||||
channels/store/storetest/user_store.go|\
|
||||
- cmd/mattermost/commands/cmdtestlib.go|\
|
||||
- cmd/mattermost/commands/db.go|\
|
||||
- cmd/mattermost/commands/export.go|\
|
||||
- cmd/mattermost/commands/import.go|\
|
||||
- cmd/mattermost/commands/jobserver.go|\
|
||||
- cmd/mattermost/commands/server.go|\
|
||||
- cmd/mattermost/commands/server_test.go|\
|
||||
- cmd/mattermost/commands/test.go|\
|
||||
- cmd/mattermost/commands/version.go|\
|
||||
+ cmd/mostlymatter/commands/cmdtestlib.go|\
|
||||
+ cmd/mostlymatter/commands/db.go|\
|
||||
+ cmd/mostlymatter/commands/export.go|\
|
||||
+ cmd/mostlymatter/commands/import.go|\
|
||||
+ cmd/mostlymatter/commands/jobserver.go|\
|
||||
+ cmd/mostlymatter/commands/server.go|\
|
||||
+ cmd/mostlymatter/commands/server_test.go|\
|
||||
+ cmd/mostlymatter/commands/test.go|\
|
||||
+ cmd/mostlymatter/commands/version.go|\
|
||||
platform/services/cache/lru_striped.go|\
|
||||
platform/services/cache/lru_striped_bench_test.go|\
|
||||
platform/services/cache/lru_striped_test.go|\
|
||||
platform/services/cache/lru_test.go|\
|
||||
platform/services/docextractor/combine.go|\
|
||||
diff --git a/server/Makefile b/server/Makefile
|
||||
index f44e976a09..aa2f7daee5 100644
|
||||
--- a/server/Makefile
|
||||
+++ b/server/Makefile
|
||||
@@ -54,11 +54,11 @@ MMCTL_PKG = github.com/mattermost/mattermost/server/v8/cmd/mmctl/commands
|
||||
MMCTL_BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
MMCTL_LDFLAGS += -X "$(MMCTL_PKG).buildDate=$(MMCTL_BUILD_DATE)"
|
||||
|
||||
# Enterprise
|
||||
BUILD_ENTERPRISE_DIR ?= ../../enterprise
|
||||
-BUILD_ENTERPRISE ?= true
|
||||
+BUILD_ENTERPRISE ?= false
|
||||
BUILD_ENTERPRISE_READY = false
|
||||
BUILD_TYPE_NAME = team
|
||||
BUILD_HASH_ENTERPRISE = none
|
||||
ifneq ($(wildcard $(BUILD_ENTERPRISE_DIR)/.),)
|
||||
MMCTL_TESTFLAGS += -ldflags '-X "$(MMCTL_PKG).EnableEnterpriseTests=true" -X "github.com/mattermost/mattermost/server/public/model.BuildEnterpriseReady=true"'
|
||||
@@ -121,20 +121,20 @@ GO_VERSION_VALIDATION_ERR_MSG = Golang version is not supported, please update t
|
||||
GO_COMPATIBILITY_TEST_VERSIONS := 1.22.7 1.23.6
|
||||
|
||||
# GOOS/GOARCH of the build host, used to determine whether we're cross-compiling or not
|
||||
BUILDER_GOOS_GOARCH="$(shell $(GO) env GOOS)_$(shell $(GO) env GOARCH)"
|
||||
|
||||
-PLATFORM_FILES="./cmd/mattermost"
|
||||
+PLATFORM_FILES="./cmd/mostlymatter"
|
||||
|
||||
# Output paths
|
||||
DIST_ROOT=dist
|
||||
-DIST_PATH=$(DIST_ROOT)/mattermost
|
||||
-DIST_PATH_LIN_AMD64=$(DIST_ROOT)/linux_amd64/mattermost
|
||||
-DIST_PATH_LIN_ARM64=$(DIST_ROOT)/linux_arm64/mattermost
|
||||
-DIST_PATH_OSX_AMD64=$(DIST_ROOT)/darwin_amd64/mattermost
|
||||
-DIST_PATH_OSX_ARM64=$(DIST_ROOT)/darwin_arm64/mattermost
|
||||
-DIST_PATH_WIN=$(DIST_ROOT)/windows/mattermost
|
||||
+DIST_PATH=$(DIST_ROOT)/mostlymatter
|
||||
+DIST_PATH_LIN_AMD64=$(DIST_ROOT)/linux_amd64/mostlymatter
|
||||
+DIST_PATH_LIN_ARM64=$(DIST_ROOT)/linux_arm64/mostlymatter
|
||||
+DIST_PATH_OSX_AMD64=$(DIST_ROOT)/darwin_amd64/mostlymatter
|
||||
+DIST_PATH_OSX_ARM64=$(DIST_ROOT)/darwin_arm64/mostlymatter
|
||||
+DIST_PATH_WIN=$(DIST_ROOT)/windows/mostlymatter
|
||||
|
||||
# Packages lists
|
||||
TE_PACKAGES=$(shell $(GO) list ./public/...) $(shell $(GO) list ./... | grep -vE 'server/v8/cmd/mmctl')
|
||||
MMCTL_PACKAGES=$(shell $(GO) list ./... | grep -E 'server/v8/cmd/mmctl')
|
||||
|
||||
@@ -728,11 +728,11 @@ clean: stop-docker ## Clean up everything except persistent server data.
|
||||
rm -f cover.out
|
||||
rm -f ecover.out
|
||||
rm -f *.out
|
||||
rm -f *.test
|
||||
rm -f channels/imports/imports.go
|
||||
- rm -f cmd/mattermost/cprofile*.out
|
||||
+ rm -f cmd/mostlymatter/cprofile*.out
|
||||
|
||||
nuke: clean clean-docker ## Clean plus removes persistent server data.
|
||||
@echo BOOM
|
||||
|
||||
rm -rf data
|
||||
diff --git a/server/channels/app/limits.go b/server/channels/app/limits.go
|
||||
index 7eccd87f3b..13b9c3ac3f 100644
|
||||
--- a/server/channels/app/limits.go
|
||||
+++ b/server/channels/app/limits.go
|
||||
@@ -8,12 +8,12 @@ import (
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
)
|
||||
|
||||
const (
|
||||
- maxUsersLimit = 2_500
|
||||
- maxUsersHardLimit = 5_000
|
||||
+ maxUsersLimit = 5_000_000
|
||||
+ maxUsersHardLimit = 10_000_000
|
||||
)
|
||||
|
||||
func (a *App) GetServerLimits() (*model.ServerLimits, *model.AppError) {
|
||||
limits := &model.ServerLimits{}
|
||||
license := a.License()
|
||||
diff --git a/server/cmd/mattermost/commands/cmdtestlib.go b/server/cmd/mostlymatter/commands/cmdtestlib.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/cmdtestlib.go
|
||||
rename to server/cmd/mostlymatter/commands/cmdtestlib.go
|
||||
diff --git a/server/cmd/mattermost/commands/db.go b/server/cmd/mostlymatter/commands/db.go
|
||||
similarity index 97%
|
||||
rename from server/cmd/mattermost/commands/db.go
|
||||
rename to server/cmd/mostlymatter/commands/db.go
|
||||
index 17c96c0c45..4a5c210479 100644
|
||||
--- a/server/cmd/mattermost/commands/db.go
|
||||
+++ b/server/cmd/mostlymatter/commands/db.go
|
||||
@@ -32,25 +32,25 @@ var InitDbCmd = &cobra.Command{
|
||||
Short: "Initialize the database",
|
||||
Long: `Initialize the database for a given DSN, executing the migrations and loading the custom defaults if any.
|
||||
|
||||
This command should be run using a database configuration DSN.`,
|
||||
Example: ` # you can use the config flag to pass the DSN
|
||||
- $ mattermost db init --config postgres://localhost/mattermost
|
||||
+ $ mostlymatter db init --config postgres://localhost/mattermost
|
||||
|
||||
# or you can use the MM_CONFIG environment variable
|
||||
- $ MM_CONFIG=postgres://localhost/mattermost mattermost db init
|
||||
+ $ MM_CONFIG=postgres://localhost/mattermost mostlymatter db init
|
||||
|
||||
# and you can set a custom defaults file to be loaded into the database
|
||||
- $ MM_CUSTOM_DEFAULTS_PATH=custom.json MM_CONFIG=postgres://localhost/mattermost mattermost db init`,
|
||||
+ $ MM_CUSTOM_DEFAULTS_PATH=custom.json MM_CONFIG=postgres://localhost/mattermost mostlymatter db init`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: initDbCmdF,
|
||||
}
|
||||
|
||||
var ResetCmd = &cobra.Command{
|
||||
Use: "reset",
|
||||
Short: "Reset the database to initial state",
|
||||
- Long: "Completely erases the database causing the loss of all data. This will reset Mattermost to its initial state.",
|
||||
+ Long: "Completely erases the database causing the loss of all data. This will reset Mostlymatter to its initial state.",
|
||||
RunE: resetCmdF,
|
||||
}
|
||||
|
||||
var MigrateCmd = &cobra.Command{
|
||||
Use: "migrate",
|
||||
diff --git a/server/cmd/mattermost/commands/exec_command_test.go b/server/cmd/mostlymatter/commands/exec_command_test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/exec_command_test.go
|
||||
rename to server/cmd/mostlymatter/commands/exec_command_test.go
|
||||
diff --git a/server/cmd/mattermost/commands/export.go b/server/cmd/mostlymatter/commands/export.go
|
||||
similarity index 92%
|
||||
rename from server/cmd/mattermost/commands/export.go
|
||||
rename to server/cmd/mostlymatter/commands/export.go
|
||||
index ece6eebacf..bc9d31fe46 100644
|
||||
--- a/server/cmd/mattermost/commands/export.go
|
||||
+++ b/server/cmd/mostlymatter/commands/export.go
|
||||
@@ -18,26 +18,26 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var ExportCmd = &cobra.Command{
|
||||
Use: "export",
|
||||
- Short: "Export data from Mattermost",
|
||||
- Long: "Export data from Mattermost in a format suitable for import into a third-party application or another Mattermost instance",
|
||||
+ Short: "Export data from Mostlymatter",
|
||||
+ Long: "Export data from Mostlymatter in a format suitable for import into a third-party application or another Mostlymatter instance",
|
||||
}
|
||||
|
||||
var ScheduleExportCmd = &cobra.Command{
|
||||
Use: "schedule",
|
||||
- Short: "Schedule an export data job in Mattermost",
|
||||
- Long: "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)",
|
||||
+ Short: "Schedule an export data job in Mostlymatter",
|
||||
+ Long: "Schedule an export data job in Mostlymatter (this will run asynchronously via a background worker)",
|
||||
Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345",
|
||||
RunE: scheduleExportCmdF,
|
||||
}
|
||||
|
||||
var BulkExportCmd = &cobra.Command{
|
||||
Use: "bulk [file]",
|
||||
Short: "Export bulk data.",
|
||||
- Long: "Export data to a file compatible with the Mattermost Bulk Import format.",
|
||||
+ Long: "Export data to a file compatible with the Mostlymatter Bulk Import format.",
|
||||
Example: "export bulk bulk_data.json",
|
||||
RunE: bulkExportCmdF,
|
||||
Args: cobra.ExactArgs(1),
|
||||
}
|
||||
|
||||
diff --git a/server/cmd/mattermost/commands/import.go b/server/cmd/mostlymatter/commands/import.go
|
||||
similarity index 98%
|
||||
rename from server/cmd/mattermost/commands/import.go
|
||||
rename to server/cmd/mostlymatter/commands/import.go
|
||||
index 434403d830..d662c8af65 100644
|
||||
--- a/server/cmd/mattermost/commands/import.go
|
||||
+++ b/server/cmd/mostlymatter/commands/import.go
|
||||
@@ -30,11 +30,11 @@ var SlackImportCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
var BulkImportCmd = &cobra.Command{
|
||||
Use: "bulk [file]",
|
||||
Short: "Import bulk data.",
|
||||
- Long: "Import data from a Mattermost Bulk Import File.",
|
||||
+ Long: "Import data from a Mostlymatter Bulk Import File.",
|
||||
Example: " import bulk bulk_data.json",
|
||||
RunE: bulkImportCmdF,
|
||||
}
|
||||
|
||||
func init() {
|
||||
diff --git a/server/cmd/mattermost/commands/init.go b/server/cmd/mostlymatter/commands/init.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/init.go
|
||||
rename to server/cmd/mostlymatter/commands/init.go
|
||||
diff --git a/server/cmd/mattermost/commands/jobserver.go b/server/cmd/mostlymatter/commands/jobserver.go
|
||||
similarity index 88%
|
||||
rename from server/cmd/mattermost/commands/jobserver.go
|
||||
rename to server/cmd/mostlymatter/commands/jobserver.go
|
||||
index 60b570bea6..5ab12fc0cb 100644
|
||||
--- a/server/cmd/mattermost/commands/jobserver.go
|
||||
+++ b/server/cmd/mostlymatter/commands/jobserver.go
|
||||
@@ -16,11 +16,11 @@ import (
|
||||
"github.com/mattermost/mattermost/server/v8/config"
|
||||
)
|
||||
|
||||
var JobserverCmd = &cobra.Command{
|
||||
Use: "jobserver",
|
||||
- Short: "Start the Mattermost job server",
|
||||
+ Short: "Start the Mostlymatter job server",
|
||||
RunE: jobserverCmdF,
|
||||
}
|
||||
|
||||
func init() {
|
||||
JobserverCmd.Flags().Bool("nojobs", false, "Do not run jobs on this jobserver.")
|
||||
@@ -44,12 +44,12 @@ func jobserverCmdF(command *cobra.Command, args []string) error {
|
||||
a.Srv().LoadLicense()
|
||||
|
||||
rctx := request.EmptyContext(a.Log())
|
||||
|
||||
// Run jobs
|
||||
- rctx.Logger().Info("Starting Mattermost job server")
|
||||
- defer rctx.Logger().Info("Stopped Mattermost job server")
|
||||
+ rctx.Logger().Info("Starting Mostlymatter job server")
|
||||
+ defer rctx.Logger().Info("Stopped Mostlymatter job server")
|
||||
|
||||
if !noJobs {
|
||||
a.Srv().Jobs.StartWorkers()
|
||||
defer a.Srv().Jobs.StopWorkers()
|
||||
}
|
||||
@@ -66,9 +66,9 @@ func jobserverCmdF(command *cobra.Command, args []string) error {
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-signalChan
|
||||
|
||||
// Cleanup anything that isn't handled by a defer statement
|
||||
- rctx.Logger().Info("Stopping Mattermost job server")
|
||||
+ rctx.Logger().Info("Stopping Mostlymatter job server")
|
||||
|
||||
return nil
|
||||
}
|
||||
diff --git a/server/cmd/mattermost/commands/main_test.go b/server/cmd/mostlymatter/commands/main_test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/main_test.go
|
||||
rename to server/cmd/mostlymatter/commands/main_test.go
|
||||
diff --git a/server/cmd/mattermost/commands/output.go b/server/cmd/mostlymatter/commands/output.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/output.go
|
||||
rename to server/cmd/mostlymatter/commands/output.go
|
||||
diff --git a/server/cmd/mattermost/commands/root.go b/server/cmd/mostlymatter/commands/root.go
|
||||
similarity index 70%
|
||||
rename from server/cmd/mattermost/commands/root.go
|
||||
rename to server/cmd/mostlymatter/commands/root.go
|
||||
index 692ac30f53..8d2081ab21 100644
|
||||
--- a/server/cmd/mattermost/commands/root.go
|
||||
+++ b/server/cmd/mostlymatter/commands/root.go
|
||||
@@ -16,13 +16,13 @@ func Run(args []string) error {
|
||||
RootCmd.SetArgs(args)
|
||||
return RootCmd.Execute()
|
||||
}
|
||||
|
||||
var RootCmd = &cobra.Command{
|
||||
- Use: "mattermost",
|
||||
+ Use: "mostlymatter",
|
||||
Short: "Open source, self-hosted Slack-alternative",
|
||||
- Long: `Mattermost offers workplace messaging across web, PC and phones with archiving, search and integration with your existing systems. Documentation available at https://docs.mattermost.com`,
|
||||
+ Long: `Mostlymatter offers workplace messaging across web, PC and phones with archiving, search and integration with your existing systems. Documentation available at https://docs.mattermost.com`,
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
checkForRootUser()
|
||||
},
|
||||
}
|
||||
|
||||
@@ -31,8 +31,8 @@ func init() {
|
||||
}
|
||||
|
||||
// checkForRootUser logs a warning if the process is running as root
|
||||
func checkForRootUser() {
|
||||
if os.Geteuid() == 0 {
|
||||
- mlog.Warn("Running Mattermost as root is not recommended. Please use a non-root user.")
|
||||
+ mlog.Warn("Running Mostlymatter as root is not recommended. Please use a non-root user.")
|
||||
}
|
||||
}
|
||||
diff --git a/server/cmd/mattermost/commands/server.go b/server/cmd/mostlymatter/commands/server.go
|
||||
similarity index 96%
|
||||
rename from server/cmd/mattermost/commands/server.go
|
||||
rename to server/cmd/mostlymatter/commands/server.go
|
||||
index 76d33bfaf3..0100abc12a 100644
|
||||
--- a/server/cmd/mattermost/commands/server.go
|
||||
+++ b/server/cmd/mostlymatter/commands/server.go
|
||||
@@ -24,11 +24,11 @@ import (
|
||||
"github.com/mattermost/mattermost/server/v8/config"
|
||||
)
|
||||
|
||||
var serverCmd = &cobra.Command{
|
||||
Use: "server",
|
||||
- Short: "Run the Mattermost server",
|
||||
+ Short: "Run the Mostlymatter server",
|
||||
RunE: serverCmdF,
|
||||
SilenceUsage: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -38,11 +38,11 @@ func init() {
|
||||
|
||||
func serverCmdF(command *cobra.Command, args []string) error {
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
|
||||
if err := utils.TranslationsPreInit(); err != nil {
|
||||
- return errors.Wrap(err, "unable to load Mattermost translation files")
|
||||
+ return errors.Wrap(err, "unable to load Mostlymatter translation files")
|
||||
}
|
||||
|
||||
customDefaults, err := loadCustomDefaults()
|
||||
if err != nil {
|
||||
mlog.Warn("Error loading custom configuration defaults: " + err.Error())
|
||||
diff --git a/server/cmd/mattermost/commands/server_test.go b/server/cmd/mostlymatter/commands/server_test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/server_test.go
|
||||
rename to server/cmd/mostlymatter/commands/server_test.go
|
||||
diff --git a/server/cmd/mattermost/commands/test.go b/server/cmd/mostlymatter/commands/test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/test.go
|
||||
rename to server/cmd/mostlymatter/commands/test.go
|
||||
diff --git a/server/cmd/mattermost/commands/utils.go b/server/cmd/mostlymatter/commands/utils.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/utils.go
|
||||
rename to server/cmd/mostlymatter/commands/utils.go
|
||||
diff --git a/server/cmd/mattermost/commands/utils_test.go b/server/cmd/mostlymatter/commands/utils_test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/utils_test.go
|
||||
rename to server/cmd/mostlymatter/commands/utils_test.go
|
||||
diff --git a/server/cmd/mattermost/commands/version.go b/server/cmd/mostlymatter/commands/version.go
|
||||
similarity index 92%
|
||||
rename from server/cmd/mattermost/commands/version.go
|
||||
rename to server/cmd/mostlymatter/commands/version.go
|
||||
index 5a5a5f160d..b0c0dc11ff 100644
|
||||
--- a/server/cmd/mattermost/commands/version.go
|
||||
+++ b/server/cmd/mostlymatter/commands/version.go
|
||||
@@ -14,11 +14,11 @@ var VersionCmd = &cobra.Command{
|
||||
Short: "Display version information",
|
||||
RunE: versionCmdF,
|
||||
}
|
||||
|
||||
func init() {
|
||||
- VersionCmd.Flags().Bool("skip-server-start", false, "Skip the server initialization and return the Mattermost version without the DB version.")
|
||||
+ VersionCmd.Flags().Bool("skip-server-start", false, "Skip the server initialization and return the Mostlymatter version without the DB version.")
|
||||
VersionCmd.Flags().MarkDeprecated("skip-server-start", "This flag is not necessary anymore and the flag will be removed in the future releases. Consider removing it from your scripts.")
|
||||
RootCmd.AddCommand(VersionCmd)
|
||||
}
|
||||
|
||||
func versionCmdF(command *cobra.Command, args []string) error {
|
||||
diff --git a/server/cmd/mattermost/commands/version_test.go b/server/cmd/mostlymatter/commands/version_test.go
|
||||
similarity index 100%
|
||||
rename from server/cmd/mattermost/commands/version_test.go
|
||||
rename to server/cmd/mostlymatter/commands/version_test.go
|
||||
diff --git a/server/cmd/mattermost/main.go b/server/cmd/mostlymatter/main.go
|
||||
similarity index 88%
|
||||
rename from server/cmd/mattermost/main.go
|
||||
rename to server/cmd/mostlymatter/main.go
|
||||
index 5fb4b4a005..1783e812f3 100644
|
||||
--- a/server/cmd/mattermost/main.go
|
||||
+++ b/server/cmd/mostlymatter/main.go
|
||||
@@ -4,11 +4,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
- "github.com/mattermost/mattermost/server/v8/cmd/mattermost/commands"
|
||||
+ "github.com/mattermost/mattermost/server/v8/cmd/mostlymatter/commands"
|
||||
// Import and register app layer slash commands
|
||||
_ "github.com/mattermost/mattermost/server/v8/channels/app/slashcommands"
|
||||
// Plugins
|
||||
_ "github.com/mattermost/mattermost/server/v8/channels/app/oauthproviders/gitlab"
|
||||
|
||||
diff --git a/server/cmd/mattermost/main_test.go b/server/cmd/mostlymatter/main_test.go
|
||||
similarity index 96%
|
||||
rename from server/cmd/mattermost/main_test.go
|
||||
rename to server/cmd/mostlymatter/main_test.go
|
||||
index 8ec8a0bea4..c661c2012c 100644
|
||||
--- a/server/cmd/mattermost/main_test.go
|
||||
+++ b/server/cmd/mostlymatter/main_test.go
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRunMain can be used to track code coverage in integration tests.
|
||||
// To run this:
|
||||
-// go test -coverpkg="<>" -ldflags '<>' -tags maincoverage -c ./cmd/mattermost/
|
||||
+// go test -coverpkg="<>" -ldflags '<>' -tags maincoverage -c ./cmd/mostlymatter/
|
||||
// ./mattermost.test -test.run="^TestRunMain$" -test.coverprofile=coverage.out
|
||||
// And then run your integration tests.
|
||||
func TestRunMain(t *testing.T) {
|
||||
main()
|
||||
}
|
||||
Двоичные данные
mostlymatter/logo.png
Обычный файл
Двоичные данные
mostlymatter/logo.png
Обычный файл
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 31 KiB |
Двоичные данные
mostlymatter/logo_with_white_background-small.png
Обычный файл
Двоичные данные
mostlymatter/logo_with_white_background-small.png
Обычный файл
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 8.1 KiB |
Двоичные данные
mostlymatter/logo_with_white_background.png
Обычный файл
Двоичные данные
mostlymatter/logo_with_white_background.png
Обычный файл
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 38 KiB |
@@ -144,7 +144,7 @@ TEMPLATES_DIR=templates
|
||||
PLUGIN_PACKAGES ?= $(PLUGIN_PACKAGES:)
|
||||
PLUGIN_PACKAGES += mattermost-plugin-calls-v1.11.4
|
||||
PLUGIN_PACKAGES += mattermost-plugin-github-v2.7.0
|
||||
PLUGIN_PACKAGES += mattermost-plugin-gitlab-v1.12.2
|
||||
PLUGIN_PACKAGES += mattermost-plugin-gitlab-v1.12.3
|
||||
PLUGIN_PACKAGES += mattermost-plugin-jira-v4.7.0
|
||||
# We need to prepackage both versions of playbooks and install the correct one based on the server license. See MM-60025.
|
||||
PLUGIN_PACKAGES += mattermost-plugin-playbooks-v1.41.1
|
||||
|
||||
@@ -48,12 +48,18 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, app.MaxEmojiFileSize)
|
||||
if err := r.ParseMultipartForm(app.MaxEmojiFileSize); err != nil {
|
||||
c.Err = model.NewAppError("createEmoji", "api.emoji.create.parse.app_error", nil, "", http.StatusBadRequest).Wrap(err)
|
||||
return
|
||||
}
|
||||
|
||||
auditRec := c.MakeAuditRecord("createEmoji", model.AuditStatusFail)
|
||||
if imageFiles := r.MultipartForm.File["image"]; len(imageFiles) > 0 && imageFiles[0].Size > app.MaxEmojiFileSize {
|
||||
c.Err = model.NewAppError("createEmoji", "api.emoji.create.too_large.app_error", nil, "", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
|
||||
auditRec := c.MakeAuditRecord(model.AuditEventCreateEmoji, model.AuditStatusFail)
|
||||
defer c.LogAuditRec(auditRec)
|
||||
|
||||
// Allow any user with CREATE_EMOJIS permission at Team level to create emojis at system level
|
||||
|
||||
@@ -164,8 +164,41 @@ func TestCreateEmoji(t *testing.T) {
|
||||
Name: model.NewId(),
|
||||
}
|
||||
|
||||
_, _, err = client.CreateEmoji(context.Background(), emoji, utils.CreateTestAnimatedGif(t, 100, 100, 10000), "image.gif")
|
||||
_, resp, err = client.CreateEmoji(context.Background(), emoji, utils.CreateTestAnimatedGif(t, 100, 100, 10000), "image.gif")
|
||||
require.Error(t, err, "should fail - emoji is too big")
|
||||
CheckRequestEntityTooLargeStatus(t, resp)
|
||||
CheckErrorID(t, err, "api.emoji.create.too_large.app_error")
|
||||
|
||||
// try to create an animated gif with too many frames
|
||||
emoji = &model.Emoji{
|
||||
CreatorId: th.BasicUser.Id,
|
||||
Name: model.NewId(),
|
||||
}
|
||||
|
||||
_, resp, err = client.CreateEmoji(context.Background(), emoji, utils.CreateTestAnimatedGif(t, 200, 200, app.MaxEmojiGIFFrames+1), "image.gif")
|
||||
require.Error(t, err, "should fail - gif has too many frames")
|
||||
CheckBadRequestStatus(t, resp)
|
||||
CheckErrorID(t, err, "api.emoji.upload.too_many_frames.app_error")
|
||||
|
||||
// try to create an animated gif with too many frames that does not need resizing
|
||||
emoji = &model.Emoji{
|
||||
CreatorId: th.BasicUser.Id,
|
||||
Name: model.NewId(),
|
||||
}
|
||||
|
||||
_, resp, err = client.CreateEmoji(context.Background(), emoji, utils.CreateTestAnimatedGif(t, app.MaxEmojiWidth, app.MaxEmojiHeight, app.MaxEmojiGIFFrames+1), "image.gif")
|
||||
require.Error(t, err, "should fail - gif has too many frames")
|
||||
CheckBadRequestStatus(t, resp)
|
||||
CheckErrorID(t, err, "api.emoji.upload.too_many_frames.app_error")
|
||||
|
||||
// try to create an animated gif with exactly the maximum allowed frames
|
||||
emoji = &model.Emoji{
|
||||
CreatorId: th.BasicUser.Id,
|
||||
Name: model.NewId(),
|
||||
}
|
||||
|
||||
_, _, err = client.CreateEmoji(context.Background(), emoji, utils.CreateTestAnimatedGif(t, 200, 200, app.MaxEmojiGIFFrames), "image.gif")
|
||||
require.NoError(t, err, "should succeed - gif has exactly the maximum allowed frames")
|
||||
|
||||
// try to create an emoji with data that isn't an image
|
||||
emoji = &model.Emoji{
|
||||
|
||||
@@ -7711,6 +7711,63 @@ func TestGetThreadsForUser(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetThreadsForUser_AfterTeamRemovalAndReinvite(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
privateChannel := th.CreatePrivateChannel()
|
||||
th.AddUserToChannel(victim, privateChannel)
|
||||
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, admin.Id))
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, victim.Id))
|
||||
}()
|
||||
|
||||
rootPost, _, err := th.Client.CreatePost(context.Background(), &model.Post{
|
||||
ChannelId: privateChannel.Id,
|
||||
Message: "private team secret",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
victimClient := th.CreateClient()
|
||||
th.LoginBasic2WithClient(victimClient)
|
||||
|
||||
_, _, err = victimClient.CreatePost(context.Background(), &model.Post{
|
||||
ChannelId: privateChannel.Id,
|
||||
RootId: rootPost.Id,
|
||||
Message: "victim reply",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
uss, _, err := victimClient.GetUserThreads(context.Background(), victim.Id, th.BasicTeam.Id, model.GetUserThreadsOpts{Extended: true})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, uss.Threads, 1, "sanity: victim should see their own thread before team removal")
|
||||
|
||||
_, err = th.SystemAdminClient.RemoveTeamMember(context.Background(), th.BasicTeam.Id, victim.Id)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = th.SystemAdminClient.AddTeamMember(context.Background(), th.BasicTeam.Id, victim.Id)
|
||||
require.NoError(t, err)
|
||||
|
||||
th.LoginBasic2WithClient(victimClient)
|
||||
|
||||
uss, _, err = victimClient.GetUserThreads(context.Background(), victim.Id, th.BasicTeam.Id, model.GetUserThreadsOpts{Extended: true})
|
||||
require.NoError(t, err)
|
||||
for _, thr := range uss.Threads {
|
||||
require.NotEqual(t, rootPost.Id, thr.PostId, "private-channel thread must not leak to re-invited user")
|
||||
}
|
||||
require.Len(t, uss.Threads, 0, "re-invited user must not receive any threads from private channels they no longer belong to")
|
||||
}
|
||||
|
||||
func TestThreadSocketEvents(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
@@ -2668,6 +2668,19 @@ func (a *App) postRemoveFromChannelMessage(c request.CTX, removerUserId string,
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeChannelMembership strips a user's channel membership and the associated
|
||||
// thread memberships. Keeping these together ensures channel access cannot be
|
||||
// revoked without also dropping the thread state that depends on it.
|
||||
func (a *App) removeChannelMembership(rctx request.CTX, userID, channelID, caller string) *model.AppError {
|
||||
if err := a.Srv().Store().Channel().RemoveMember(rctx, channelID, userID); err != nil {
|
||||
return model.NewAppError(caller, "app.channel.remove_member.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
if err := a.Srv().Store().Thread().DeleteMembershipsForChannel(userID, channelID); err != nil {
|
||||
return model.NewAppError(caller, model.NoTranslation, nil, "failed to delete threadmemberships upon leaving channel", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) removeUserFromChannel(c request.CTX, userIDToRemove string, removerUserId string, channel *model.Channel) *model.AppError {
|
||||
user, nErr := a.Srv().Store().User().Get(context.Background(), userIDToRemove)
|
||||
if nErr != nil {
|
||||
@@ -2702,15 +2715,12 @@ func (a *App) removeUserFromChannel(c request.CTX, userIDToRemove string, remove
|
||||
return err
|
||||
}
|
||||
|
||||
if err := a.Srv().Store().Channel().RemoveMember(c, channel.Id, userIDToRemove); err != nil {
|
||||
return model.NewAppError("removeUserFromChannel", "app.channel.remove_member.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
if appErr := a.removeChannelMembership(c, userIDToRemove, channel.Id, "removeUserFromChannel"); appErr != nil {
|
||||
return appErr
|
||||
}
|
||||
if err := a.Srv().Store().ChannelMemberHistory().LogLeaveEvent(userIDToRemove, channel.Id, model.GetMillis()); err != nil {
|
||||
return model.NewAppError("removeUserFromChannel", "app.channel_member_history.log_leave_event.internal_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
if err := a.Srv().Store().Thread().DeleteMembershipsForChannel(userIDToRemove, channel.Id); err != nil {
|
||||
return model.NewAppError("removeUserFromChannel", model.NoTranslation, nil, "failed to delete threadmemberships upon leaving channel", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
|
||||
if isGuest {
|
||||
currentMembers, err := a.GetChannelMembersForUser(c, channel.TeamId, userIDToRemove)
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/mattermost/mattermost/server/public/shared/request"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/store"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/imgutils"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -34,6 +35,7 @@ const (
|
||||
MaxEmojiHeight = 128
|
||||
MaxEmojiOriginalWidth = 1028
|
||||
MaxEmojiOriginalHeight = 1028
|
||||
MaxEmojiGIFFrames = 70
|
||||
)
|
||||
|
||||
func (a *App) CreateEmoji(c request.CTX, sessionUserId string, emoji *model.Emoji, multiPartImageData *multipart.Form) (*model.Emoji, *model.AppError) {
|
||||
@@ -122,6 +124,24 @@ func (a *App) uploadEmojiImage(c request.CTX, id string, filename string, file i
|
||||
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.seek.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
|
||||
// Enforce the frame limit on every animated GIF, regardless of whether it
|
||||
// needs resizing, so the cap applies to the direct-write path too.
|
||||
isGIF := model.NewInfo(filename).MimeType == "image/gif"
|
||||
if isGIF {
|
||||
frameCount, err := imgutils.CountGIFFrames(file)
|
||||
if err != nil {
|
||||
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.image.app_error", nil, "", http.StatusBadRequest).Wrap(err)
|
||||
}
|
||||
if frameCount > MaxEmojiGIFFrames {
|
||||
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.too_many_frames.app_error", map[string]any{
|
||||
"MaxFrames": MaxEmojiGIFFrames,
|
||||
}, "", http.StatusBadRequest)
|
||||
}
|
||||
if _, err = file.Seek(0, io.SeekStart); err != nil {
|
||||
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.seek.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
}
|
||||
|
||||
if config.Width <= MaxEmojiWidth && config.Height <= MaxEmojiHeight {
|
||||
// No need to resize the image
|
||||
_, appErr := a.WriteFile(file, getEmojiImagePath(id))
|
||||
@@ -131,8 +151,7 @@ func (a *App) uploadEmojiImage(c request.CTX, id string, filename string, file i
|
||||
// Create a buffer for the resized image
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
info := model.NewInfo(filename)
|
||||
if info.MimeType == "image/gif" {
|
||||
if isGIF {
|
||||
g, err := gif.DecodeAll(file)
|
||||
if err != nil {
|
||||
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_decode_error", nil, "", http.StatusBadRequest).Wrap(err)
|
||||
|
||||
@@ -858,12 +858,14 @@ func (a *App) UploadFileX(c request.CTX, channelID, name string, input io.Reader
|
||||
|
||||
if *a.Config().FileSettings.ExtractContent && t.ExtractContent {
|
||||
infoCopy := *t.fileinfo
|
||||
a.Srv().GoBuffered(func() {
|
||||
if !a.Srv().GoExtraction(func() {
|
||||
err := a.ExtractContentFromFileInfo(c, &infoCopy)
|
||||
if err != nil {
|
||||
c.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
})
|
||||
}) {
|
||||
c.Logger().Warn("Content extraction queue is full, skipping inline extraction; this file's content will not be searchable until an admin runs a content extraction job (e.g. mmctl extract)", mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
}
|
||||
|
||||
return t.fileinfo, nil
|
||||
@@ -1125,12 +1127,14 @@ func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTe
|
||||
// and something we can do without.
|
||||
if *a.Config().FileSettings.ExtractContent && extractContent {
|
||||
infoCopy := *info
|
||||
a.Srv().GoBuffered(func() {
|
||||
if !a.Srv().GoExtraction(func() {
|
||||
err := a.ExtractContentFromFileInfo(c, &infoCopy)
|
||||
if err != nil {
|
||||
c.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
})
|
||||
}) {
|
||||
c.Logger().Warn("Content extraction queue is full, skipping inline extraction; this file's content will not be searchable until an admin runs a content extraction job (e.g. mmctl extract)", mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
}
|
||||
|
||||
return info, data, nil
|
||||
@@ -1584,10 +1588,15 @@ func (a *App) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileI
|
||||
if aerr != nil {
|
||||
return errors.Wrap(aerr, "failed to open file for extract file content")
|
||||
}
|
||||
defer file.Close()
|
||||
// Ownership of closing the file is handed to docextractor.Extract via
|
||||
// ReaderCloser: with a timeout configured, extraction may continue on a
|
||||
// detached goroutine after Extract returns, so closing the file here would
|
||||
// race with that goroutine still reading it.
|
||||
text, err := docextractor.Extract(rctx.Logger(), fileInfo.Name, file, docextractor.ExtractSettings{
|
||||
ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
|
||||
MaxFileSize: *a.Config().FileSettings.MaxFileSize,
|
||||
Timeout: time.Duration(*a.Config().FileSettings.ExtractContentTimeout) * time.Second,
|
||||
ReaderCloser: file,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to extract file content")
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
package platform
|
||||
|
||||
import "sync/atomic"
|
||||
import (
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Go creates a goroutine, but maintains a record of it to ensure that execution completes before
|
||||
// the server is shutdown.
|
||||
@@ -45,3 +48,55 @@ func (ps *PlatformService) GoBuffered(f func()) {
|
||||
<-ps.goroutineBuffered
|
||||
}()
|
||||
}
|
||||
|
||||
// startExtractionWorkers launches the fixed-size pool of workers that run
|
||||
// document extraction tasks submitted through GoExtraction.
|
||||
func (ps *PlatformService) startExtractionWorkers() {
|
||||
numWorkers := runtime.NumCPU()
|
||||
for range numWorkers {
|
||||
ps.extractionWG.Go(func() {
|
||||
for {
|
||||
select {
|
||||
case <-ps.extractionStop:
|
||||
return
|
||||
case f := <-ps.extractionQueue:
|
||||
f()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// stopExtractionWorkers signals the extraction workers to exit and waits for
|
||||
// any in-flight extraction to finish. Queued-but-not-started tasks are drained
|
||||
// and discarded so a worker cannot dequeue and run them after shutdown has been
|
||||
// signaled.
|
||||
func (ps *PlatformService) stopExtractionWorkers() {
|
||||
close(ps.extractionStop)
|
||||
|
||||
drain:
|
||||
for {
|
||||
select {
|
||||
case <-ps.extractionQueue:
|
||||
default:
|
||||
break drain
|
||||
}
|
||||
}
|
||||
|
||||
ps.extractionWG.Wait()
|
||||
}
|
||||
|
||||
// GoExtraction submits f to the bounded document extraction worker pool. It
|
||||
// never blocks the caller: if every worker is busy and the queue is full it
|
||||
// returns false without running f. Skipped files stay unextracted until an
|
||||
// admin runs a content extraction job (e.g. mmctl extract); there is no
|
||||
// scheduler that picks them up automatically. This keeps expensive extractions
|
||||
// from stalling the request goroutines that dispatch them.
|
||||
func (ps *PlatformService) GoExtraction(f func()) bool {
|
||||
select {
|
||||
case ps.extractionQueue <- f:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
79
server/channels/app/platform/goroutines_test.go
Обычный файл
79
server/channels/app/platform/goroutines_test.go
Обычный файл
@@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package platform
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGoExtraction(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
|
||||
t.Run("runs submitted work on the pool", func(t *testing.T) {
|
||||
const tasks = 5
|
||||
ps := &PlatformService{
|
||||
extractionQueue: make(chan func(), tasks),
|
||||
extractionStop: make(chan struct{}),
|
||||
}
|
||||
ps.startExtractionWorkers()
|
||||
defer ps.stopExtractionWorkers()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(tasks)
|
||||
for range tasks {
|
||||
require.True(t, ps.GoExtraction(func() {
|
||||
wg.Done()
|
||||
}))
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
require.Fail(t, "submitted extraction tasks did not run")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("never blocks and skips work once the queue is saturated", func(t *testing.T) {
|
||||
// No workers are started, so nothing drains the queue.
|
||||
ps := &PlatformService{
|
||||
extractionQueue: make(chan func(), 2),
|
||||
extractionStop: make(chan struct{}),
|
||||
}
|
||||
|
||||
require.True(t, ps.GoExtraction(func() {}))
|
||||
require.True(t, ps.GoExtraction(func() {}))
|
||||
// The queue is now full; further submissions must be rejected rather
|
||||
// than block the caller.
|
||||
require.False(t, ps.GoExtraction(func() {}))
|
||||
})
|
||||
|
||||
t.Run("stop waits for in-flight extraction to finish", func(t *testing.T) {
|
||||
ps := &PlatformService{
|
||||
extractionQueue: make(chan func(), 1),
|
||||
extractionStop: make(chan struct{}),
|
||||
}
|
||||
ps.startExtractionWorkers()
|
||||
|
||||
var finished bool
|
||||
started := make(chan struct{})
|
||||
require.True(t, ps.GoExtraction(func() {
|
||||
close(started)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
finished = true
|
||||
}))
|
||||
|
||||
<-started
|
||||
ps.stopExtractionWorkers()
|
||||
require.True(t, finished, "stopExtractionWorkers should wait for the running task to complete")
|
||||
})
|
||||
}
|
||||
@@ -107,6 +107,13 @@ type PlatformService struct {
|
||||
goroutineExitSignal chan struct{}
|
||||
goroutineBuffered chan struct{}
|
||||
|
||||
// Document content extraction runs on a dedicated, bounded worker pool so
|
||||
// that expensive extractions cannot saturate the generic worker pool and
|
||||
// block the request goroutines that dispatch them.
|
||||
extractionQueue chan func()
|
||||
extractionStop chan struct{}
|
||||
extractionWG sync.WaitGroup
|
||||
|
||||
additionalClusterHandlers map[model.ClusterEvent]einterfaces.ClusterMessageHandler
|
||||
|
||||
shareChannelServiceMux sync.RWMutex
|
||||
@@ -136,6 +143,8 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
|
||||
hashSeed: maphash.MakeSeed(),
|
||||
goroutineExitSignal: make(chan struct{}, 1),
|
||||
goroutineBuffered: make(chan struct{}, runtime.NumCPU()),
|
||||
extractionQueue: make(chan func(), runtime.NumCPU()),
|
||||
extractionStop: make(chan struct{}),
|
||||
WebSocketRouter: &WebSocketRouter{
|
||||
handlers: make(map[string]webSocketHandler),
|
||||
},
|
||||
@@ -401,6 +410,8 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
|
||||
ps.searchConfigListenerId = searchConfigListenerId
|
||||
ps.searchLicenseListenerId = searchLicenseListenerId
|
||||
|
||||
ps.startExtractionWorkers()
|
||||
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
@@ -517,6 +528,10 @@ func (ps *PlatformService) Shutdown() error {
|
||||
|
||||
ps.RemoveLicenseListener(ps.licenseListenerId)
|
||||
|
||||
// Stop the document extraction workers and wait for any in-flight
|
||||
// extraction to finish before closing the store it depends on.
|
||||
ps.stopExtractionWorkers()
|
||||
|
||||
// we need to wait the goroutines to finish before closing the store
|
||||
// and this needs to be called after hub stop because hub generates goroutines
|
||||
// when it is active. If we wait first we have no mechanism to prevent adding
|
||||
|
||||
@@ -813,6 +813,14 @@ func (s *Server) GoBuffered(f func()) {
|
||||
s.platform.GoBuffered(f)
|
||||
}
|
||||
|
||||
// GoExtraction submits f to the bounded document extraction worker pool without
|
||||
// blocking the caller. It returns false if the pool is saturated and f was not
|
||||
// run; skipped files stay unextracted until an admin runs a content extraction
|
||||
// job (e.g. mmctl extract).
|
||||
func (s *Server) GoExtraction(f func()) bool {
|
||||
return s.platform.GoExtraction(f)
|
||||
}
|
||||
|
||||
var corsAllowedMethods = []string{
|
||||
"POST",
|
||||
"GET",
|
||||
|
||||
@@ -1237,8 +1237,8 @@ func (a *App) LeaveTeam(c request.CTX, team *model.Team, user *model.User, reque
|
||||
for _, channel := range channelList {
|
||||
if !channel.IsGroupOrDirect() {
|
||||
a.invalidateCacheForChannelMembers(channel.Id)
|
||||
if nErr = a.Srv().Store().Channel().RemoveMember(c, channel.Id, user.Id); nErr != nil {
|
||||
return model.NewAppError("LeaveTeam", "app.channel.remove_member.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
||||
if appErr := a.removeChannelMembership(c, user.Id, channel.Id, "LeaveTeam"); appErr != nil {
|
||||
return appErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1139,6 +1139,256 @@ func TestLeaveTeamPanic(t *testing.T) {
|
||||
}, "unexpected panic from LeaveTeam")
|
||||
}
|
||||
|
||||
func TestLeaveTeamCleansUpThreadMemberships(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
privateChannel := th.CreatePrivateChannel(th.Context, th.BasicTeam)
|
||||
th.AddUserToChannel(victim, privateChannel)
|
||||
|
||||
rootPost, _, appErr := th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: admin.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
Message: "private team secret",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, admin.Id))
|
||||
}()
|
||||
|
||||
_, _, appErr = th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: victim.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
RootId: rootPost.Id,
|
||||
Message: "victim reply",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, victim.Id))
|
||||
}()
|
||||
|
||||
_, sErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
require.NoError(t, sErr, "victim should follow the thread after replying")
|
||||
|
||||
appErr = th.App.LeaveTeam(th.Context, th.BasicTeam, victim, victim.Id)
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, gErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
var errNotFound *store.ErrNotFound
|
||||
require.ErrorAs(t, gErr, &errNotFound, "thread membership must be deleted when user leaves the team")
|
||||
}
|
||||
|
||||
func TestLeaveTeamCleansUpThreadMembershipsAcrossChannels(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
privateA := th.CreatePrivateChannel(th.Context, th.BasicTeam)
|
||||
privateB := th.CreatePrivateChannel(th.Context, th.BasicTeam)
|
||||
openC := th.CreateChannel(th.Context, th.BasicTeam)
|
||||
th.AddUserToChannel(victim, privateA)
|
||||
th.AddUserToChannel(victim, privateB)
|
||||
th.AddUserToChannel(victim, openC)
|
||||
|
||||
rootIDs := make([]string, 0, 3)
|
||||
for _, ch := range []*model.Channel{privateA, privateB, openC} {
|
||||
root, _, appErr := th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: admin.Id,
|
||||
ChannelId: ch.Id,
|
||||
Message: "root in " + ch.Id,
|
||||
}, ch, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
_, _, appErr = th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: victim.Id,
|
||||
ChannelId: ch.Id,
|
||||
RootId: root.Id,
|
||||
Message: "reply",
|
||||
}, ch, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
rootIDs = append(rootIDs, root.Id)
|
||||
}
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, admin.Id))
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, victim.Id))
|
||||
}()
|
||||
|
||||
for _, rid := range rootIDs {
|
||||
_, sErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rid)
|
||||
require.NoError(t, sErr, "sanity: victim should follow each thread")
|
||||
}
|
||||
|
||||
appErr := th.App.LeaveTeam(th.Context, th.BasicTeam, victim, victim.Id)
|
||||
require.Nil(t, appErr)
|
||||
|
||||
var errNotFound *store.ErrNotFound
|
||||
for _, rid := range rootIDs {
|
||||
_, gErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rid)
|
||||
require.ErrorAs(t, gErr, &errNotFound, "thread membership for %s must be deleted on team leave", rid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaveTeamPreservesDMThreadMemberships(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
dmChannel, appErr := th.App.GetOrCreateDirectChannel(th.Context, admin.Id, victim.Id)
|
||||
require.Nil(t, appErr)
|
||||
|
||||
dmRoot, _, appErr := th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: admin.Id,
|
||||
ChannelId: dmChannel.Id,
|
||||
Message: "dm root",
|
||||
}, dmChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
_, _, appErr = th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: victim.Id,
|
||||
ChannelId: dmChannel.Id,
|
||||
RootId: dmRoot.Id,
|
||||
Message: "dm reply",
|
||||
}, dmChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, admin.Id))
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, victim.Id))
|
||||
}()
|
||||
|
||||
_, sErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, dmRoot.Id)
|
||||
require.NoError(t, sErr, "sanity: victim should follow the DM thread")
|
||||
|
||||
appErr = th.App.LeaveTeam(th.Context, th.BasicTeam, victim, victim.Id)
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, gErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, dmRoot.Id)
|
||||
require.NoError(t, gErr, "DM thread membership must survive leaving an unrelated team")
|
||||
}
|
||||
|
||||
func TestGetThreadsForUser_ReadPathRejectsOrphanThreadMembership(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
privateChannel := th.CreatePrivateChannel(th.Context, th.BasicTeam)
|
||||
th.AddUserToChannel(victim, privateChannel)
|
||||
|
||||
rootPost, _, appErr := th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: admin.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
Message: "private team secret",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
defer func() {
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, admin.Id))
|
||||
require.NoError(t, th.App.Srv().Store().Post().PermanentDeleteByUser(th.Context, victim.Id))
|
||||
}()
|
||||
|
||||
_, _, appErr = th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: victim.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
RootId: rootPost.Id,
|
||||
Message: "victim reply",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, sErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
require.NoError(t, sErr, "sanity: victim should follow the thread after replying")
|
||||
|
||||
require.NoError(t, th.App.Srv().Store().Channel().RemoveMember(th.Context, privateChannel.Id, victim.Id))
|
||||
|
||||
_, sErr2 := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
require.NoError(t, sErr2, "sanity: synthetic orphan ThreadMembership must remain")
|
||||
|
||||
threads, gErr := th.App.Srv().Store().Thread().GetThreadsForUser(victim.Id, th.BasicTeam.Id, model.GetUserThreadsOpts{})
|
||||
require.NoError(t, gErr)
|
||||
for _, thr := range threads {
|
||||
require.NotEqual(t, rootPost.Id, thr.PostId, "read path must not surface threads from channels the user no longer belongs to")
|
||||
}
|
||||
require.Empty(t, threads, "GetThreadsForUser must filter out orphan ThreadMembership rows")
|
||||
|
||||
totalThreads, gErr := th.App.Srv().Store().Thread().GetTotalThreads(victim.Id, th.BasicTeam.Id, model.GetUserThreadsOpts{})
|
||||
require.NoError(t, gErr)
|
||||
require.Zero(t, totalThreads, "GetTotalThreads must not count orphan ThreadMembership rows")
|
||||
|
||||
totalUnread, gErr := th.App.Srv().Store().Thread().GetTotalUnreadThreads(victim.Id, th.BasicTeam.Id, model.GetUserThreadsOpts{})
|
||||
require.NoError(t, gErr)
|
||||
require.Zero(t, totalUnread, "GetTotalUnreadThreads must not count orphan ThreadMembership rows")
|
||||
}
|
||||
|
||||
func TestPermanentDeleteChannelRemovesThreadMemberships(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.ServiceSettings.ThreadAutoFollow = true
|
||||
*cfg.ServiceSettings.CollapsedThreads = model.CollapsedThreadsDefaultOn
|
||||
})
|
||||
th.App.Srv().SetLicense(model.NewTestLicenseSKU(model.LicenseShortSkuProfessional))
|
||||
|
||||
admin := th.BasicUser
|
||||
victim := th.BasicUser2
|
||||
|
||||
privateChannel := th.CreatePrivateChannel(th.Context, th.BasicTeam)
|
||||
th.AddUserToChannel(victim, privateChannel)
|
||||
|
||||
rootPost, _, appErr := th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: admin.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
Message: "doomed root",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, _, appErr = th.App.CreatePost(th.Context, &model.Post{
|
||||
UserId: victim.Id,
|
||||
ChannelId: privateChannel.Id,
|
||||
RootId: rootPost.Id,
|
||||
Message: "doomed reply",
|
||||
}, privateChannel, model.CreatePostFlags{SetOnline: true})
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, sErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
require.NoError(t, sErr, "victim should follow the thread after replying")
|
||||
|
||||
appErr = th.App.PermanentDeleteChannel(th.Context, privateChannel)
|
||||
require.Nil(t, appErr)
|
||||
|
||||
_, gErr := th.App.Srv().Store().Thread().GetMembershipForUser(victim.Id, rootPost.Id)
|
||||
var errNotFound *store.ErrNotFound
|
||||
require.ErrorAs(t, gErr, &errNotFound, "thread membership must be deleted with the channel")
|
||||
}
|
||||
|
||||
func TestAppUpdateTeamScheme(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t).InitBasic()
|
||||
|
||||
@@ -339,12 +339,14 @@ func (a *App) UploadData(c request.CTX, us *model.UploadSession, rd io.Reader) (
|
||||
|
||||
if *a.Config().FileSettings.ExtractContent {
|
||||
infoCopy := *info
|
||||
a.Srv().Go(func() {
|
||||
if !a.Srv().GoExtraction(func() {
|
||||
err := a.ExtractContentFromFileInfo(c, &infoCopy)
|
||||
if err != nil {
|
||||
c.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
})
|
||||
}) {
|
||||
c.Logger().Warn("Content extraction queue is full, skipping inline extraction; this file's content will not be searchable until an admin runs a content extraction job (e.g. mmctl extract)", mlog.String("fileInfoId", infoCopy.Id))
|
||||
}
|
||||
}
|
||||
|
||||
// delete upload session
|
||||
|
||||
@@ -285,6 +285,8 @@ channels/db/migrations/mysql/000143_backfill_roles_schemeid.down.sql
|
||||
channels/db/migrations/mysql/000143_backfill_roles_schemeid.up.sql
|
||||
channels/db/migrations/mysql/000144_add_roles_schemeid_index.down.sql
|
||||
channels/db/migrations/mysql/000144_add_roles_schemeid_index.up.sql
|
||||
channels/db/migrations/mysql/000195_threadmemberships_cleanup_v2.down.sql
|
||||
channels/db/migrations/mysql/000195_threadmemberships_cleanup_v2.up.sql
|
||||
channels/db/migrations/postgres/000001_create_teams.down.sql
|
||||
channels/db/migrations/postgres/000001_create_teams.up.sql
|
||||
channels/db/migrations/postgres/000002_create_team_members.down.sql
|
||||
@@ -571,3 +573,5 @@ channels/db/migrations/postgres/000143_backfill_roles_schemeid.down.sql
|
||||
channels/db/migrations/postgres/000143_backfill_roles_schemeid.up.sql
|
||||
channels/db/migrations/postgres/000144_add_roles_schemeid_index.down.sql
|
||||
channels/db/migrations/postgres/000144_add_roles_schemeid_index.up.sql
|
||||
channels/db/migrations/postgres/000195_threadmemberships_cleanup_v2.down.sql
|
||||
channels/db/migrations/postgres/000195_threadmemberships_cleanup_v2.up.sql
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
-- Skipping it because the forward migration is destructive
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Drop ThreadMembership rows whose user is no longer a member of the thread's channel.
|
||||
DELETE tm
|
||||
FROM ThreadMemberships AS tm
|
||||
JOIN Threads ON Threads.PostId = tm.PostId
|
||||
LEFT JOIN ChannelMembers ON ChannelMembers.UserId = tm.UserId
|
||||
AND Threads.ChannelId = ChannelMembers.ChannelId
|
||||
WHERE ChannelMembers.ChannelId IS NULL;
|
||||
@@ -0,0 +1 @@
|
||||
-- Skipping it because the forward migration is destructive
|
||||
@@ -0,0 +1,13 @@
|
||||
-- Drop ThreadMembership rows whose user is no longer a member of the thread's channel.
|
||||
DELETE FROM threadmemberships WHERE (postid, userid) IN (
|
||||
SELECT
|
||||
threadmemberships.postid,
|
||||
threadmemberships.userid
|
||||
FROM
|
||||
threadmemberships
|
||||
JOIN threads ON threads.postid = threadmemberships.postid
|
||||
LEFT JOIN channelmembers ON channelmembers.userid = threadmemberships.userid
|
||||
AND threads.channelid = channelmembers.channelid
|
||||
WHERE
|
||||
channelmembers.channelid IS NULL
|
||||
);
|
||||
@@ -70,6 +70,17 @@ type SqlThreadStore struct {
|
||||
func (s *SqlThreadStore) ClearCaches() {
|
||||
}
|
||||
|
||||
// channelMembershipPredicate filters out ThreadMemberships whose user is no
|
||||
// longer a member of the thread's channel. DM/GM threads have an empty
|
||||
// ThreadTeamId and are exempt because their access is intrinsic to the
|
||||
// channel members.
|
||||
func channelMembershipPredicate() sq.Sqlizer {
|
||||
return sq.Or{
|
||||
sq.Eq{"Threads.ThreadTeamId": ""},
|
||||
sq.Expr("EXISTS (SELECT 1 FROM ChannelMembers WHERE ChannelMembers.ChannelId = Threads.ChannelId AND ChannelMembers.UserId = ThreadMemberships.UserId)"),
|
||||
}
|
||||
}
|
||||
|
||||
func newSqlThreadStore(sqlStore *SqlStore) store.ThreadStore {
|
||||
s := SqlThreadStore{
|
||||
SqlStore: sqlStore,
|
||||
@@ -131,7 +142,8 @@ func (s *SqlThreadStore) getTotalThreadsQuery(userId, teamId string, opts model.
|
||||
Where(sq.Eq{
|
||||
"ThreadMemberships.UserId": userId,
|
||||
"ThreadMemberships.Following": true,
|
||||
})
|
||||
}).
|
||||
Where(channelMembershipPredicate())
|
||||
|
||||
if teamId != "" {
|
||||
if opts.ExcludeDirect {
|
||||
@@ -198,7 +210,8 @@ func (s *SqlThreadStore) GetTotalUnreadMentions(userId, teamId string, opts mode
|
||||
Where(sq.Eq{
|
||||
"ThreadMemberships.UserId": userId,
|
||||
"ThreadMemberships.Following": true,
|
||||
})
|
||||
}).
|
||||
Where(channelMembershipPredicate())
|
||||
|
||||
if teamId != "" {
|
||||
if opts.ExcludeDirect {
|
||||
@@ -234,15 +247,13 @@ func (s *SqlThreadStore) GetTotalUnreadUrgentMentions(userId, teamId string, opt
|
||||
Select("COALESCE(SUM(ThreadMemberships.UnreadMentions),0)").
|
||||
From("ThreadMemberships").
|
||||
Join("PostsPriority ON PostsPriority.PostId = ThreadMemberships.PostId").
|
||||
Join("Threads ON Threads.PostId = ThreadMemberships.PostId").
|
||||
Where(sq.Eq{
|
||||
"ThreadMemberships.UserId": userId,
|
||||
"ThreadMemberships.Following": true,
|
||||
"PostsPriority.Priority": model.PostPriorityUrgent,
|
||||
})
|
||||
|
||||
if teamId != "" || !opts.Deleted {
|
||||
query = query.Join("Threads ON Threads.PostId = ThreadMemberships.PostId")
|
||||
}
|
||||
}).
|
||||
Where(channelMembershipPredicate())
|
||||
|
||||
if teamId != "" {
|
||||
if opts.ExcludeDirect {
|
||||
@@ -298,7 +309,8 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
|
||||
|
||||
query = query.
|
||||
Where(sq.Eq{"ThreadMemberships.UserId": userId}).
|
||||
Where(sq.Eq{"ThreadMemberships.Following": true})
|
||||
Where(sq.Eq{"ThreadMemberships.Following": true}).
|
||||
Where(channelMembershipPredicate())
|
||||
|
||||
if opts.IncludeIsUrgent {
|
||||
urgencyCase := sq.
|
||||
@@ -405,6 +417,7 @@ func (s *SqlThreadStore) GetTeamsUnreadForUser(userID string, teamIDs []string,
|
||||
sq.Eq{"ThreadMemberships.Following": true},
|
||||
sq.Eq{"Threads.ThreadTeamId": teamIDs},
|
||||
sq.Eq{"COALESCE(Threads.ThreadDeleteAt, 0)": 0},
|
||||
channelMembershipPredicate(),
|
||||
}
|
||||
|
||||
var eg errgroup.Group
|
||||
|
||||
@@ -717,6 +717,12 @@ func testGetTeamsUnreadForUser(t *testing.T, rctx request.CTX, ss store.Store) {
|
||||
Type: model.ChannelTypeOpen,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: channel1.Id,
|
||||
UserId: userID,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
post, err := ss.Post().Save(rctx, &model.Post{
|
||||
ChannelId: channel1.Id,
|
||||
UserId: userID,
|
||||
@@ -759,6 +765,12 @@ func testGetTeamsUnreadForUser(t *testing.T, rctx request.CTX, ss store.Store) {
|
||||
Type: model.ChannelTypeOpen,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: channel2.Id,
|
||||
UserId: userID,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
post2, err := ss.Post().Save(rctx, &model.Post{
|
||||
ChannelId: channel2.Id,
|
||||
@@ -869,6 +881,12 @@ func testVarious(t *testing.T, rctx request.CTX, ss store.Store) {
|
||||
Type: model.ChannelTypeOpen,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: team1channel1.Id,
|
||||
UserId: user1ID,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
team2channel1, err := ss.Channel().Save(rctx, &model.Channel{
|
||||
TeamId: team2.Id,
|
||||
@@ -877,6 +895,12 @@ func testVarious(t *testing.T, rctx request.CTX, ss store.Store) {
|
||||
Type: model.ChannelTypeOpen,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: team2channel1.Id,
|
||||
UserId: user1ID,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
dm1, err := ss.Channel().CreateDirectChannel(rctx, &model.User{Id: user1ID}, &model.User{Id: user2ID})
|
||||
require.NoError(t, err)
|
||||
@@ -1345,6 +1369,17 @@ func testMarkAllAsReadByChannels(t *testing.T, rctx request.CTX, ss store.Store)
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, ch := range []*model.Channel{channel1, channel2} {
|
||||
for _, uid := range []string{userAID, userBID} {
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: ch.Id,
|
||||
UserId: uid,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
createThreadMembership := func(userID, postID string) {
|
||||
t.Helper()
|
||||
opts := store.ThreadMembershipOpts{
|
||||
@@ -1535,6 +1570,17 @@ func testMarkAllAsReadByTeam(t *testing.T, rctx request.CTX, ss store.Store) {
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, ch := range []*model.Channel{team1channel1, team1channel2, team2channel1, team2channel2} {
|
||||
for _, uid := range []string{userAID, userBID} {
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: ch.Id,
|
||||
UserId: uid,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
team1channel1post1, err := ss.Post().Save(rctx, &model.Post{
|
||||
ChannelId: team1channel1.Id,
|
||||
UserId: postingUserId,
|
||||
@@ -2110,6 +2156,13 @@ func testUpdateTeamIdForChannelThreads(t *testing.T, rctx request.CTX, ss store.
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: channel1.Id,
|
||||
UserId: userA.Id,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, clean := createThreadMembership(userA.Id, rootPost1.Id, true)
|
||||
defer clean()
|
||||
|
||||
@@ -2134,6 +2187,13 @@ func testUpdateTeamIdForChannelThreads(t *testing.T, rctx request.CTX, ss store.
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = ss.Channel().SaveMember(rctx, &model.ChannelMember{
|
||||
ChannelId: channel1.Id,
|
||||
UserId: userA.Id,
|
||||
NotifyProps: model.GetDefaultChannelNotifyProps(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
newTeamID := model.NewId()
|
||||
|
||||
_, clean := createThreadMembership(userA.Id, rootPost1.Id, true)
|
||||
|
||||
@@ -198,9 +198,14 @@ func (fs *FileStore) RemoveFile(name string) error {
|
||||
mlog.Debug("Skipping removal of configuration file with absolute path", mlog.String("filename", name))
|
||||
return nil
|
||||
}
|
||||
resolvedPath := filepath.Join(filepath.Dir(fs.path), name)
|
||||
|
||||
err := os.Remove(resolvedPath)
|
||||
root, err := os.OpenRoot(filepath.Dir(fs.path))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to open config directory")
|
||||
}
|
||||
defer root.Close()
|
||||
|
||||
err = root.Remove(name)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1226,6 +1226,55 @@ func TestFileRemoveFile(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.True(t, has)
|
||||
})
|
||||
|
||||
t.Run("reject invalid relative path", func(t *testing.T) {
|
||||
path, tearDown := setupConfigFile(t, minimalConfig)
|
||||
defer tearDown()
|
||||
|
||||
fs, err := NewFileStore(path, false)
|
||||
require.NoError(t, err)
|
||||
defer fs.Close()
|
||||
|
||||
baseDir := filepath.Dir(path)
|
||||
parentDir := filepath.Dir(baseDir)
|
||||
outsideFile := filepath.Join(parentDir, "invalid-target-file")
|
||||
|
||||
err = os.WriteFile(outsideFile, []byte("outside"), 0600)
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(outsideFile)
|
||||
|
||||
relativePath, err := filepath.Rel(baseDir, outsideFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fs.RemoveFile(relativePath)
|
||||
require.Error(t, err)
|
||||
|
||||
_, statErr := os.Stat(outsideFile)
|
||||
require.NoError(t, statErr)
|
||||
})
|
||||
|
||||
t.Run("remove valid relative file", func(t *testing.T) {
|
||||
path, tearDown := setupConfigFile(t, minimalConfig)
|
||||
defer tearDown()
|
||||
|
||||
fs, err := NewFileStore(path, false)
|
||||
require.NoError(t, err)
|
||||
defer fs.Close()
|
||||
|
||||
nestedDir := filepath.Join(filepath.Dir(path), "certs")
|
||||
err = os.MkdirAll(nestedDir, 0700)
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := filepath.Join("certs", "valid-cert.pem")
|
||||
err = fs.SetFile(filename, []byte("cert-data"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fs.RemoveFile(filename)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, statErr := os.Stat(filepath.Join(filepath.Dir(path), filename))
|
||||
require.ErrorIs(t, statErr, os.ErrNotExist)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileStoreString(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.enterprise for license information.
|
||||
|
||||
package global_relay_export
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/shared/request"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
|
||||
)
|
||||
|
||||
// --- fault-injection helpers ------------------------------------------------
|
||||
|
||||
// scriptedReader serves data[pos:], honouring Seek so a resumed read (re-open + Seek past
|
||||
// the bytes already streamed) continues from the right offset, exactly like the S3/local
|
||||
// backends. When failAfter >= 0 it returns an error once pos reaches that offset, modelling
|
||||
// a transient read failure mid-stream (e.g. an S3 timeout). failAfter == 0 fails immediately
|
||||
// with no progress; failAfter < 0 reads cleanly to EOF.
|
||||
type scriptedReader struct {
|
||||
data []byte
|
||||
pos int
|
||||
failAfter int
|
||||
}
|
||||
|
||||
var _ filestore.ReadCloseSeeker = (*scriptedReader)(nil)
|
||||
|
||||
func (r *scriptedReader) Read(p []byte) (int, error) {
|
||||
if r.failAfter >= 0 && r.pos >= r.failAfter {
|
||||
return 0, errors.New("simulated transient S3 read failure mid-stream")
|
||||
}
|
||||
if r.pos >= len(r.data) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
end := len(r.data)
|
||||
if r.failAfter >= 0 && r.failAfter < end {
|
||||
end = r.failAfter
|
||||
}
|
||||
n := copy(p, r.data[r.pos:end])
|
||||
r.pos += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *scriptedReader) Close() error { return nil }
|
||||
|
||||
func (r *scriptedReader) Seek(offset int64, whence int) (int64, error) {
|
||||
if whence != io.SeekStart {
|
||||
return 0, fmt.Errorf("scriptedReader: unsupported whence %d", whence)
|
||||
}
|
||||
r.pos = int(offset)
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
// scriptedBackend is a filestore.FileBackend whose Reader is driven by a per-path factory.
|
||||
// generateEmail exercises Reader plus FileExists (consulted on a read failure to tell a
|
||||
// genuinely-missing object apart from a transient read error); the rest of the embedded
|
||||
// interface is nil and unused.
|
||||
type scriptedBackend struct {
|
||||
filestore.FileBackend
|
||||
readers map[string]func() (filestore.ReadCloseSeeker, error)
|
||||
// missing lists paths that FileExists should report as absent, modelling an object that
|
||||
// opens lazily (S3/MinIO) but no longer exists. Paths not listed report as present.
|
||||
missing map[string]bool
|
||||
}
|
||||
|
||||
func (b *scriptedBackend) Reader(path string) (filestore.ReadCloseSeeker, error) {
|
||||
if fn, ok := b.readers[path]; ok {
|
||||
return fn()
|
||||
}
|
||||
return nil, fmt.Errorf("scriptedBackend: no reader registered for %q", path)
|
||||
}
|
||||
|
||||
func (b *scriptedBackend) FileExists(path string) (bool, error) {
|
||||
return !b.missing[path], nil
|
||||
}
|
||||
|
||||
// scriptedReaderFactory returns a Reader factory that fails at the given offsets on
|
||||
// successive opens: the i-th open uses failAt[i] (see scriptedReader.failAfter). Opens past
|
||||
// the end of failAt read cleanly to EOF. The same data is served every open, so a resumed
|
||||
// read reconstructs the full content.
|
||||
func scriptedReaderFactory(data []byte, failAt ...int) func() (filestore.ReadCloseSeeker, error) {
|
||||
var call int
|
||||
return func() (filestore.ReadCloseSeeker, error) {
|
||||
failAfter := -1
|
||||
if call < len(failAt) {
|
||||
failAfter = failAt[call]
|
||||
}
|
||||
call++
|
||||
return &scriptedReader{data: data, failAfter: failAfter}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// healthyReader serves the given content cleanly in a single uninterrupted read.
|
||||
func healthyReader(content []byte) func() (filestore.ReadCloseSeeker, error) {
|
||||
return scriptedReaderFactory(content)
|
||||
}
|
||||
|
||||
// openFails models a file that cannot be opened at all (e.g. deleted from the store).
|
||||
func openFails() (filestore.ReadCloseSeeker, error) {
|
||||
return nil, errors.New("file does not exist")
|
||||
}
|
||||
|
||||
// assertAttachmentPresent checks that content was written into the email as an attachment.
|
||||
// gomail stores attachments base64-encoded, and MIME wraps that base64 across lines, so we
|
||||
// drop the line breaks and look for the unwrapped encoding — letting content be any length.
|
||||
func assertAttachmentPresent(t *testing.T, out *bytes.Buffer, content []byte, msg string) {
|
||||
t.Helper()
|
||||
unwrapped := strings.NewReplacer("\r", "", "\n", "").Replace(out.String())
|
||||
require.Contains(t, unwrapped, base64.StdEncoding.EncodeToString(content), msg)
|
||||
}
|
||||
|
||||
func newChannelExport(files ...*model.FileInfo) *ChannelExport {
|
||||
return &ChannelExport{
|
||||
ChannelId: "channelid1234567890123456",
|
||||
ChannelName: "test-channel",
|
||||
ChannelDisplayName: "Test Channel",
|
||||
ChannelType: model.ChannelTypeDirect,
|
||||
Participants: []ParticipantRow{
|
||||
{JoinExport: shared.JoinExport{UserId: "userid", UserEmail: "participant@example.com"}},
|
||||
},
|
||||
uploadedFiles: files,
|
||||
}
|
||||
}
|
||||
|
||||
// runGenerateEmail calls generateEmail inside a testing/synctest bubble so the exponential
|
||||
// backoff between read retries uses a fake clock and the retries don't actually sleep. Any
|
||||
// panic is recovered so a regression (e.g. the MM-69242 nil-pointer that crashes the whole
|
||||
// server) shows up as a clear test failure rather than crashing the test binary.
|
||||
func runGenerateEmail(t *testing.T, backend filestore.FileBackend, ce *ChannelExport) (warnings int, out *bytes.Buffer, genErr error) {
|
||||
t.Helper()
|
||||
return runGenerateEmailCtx(context.Background(), t, backend, ce)
|
||||
}
|
||||
|
||||
// runGenerateEmailCtx is runGenerateEmail with a caller-supplied context, used to exercise
|
||||
// job cancellation mid-retry.
|
||||
func runGenerateEmailCtx(ctx context.Context, t *testing.T, backend filestore.FileBackend, ce *ChannelExport) (warnings int, out *bytes.Buffer, genErr error) {
|
||||
t.Helper()
|
||||
templatesDir, ok := fileutils.FindDir("templates")
|
||||
require.True(t, ok, "could not locate the server templates dir")
|
||||
templatesContainer, err := templates.New(templatesDir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, templatesContainer)
|
||||
|
||||
// The logger (and its async logr goroutine) is created OUTSIDE the bubble so it
|
||||
// isn't tracked as a bubble goroutine.
|
||||
rctx := request.TestContext(t).WithContext(ctx)
|
||||
out = &bytes.Buffer{}
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var recovered any
|
||||
func() {
|
||||
defer func() { recovered = recover() }()
|
||||
warnings, genErr = generateEmail(rctx, backend, ce, templatesContainer, out)
|
||||
}()
|
||||
if recovered != nil {
|
||||
t.Fatalf("MM-69242 regression: generateEmail panicked (in production this crashes "+
|
||||
"the entire server): %v", recovered)
|
||||
}
|
||||
})
|
||||
|
||||
return warnings, out, genErr
|
||||
}
|
||||
|
||||
// --- tests ------------------------------------------------------------------
|
||||
|
||||
// A transient read failure that clears within the retry budget must not fail the export:
|
||||
// the attachment is resumed on a later attempt and its full content lands in the email.
|
||||
func TestGenerateEmail_TransientReadRecoversOnRetry(t *testing.T) {
|
||||
flakyContent := []byte("flaky attachment content that must survive a mid-stream failure and resume")
|
||||
healthyContent := []byte("healthy attachment content")
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "flaky.bin", Path: "data/flaky.bin"},
|
||||
&model.FileInfo{Id: "file2", Name: "healthy.bin", Path: "data/healthy.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
// First open fails after 20 bytes; the retry re-opens, seeks to 20, and finishes —
|
||||
// proving the resume reconstructs the full content, not just non-empty output.
|
||||
"data/flaky.bin": scriptedReaderFactory(flakyContent, 20),
|
||||
"data/healthy.bin": healthyReader(healthyContent),
|
||||
}}
|
||||
|
||||
warnings, out, genErr := runGenerateEmail(t, backend, ce)
|
||||
|
||||
require.NoError(t, genErr, "a transient failure that clears within the retry budget should succeed")
|
||||
require.Equal(t, 0, warnings)
|
||||
assertAttachmentPresent(t, out, healthyContent, "the healthy attachment's content should be in the email")
|
||||
assertAttachmentPresent(t, out, flakyContent, "the resumed attachment's full content should be in the email")
|
||||
}
|
||||
|
||||
// A read failure that persists past the retry budget fails the batch (so the job retries)
|
||||
// instead of shipping an incomplete export — and, critically, never panics. The healthy
|
||||
// attachment after the failing one is exactly what triggered the MM-69242 nil-deref.
|
||||
func TestGenerateEmail_PersistentReadFailsBatch(t *testing.T) {
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "fails.bin", Path: "data/fails.bin"},
|
||||
&model.FileInfo{Id: "file2", Name: "healthy.bin", Path: "data/healthy.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
// Fails immediately (no progress) on every attempt, exhausting the stall budget.
|
||||
"data/fails.bin": scriptedReaderFactory(make([]byte, 200), 0, 0, 0),
|
||||
"data/healthy.bin": healthyReader([]byte("healthy attachment content")),
|
||||
}}
|
||||
|
||||
warnings, _, genErr := runGenerateEmail(t, backend, ce)
|
||||
|
||||
require.Error(t, genErr, "a persistent attachment read failure should fail the batch (so the job retries)")
|
||||
require.Equal(t, 0, warnings, "a read failure is an error, not a skipped/missing-file warning")
|
||||
}
|
||||
|
||||
// A genuinely missing attachment (open fails AND FileExists confirms it's gone) keeps the
|
||||
// prior MM-62493 behavior: warn, increment the warning count, skip it, don't fail the batch.
|
||||
func TestGenerateEmail_MissingAttachmentSkipped(t *testing.T) {
|
||||
healthyContent := []byte("healthy attachment content")
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "missing.bin", Path: "data/missing.bin"},
|
||||
&model.FileInfo{Id: "file2", Name: "healthy.bin", Path: "data/healthy.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{
|
||||
readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
"data/missing.bin": openFails,
|
||||
"data/healthy.bin": healthyReader(healthyContent),
|
||||
},
|
||||
missing: map[string]bool{"data/missing.bin": true}, // FileExists confirms it's gone
|
||||
}
|
||||
|
||||
warnings, out, genErr := runGenerateEmail(t, backend, ce)
|
||||
|
||||
require.NoError(t, genErr, "a missing file should be skipped, not fail the batch")
|
||||
require.Equal(t, 1, warnings, "the missing file should be counted as a warning")
|
||||
assertAttachmentPresent(t, out, healthyContent, "the surviving attachment should still be exported")
|
||||
}
|
||||
|
||||
// A transient OPEN failure (backend.Reader errors, but the file still exists) must NOT be
|
||||
// silently skipped as "missing": it is retried and, if it persists, fails the batch so the
|
||||
// job retries — otherwise a transient infrastructure hiccup at open time could drop an
|
||||
// attachment from a compliance export and still report success (MM-69338).
|
||||
func TestGenerateEmail_TransientOpenFailureFailsBatch(t *testing.T) {
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "openflaky.bin", Path: "data/openflaky.bin"},
|
||||
&model.FileInfo{Id: "file2", Name: "healthy.bin", Path: "data/healthy.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{
|
||||
readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
"data/openflaky.bin": openFails, // open fails on every attempt...
|
||||
"data/healthy.bin": healthyReader([]byte("healthy attachment content")),
|
||||
},
|
||||
// ...but FileExists reports the object still present, so it's a transient hiccup, not a
|
||||
// deletion (no entry in `missing` ⇒ FileExists returns true).
|
||||
}
|
||||
|
||||
warnings, _, genErr := runGenerateEmail(t, backend, ce)
|
||||
|
||||
require.Error(t, genErr, "a transient open failure on an existing file should fail the batch, not be skipped")
|
||||
require.Equal(t, 0, warnings, "a transient open failure is an error, not a skipped/missing-file warning")
|
||||
}
|
||||
|
||||
// On S3/MinIO a deleted object is not detected when the reader is opened (minio-go's
|
||||
// GetObject is lazy); the "no such key" only surfaces as a read error on the first Read. Such
|
||||
// a file must be treated as missing — skipped with a warning, batch not failed and not
|
||||
// retried — exactly like a local-backend open failure (MM-62493). Modeled
|
||||
// here by a reader that opens but fails its first read, with FileExists reporting it absent.
|
||||
func TestGenerateEmail_ReadNotFoundSkipped(t *testing.T) {
|
||||
healthyContent := []byte("healthy attachment content")
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "s3missing.bin", Path: "data/s3missing.bin"},
|
||||
&model.FileInfo{Id: "file2", Name: "healthy.bin", Path: "data/healthy.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{
|
||||
readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
"data/s3missing.bin": scriptedReaderFactory(make([]byte, 200), 0), // opens, first read fails
|
||||
"data/healthy.bin": healthyReader(healthyContent),
|
||||
},
|
||||
missing: map[string]bool{"data/s3missing.bin": true}, // FileExists reports it gone
|
||||
}
|
||||
|
||||
warnings, out, genErr := runGenerateEmail(t, backend, ce)
|
||||
|
||||
require.NoError(t, genErr, "a read-time not-found must be skipped, not fail the batch")
|
||||
require.Equal(t, 1, warnings, "the missing file should be counted as a warning")
|
||||
assertAttachmentPresent(t, out, healthyContent, "the surviving attachment should still be exported")
|
||||
}
|
||||
|
||||
// A job cancelled while a read is backing off must abort promptly with the context error
|
||||
// (not sleep through the backoff, not panic).
|
||||
func TestGenerateEmail_ContextCancelledDuringRetry(t *testing.T) {
|
||||
ce := newChannelExport(
|
||||
&model.FileInfo{Id: "file1", Name: "fails.bin", Path: "data/fails.bin"},
|
||||
)
|
||||
backend := &scriptedBackend{readers: map[string]func() (filestore.ReadCloseSeeker, error){
|
||||
"data/fails.bin": scriptedReaderFactory(make([]byte, 200), 0, 0, 0),
|
||||
}}
|
||||
|
||||
// Cancelled before the first backoff: the retry loop hits the first read failure, then
|
||||
// the cancellation wins the backoff select immediately.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
warnings, _, genErr := runGenerateEmailCtx(ctx, t, backend, ce)
|
||||
|
||||
require.Error(t, genErr, "a cancelled job should fail rather than ship a partial export")
|
||||
require.ErrorIs(t, genErr, context.Canceled)
|
||||
require.Equal(t, 0, warnings)
|
||||
}
|
||||
@@ -6,6 +6,7 @@ package global_relay_export
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
@@ -32,6 +33,17 @@ const (
|
||||
GlobalRelayChannelIDHeader = "X-Mattermost-ChannelID"
|
||||
GlobalRelayChannelTypeHeader = "X-Mattermost-ChannelType"
|
||||
MaxEmailsPerConnection = 400
|
||||
|
||||
// maxAttachmentReadAttempts bounds how many consecutive read attempts that make no
|
||||
// forward progress we tolerate before giving up. Attempts that do make progress reset
|
||||
// this budget (and the backoff), so a large attachment can still complete over a flaky
|
||||
// connection. If the budget is exhausted, the batch is failed and the job is retried.
|
||||
maxAttachmentReadAttempts = 3
|
||||
|
||||
// attachmentReadBackoff is the initial delay before retrying a stalled attachment read;
|
||||
// it doubles after each stalled attempt (exponential backoff) and resets once a retry
|
||||
// makes progress.
|
||||
attachmentReadBackoff = 1 * time.Second
|
||||
)
|
||||
|
||||
// MaxEmailBytes is a var because it needs to be set in tests. Otherwise it shouldn't be touched.
|
||||
@@ -264,7 +276,7 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
|
||||
|
||||
htmlBody, err := channelExportToHTML(rctx, channelExport, templates)
|
||||
if err != nil {
|
||||
return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
|
||||
return warningCount, fmt.Errorf("unable to render the channel export to HTML: %w", err)
|
||||
}
|
||||
|
||||
subject := fmt.Sprintf("Mattermost Compliance Export: %s", channelExport.ChannelDisplayName)
|
||||
@@ -295,34 +307,177 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
|
||||
m.SetBody("text/plain", txtBody)
|
||||
m.AddAlternative("text/html", htmlMessage)
|
||||
|
||||
// attachmentReadErr captures a genuine attachment read/write failure that we must
|
||||
// NOT surface to gomail. gomail v2.3.1 stores any error returned by a copy closure
|
||||
// and then nil-derefs while writing the *next* attachment, which panics and
|
||||
// (because workers don't recover) crashes the whole server (MM-69242). So the
|
||||
// closure always returns nil and we fail the batch here, after WriteTo.
|
||||
var attachmentReadErr error
|
||||
|
||||
for _, fileInfo := range channelExport.uploadedFiles {
|
||||
path := fileInfo.Path
|
||||
|
||||
m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
|
||||
var reader filestore.ReadCloseSeeker
|
||||
reader, err = fileAttachmentBackend.Reader(path)
|
||||
if err != nil {
|
||||
missing, readErr := streamAttachmentForExport(rctx, fileAttachmentBackend, path, writer)
|
||||
switch {
|
||||
case missing:
|
||||
// The attachment no longer exists in the store (confirmed via FileExists).
|
||||
// Warn and skip so a single deleted file can't block the export (MM-62493).
|
||||
rctx.Logger().Warn("File not found for export", mlog.String("filename", path))
|
||||
warningCount += 1
|
||||
return nil
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
_, err = io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to add attachment to the Global Relay export: %w", err)
|
||||
case readErr != nil:
|
||||
// A read/write failure that persisted across retries. Record it and fail the
|
||||
// batch after WriteTo so the job retries instead of shipping an incomplete export.
|
||||
rctx.Logger().Error("Failed to read attachment for Global Relay export after retries",
|
||||
mlog.String("filename", path), mlog.Err(readErr))
|
||||
attachmentReadErr = errors.Join(attachmentReadErr, fmt.Errorf("attachment %q: %w", path, readErr))
|
||||
}
|
||||
// Always return nil: an error here poisons gomail's writer and panics on the
|
||||
// next attachment (MM-69242). We fail the batch after WriteTo instead.
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
_, err = m.WriteTo(w)
|
||||
if err != nil {
|
||||
return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
|
||||
if _, err = m.WriteTo(w); err != nil {
|
||||
return warningCount, fmt.Errorf("unable to write the eml message: %w", err)
|
||||
}
|
||||
if attachmentReadErr != nil {
|
||||
return warningCount, fmt.Errorf("unable to read one or more attachments for the eml message: %w", attachmentReadErr)
|
||||
}
|
||||
return warningCount, nil
|
||||
}
|
||||
|
||||
// errAttachmentStreamFatal wraps a streaming failure that retrying cannot fix — a failure
|
||||
// writing to the output (gomail) stream, or a failed resume Seek. The caller fails the batch
|
||||
// rather than retrying or skipping.
|
||||
var errAttachmentStreamFatal = errors.New("attachment stream cannot be retried")
|
||||
|
||||
// streamAttachmentForExport streams the attachment at path directly into dst (the gomail
|
||||
// writer), retrying transient failures (e.g. an S3 timeout, whether it surfaces when opening
|
||||
// the reader or mid-read). Each retry re-opens the backend reader and Seeks past the bytes
|
||||
// already written, so a retry resumes rather than re-downloads: memory stays constant (an S3
|
||||
// Seek is a ranged GET, not a fresh download) instead of buffering a whole, up to
|
||||
// ~MaxEmailBytes, attachment. Only attempts that make NO forward progress count against the
|
||||
// retry budget, so a large attachment can still complete over a flaky connection as long as
|
||||
// each retry advances; a cancelled job aborts promptly via the context.
|
||||
//
|
||||
// An open or read failure is classified, not assumed missing: it returns missing=true only
|
||||
// when FileExists confirms the object is genuinely gone (the caller skips it, preserving
|
||||
// MM-62493). A failure on a file that still exists — a transient infrastructure hiccup at open
|
||||
// or read time, indistinguishable from a deletion by error alone — is retried and, if it
|
||||
// persists, returned as an error so the batch fails rather than silently dropping an
|
||||
// attachment from a compliance export (MM-69338).
|
||||
//
|
||||
// NOTE: a failed stream may have already written a partial attachment to dst. That is safe
|
||||
// only because the caller fails the whole batch on a non-nil error, so the incomplete output
|
||||
// is discarded and the job retries; the closure must NOT return this error to gomail (MM-69242).
|
||||
func streamAttachmentForExport(rctx request.CTX, backend filestore.FileBackend, path string, dst io.Writer) (missing bool, err error) {
|
||||
var written int64
|
||||
backoff := attachmentReadBackoff
|
||||
for stalled := 0; stalled < maxAttachmentReadAttempts; {
|
||||
var n int64
|
||||
n, err = streamAttachmentOnce(backend, path, dst, written)
|
||||
written += n
|
||||
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
if errors.Is(err, errAttachmentStreamFatal) {
|
||||
// Output-stream write failure or a failed resume Seek: retrying can't help, so
|
||||
// fail the batch.
|
||||
return false, err
|
||||
}
|
||||
|
||||
// An open or read failure — both retryable, but first tell a genuinely-missing file
|
||||
// apart from a transient hiccup. If the object is gone (and we've emitted nothing yet),
|
||||
// skip it so a single deleted file can't block the export forever (preserves MM-62493).
|
||||
// Anything else — it still exists, or the existence check itself failed — is treated as
|
||||
// transient: retried, then failed, so a transient open/read error can't silently drop an
|
||||
// attachment from a compliance export (MM-69338). On S3/MinIO a deleted object isn't even
|
||||
// detected on open (minio-go's GetObject is lazy), so this read-time check is what makes
|
||||
// the skip work there at all.
|
||||
if written == 0 {
|
||||
if exists, existsErr := backend.FileExists(path); existsErr == nil && !exists {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
if n > 0 {
|
||||
// Made forward progress: the next attempt resumes further along. Reset the
|
||||
// stall budget and backoff so a flaky connection can still finish a large file.
|
||||
stalled = 0
|
||||
backoff = attachmentReadBackoff
|
||||
} else {
|
||||
stalled++
|
||||
if stalled >= maxAttachmentReadAttempts {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Transient failure: back off (exponentially) before retrying, but bail out promptly
|
||||
// if the job is being cancelled rather than sleeping through it.
|
||||
rctx.Logger().Warn("Transient error streaming attachment for Global Relay export; backing off before retry",
|
||||
mlog.String("filename", path), mlog.Int("bytesRead", written),
|
||||
mlog.Duration("backoff", backoff), mlog.Err(err))
|
||||
|
||||
select {
|
||||
case <-time.After(backoff):
|
||||
case <-rctx.Context().Done():
|
||||
return false, rctx.Context().Err()
|
||||
}
|
||||
|
||||
backoff *= 2
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
// streamAttachmentOnce makes a single open+copy attempt, resuming past resumeFrom bytes so a
|
||||
// retry continues rather than re-downloads. It returns the bytes copied in this attempt. A nil
|
||||
// error means the attachment streamed fully. An error wrapping errAttachmentStreamFatal is not
|
||||
// retryable (output-stream write failure or a failed resume Seek); any other error is a
|
||||
// retryable open/read failure that the caller classifies as missing-vs-transient.
|
||||
func streamAttachmentOnce(backend filestore.FileBackend, path string, dst io.Writer, resumeFrom int64) (int64, error) {
|
||||
reader, err := backend.Reader(path)
|
||||
if err != nil {
|
||||
// Open failure: retryable. The caller checks FileExists to tell a deleted file
|
||||
// (skip) from a transient hiccup (retry, then fail the batch).
|
||||
return 0, err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
if resumeFrom > 0 {
|
||||
// Resume where the previous attempt left off instead of re-reading from the start.
|
||||
if _, err = reader.Seek(resumeFrom, io.SeekStart); err != nil {
|
||||
return 0, fmt.Errorf("%w: seeking to resume offset %d: %w", errAttachmentStreamFatal, resumeFrom, err)
|
||||
}
|
||||
}
|
||||
|
||||
rd := &readErrorReader{Reader: reader}
|
||||
n, err := io.Copy(dst, rd)
|
||||
if err != nil && rd.readErr == nil {
|
||||
// io.Copy failed writing to the output stream, not reading the attachment.
|
||||
return n, fmt.Errorf("%w: %w", errAttachmentStreamFatal, err)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// readErrorReader wraps a reader and remembers the last non-EOF read error. It lets the
|
||||
// caller of io.Copy tell a failed attachment read (retryable) apart from a failed write to
|
||||
// the output stream (not retryable), which io.Copy collapses into a single error.
|
||||
type readErrorReader struct {
|
||||
io.Reader
|
||||
readErr error
|
||||
}
|
||||
|
||||
func (r *readErrorReader) Read(p []byte) (int, error) {
|
||||
n, err := r.Reader.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
r.readErr = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func getParticipantEmails(channelExport *ChannelExport) []string {
|
||||
participantEmails := make([]string, 0, len(channelExport.Participants))
|
||||
for _, participant := range channelExport.Participants {
|
||||
|
||||
@@ -2056,6 +2056,10 @@
|
||||
"id": "api.emoji.upload.seek.app_error",
|
||||
"translation": "Unable to seek to file start."
|
||||
},
|
||||
{
|
||||
"id": "api.emoji.upload.too_many_frames.app_error",
|
||||
"translation": "Unable to create emoji. Animated GIF must have at most {{.MaxFrames}} frames."
|
||||
},
|
||||
{
|
||||
"id": "api.error_get_first_admin_complete_setup",
|
||||
"translation": "Error trying to retrieve first admin complete setup from the store."
|
||||
@@ -9452,6 +9456,10 @@
|
||||
"id": "model.config.is_valid.export.retention_days_too_low.app_error",
|
||||
"translation": "Invalid value for RetentionDays. Value should be greater than 0"
|
||||
},
|
||||
{
|
||||
"id": "model.config.is_valid.extract_content_timeout.app_error",
|
||||
"translation": "Invalid content extraction timeout for file settings. Must be a whole number of seconds greater than or equal to zero."
|
||||
},
|
||||
{
|
||||
"id": "model.config.is_valid.file_driver.app_error",
|
||||
"translation": "Invalid driver name for file settings. Must be 'local' or 'amazons3'."
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
@@ -15,6 +17,23 @@ type ExtractSettings struct {
|
||||
MaxFileSize int64
|
||||
MMPreviewURL string
|
||||
MMPreviewSecret string
|
||||
// Timeout bounds how long a caller waits for a single extraction. A value
|
||||
// <= 0 disables it. NOTE: this bounds wall-clock wait time (and thus how
|
||||
// long an extraction occupies its caller's worker slot), NOT CPU work.
|
||||
// The docconv converters are not context-aware, so on timeout the
|
||||
// converter keeps running to completion on a detached goroutine and keeps
|
||||
// consuming CPU until it finishes on its own. Under sustained load,
|
||||
// detached extractions can therefore accumulate and run concurrently. The
|
||||
// primary bound on the work of any single extraction is MaxFileSize, which
|
||||
// limits how much input the converter reads.
|
||||
Timeout time.Duration
|
||||
// ReaderCloser, when set, transfers ownership of closing the input reader
|
||||
// to this package. It is closed only after extraction has actually
|
||||
// finished reading. This matters with Timeout set: on timeout the caller
|
||||
// returns while the converter may still be reading on a detached
|
||||
// goroutine, so the caller must NOT close the reader itself or it would
|
||||
// race with (and close the file out from under) that goroutine.
|
||||
ReaderCloser io.Closer
|
||||
}
|
||||
|
||||
// Extract extract the text from a document using the system default extractors
|
||||
@@ -45,7 +64,71 @@ func ExtractWithExtraExtractors(logger mlog.LoggerIFace, filename string, r io.R
|
||||
enabledExtractors.Add(&plainExtractor{})
|
||||
|
||||
if enabledExtractors.Match(filename) {
|
||||
return enabledExtractors.Extract(filename, r, settings.MaxFileSize)
|
||||
return extractWithTimeout(enabledExtractors, filename, r, settings)
|
||||
}
|
||||
|
||||
// No extractor matched, so nothing will read r; close it here since
|
||||
// extractWithTimeout (which otherwise owns the close) is never reached.
|
||||
if settings.ReaderCloser != nil {
|
||||
settings.ReaderCloser.Close()
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// extractWithTimeout runs the extraction and stops waiting for it once
|
||||
// settings.Timeout elapses. Because the underlying docconv converters are not
|
||||
// context-aware, the extraction runs on a detached goroutine: on timeout we
|
||||
// stop waiting and return an error, releasing the caller (and its worker slot)
|
||||
// even though the converter keeps running.
|
||||
//
|
||||
// This decouples extraction from the caller, but it does NOT cap CPU: the
|
||||
// detached converter continues to completion in the background, so a sustained
|
||||
// stream of expensive documents can leave several detached extractions running
|
||||
// at once. The per-extraction work is bounded instead by MaxFileSize (input
|
||||
// size). Load-shedding on the number of in-flight detached extractions is a
|
||||
// possible future improvement; it is intentionally not done here so it does
|
||||
// not also throttle the backfill job that re-extracts skipped content.
|
||||
func extractWithTimeout(e Extractor, filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
|
||||
if settings.Timeout <= 0 {
|
||||
if settings.ReaderCloser != nil {
|
||||
defer settings.ReaderCloser.Close()
|
||||
}
|
||||
return e.Extract(filename, r, settings.MaxFileSize)
|
||||
}
|
||||
|
||||
type extractResult struct {
|
||||
text string
|
||||
err error
|
||||
}
|
||||
resultCh := make(chan extractResult, 1)
|
||||
go func() {
|
||||
// This goroutine owns the reader for the lifetime of the extraction.
|
||||
// After the timeout fires the caller returns, but the converter may
|
||||
// still be reading r here, so the reader is closed only once this
|
||||
// goroutine is done with it - never by the caller.
|
||||
if settings.ReaderCloser != nil {
|
||||
defer settings.ReaderCloser.Close()
|
||||
}
|
||||
// This goroutine is detached, so an unrecovered panic in an extractor
|
||||
// would crash the whole server. Convert it into an error instead.
|
||||
// resultCh is buffered (cap 1), so this send never blocks even if the
|
||||
// caller already timed out and stopped receiving.
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
resultCh <- extractResult{err: fmt.Errorf("panic during document text extraction: %v", rec)}
|
||||
}
|
||||
}()
|
||||
text, err := e.Extract(filename, r, settings.MaxFileSize)
|
||||
resultCh <- extractResult{text: text, err: err}
|
||||
}()
|
||||
|
||||
timer := time.NewTimer(settings.Timeout)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case res := <-resultCh:
|
||||
return res.text, res.err
|
||||
case <-timer.C:
|
||||
return "", fmt.Errorf("document text extraction timed out after %s", settings.Timeout)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -213,6 +215,144 @@ func TestExtractWithExtraExtractors(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
type slowExtractor struct {
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func (se *slowExtractor) Name() string { return "slowExtractor" }
|
||||
|
||||
func (se *slowExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (se *slowExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
time.Sleep(se.delay)
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func TestExtractTimeout(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
data := []byte("hello world")
|
||||
|
||||
t.Run("aborts a slow extraction once the timeout elapses", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 50 * time.Millisecond}, []Extractor{&slowExtractor{delay: 10 * time.Second}})
|
||||
elapsed := time.Since(start)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
assert.Contains(t, err.Error(), "timed out")
|
||||
assert.Less(t, elapsed, 5*time.Second, "should return shortly after the timeout, not wait for the extraction")
|
||||
})
|
||||
|
||||
t.Run("returns the result when extraction finishes within the timeout", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 5 * time.Second}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "done", text)
|
||||
})
|
||||
|
||||
t.Run("a zero timeout disables the bound", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 0}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "done", text)
|
||||
})
|
||||
|
||||
t.Run("a panic in the detached extraction is converted to an error", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&panickingExtractor{}})
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
require.Contains(t, err.Error(), "panic")
|
||||
})
|
||||
}
|
||||
|
||||
type panickingExtractor struct{}
|
||||
|
||||
func (pe *panickingExtractor) Name() string { return "panickingExtractor" }
|
||||
|
||||
func (pe *panickingExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (pe *panickingExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
type recordingCloser struct {
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (c *recordingCloser) Close() error {
|
||||
c.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// blockingExtractor blocks inside Extract until release is closed, simulating a
|
||||
// converter that is still using the reader after an extraction timeout fires.
|
||||
type blockingExtractor struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (be *blockingExtractor) Name() string { return "blockingExtractor" }
|
||||
|
||||
func (be *blockingExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (be *blockingExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
close(be.started)
|
||||
<-be.release
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func TestExtractReaderCloserOwnership(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
|
||||
t.Run("reader is closed only after the detached extraction finishes on timeout", func(t *testing.T) {
|
||||
closer := &recordingCloser{}
|
||||
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{})}
|
||||
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: closer}
|
||||
|
||||
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), settings, []Extractor{be})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "timed out")
|
||||
|
||||
// Wait (with a deadline) for the detached extraction to start so the
|
||||
// test fails fast instead of hanging if it never runs.
|
||||
select {
|
||||
case <-be.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "extraction did not start within the deadline")
|
||||
}
|
||||
// The extraction goroutine is still running, so closing the reader now
|
||||
// would race with it; it must stay open.
|
||||
require.False(t, closer.closed.Load(), "reader must not be closed while the extraction goroutine is still running")
|
||||
|
||||
close(be.release)
|
||||
require.Eventually(t, closer.closed.Load, 2*time.Second, 5*time.Millisecond, "reader should be closed once the extraction goroutine finishes")
|
||||
})
|
||||
|
||||
t.Run("reader is closed on the synchronous path", func(t *testing.T) {
|
||||
closer := &recordingCloser{}
|
||||
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), ExtractSettings{ReaderCloser: closer}, []Extractor{&slowExtractor{delay: 0}})
|
||||
require.NoError(t, err)
|
||||
require.True(t, closer.closed.Load(), "reader should be closed after synchronous extraction")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDocumentMaxFileSize(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
|
||||
data, err := testutils.ReadTestFile("sample-doc.docx")
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("a generous limit extracts the document content", func(t *testing.T) {
|
||||
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 10 * 1024 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a tiny limit prevents the document content from being extracted", func(t *testing.T) {
|
||||
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 16})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, text, "simple")
|
||||
})
|
||||
}
|
||||
|
||||
func TestArchiveMaxFileSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"code.sajari.com/docconv/v2"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
)
|
||||
|
||||
type documentExtractor struct{}
|
||||
@@ -36,7 +38,7 @@ func (de *documentExtractor) Match(filename string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (de *documentExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (out string, outErr error) {
|
||||
func (de *documentExtractor) Extract(filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
out = ""
|
||||
@@ -50,7 +52,14 @@ func (de *documentExtractor) Extract(filename string, r io.ReadSeeker, _ int64)
|
||||
return "", errors.New("unknown converter")
|
||||
}
|
||||
|
||||
text, _, err := converter(r)
|
||||
// Bound how much data the converter is allowed to read so a small upload
|
||||
// cannot expand into an unbounded amount of in-memory work.
|
||||
var reader io.Reader = r
|
||||
if maxFileSize > 0 {
|
||||
reader = utils.NewLimitedReaderWithError(r, maxFileSize)
|
||||
}
|
||||
|
||||
text, _, err := converter(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ledongthuc/pdf"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
)
|
||||
|
||||
type pdfExtractor struct{}
|
||||
@@ -29,7 +31,7 @@ func (pe *pdfExtractor) Match(filename string) bool {
|
||||
return supportedExtensions[extension]
|
||||
}
|
||||
|
||||
func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (out string, outErr error) {
|
||||
func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
out = ""
|
||||
@@ -42,7 +44,14 @@ func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker, _ int64) (out
|
||||
}
|
||||
defer f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
size, err := io.Copy(f, r)
|
||||
|
||||
// Bound how much data is copied to disk so a small upload cannot expand
|
||||
// into an unbounded amount of temporary storage.
|
||||
var src io.Reader = r
|
||||
if maxFileSize > 0 {
|
||||
src = utils.NewLimitedReaderWithError(r, maxFileSize)
|
||||
}
|
||||
size, err := io.Copy(f, src)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error copying data into temporary file: %v", err)
|
||||
}
|
||||
|
||||
@@ -50,3 +50,30 @@ func TestWrongPdfFile(t *testing.T) {
|
||||
_, err = extractor.Extract("sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestPdfMaxFileSize(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
content, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(content), 16, "fixture must be larger than the tight limit under test")
|
||||
|
||||
t.Run("a zero limit means unlimited and extracts the content", func(t *testing.T) {
|
||||
text, err := extractor.Extract("sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a generous limit extracts the content", func(t *testing.T) {
|
||||
text, err := extractor.Extract("sample-doc.pdf", bytes.NewReader(content), 10*1024*1024)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a tight limit prevents extraction", func(t *testing.T) {
|
||||
// The reader errors once it reads past the limit, so io.Copy to the
|
||||
// temp file fails and no text is extracted.
|
||||
text, err := extractor.Extract("sample-doc.pdf", bytes.NewReader(content), 16)
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1767,6 +1767,7 @@ type FileSettings struct {
|
||||
Directory *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
|
||||
EnablePublicLink *bool `access:"site_public_links,cloud_restrictable"`
|
||||
ExtractContent *bool `access:"environment_file_storage,write_restrictable"`
|
||||
ExtractContentTimeout *int `access:"environment_file_storage,write_restrictable"` // In seconds. 0 disables the timeout.
|
||||
ArchiveRecursion *bool `access:"environment_file_storage,write_restrictable"`
|
||||
PublicLinkSalt *string `access:"site_public_links,cloud_restrictable"` // telemetry: none
|
||||
InitialFont *string `access:"environment_file_storage,cloud_restrictable"` // telemetry: none
|
||||
@@ -1844,6 +1845,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
|
||||
s.ExtractContent = NewPointer(true)
|
||||
}
|
||||
|
||||
if s.ExtractContentTimeout == nil {
|
||||
s.ExtractContentTimeout = NewPointer(10)
|
||||
}
|
||||
|
||||
if s.ArchiveRecursion == nil {
|
||||
s.ArchiveRecursion = NewPointer(false)
|
||||
}
|
||||
@@ -4230,6 +4235,10 @@ func (s *FileSettings) isValid() *AppError {
|
||||
return NewAppError("Config.IsValid", "model.config.is_valid.max_file_size.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if *s.ExtractContentTimeout < 0 {
|
||||
return NewAppError("Config.IsValid", "model.config.is_valid.extract_content_timeout.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if !(*s.DriverName == ImageDriverLocal || *s.DriverName == ImageDriverS3) {
|
||||
return NewAppError("Config.IsValid", "model.config.is_valid.file_driver.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
@@ -296,6 +296,38 @@ func TestFileSettingsDirectoryWhitespaceValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSettingsExtractContentTimeout(t *testing.T) {
|
||||
t.Run("default is valid", func(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
cfg.SetDefaults()
|
||||
require.NotNil(t, cfg.FileSettings.ExtractContentTimeout)
|
||||
assert.Equal(t, 10, *cfg.FileSettings.ExtractContentTimeout)
|
||||
assert.Nil(t, cfg.FileSettings.isValid())
|
||||
})
|
||||
|
||||
t.Run("zero disables the timeout and is valid", func(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
cfg.SetDefaults()
|
||||
cfg.FileSettings.ExtractContentTimeout = NewPointer(0)
|
||||
assert.Nil(t, cfg.FileSettings.isValid())
|
||||
})
|
||||
|
||||
t.Run("a positive value is valid", func(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
cfg.SetDefaults()
|
||||
cfg.FileSettings.ExtractContentTimeout = NewPointer(10)
|
||||
assert.Nil(t, cfg.FileSettings.isValid())
|
||||
})
|
||||
|
||||
t.Run("a negative value is rejected", func(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
cfg.SetDefaults()
|
||||
cfg.FileSettings.ExtractContentTimeout = NewPointer(-1)
|
||||
err := cfg.FileSettings.isValid()
|
||||
require.NotNil(t, err)
|
||||
assert.Equal(t, "model.config.is_valid.extract_content_timeout.app_error", err.Id)
|
||||
})
|
||||
}
|
||||
func TestConfigDefaultSignatureAlgorithm(t *testing.T) {
|
||||
c1 := Config{}
|
||||
c1.SetDefaults()
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
// It should be maintained in chronological order with most current
|
||||
// release at the front of the list.
|
||||
var versions = []string{
|
||||
"10.11.21",
|
||||
"10.11.20",
|
||||
"10.11.19",
|
||||
"10.11.18",
|
||||
|
||||
@@ -1081,6 +1081,18 @@ const AdminDefinition: AdminDefinitionType = {
|
||||
it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)),
|
||||
),
|
||||
},
|
||||
{
|
||||
type: 'number',
|
||||
key: 'FileSettings.ExtractContentTimeout',
|
||||
label: defineMessage({id: 'admin.image.extractContentTimeoutTitle', defaultMessage: 'Document content extraction timeout (seconds):'}),
|
||||
help_text: defineMessage({id: 'admin.image.extractContentTimeoutDescription', defaultMessage: 'Maximum number of seconds spent extracting the searchable content of a single uploaded document. Extractions that exceed this limit are aborted to protect server performance. Set to 0 to disable the timeout.'}),
|
||||
placeholder: defineMessage({id: 'admin.image.extractContentTimeoutExample', defaultMessage: '10'}),
|
||||
validate: validators.minValue(0, defineMessage({id: 'admin.image.extractContentTimeout.minValue', defaultMessage: 'Timeout must be 0 or greater.'})),
|
||||
isDisabled: it.any(
|
||||
it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)),
|
||||
it.configIsFalse('FileSettings', 'ExtractContent'),
|
||||
),
|
||||
},
|
||||
{
|
||||
type: 'bool',
|
||||
key: 'FileSettings.ArchiveRecursion',
|
||||
|
||||
@@ -1349,6 +1349,10 @@
|
||||
"admin.image.enableProxyDescription": "When true, enables an image proxy for loading all Markdown images.",
|
||||
"admin.image.exportDirectoryDescription": "Directory to which files are written. If blank, defaults to ./data/.",
|
||||
"admin.image.extractContentDescription": "When enabled, supported document types are searchable by their content. Search results for existing documents may be incomplete <link>until a data migration is executed</link>.",
|
||||
"admin.image.extractContentTimeout.minValue": "Timeout must be 0 or greater.",
|
||||
"admin.image.extractContentTimeoutDescription": "Maximum number of seconds spent extracting the searchable content of a single uploaded document. Extractions that exceed this limit are aborted to protect server performance. Set to 0 to disable the timeout.",
|
||||
"admin.image.extractContentTimeoutExample": "10",
|
||||
"admin.image.extractContentTimeoutTitle": "Document content extraction timeout (seconds):",
|
||||
"admin.image.extractContentTitle": "Enable document search by content:",
|
||||
"admin.image.localDescription": "Directory to which files and images are written. If blank, defaults to ./data/.",
|
||||
"admin.image.localExample": "E.g.: \"./data/\"",
|
||||
|
||||
@@ -548,6 +548,7 @@ export type FileSettings = {
|
||||
Directory: string;
|
||||
EnablePublicLink: boolean;
|
||||
ExtractContent: boolean;
|
||||
ExtractContentTimeout: number;
|
||||
ArchiveRecursion: boolean;
|
||||
PublicLinkSalt: string;
|
||||
InitialFont: string;
|
||||
|
||||
Ссылка в новой задаче
Block a user