From a94d2141dad11c7d9fc4a34203dd51ce3d15d756 Mon Sep 17 00:00:00 2001 From: Tom De Moor Date: Mon, 19 Sep 2022 18:03:39 +0200 Subject: [PATCH 01/12] Translated using Weblate (Dutch) Currently translated at 100.0% (2381 of 2381 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/nl/ --- i18n/nl.json | 62 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/i18n/nl.json b/i18n/nl.json index 96f6766ccf..6f883eaa1a 100644 --- a/i18n/nl.json +++ b/i18n/nl.json @@ -4321,7 +4321,7 @@ }, { "id": "model.post.is_valid.channel_id.app_error", - "translation": "Ongeldige kanaal id" + "translation": "Ongeldige kanaal id." }, { "id": "model.post.is_valid.create_at.app_error", @@ -4569,7 +4569,7 @@ }, { "id": "oauth.gitlab.tos.error", - "translation": "GitLab's gebruiksvoorwaarden zijn bijgewerkt. Ga naar gitlab.com om deze te aanvaarden probeer opnieuw in te loggen op Mattermost." + "translation": "GitLab's gebruiksvoorwaarden zijn bijgewerkt. Ga naar {{.URL}} om deze te aanvaarden probeer opnieuw in te loggen op Mattermost." }, { "id": "plugin.api.update_user_status.bad_status", @@ -7653,15 +7653,15 @@ }, { "id": "api.templates.payment_failed.title", - "translation": "Mislukte betaling" + "translation": "De betaling is niet gelukt" }, { "id": "api.templates.payment_failed.subject", - "translation": "Actie vereist: Betaling mislukt voor Mattermost Cloud" + "translation": "Actie vereist: Betaling mislukt voor Mattermost {{.Plan}}" }, { "id": "api.templates.payment_failed.info3", - "translation": "Om een ononderbroken abonnement op Mattermost Cloud te garanderen, kan je contact opnemen met jouw financiële instelling om het onderliggende probleem op te lossen of jouw betalingsinformatie bijwerken. Zodra de betalingsinformatie is bijgewerkt, zal Mattermost proberen een eventueel openstaand saldo te vereffenen." + "translation": "Om een ononderbroken toegang op Mattermost {{.Plan}} te garanderen, kan je contact opnemen met jouw financiële instelling om het onderliggende probleem op te lossen of jouw betalingsinformatie bijwerken. Zodra de betalingsinformatie is bijgewerkt, zal Mattermost proberen een eventueel openstaand saldo te vereffenen." }, { "id": "api.templates.payment_failed.info2", @@ -9485,5 +9485,57 @@ { "id": "api.templates.delinquency_90.subtitle1", "translation": "Als je Cloud Professional- of Enterprise-functies gebruikt voor belangrijke zakelijke activiteiten, zullen deze niet langer beschikbaar zijn en zal je verminderde prestaties ervaren." + }, + { + "id": "ent.saml.configure.certificate_parse_error.app_error", + "translation": "SAML kon het Identity Provider Public Certificate niet succesvol laden, neem contact op met jouw systeembeheerder." + }, + { + "id": "app.user.get_badge_count.app_error", + "translation": "We konden het aantal badges van de gebruiker niet ophalen." + }, + { + "id": "app.notify_admin.send_notification_post.app_error", + "translation": "Kon geen kennisgevingsbericht verzenden." + }, + { + "id": "app.notify_admin.save.app_error", + "translation": "Kon geen notificatie-data opslaan." + }, + { + "id": "app.cloud.upgrade_plan_bot_message_single", + "translation": "{{.UsersNum}} lid van de {{.WorkspaceName}} werkruimte heeft een werkruimte upgrade aangevraagd voor: " + }, + { + "id": "app.cloud.upgrade_plan_bot_message", + "translation": "{{.UsersNum}} leden van de {{.WorkspaceName}} werkruimte hebben een werkruimte upgrade aangevraagd voor: " + }, + { + "id": "app.cloud.trial_plan_bot_message_single", + "translation": "{{.UsersNum}} lid van de {{.WorkspaceName}} werkruimte heeft gevraagd om de Enterprise-proefperiode te starten om toegang te krijgen tot: " + }, + { + "id": "app.cloud.trial_plan_bot_message", + "translation": "{{.UsersNum}} leden van de {{.WorkspaceName}} werkruimte hebben gevraagd om de Enterprise-proefperiode te starten om toegang te krijgen tot: " + }, + { + "id": "app.cloud.get_subscription_delinquency_date.app_error", + "translation": "Abonnement is niet vervallen" + }, + { + "id": "app.cloud.get_subscription.app_error", + "translation": "Kon het cloud abonnement niet ophalen" + }, + { + "id": "app.cloud.get_current_plan_name.app_error", + "translation": "Kan huidige plannaam niet ophalen" + }, + { + "id": "app.cloud.get_cloud_products.app_error", + "translation": "Kon cloud producten niet ophalen" + }, + { + "id": "api.templates.delinquency_90.title", + "translation": "Jouw Mattermost-werkruimte werd gedowngraded" } ] From 1d522502648361a60e90e54122bfe4930542be06 Mon Sep 17 00:00:00 2001 From: Kaya Zeren Date: Mon, 19 Sep 2022 18:03:40 +0200 Subject: [PATCH 02/12] Translated using Weblate (Turkish) Currently translated at 100.0% (2381 of 2381 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/tr/ --- i18n/tr.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/i18n/tr.json b/i18n/tr.json index 05ef8c4e65..563015338b 100644 --- a/i18n/tr.json +++ b/i18n/tr.json @@ -9529,5 +9529,13 @@ { "id": "app.cloud.get_current_plan_name.app_error", "translation": "Geçerli tarifenin adı alınamadı" + }, + { + "id": "ent.saml.configure.certificate_parse_error.app_error", + "translation": "SAML, herkese açık kimlik hizmeti sağlayıcı sertifikasını yükleyemedi. Lütfen sistem yöneticinizle görüşün." + }, + { + "id": "app.user.get_badge_count.app_error", + "translation": "Kullanıcının nişan sayısını alamadık." } ] From 3df269f84e3f049942848e12a2dad9d34a84991d Mon Sep 17 00:00:00 2001 From: kaakaa Date: Mon, 19 Sep 2022 18:03:40 +0200 Subject: [PATCH 03/12] Translated using Weblate (Japanese) Currently translated at 100.0% (2381 of 2381 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/ja/ --- i18n/ja.json | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/i18n/ja.json b/i18n/ja.json index 743dfa5997..b4d3379d92 100644 --- a/i18n/ja.json +++ b/i18n/ja.json @@ -4565,7 +4565,7 @@ }, { "id": "oauth.gitlab.tos.error", - "translation": "GitLabの利用規約が更新されました。gitlab.comで新しい利用規約に同意してから、もう一度Mattermostへログインしてみてください。" + "translation": "GitLabの利用規約が更新されました。{{.URL}}で新しい利用規約に同意してから、もう一度Mattermostへログインしてみてください。" }, { "id": "plugin.api.update_user_status.bad_status", @@ -7705,15 +7705,15 @@ }, { "id": "api.templates.payment_failed.title", - "translation": "支払いができませんでした" + "translation": "支払いが正常に処理されませんでした" }, { "id": "api.templates.payment_failed.subject", - "translation": "注意: Mattermost Cloudへの支払いができませんでした" + "translation": "注意: Mattermost {{.Plan}}の支払いができませんでした" }, { "id": "api.templates.payment_failed.info3", - "translation": "Mattermost Cloudへの支払いを続けるために、支払いの問題についてあなたの金融機関に問い合わせるか、支払い情報を更新してください。支払い情報が更新されると、Mattermostは未払いの支払いを解決しようとします。" + "translation": "Mattermost {{.Plan}}へアクセスし続けるために、支払いの問題についてあなたの金融機関に問い合わせるか、支払い情報を更新してください。支払い情報が更新されると、Mattermostは未払いの支払いを解決しようとします。" }, { "id": "api.templates.payment_failed.info2", @@ -9494,5 +9494,41 @@ { "id": "api.cloud.delinquency_email.missing_email_to_trigger", "translation": "滞納メールを送信するための必須項目が不足しています。" + }, + { + "id": "ent.saml.configure.certificate_parse_error.app_error", + "translation": "SAMLがIDプロバイダ公開証明書を正常に読み込めなかったため、システム管理者に連絡してください。" + }, + { + "id": "app.user.get_badge_count.app_error", + "translation": "ユーザーのバッジ数を取得できませんでした。" + }, + { + "id": "app.notify_admin.send_notification_post.app_error", + "translation": "通知ポストを送信できませんでした。" + }, + { + "id": "app.notify_admin.save.app_error", + "translation": "通知データを保存できませんでした。" + }, + { + "id": "app.cloud.upgrade_plan_bot_message_single", + "translation": "{{.WorkspaceName}}ワークスペースの{{.UsersNum}}メンバーから、ワークスペースをアップグレードするようリクエストがありました: " + }, + { + "id": "app.cloud.upgrade_plan_bot_message", + "translation": "{{.WorkspaceName}}ワークスペースの{{.UsersNum}}メンバーから、ワークスペースをアップグレードするようリクエストがありました: " + }, + { + "id": "app.cloud.trial_plan_bot_message_single", + "translation": "{{.WorkspaceName}}ワークスペースの{{.UsersNum}}メンバーから、以下にアクセスするためにEnterpriseトライアルを開始するようリクエストがありました: " + }, + { + "id": "app.cloud.trial_plan_bot_message", + "translation": "{{.WorkspaceName}}ワークスペースの{{.UsersNum}}メンバーから、以下にアクセスするためにEnterpriseトライアルを開始するようリクエストがありました: " + }, + { + "id": "app.cloud.get_current_plan_name.app_error", + "translation": "現在のプラン名を取得できませんでした" } ] From ad0705bbca5f4ecd323b37afb09107a92756c9b3 Mon Sep 17 00:00:00 2001 From: Tim Scheuermann Date: Wed, 21 Sep 2022 12:15:33 +0200 Subject: [PATCH 04/12] MM-46990 Report correct import error (#21025) --- i18n/en.json | 4 ++++ jobs/base_workers.go | 2 +- jobs/jobs.go | 3 +++ jobs/jobs_test.go | 22 ++++++++++++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/i18n/en.json b/i18n/en.json index aeeca11e0a..eb49526f56 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -5607,6 +5607,10 @@ "id": "app.job.download_export_results_not_enabled", "translation": "DownloadExportResults in config.json is false. Please set this to true to download the results of this job." }, + { + "id": "app.job.error", + "translation": "Error during job execution." + }, { "id": "app.job.get.app_error", "translation": "Unable to get the job." diff --git a/jobs/base_workers.go b/jobs/base_workers.go index 60fa0465ef..58bdc98ba7 100644 --- a/jobs/base_workers.go +++ b/jobs/base_workers.go @@ -81,7 +81,7 @@ func (worker *SimpleWorker) DoJob(job *model.Job) { err := worker.execute(job) if err != nil { mlog.Error("SimpleWorker: job execution error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err)) - worker.setJobError(job, model.NewAppError("DoJob", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)) + worker.setJobError(job, model.NewAppError("DoJob", "app.job.error", nil, "", http.StatusInternalServerError).Wrap(err)) return } diff --git a/jobs/jobs.go b/jobs/jobs.go index fdc9dbd4c1..e23ca766c6 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -122,6 +122,9 @@ func (srv *JobServer) SetJobError(job *model.Job, jobError *model.AppError) *mod if jobError.DetailedError != "" { job.Data["error"] += " — " + jobError.DetailedError } + if wrapped := jobError.Unwrap(); wrapped != nil { + job.Data["error"] += " — " + wrapped.Error() + } updated, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress) if err != nil { return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err) diff --git a/jobs/jobs_test.go b/jobs/jobs_test.go index bb28db69a8..04c315d85e 100644 --- a/jobs/jobs_test.go +++ b/jobs/jobs_test.go @@ -4,6 +4,8 @@ package jobs import ( + "errors" + "net/http" "testing" "github.com/stretchr/testify/require" @@ -390,6 +392,26 @@ func TestSetJobError(t *testing.T) { err := jobServer.SetJobError(job, jobError) require.Nil(t, err) }) + + t.Run("error message set correctly", func(t *testing.T) { + jobServer, mockStore, _ := makeJobServer(t) + + jobError := model.NewAppError("anywhere", "not.a.valid.id", nil, "details", http.StatusTeapot).Wrap(errors.New("wrapped")) + + job := &model.Job{ + Id: "job_id", + Type: "job_type", + Progress: -1, + Data: map[string]string{}, + } + + mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, nil) + mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusCancelRequested).Return(true, nil) + + err := jobServer.SetJobError(job, jobError) + require.Nil(t, err) + require.Equal(t, "not.a.valid.id — details — wrapped", job.Data["error"]) + }) }) } From 943fa33f64f337655999dc1f4f8abca4c346198f Mon Sep 17 00:00:00 2001 From: mattermod Date: Fri, 23 Sep 2022 12:23:38 +0000 Subject: [PATCH 05/12] Update latest version to 7.3.0 --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 90c15906c4..ab99fe0dcd 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -8,7 +8,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV PATH="/mattermost/bin:${PATH}" ARG PUID=2000 ARG PGID=2000 -ARG MM_PACKAGE="https://releases.mattermost.com/7.2.0/mattermost-7.2.0-linux-amd64.tar.gz?src=docker" +ARG MM_PACKAGE="https://releases.mattermost.com/7.3.0/mattermost-7.3.0-linux-amd64.tar.gz?src=docker" # # Install needed packages and indirect dependencies RUN apt-get update \ From 2ea14ef395fad8919b2f4137642a7f50b370ffba Mon Sep 17 00:00:00 2001 From: Shivashis Padhi Date: Fri, 23 Sep 2022 10:12:24 -0400 Subject: [PATCH 06/12] [MM-47002] Fix new private channels not showing up in least active channels insights (#21031) Automatic Merge --- store/sqlstore/channel_store.go | 4 ++-- store/storetest/channel_store.go | 35 +++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index 19686d967d..ee33d608ac 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -4367,7 +4367,7 @@ func (s SqlChannelStore) GetTopInactiveChannelsForTeamSince(teamID string, userI FROM Channels LEFT JOIN Posts on Posts.ChannelId = Channels.Id AND Posts.Type = '' AND Posts.CreateAt > ? AND Posts.DeleteAt = 0 - LEFT JOIN ChannelMembers on Posts.ChannelId = ChannelMembers.ChannelId + LEFT JOIN ChannelMembers on Channels.Id = ChannelMembers.ChannelId WHERE Channels.TeamId = ? AND Channels.CreateAt < ? @@ -4416,7 +4416,7 @@ func (s SqlChannelStore) GetTopInactiveChannelsForUserSince(teamID string, userI FROM Channels LEFT JOIN Posts on Posts.ChannelId = Channels.Id AND Posts.Type = '' AND Posts.CreateAt > ? AND Posts.DeleteAt = 0 - LEFT JOIN ChannelMembers on Posts.ChannelId = ChannelMembers.ChannelId + LEFT JOIN ChannelMembers on Channels.Id = ChannelMembers.ChannelId WHERE Channels.DeleteAt = 0 AND Channels.CreateAt < ? diff --git a/store/storetest/channel_store.go b/store/storetest/channel_store.go index 7b2757ad14..3f4dd4cedd 100644 --- a/store/storetest/channel_store.go +++ b/store/storetest/channel_store.go @@ -8007,6 +8007,16 @@ func testGetTopInactiveChannels(t *testing.T, ss store.Store) { channelPrivate, nErr := ss.Channel().Save(&c3, -1) require.NoError(t, nErr) + // create private channel with post + c3NoPost := model.Channel{} + c3NoPost.TeamId = team.Id + c3NoPost.DisplayName = "Channel3" + model.NewId() + c3NoPost.Name = NewTestId() + c3NoPost.Type = model.ChannelTypePrivate + c3NoPost.CreateAt = 1 + channelPrivateNoPost, nErr := ss.Channel().Save(&c3NoPost, -1) + require.NoError(t, nErr) + // create dm channel u1 := model.User{} u1.Email = MakeEmail() @@ -8029,6 +8039,9 @@ func testGetTopInactiveChannels(t *testing.T, ss store.Store) { cm1 := &model.ChannelMember{ChannelId: channelPrivate.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()} _, err = ss.Channel().SaveMember(cm1) require.NoError(t, err) + cm1NoPost := &model.ChannelMember{ChannelId: channelPrivateNoPost.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()} + _, err = ss.Channel().SaveMember(cm1NoPost) + require.NoError(t, err) cm1Public := &model.ChannelMember{ChannelId: channelPublic1.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()} _, err = ss.Channel().SaveMember(cm1Public) require.NoError(t, err) @@ -8085,25 +8098,27 @@ func testGetTopInactiveChannels(t *testing.T, ss store.Store) { t.Run("top inactive channels for team - u1 ", func(t *testing.T) { topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForTeamSince(team.Id, u1.Id, 2, 0, 10) require.NoError(t, err) - require.Len(t, topInactiveChannels.Items, 3) - require.Equal(t, topInactiveChannels.Items[0].ID, channelSaved0.Id) - require.Equal(t, topInactiveChannels.Items[0].LastActivityAt, postToCheckLastUpdateAt.CreateAt) - require.Equal(t, topInactiveChannels.Items[1].ID, channelPrivate.Id) - require.Equal(t, topInactiveChannels.Items[2].ID, channelPublic1.Id) + require.Len(t, topInactiveChannels.Items, 4) + require.Equal(t, topInactiveChannels.Items[0].ID, channelPrivateNoPost.Id) + require.Equal(t, topInactiveChannels.Items[1].ID, channelSaved0.Id) + require.Equal(t, topInactiveChannels.Items[1].LastActivityAt, postToCheckLastUpdateAt.CreateAt) + require.Equal(t, topInactiveChannels.Items[2].ID, channelPrivate.Id) + require.Equal(t, topInactiveChannels.Items[3].ID, channelPublic1.Id) // test bot posts are counted - require.Equal(t, topInactiveChannels.Items[2].MessageCount, int64(4)) + require.Equal(t, topInactiveChannels.Items[3].MessageCount, int64(4)) // participants - require.Equal(t, topInactiveChannels.Items[1].Participants[0], u1.Id) require.Equal(t, topInactiveChannels.Items[2].Participants[0], u1.Id) + require.Equal(t, topInactiveChannels.Items[3].Participants[0], u1.Id) }) t.Run("top inactive channels for user - u1 ", func(t *testing.T) { topInactiveChannels, err := ss.Channel().GetTopInactiveChannelsForUserSince(team.Id, u1.Id, 2, 0, 10) require.NoError(t, err) - require.Len(t, topInactiveChannels.Items, 2) - require.Equal(t, topInactiveChannels.Items[0].ID, channelPrivate.Id) - require.Equal(t, topInactiveChannels.Items[1].ID, channelPublic1.Id) + require.Len(t, topInactiveChannels.Items, 3) + require.Equal(t, topInactiveChannels.Items[0].ID, channelPrivateNoPost.Id) + require.Equal(t, topInactiveChannels.Items[1].ID, channelPrivate.Id) + require.Equal(t, topInactiveChannels.Items[2].ID, channelPublic1.Id) }) // for u2 From ec4df746ea6483e8e8b9599ea2c885ecffe8b2e6 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Mon, 26 Sep 2022 18:03:46 +0200 Subject: [PATCH 07/12] Translated using Weblate (English (Australia)) Currently translated at 100.0% (2381 of 2381 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/en_AU/ --- i18n/en_AU.json | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/i18n/en_AU.json b/i18n/en_AU.json index eafc534942..e1dc7cb1c0 100644 --- a/i18n/en_AU.json +++ b/i18n/en_AU.json @@ -253,7 +253,7 @@ }, { "id": "oauth.gitlab.tos.error", - "translation": "GitLab's Terms of Service have updated. Please go to gitlab.com to accept them and then try logging into Mattermost again." + "translation": "GitLab's Terms of Service have updated. Please go to {{.URL}} to accept them and then try logging into Mattermost again." }, { "id": "model.websocket_client.connect_fail.app_error", @@ -4989,15 +4989,15 @@ }, { "id": "api.templates.payment_failed.title", - "translation": "Failed Payment" + "translation": "The payment wasn't successful" }, { "id": "api.templates.payment_failed.subject", - "translation": "Action required: Payment failed for Mattermost Cloud" + "translation": "Action required: Payment failed for Mattermost {{.Plan}}" }, { "id": "api.templates.payment_failed.info3", - "translation": "To ensure uninterrupted subscription to Mattermost Cloud, please either contact your financial institution to fix the underlying problem or update your payment information. Once payment information is updated, Mattermost will attempt to settle any outstanding balance." + "translation": "To ensure uninterrupted access to Mattermost {{.Plan}}, please either contact your financial institution to fix the underlying problem or update your payment information. Once payment information is updated, Mattermost will attempt to settle any outstanding balance." }, { "id": "api.templates.payment_failed.info2", @@ -9525,5 +9525,17 @@ { "id": "app.cloud.trial_plan_bot_message", "translation": "{{.UsersNum}} members of the {{.WorkspaceName}} workspace have requested starting the Enterprise trial for access to: " + }, + { + "id": "ent.saml.configure.certificate_parse_error.app_error", + "translation": "SAML could not load Identity Provider Public Certificate successfully, please contact your system administrator." + }, + { + "id": "app.user.get_badge_count.app_error", + "translation": "The user's badge couldn't be retrieved." + }, + { + "id": "app.cloud.get_current_plan_name.app_error", + "translation": "Unable to get current plan name" } ] From 1690286e6b65d679f8910dfd46c2a5a82dd8fbb3 Mon Sep 17 00:00:00 2001 From: kaakaa Date: Mon, 26 Sep 2022 18:03:46 +0200 Subject: [PATCH 08/12] Translated using Weblate (Japanese) Currently translated at 100.0% (2382 of 2382 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/ja/ --- i18n/ja.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/i18n/ja.json b/i18n/ja.json index b4d3379d92..5918ca411b 100644 --- a/i18n/ja.json +++ b/i18n/ja.json @@ -9530,5 +9530,9 @@ { "id": "app.cloud.get_current_plan_name.app_error", "translation": "現在のプラン名を取得できませんでした" + }, + { + "id": "app.job.error", + "translation": "ジョブ実行中にエラーが発生しました。" } ] From 963643fd6c14e6f4517f70a2b936be44ad0ce416 Mon Sep 17 00:00:00 2001 From: master7 Date: Mon, 26 Sep 2022 18:03:47 +0200 Subject: [PATCH 09/12] Translated using Weblate (Polish) Currently translated at 100.0% (2382 of 2382 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/pl/ --- i18n/pl.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/i18n/pl.json b/i18n/pl.json index a5593cfa3c..de2e7d8a68 100644 --- a/i18n/pl.json +++ b/i18n/pl.json @@ -9538,5 +9538,9 @@ { "id": "app.user.get_badge_count.app_error", "translation": "Nie mogliśmy uzyskać liczby odznak dla użytkownika." + }, + { + "id": "app.job.error", + "translation": "Błąd podczas wykonywania zadania." } ] From fcfdadf106b4823e47f4ab593a32180eade26481 Mon Sep 17 00:00:00 2001 From: MArtin Johnson Date: Mon, 26 Sep 2022 18:03:47 +0200 Subject: [PATCH 10/12] Translated using Weblate (Swedish) Currently translated at 100.0% (2382 of 2382 strings) Translation: mattermost-languages-shipped/mattermost-server Translate-URL: https://translate.mattermost.com/projects/mattermost/mattermost-server_master/sv/ --- i18n/sv.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/i18n/sv.json b/i18n/sv.json index e3ede34fd8..2374ce62ad 100644 --- a/i18n/sv.json +++ b/i18n/sv.json @@ -9537,5 +9537,9 @@ { "id": "api.cloud.delinquency_email.missing_email_to_trigger", "translation": "Information i obligatoriska fält saknas för att kunna skicka e-postmeddelanden om utebliven betalning." + }, + { + "id": "app.job.error", + "translation": "Fel under utförandet av jobbet." } ] From b9834a2fc2a39076684ec7944e0425205676c3fa Mon Sep 17 00:00:00 2001 From: Orlando Romo <34588445+orlandorode97@users.noreply.github.com> Date: Tue, 27 Sep 2022 13:00:42 -0500 Subject: [PATCH 11/12] [MM-42191]: Include deleted posts (#19985) * MM-42191: Include deleted posts: Add includeDeleted query parameter for getPostsForChannel * MM-42191: Fix error typo for includeDeleted query parameter * MM-42191: Include deleted posts: Set permission error when deleted posts are requested by non system admins * MM-42191: Include deleted posts: Refactor replyCountSubQuery and conditions when includeDeleted is not presented, refactor getRootPosts * MM-42191: Include deleted posts: Refactor getRootPosts function along with skipFetchThreads and includeDeleted * MM-42191: Include deleted posts: Rename includeDeleted to include_deleted param * MM-42191: Include deleted posts: Fix failed posts unit tests * MM-42191: Include deleted posts: Add missing include deleted option in multiple queries * MM-42191: Include deleted posts: Add tests for include deleted option in TestGetPostsForChannel, TestGetPostsBefore, TestGetPostsAfter * MM-42191: include deleted posts: Add tests cases for post store test * MM-42191: Include deleted posts: Add extra unit test to ensure not returning deleted posts when IncludeDelete is false Co-authored-by: Mattermod --- api4/commands_test.go | 6 +- api4/post.go | 12 ++- api4/post_test.go | 158 ++++++++++++++++++++++++---------- api4/team_test.go | 2 +- model/client4.go | 16 +++- model/permission.go | 8 ++ model/post.go | 1 + store/sqlstore/post_store.go | 76 +++++++++++----- store/storetest/post_store.go | 67 ++++++++++++++ 9 files changed, 269 insertions(+), 77 deletions(-) diff --git a/api4/commands_test.go b/api4/commands_test.go index 24a2c2714d..59f6b58e0f 100644 --- a/api4/commands_test.go +++ b/api4/commands_test.go @@ -33,7 +33,7 @@ func TestEchoCommand(t *testing.T) { time.Sleep(100 * time.Millisecond) - p1, _, err := client.GetPostsForChannel(channel1.Id, 0, 2, "", false) + p1, _, err := client.GetPostsForChannel(channel1.Id, 0, 2, "", false, false) require.NoError(t, err) require.Len(t, p1.Order, 2, "Echo command failed to send") } @@ -352,7 +352,7 @@ func TestMeCommand(t *testing.T) { time.Sleep(100 * time.Millisecond) - p1, _, err := client.GetPostsForChannel(channel.Id, 0, 2, "", false) + p1, _, err := client.GetPostsForChannel(channel.Id, 0, 2, "", false, false) require.NoError(t, err) require.Len(t, p1.Order, 2, "Command failed to send") @@ -454,7 +454,7 @@ func TestShrugCommand(t *testing.T) { time.Sleep(100 * time.Millisecond) - p1, _, err := client.GetPostsForChannel(channel.Id, 0, 2, "", false) + p1, _, err := client.GetPostsForChannel(channel.Id, 0, 2, "", false, false) require.NoError(t, err) require.Len(t, p1.Order, 2, "Command failed to send") require.Equal(t, `¯\\\_(ツ)\_/¯`, p1.Posts[p1.Order[0]].Message, "invalid shrug response") diff --git a/api4/post.go b/api4/post.go index 4725b4cfaa..479d249967 100644 --- a/api4/post.go +++ b/api4/post.go @@ -183,10 +183,16 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { skipFetchThreads := r.URL.Query().Get("skipFetchThreads") == "true" collapsedThreads := r.URL.Query().Get("collapsedThreads") == "true" collapsedThreadsExtended := r.URL.Query().Get("collapsedThreadsExtended") == "true" + includeDeleted := r.URL.Query().Get("include_deleted") == "true" channelId := c.Params.ChannelId page := c.Params.Page perPage := c.Params.PerPage + if !c.IsSystemAdmin() && includeDeleted { + c.SetPermissionError(model.PermissionReadDeletedPosts) + return + } + if !c.App.SessionHasPermissionToChannel(c.AppContext, *c.AppContext.Session(), channelId, model.PermissionReadChannel) { c.SetPermissionError(model.PermissionReadChannel) return @@ -217,7 +223,7 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { return } - list, err = c.App.GetPostsAfterPost(model.GetPostsOptions{ChannelId: channelId, PostId: afterPost, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, UserId: c.AppContext.Session().UserId}) + list, err = c.App.GetPostsAfterPost(model.GetPostsOptions{ChannelId: channelId, PostId: afterPost, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, UserId: c.AppContext.Session().UserId, IncludeDeleted: includeDeleted}) } else if beforePost != "" { etag = c.App.GetPostsEtag(channelId, collapsedThreads) @@ -225,7 +231,7 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { return } - list, err = c.App.GetPostsBeforePost(model.GetPostsOptions{ChannelId: channelId, PostId: beforePost, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, CollapsedThreadsExtended: collapsedThreadsExtended, UserId: c.AppContext.Session().UserId}) + list, err = c.App.GetPostsBeforePost(model.GetPostsOptions{ChannelId: channelId, PostId: beforePost, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, CollapsedThreadsExtended: collapsedThreadsExtended, UserId: c.AppContext.Session().UserId, IncludeDeleted: includeDeleted}) } else { etag = c.App.GetPostsEtag(channelId, collapsedThreads) @@ -233,7 +239,7 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { return } - list, err = c.App.GetPostsPage(model.GetPostsOptions{ChannelId: channelId, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, CollapsedThreadsExtended: collapsedThreadsExtended, UserId: c.AppContext.Session().UserId}) + list, err = c.App.GetPostsPage(model.GetPostsOptions{ChannelId: channelId, Page: page, PerPage: perPage, SkipFetchThreads: skipFetchThreads, CollapsedThreads: collapsedThreads, CollapsedThreadsExtended: collapsedThreadsExtended, UserId: c.AppContext.Session().UserId, IncludeDeleted: includeDeleted}) } if err != nil { diff --git a/api4/post_test.go b/api4/post_test.go index 6447fb890b..31f1185ea9 100644 --- a/api4/post_test.go +++ b/api4/post_test.go @@ -1068,17 +1068,17 @@ func TestGetPostsForChannel(t *testing.T) { post4 := th.CreatePost() th.TestForAllClients(t, func(t *testing.T, c *model.Client4) { - posts, resp, err := c.GetPostsForChannel(th.BasicChannel.Id, 0, 60, "", false) + posts, resp, err := c.GetPostsForChannel(th.BasicChannel.Id, 0, 60, "", false, false) require.NoError(t, err) require.Equal(t, post4.Id, posts.Order[0], "wrong order") require.Equal(t, post3.Id, posts.Order[1], "wrong order") require.Equal(t, post2.Id, posts.Order[2], "wrong order") require.Equal(t, post1.Id, posts.Order[3], "wrong order") - posts, resp, _ = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, resp.Etag, false) + posts, resp, _ = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, resp.Etag, false, false) CheckEtag(t, posts, resp) - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "wrong number returned") @@ -1087,11 +1087,11 @@ func TestGetPostsForChannel(t *testing.T) { _, ok = posts.Posts[post1.Id] require.True(t, ok, "missing root post") - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 1, 1, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 1, 1, "", false, false) require.NoError(t, err) require.Equal(t, post3.Id, posts.Order[0], "wrong order") - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 10000, 10000, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 10000, 10000, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should be no posts") }) @@ -1121,21 +1121,21 @@ func TestGetPostsForChannel(t *testing.T) { require.True(t, f, "missing post") } - _, resp, err := c.GetPostsForChannel("", 0, 60, "", false) + _, resp, err := c.GetPostsForChannel("", 0, 60, "", false, false) require.Error(t, err) CheckBadRequestStatus(t, resp) - _, resp, err = c.GetPostsForChannel("junk", 0, 60, "", false) + _, resp, err = c.GetPostsForChannel("junk", 0, 60, "", false, false) require.Error(t, err) CheckBadRequestStatus(t, resp) }) - _, resp, err := client.GetPostsForChannel(model.NewId(), 0, 60, "", false) + _, resp, err := client.GetPostsForChannel(model.NewId(), 0, 60, "", false, false) require.Error(t, err) CheckForbiddenStatus(t, resp) client.Logout() - _, resp, err = client.GetPostsForChannel(model.NewId(), 0, 60, "", false) + _, resp, err = client.GetPostsForChannel(model.NewId(), 0, 60, "", false, false) require.Error(t, err) CheckUnauthorizedStatus(t, resp) @@ -1151,12 +1151,12 @@ func TestGetPostsForChannel(t *testing.T) { var posts *model.PostList th.TestForAllClients(t, func(t *testing.T, c *model.Client4) { // get the system post IDs posted before the created posts above - posts, _, err = c.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false) + posts, _, err = c.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false, false) require.NoError(t, err) systemPostId1 := posts.Order[1] // similar to '/posts' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 60, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 60, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 12, "expected 12 posts") require.Equal(t, post10.Id, posts.Order[0], "posts not in order") @@ -1165,7 +1165,7 @@ func TestGetPostsForChannel(t *testing.T) { require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") // similar to '/posts?per_page=3' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post10.Id, posts.Order[0], "posts not in order") @@ -1174,7 +1174,7 @@ func TestGetPostsForChannel(t *testing.T) { require.Equal(t, post7.Id, posts.PrevPostId, "should return post7.Id as PrevPostId") // similar to '/posts?per_page=3&page=1' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 1, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 1, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post7.Id, posts.Order[0], "posts not in order") @@ -1183,7 +1183,7 @@ func TestGetPostsForChannel(t *testing.T) { require.Equal(t, post4.Id, posts.PrevPostId, "should return post4.Id as PrevPostId") // similar to '/posts?per_page=3&page=2' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 2, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 2, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post4.Id, posts.Order[0], "posts not in order") @@ -1192,7 +1192,7 @@ func TestGetPostsForChannel(t *testing.T) { require.Equal(t, post1.Id, posts.PrevPostId, "should return post1.Id as PrevPostId") // similar to '/posts?per_page=3&page=3' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 3, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 3, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post1.Id, posts.Order[0], "posts not in order") @@ -1201,7 +1201,7 @@ func TestGetPostsForChannel(t *testing.T) { require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") // similar to '/posts?per_page=3&page=4' - posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 4, 3, "", false) + posts, _, err = c.GetPostsForChannel(th.BasicChannel.Id, 4, 3, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, "", posts.NextPostId, "should return an empty NextPostId") @@ -1220,15 +1220,37 @@ func TestGetPostsForChannel(t *testing.T) { }) // the endpoint should work fine when viewing archived channels is enabled - _, _, err = c.GetPostsForChannel(channel.Id, 0, 10, "", false) + _, _, err = c.GetPostsForChannel(channel.Id, 0, 10, "", false, false) require.NoError(t, err) // the endpoint should return forbidden if viewing archived channels is disabled th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.ExperimentalViewArchivedChannels = false }) - _, resp, err = c.GetPostsForChannel(channel.Id, 0, 10, "", false) + _, resp, err = c.GetPostsForChannel(channel.Id, 0, 10, "", false, false) require.Error(t, err) CheckForbiddenStatus(t, resp) }, "Should forbid to retrieve posts if the channel is archived and users are not allowed to view archived messages") + + client.DeletePost(post10.Id) + client.DeletePost(post8.Id) + + // include deleted posts for non-admin users. + _, resp, err = client.GetPostsForChannel(th.BasicChannel.Id, 0, 100, "", false, true) + require.Error(t, err) + CheckForbiddenStatus(t, resp) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + // include deleted posts for admin users. + posts, resp, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 100, "", false, true) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 12, "expected 12 posts") + + // not include deleted posts for admin users. + posts, resp, err = c.GetPostsForChannel(th.BasicChannel.Id, 0, 100, "", false, false) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 10, "expected 10 posts") + }) } func TestGetFlaggedPostsForUser(t *testing.T) { @@ -1440,7 +1462,7 @@ func TestGetPostsBefore(t *testing.T) { post4 := th.CreatePost() post5 := th.CreatePost() - posts, _, err := client.GetPostsBefore(th.BasicChannel.Id, post3.Id, 0, 100, "", false) + posts, _, err := client.GetPostsBefore(th.BasicChannel.Id, post3.Id, 0, 100, "", false, false) require.NoError(t, err) found := make([]bool, 2) @@ -1462,18 +1484,18 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, post3.Id, posts.NextPostId, "should match NextPostId") require.Equal(t, "", posts.PrevPostId, "should match empty PrevPostId") - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post4.Id, 1, 1, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post4.Id, 1, 1, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 1, "too many posts returned") require.Equal(t, post2.Id, posts.Order[0], "should match returned post") require.Equal(t, post3.Id, posts.NextPostId, "should match NextPostId") require.Equal(t, post1.Id, posts.PrevPostId, "should match PrevPostId") - _, resp, err := client.GetPostsBefore(th.BasicChannel.Id, "junk", 1, 1, "", false) + _, resp, err := client.GetPostsBefore(th.BasicChannel.Id, "junk", 1, 1, "", false, false) require.Error(t, err) CheckBadRequestStatus(t, resp) - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post5.Id, 0, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post5.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 3, "should match length of posts returned") require.Equal(t, post4.Id, posts.Order[0], "should match returned post") @@ -1482,12 +1504,12 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, post1.Id, posts.PrevPostId, "should match PrevPostId") // get the system post IDs posted before the created posts above - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false, false) require.NoError(t, err) systemPostId2 := posts.Order[0] systemPostId1 := posts.Order[1] - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post5.Id, 1, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post5.Id, 1, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 3, "should match length of posts returned") require.Equal(t, post1.Id, posts.Order[0], "should match returned post") @@ -1502,10 +1524,10 @@ func TestGetPostsBefore(t *testing.T) { th.CreatePost() // post7 post8 := th.CreatePost() post9 := th.CreatePost() - th.CreatePost() // post10 + post10 := th.CreatePost() // post10 // similar to '/posts?before=post9' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 0, 60, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 0, 60, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 10, "expected 10 posts") require.Equal(t, post8.Id, posts.Order[0], "posts not in order") @@ -1514,7 +1536,7 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") // similar to '/posts?before=post9&per_page=3' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 0, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post8.Id, posts.Order[0], "posts not in order") @@ -1523,7 +1545,7 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, post5.Id, posts.PrevPostId, "should return post5.Id as PrevPostId") // similar to '/posts?before=post9&per_page=3&page=1' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 1, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 1, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post5.Id, posts.Order[0], "posts not in order") @@ -1532,7 +1554,7 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, post2.Id, posts.PrevPostId, "should return post2.Id as PrevPostId") // similar to '/posts?before=post9&per_page=3&page=2' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 2, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 2, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post2.Id, posts.Order[0], "posts not in order") @@ -1541,7 +1563,7 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, systemPostId1, posts.PrevPostId, "should return systemPostId1 as PrevPostId") // similar to '/posts?before=post1&per_page=3' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 3, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 2, "expected 2 posts") require.Equal(t, systemPostId2, posts.Order[0], "posts not in order") @@ -1550,14 +1572,14 @@ func TestGetPostsBefore(t *testing.T) { require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") // similar to '/posts?before=systemPostId1' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, systemPostId1, 0, 60, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, systemPostId1, 0, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, systemPostId1, posts.NextPostId, "should return systemPostId1 as NextPostId") require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") // similar to '/posts?before=systemPostId1&per_page=60&page=1' - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, systemPostId1, 1, 60, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, systemPostId1, 1, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 posts") require.Equal(t, "", posts.NextPostId, "should return an empty NextPostId") @@ -1565,11 +1587,33 @@ func TestGetPostsBefore(t *testing.T) { // similar to '/posts?before=non-existent-post' nonExistentPostId := model.NewId() - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, nonExistentPostId, 0, 60, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, nonExistentPostId, 0, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, nonExistentPostId, posts.NextPostId, "should return nonExistentPostId as NextPostId") require.Equal(t, "", posts.PrevPostId, "should return an empty PrevPostId") + + client.DeletePost(post9.Id) + client.DeletePost(post8.Id) + + // include deleted posts for non-admin users. + _, resp, err = client.GetPostsBefore(th.BasicChannel.Id, post9.Id, 0, 60, "", false, true) + require.Error(t, err) + CheckForbiddenStatus(t, resp) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + // include deleted posts for admin users. + posts, resp, err = c.GetPostsBefore(th.BasicChannel.Id, post10.Id, 0, 60, "", false, true) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 11, "expected 11 posts") + + // not include deleted posts for admin users. + posts, resp, err = c.GetPostsBefore(th.BasicChannel.Id, post10.Id, 0, 60, "", false, false) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 9, "expected 9 posts") + }) } func TestGetPostsAfter(t *testing.T) { @@ -1583,7 +1627,7 @@ func TestGetPostsAfter(t *testing.T) { post4 := th.CreatePost() post5 := th.CreatePost() - posts, _, err := client.GetPostsAfter(th.BasicChannel.Id, post3.Id, 0, 100, "", false) + posts, _, err := client.GetPostsAfter(th.BasicChannel.Id, post3.Id, 0, 100, "", false, false) require.NoError(t, err) found := make([]bool, 2) @@ -1603,18 +1647,18 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, "", posts.NextPostId, "should match empty NextPostId") require.Equal(t, post3.Id, posts.PrevPostId, "should match PrevPostId") - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 1, 1, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 1, 1, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 1, "too many posts returned") require.Equal(t, post4.Id, posts.Order[0], "should match returned post") require.Equal(t, post5.Id, posts.NextPostId, "should match NextPostId") require.Equal(t, post3.Id, posts.PrevPostId, "should match PrevPostId") - _, resp, err := client.GetPostsAfter(th.BasicChannel.Id, "junk", 1, 1, "", false) + _, resp, err := client.GetPostsAfter(th.BasicChannel.Id, "junk", 1, 1, "", false, false) require.Error(t, err) CheckBadRequestStatus(t, resp) - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post1.Id, 0, 3, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post1.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 3, "should match length of posts returned") require.Equal(t, post4.Id, posts.Order[0], "should match returned post") @@ -1622,7 +1666,7 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, post5.Id, posts.NextPostId, "should match NextPostId") require.Equal(t, post1.Id, posts.PrevPostId, "should match PrevPostId") - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post1.Id, 1, 3, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post1.Id, 1, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Posts, 1, "should match length of posts returned") require.Equal(t, post5.Id, posts.Order[0], "should match returned post") @@ -1638,7 +1682,7 @@ func TestGetPostsAfter(t *testing.T) { post10 := th.CreatePost() // similar to '/posts?after=post2' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 0, 60, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 0, 60, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 8, "expected 8 posts") require.Equal(t, post10.Id, posts.Order[0], "should match order") @@ -1647,7 +1691,7 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, post2.Id, posts.PrevPostId, "should return post2.Id as PrevPostId") // similar to '/posts?after=post2&per_page=3' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 0, 3, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 0, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post5.Id, posts.Order[0], "should match order") @@ -1656,7 +1700,7 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, post2.Id, posts.PrevPostId, "should return post2.Id as PrevPostId") // similar to '/posts?after=post2&per_page=3&page=1' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 1, 3, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 1, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 3, "expected 3 posts") require.Equal(t, post8.Id, posts.Order[0], "should match order") @@ -1665,7 +1709,7 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, post5.Id, posts.PrevPostId, "should return post5.Id as PrevPostId") // similar to '/posts?after=post2&per_page=3&page=2' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 2, 3, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post2.Id, 2, 3, "", false, false) require.NoError(t, err) require.Len(t, posts.Order, 2, "expected 2 posts") require.Equal(t, post10.Id, posts.Order[0], "should match order") @@ -1674,14 +1718,14 @@ func TestGetPostsAfter(t *testing.T) { require.Equal(t, post8.Id, posts.PrevPostId, "should return post8.Id as PrevPostId") // similar to '/posts?after=post10' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post10.Id, 0, 60, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post10.Id, 0, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, "", posts.NextPostId, "should return an empty NextPostId") require.Equal(t, post10.Id, posts.PrevPostId, "should return post10.Id as PrevPostId") // similar to '/posts?after=post10&page=1' - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post10.Id, 1, 60, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, post10.Id, 1, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, "", posts.NextPostId, "should return an empty NextPostId") @@ -1689,11 +1733,33 @@ func TestGetPostsAfter(t *testing.T) { // similar to '/posts?after=non-existent-post' nonExistentPostId := model.NewId() - posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, nonExistentPostId, 0, 60, "", false) + posts, _, err = client.GetPostsAfter(th.BasicChannel.Id, nonExistentPostId, 0, 60, "", false, false) require.NoError(t, err) require.Empty(t, posts.Order, "should return 0 post") require.Equal(t, "", posts.NextPostId, "should return an empty NextPostId") require.Equal(t, nonExistentPostId, posts.PrevPostId, "should return nonExistentPostId as PrevPostId") + + client.DeletePost(post10.Id) + client.DeletePost(post9.Id) + + // include deleted posts for non-admin users. + _, resp, err = client.GetPostsAfter(th.BasicChannel.Id, post1.Id, 0, 60, "", false, true) + require.Error(t, err) + CheckForbiddenStatus(t, resp) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + // include deleted posts for admin users. + posts, resp, err = c.GetPostsAfter(th.BasicChannel.Id, post1.Id, 0, 60, "", false, true) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 9, "expected 9 posts") + + // not include deleted posts for admin users. + posts, resp, err = c.GetPostsAfter(th.BasicChannel.Id, post1.Id, 0, 60, "", false, false) + require.NoError(t, err) + CheckOKStatus(t, resp) + require.Len(t, posts.Order, 7, "expected 7 posts") + }) } func TestGetPostsForChannelAroundLastUnread(t *testing.T) { @@ -1798,7 +1864,7 @@ func TestGetPostsForChannelAroundLastUnread(t *testing.T) { require.Len(t, posts.Order, 12, "Should return 12 posts only since there's no unread post") // get the first system post generated before the created posts above - posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false) + posts, _, err = client.GetPostsBefore(th.BasicChannel.Id, post1.Id, 0, 2, "", false, false) require.NoError(t, err) systemPost0 := posts.Posts[posts.Order[0]] postIdNames[systemPost0.Id] = "system post 0" diff --git a/api4/team_test.go b/api4/team_test.go index a8b7a45702..e4dcc431c2 100644 --- a/api4/team_test.go +++ b/api4/team_test.go @@ -3028,7 +3028,7 @@ func TestImportTeam(t *testing.T) { require.NoError(t, err) require.Equal(t, importedChannel.Name, "general", "names did not match expected: general") - posts, _, err := th.SystemAdminClient.GetPostsForChannel(importedChannel.Id, 0, 60, "", false) + posts, _, err := th.SystemAdminClient.GetPostsForChannel(importedChannel.Id, 0, 60, "", false, false) require.NoError(t, err) require.Equal(t, posts.Posts[posts.Order[3]].Message, "This is a test post to test the import process", "missing posts in the import process") }) diff --git a/model/client4.go b/model/client4.go index 9e069fa535..f00852a77e 100644 --- a/model/client4.go +++ b/model/client4.go @@ -3925,11 +3925,15 @@ func (c *Client4) GetPostThreadWithOpts(postID string, etag string, opts GetPost } // GetPostsForChannel gets a page of posts with an array for ordering for a channel. -func (c *Client4) GetPostsForChannel(channelId string, page, perPage int, etag string, collapsedThreads bool) (*PostList, *Response, error) { +func (c *Client4) GetPostsForChannel(channelId string, page, perPage int, etag string, collapsedThreads bool, includeDeleted bool) (*PostList, *Response, error) { query := fmt.Sprintf("?page=%v&per_page=%v", page, perPage) if collapsedThreads { query += "&collapsedThreads=true" } + + if includeDeleted { + query += "&include_deleted=true" + } r, err := c.DoAPIGet(c.channelRoute(channelId)+"/posts"+query, etag) if err != nil { return nil, BuildResponse(r), err @@ -4050,11 +4054,14 @@ func (c *Client4) GetPostsSince(channelId string, time int64, collapsedThreads b } // GetPostsAfter gets a page of posts that were posted after the post provided. -func (c *Client4) GetPostsAfter(channelId, postId string, page, perPage int, etag string, collapsedThreads bool) (*PostList, *Response, error) { +func (c *Client4) GetPostsAfter(channelId, postId string, page, perPage int, etag string, collapsedThreads bool, includeDeleted bool) (*PostList, *Response, error) { query := fmt.Sprintf("?page=%v&per_page=%v&after=%v", page, perPage, postId) if collapsedThreads { query += "&collapsedThreads=true" } + if includeDeleted { + query += "&include_deleted=true" + } r, err := c.DoAPIGet(c.channelRoute(channelId)+"/posts"+query, etag) if err != nil { return nil, BuildResponse(r), err @@ -4071,11 +4078,14 @@ func (c *Client4) GetPostsAfter(channelId, postId string, page, perPage int, eta } // GetPostsBefore gets a page of posts that were posted before the post provided. -func (c *Client4) GetPostsBefore(channelId, postId string, page, perPage int, etag string, collapsedThreads bool) (*PostList, *Response, error) { +func (c *Client4) GetPostsBefore(channelId, postId string, page, perPage int, etag string, collapsedThreads bool, includeDeleted bool) (*PostList, *Response, error) { query := fmt.Sprintf("?page=%v&per_page=%v&before=%v", page, perPage, postId) if collapsedThreads { query += "&collapsedThreads=true" } + if includeDeleted { + query += "&include_deleted=true" + } r, err := c.DoAPIGet(c.channelRoute(channelId)+"/posts"+query, etag) if err != nil { return nil, BuildResponse(r), err diff --git a/model/permission.go b/model/permission.go index bf1f40238c..65f8edaca8 100644 --- a/model/permission.go +++ b/model/permission.go @@ -73,6 +73,7 @@ var PermissionDeleteOthersEmojis *Permission var PermissionCreatePost *Permission var PermissionCreatePostPublic *Permission var PermissionCreatePostEphemeral *Permission +var PermissionReadDeletedPosts *Permission var PermissionEditPost *Permission var PermissionEditOthersPosts *Permission var PermissionDeletePost *Permission @@ -708,6 +709,12 @@ func initializePermissions() { "authentication.permissions.create_post_ephemeral.description", PermissionScopeChannel, } + PermissionReadDeletedPosts = &Permission{ + "read_deleted_posts", + "authentication.permissions.read_deleted_posts.name", + "authentication.permissions.read_deleted_posts.description", + PermissionScopeChannel, + } PermissionEditPost = &Permission{ "edit_post", "authentication.permissions.edit_post.name", @@ -2302,6 +2309,7 @@ func initializePermissions() { PermissionCreatePost, PermissionCreatePostPublic, PermissionCreatePostEphemeral, + PermissionReadDeletedPosts, PermissionEditPost, PermissionEditOthersPosts, PermissionDeletePost, diff --git a/model/post.go b/model/post.go index f0ebcfbea2..4916779b52 100644 --- a/model/post.go +++ b/model/post.go @@ -304,6 +304,7 @@ type GetPostsOptions struct { FromPost string // PostId after which to send the items FromCreateAt int64 // CreateAt after which to send the items Direction string // Only accepts up|down. Indicates the order in which to send the items. + IncludeDeleted bool } type PostCountOptions struct { diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 2f3fd2b6b9..747b9d0f81 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -1168,13 +1168,13 @@ func (s *SqlPostStore) GetPosts(options model.GetPostsOptions, _ bool, sanitizeO rpc := make(chan store.StoreResult, 1) go func() { - posts, err := s.getRootPosts(options.ChannelId, offset, options.PerPage, options.SkipFetchThreads) + posts, err := s.getRootPosts(options.ChannelId, offset, options.PerPage, options.SkipFetchThreads, options.IncludeDeleted) rpc <- store.StoreResult{Data: posts, NErr: err} close(rpc) }() cpc := make(chan store.StoreResult, 1) go func() { - posts, err := s.getParentsPosts(options.ChannelId, offset, options.PerPage, options.SkipFetchThreads) + posts, err := s.getParentsPosts(options.ChannelId, offset, options.PerPage, options.SkipFetchThreads, options.IncludeDeleted) cpc <- store.StoreResult{Data: posts, NErr: err} close(cpc) }() @@ -1440,13 +1440,18 @@ func (s *SqlPostStore) getPostsAround(before bool, options model.GetPostsOptions ) } query := s.getQueryBuilder().Select(columns...) - replyCountSubQuery := s.getQueryBuilder().Select("COUNT(*)").From("Posts").Where(sq.Expr("Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0")) + replyCountSubQuery := s.getQueryBuilder().Select("COUNT(*)").From("Posts").Where(sq.Expr("Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END)")) conditions := sq.And{ sq.Expr(`CreateAt `+direction+` (SELECT CreateAt FROM Posts WHERE Id = ?)`, options.PostId), sq.Eq{"p.ChannelId": options.ChannelId}, - sq.Eq{"p.DeleteAt": int(0)}, } + + if !options.IncludeDeleted { + replyCountSubQuery = replyCountSubQuery.Where(sq.Expr("Posts.DeleteAt = 0")) + conditions = append(conditions, sq.Eq{"p.DeleteAt": int(0)}) + } + if options.CollapsedThreads { conditions = append(conditions, sq.Eq{"RootId": ""}) query = query.LeftJoin("Threads ON Threads.PostId = p.Id").LeftJoin("ThreadMemberships ON ThreadMemberships.PostId = p.Id AND ThreadMemberships.UserId=?", options.UserId) @@ -1492,10 +1497,13 @@ func (s *SqlPostStore) getPostsAround(before bool, options model.GetPostsOptions Where(sq.And{ idQuery, sq.Eq{"p.ChannelId": options.ChannelId}, - sq.Eq{"p.DeleteAt": 0}, }). OrderBy("CreateAt DESC") + if !options.IncludeDeleted { + rootQuery = rootQuery.Where(sq.Eq{"p.DeleteAt": 0}) + } + rootQueryString, rootArgs, nErr := rootQuery.ToSql() if nErr != nil { @@ -1620,14 +1628,21 @@ func (s *SqlPostStore) GetPostAfterTime(channelId string, time int64, collapsedT return &post, nil } -func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, error) { +func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, skipFetchThreads bool, includeDeleted bool) ([]*model.Post, error) { posts := []*model.Post{} var fetchQuery string if skipFetchThreads { - fetchQuery = "SELECT p.*, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.ChannelId = ? AND p.DeleteAt = 0 ORDER BY p.CreateAt DESC LIMIT ? OFFSET ?" + fetchQuery = "SELECT p.*, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END)) as ReplyCount FROM Posts p WHERE p.ChannelId = ? ORDER BY p.CreateAt DESC LIMIT ? OFFSET ?" + if !includeDeleted { + fetchQuery = "SELECT p.*, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.ChannelId = ? AND p.DeleteAt = 0 ORDER BY p.CreateAt DESC LIMIT ? OFFSET ?" + } } else { - fetchQuery = "SELECT * FROM Posts WHERE Posts.ChannelId = ? AND Posts.DeleteAt = 0 ORDER BY Posts.CreateAt DESC LIMIT ? OFFSET ?" + fetchQuery = "SELECT * FROM Posts WHERE Posts.ChannelId = ? ORDER BY Posts.CreateAt DESC LIMIT ? OFFSET ?" + if !includeDeleted { + fetchQuery = "SELECT * FROM Posts WHERE Posts.ChannelId = ? AND Posts.DeleteAt = 0 ORDER BY Posts.CreateAt DESC LIMIT ? OFFSET ?" + } } + err := s.GetReplicaX().Select(&posts, fetchQuery, channelId, limit, offset) if err != nil { return nil, errors.Wrap(err, "failed to find Posts") @@ -1635,9 +1650,14 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski return posts, nil } -func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, error) { +func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, skipFetchThreads bool, includeDeleted bool) ([]*model.Post, error) { if s.DriverName() == model.DatabaseDriverPostgres { - return s.getParentsPostsPostgreSQL(channelId, offset, limit, skipFetchThreads) + return s.getParentsPostsPostgreSQL(channelId, offset, limit, skipFetchThreads, includeDeleted) + } + + deleteAtCondition := "AND DeleteAt = 0" + if includeDeleted { + deleteAtCondition = "" } // query parent Ids first @@ -1651,9 +1671,8 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, FROM Posts WHERE - Posts.ChannelId = ? - AND Posts.DeleteAt = 0 - ORDER BY Posts.CreateAt DESC + ChannelId = ? ` + deleteAtCondition + ` + ORDER BY CreateAt DESC LIMIT ? OFFSET ?) q WHERE q.RootId != ''` @@ -1669,7 +1688,11 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, var where sq.Sqlizer where = sq.Eq{"p.Id": roots} if skipFetchThreads { - cols = append(cols, "(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount") + col := "(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END)) as ReplyCount" + if !includeDeleted { + col = "(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount" + } + cols = append(cols, col) } else { where = sq.Or{ where, @@ -1683,10 +1706,13 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, Where(sq.And{ where, sq.Eq{"p.ChannelId": channelId}, - sq.Eq{"p.DeleteAt": 0}, }). OrderBy("p.CreateAt") + if !includeDeleted { + query = query.Where(sq.Eq{"p.DeleteAt": 0}) + } + sql, args, err := query.ToSql() if err != nil { return nil, errors.Wrap(err, "ParentPosts_Tosql") @@ -1700,15 +1726,25 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, return posts, nil } -func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, error) { +func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, limit int, skipFetchThreads bool, includeDeleted bool) ([]*model.Post, error) { posts := []*model.Post{} replyCountQuery := "" onStatement := "q1.RootId = q2.Id" if skipFetchThreads { - replyCountQuery = ` ,(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery = ` ,(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END)) as ReplyCount` + if !includeDeleted { + replyCountQuery = ` ,(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + } } else { onStatement += " OR q1.RootId = q2.RootId" } + + deleteAtQueryCondition := "AND q2.DeleteAt = 0" + deleteAtSubQueryCondition := "AND Posts.DeleteAt = 0" + if includeDeleted { + deleteAtQueryCondition, deleteAtSubQueryCondition = "", "" + } + err := s.GetReplicaX().Select(&posts, `SELECT q2.*`+replyCountQuery+` FROM @@ -1722,15 +1758,13 @@ func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, l FROM Posts WHERE - Posts.ChannelId = ? - AND Posts.DeleteAt = 0 + Posts.ChannelId = ? `+deleteAtSubQueryCondition+` ORDER BY Posts.CreateAt DESC LIMIT ? OFFSET ?) q3 WHERE q3.RootId != '') q1 ON `+onStatement+` WHERE - q2.ChannelId = ? - AND q2.DeleteAt = 0 + q2.ChannelId = ? `+deleteAtQueryCondition+` ORDER BY q2.CreateAt`, channelId, limit, offset, channelId) if err != nil { return nil, errors.Wrapf(err, "failed to find Posts with channelId=%s", channelId) diff --git a/store/storetest/post_store.go b/store/storetest/post_store.go index 41b8c138d3..dd1ba40ae4 100644 --- a/store/storetest/post_store.go +++ b/store/storetest/post_store.go @@ -1946,6 +1946,73 @@ func testPostStoreGetPosts(t *testing.T, ss store.Store) { assert.Equal(t, int64(1), postList.Posts[post5.Id].ReplyCount) assert.Equal(t, int64(1), postList.Posts[post6.Id].ReplyCount) }) + + t.Run("should return all posts in a channel included deleted posts", func(t *testing.T) { + err := ss.Post().Delete(post1.Id, 1, userId) + require.NoError(t, err) + + postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 30, SkipFetchThreads: false, IncludeDeleted: true}, false, map[string]bool{}) + require.NoError(t, err) + + assert.Equal(t, []string{ + post6.Id, + post5.Id, + post4.Id, + post3.Id, + post2.Id, + post1.Id, + }, postList.Order) + + assert.Len(t, postList.Posts, 6) + assert.NotNil(t, postList.Posts[post1.Id]) + assert.NotNil(t, postList.Posts[post2.Id]) + assert.NotNil(t, postList.Posts[post3.Id]) + assert.NotNil(t, postList.Posts[post4.Id]) + assert.NotNil(t, postList.Posts[post5.Id]) + assert.NotNil(t, postList.Posts[post6.Id]) + }) + + t.Run("should return all posts in a channel included deleted posts without threads", func(t *testing.T) { + err := ss.Post().Delete(post5.Id, 1, userId) + require.NoError(t, err) + + postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 30, SkipFetchThreads: true, IncludeDeleted: true}, false, map[string]bool{}) + require.NoError(t, err) + + assert.Equal(t, []string{ + post6.Id, + post5.Id, + post4.Id, + post3.Id, + post2.Id, + post1.Id, + }, postList.Order) + + assert.Len(t, postList.Posts, 6) + assert.NotNil(t, postList.Posts[post5.Id]) + assert.NotNil(t, postList.Posts[post6.Id]) + assert.Equal(t, int64(1), postList.Posts[post5.Id].ReplyCount) + assert.Equal(t, int64(1), postList.Posts[post6.Id].ReplyCount) + }) + + t.Run("should return the lasts posts created in channel without include deleted posts", func(t *testing.T) { + err := ss.Post().Delete(post6.Id, 1, userId) + require.NoError(t, err) + + postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 30, SkipFetchThreads: true, IncludeDeleted: false}, false, map[string]bool{}) + require.NoError(t, err) + + assert.Equal(t, []string{ + post4.Id, + post3.Id, + post2.Id, + }, postList.Order) + + assert.Len(t, postList.Posts, 3) + assert.NotNil(t, postList.Posts[post2.Id]) + assert.NotNil(t, postList.Posts[post3.Id]) + assert.NotNil(t, postList.Posts[post4.Id]) + }) } func testPostStoreGetPostBeforeAfter(t *testing.T, ss store.Store) { From f5f036d94b3ab815255ea36a1707bcd7550aefcc Mon Sep 17 00:00:00 2001 From: Vishal Date: Wed, 28 Sep 2022 22:22:53 +0530 Subject: [PATCH 12/12] [MM-44489] Cloud limits: enforcing files (#20703) * Update last accessible file time * Filter fileInfos * Set inaccessible header * Fix lint issue * Fix lint issue * Fix i18n * add nil check * Fix merge conflicts * Add helper functions to clear out inaccessible files content * Remove content for inaccessible files * Fix typo * wip * Remove InaccessibleContent field, instead use Archived * Add store tests * Add tests * Add separate funcs to ignore cloud limits * Use separate query for MySql * Use GetReplicaX Co-authored-by: Mattermod --- api4/file.go | 13 ++ app/app_iface.go | 8 +- app/file.go | 129 ++++++++++++- app/file_helper.go | 210 +++++++++++++++++++++ app/file_helper_test.go | 206 ++++++++++++++++++++ app/file_test.go | 65 +++++++ app/import_functions.go | 4 +- app/opentracing/opentracing_layer.go | 54 +++++- app/post.go | 25 ++- app/post_metadata.go | 6 +- app/post_test.go | 4 +- app/server.go | 7 + i18n/en.json | 8 + jobs/last_accessible_file/scheduler.go | 24 +++ jobs/last_accessible_file/worker.go | 28 +++ model/client4.go | 1 + model/file_info.go | 14 ++ model/file_info_list.go | 2 + model/job.go | 2 + model/system.go | 1 + store/opentracinglayer/opentracinglayer.go | 18 ++ store/retrylayer/retrylayer.go | 21 +++ store/sqlstore/file_info_store.go | 46 +++++ store/store.go | 2 + store/storetest/file_info_store.go | 70 +++++++ store/storetest/mocks/FileInfoStore.go | 21 +++ store/timerlayer/timerlayer.go | 16 ++ 27 files changed, 984 insertions(+), 21 deletions(-) create mode 100644 app/file_helper.go create mode 100644 app/file_helper_test.go create mode 100644 jobs/last_accessible_file/scheduler.go create mode 100644 jobs/last_accessible_file/worker.go diff --git a/api4/file.go b/api4/file.go index c1cf36d4a5..f81da72bf8 100644 --- a/api4/file.go +++ b/api4/file.go @@ -482,6 +482,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } auditRec.AddMeta("file", info) @@ -514,6 +515,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -555,6 +557,7 @@ func getFileLink(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } auditRec.AddMeta("file", info) @@ -589,6 +592,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -622,6 +626,7 @@ func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -650,6 +655,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -824,3 +830,10 @@ func searchFiles(c *Context, w http.ResponseWriter, r *http.Request, teamID stri c.Logger.Warn("Error while writing response", mlog.Err(err)) } } + +func setInaccessibleFileHeader(w http.ResponseWriter, appErr *model.AppError) { + // File is inaccessible due to cloud plan's limit. + if appErr.Id == "app.file.cloud.get.app_error" { + w.Header().Set(model.HeaderFirstInaccessibleFileTime, "1") + } +} diff --git a/app/app_iface.go b/app/app_iface.go index 952362d63e..25fbb1694c 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -76,6 +76,9 @@ type AppIface interface { CheckProviderAttributes(user *model.User, patch *model.UserPatch) string // ClientConfigWithComputed gets the configuration in a format suitable for sending to the client. ClientConfigWithComputed() map[string]string + // ComputeLastAccessibleFileTime updates cache with CreateAt time of the last accessible file as per the cloud plan's limit. + // Use GetLastAccessibleFileTime() to access the result. + ComputeLastAccessibleFileTime() error // ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit. // Use GetLastAccessiblePostTime() to access the result. ComputeLastAccessiblePostTime() error @@ -176,6 +179,8 @@ type AppIface interface { // GetEnvironmentConfig returns a map of configuration keys whose values have been overridden by an environment variable. // If filter is not nil and returns false for a struct field, that field will be omitted. GetEnvironmentConfig(filter func(reflect.StructField) bool) map[string]any + // GetFileInfosForPost also returns firstInaccessibleFileTime based on cloud plan's limit. + GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) // GetFilteredUsersStats is used to get a count of users based on the set of filters supported by UserCountOptions. GetFilteredUsersStats(options *model.UserCountOptions) (*model.UsersStats, *model.AppError) // GetGroupsByTeam returns the paged list and the total count of group associated to the given team. @@ -186,6 +191,8 @@ type AppIface interface { // relationship with a user. That means any user sharing any channel, including // direct and group channels. GetKnownUsers(userID string) ([]string, *model.AppError) + // GetLastAccessibleFileTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit + GetLastAccessibleFileTime() (int64, *model.AppError) // GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit GetLastAccessiblePostTime() (int64, *model.AppError) // GetLdapGroup retrieves a single LDAP group by the given LDAP group id. @@ -634,7 +641,6 @@ type AppIface interface { GetFile(fileID string) ([]byte, *model.AppError) GetFileInfo(fileID string) (*model.FileInfo, *model.AppError) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, *model.AppError) - GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) GetFileInfosForPostWithMigration(postID string, includeDeleted bool) ([]*model.FileInfo, *model.AppError) GetFlaggedPosts(userID string, offset int, limit int) (*model.PostList, *model.AppError) GetFlaggedPostsForChannel(userID, channelID string, offset int, limit int) (*model.PostList, *model.AppError) diff --git a/app/file.go b/app/file.go index c461d1765b..d2520c5f62 100644 --- a/app/file.go +++ b/app/file.go @@ -12,12 +12,14 @@ import ( "fmt" "image" "io" + "math" "net/http" "net/url" "os" "path" "path/filepath" "regexp" + "strconv" "strings" "sync" "time" @@ -1082,11 +1084,30 @@ func (s *Server) getFileInfo(fileID string) (*model.FileInfo, *model.AppError) { } func (a *App) GetFileInfo(fileID string) (*model.FileInfo, *model.AppError) { - fileInfo, err := a.Srv().getFileInfo(fileID) - if err == nil { + fileInfo, appErr := a.Srv().getFileInfo(fileID) + if appErr != nil { + return nil, appErr + } + + firstInaccessibleFileTime, appErr := a.isInaccessibleFile(fileInfo) + if appErr != nil { + return nil, appErr + } + if firstInaccessibleFileTime > 0 { + return nil, model.NewAppError("GetFileInfo", "app.file.cloud.get.app_error", nil, "", http.StatusForbidden) + } + + a.generateMiniPreview(fileInfo) + return fileInfo, appErr +} + +func (a *App) getFileInfoIgnoreCloudLimit(fileID string) (*model.FileInfo, *model.AppError) { + fileInfo, appErr := a.Srv().getFileInfo(fileID) + if appErr == nil { a.generateMiniPreview(fileInfo) } - return fileInfo, err + + return fileInfo, appErr } func (a *App) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, *model.AppError) { @@ -1104,6 +1125,16 @@ func (a *App) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([ } } + filterOptions := filterFileOptions{} + if opt != nil && (opt.SortBy == "" || opt.SortBy == model.FileinfoSortByCreated) { + filterOptions.assumeSortedCreatedAt = true + } + + fileInfos, _, appErr := a.getFilteredAccessibleFiles(fileInfos, filterOptions) + if appErr != nil { + return nil, appErr + } + a.generateMiniPreviewForInfos(fileInfos) return fileInfos, nil @@ -1123,6 +1154,20 @@ func (a *App) GetFile(fileID string) ([]byte, *model.AppError) { return data, nil } +func (a *App) getFileIgnoreCloudLimit(fileID string) ([]byte, *model.AppError) { + info, err := a.getFileInfoIgnoreCloudLimit(fileID) + if err != nil { + return nil, err + } + + data, err := a.ReadFile(info.Path) + if err != nil { + return nil, err + } + + return data, nil +} + func (a *App) CopyFileInfos(userID string, fileIDs []string) ([]string, *model.AppError) { var newFileIds []string @@ -1252,7 +1297,7 @@ func (a *App) SearchFilesInTeamForUser(c *request.Context, terms string, userId } } - return fileInfoSearchResults, nil + return fileInfoSearchResults, a.filterInaccessibleFiles(fileInfoSearchResults, filterFileOptions{assumeSortedCreatedAt: true}) } func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { @@ -1288,3 +1333,79 @@ func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { } return nil } + +// GetLastAccessibleFileTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit +func (a *App) GetLastAccessibleFileTime() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + system, err := a.Srv().Store.System().GetByName(model.SystemLastAccessibleFileTime) + if err != nil { + var nfErr *store.ErrNotFound + switch { + case errors.As(err, &nfErr): + // All files are accessible + return 0, nil + default: + return 0, model.NewAppError("GetLastAccessibleFileTime", "app.system.get_by_name.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + lastAccessibleFileTime, err := strconv.ParseInt(system.Value, 10, 64) + if err != nil { + return 0, model.NewAppError("GetLastAccessibleFileTime", "common.parse_error_int64", map[string]interface{}{"Value": system.Value}, err.Error(), http.StatusInternalServerError) + } + + return lastAccessibleFileTime, nil +} + +// ComputeLastAccessibleFileTime updates cache with CreateAt time of the last accessible file as per the cloud plan's limit. +// Use GetLastAccessibleFileTime() to access the result. +func (a *App) ComputeLastAccessibleFileTime() error { + limit, appErr := a.getCloudFilesSizeLimit() + if appErr != nil { + return appErr + } + + createdAt, err := a.Srv().GetStore().FileInfo().GetUptoNSizeFileTime(limit) + if err != nil { + var nfErr *store.ErrNotFound + if !errors.As(err, &nfErr) { + return model.NewAppError("ComputeLastAccessibleFileTime", "app.last_accessible_file.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + // Update Cache + err = a.Srv().Store.System().SaveOrUpdate(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: strconv.FormatInt(createdAt, 10), + }) + if err != nil { + return model.NewAppError("ComputeLastAccessibleFileTime", "app.system.save.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +// getCloudFilesSizeLimit returns size in bytes +func (a *App) getCloudFilesSizeLimit() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + // limits is in bits + limits, err := a.Cloud().GetCloudLimits("") + if err != nil { + return 0, model.NewAppError("getCloudFilesSizeLimit", "api.cloud.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if limits == nil || limits.Files == nil || limits.Files.TotalStorage == nil { + // Cloud limit is not applicable + return 0, nil + } + + return int64(math.Ceil(float64(*limits.Files.TotalStorage) / 8)), nil +} diff --git a/app/file_helper.go b/app/file_helper.go new file mode 100644 index 0000000000..a3059ff30b --- /dev/null +++ b/app/file_helper.go @@ -0,0 +1,210 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/v6/model" +) + +// removeInaccessibleContentFromFilesSlice removes content from the files beyond the cloud plan's limit +// and also returns the firstInaccessibleFileTime +func (a *App) removeInaccessibleContentFromFilesSlice(files []*model.FileInfo) (int64, *model.AppError) { + if len(files) == 0 { + return 0, nil + } + + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return 0, model.NewAppError("removeInaccessibleFileListContent", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } + if lastAccessibleFileTime == 0 { + // No need to remove content, all files are accessible + return 0, nil + } + + var firstInaccessibleFileTime int64 = 0 + for _, file := range files { + if createAt := file.CreateAt; createAt < lastAccessibleFileTime { + file.MakeContentInaccessible() + if createAt > firstInaccessibleFileTime { + firstInaccessibleFileTime = createAt + } + } + } + + return firstInaccessibleFileTime, nil +} + +// filterInaccessibleFiles filters out the files, past the cloud limit +func (a *App) filterInaccessibleFiles(fileList *model.FileInfoList, options filterFileOptions) *model.AppError { + if fileList == nil || fileList.FileInfos == nil || len(fileList.FileInfos) == 0 { + return nil + } + + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return model.NewAppError("filterInaccessibleFiles", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } + if lastAccessibleFileTime == 0 { + // No need to filter, all files are accessible + return nil + } + + if len(fileList.FileInfos) == len(fileList.Order) && options.assumeSortedCreatedAt { + lenFiles := len(fileList.FileInfos) + getCreateAt := func(i int) int64 { return fileList.FileInfos[fileList.Order[i]].CreateAt } + + bounds := getTimeSortedPostAccessibleBounds(lastAccessibleFileTime, lenFiles, getCreateAt) + + if bounds.allAccessible(lenFiles) { + return nil + } + if bounds.noAccessible() { + if lenFiles > 0 { + firstFileCreatedAt := fileList.FileInfos[fileList.Order[0]].CreateAt + lastFileCreatedAt := fileList.FileInfos[fileList.Order[lenFiles-1]].CreateAt + fileList.FirstInaccessibleFileTime = max(firstFileCreatedAt, lastFileCreatedAt) + } + fileList.FileInfos = map[string]*model.FileInfo{} + fileList.Order = []string{} + return nil + } + startInaccessibleIndex, endInaccessibleIndex := bounds.getInaccessibleRange(len(fileList.Order)) + startInaccessibleCreatedAt := fileList.FileInfos[fileList.Order[startInaccessibleIndex]].CreateAt + endInaccessibleCreatedAt := fileList.FileInfos[fileList.Order[endInaccessibleIndex]].CreateAt + fileList.FirstInaccessibleFileTime = max(startInaccessibleCreatedAt, endInaccessibleCreatedAt) + + files := fileList.FileInfos + order := fileList.Order + accessibleCount := bounds.end - bounds.start + 1 + inaccessibleCount := lenFiles - accessibleCount + // Linearly cover shorter route to traverse files map + if inaccessibleCount < accessibleCount { + for i := 0; i < bounds.start; i++ { + delete(files, order[i]) + } + for i := bounds.end + 1; i < lenFiles; i++ { + delete(files, order[i]) + } + } else { + accessibleFiles := make(map[string]*model.FileInfo, accessibleCount) + for i := bounds.start; i <= bounds.end; i++ { + accessibleFiles[order[i]] = files[order[i]] + } + fileList.FileInfos = accessibleFiles + } + + fileList.Order = fileList.Order[bounds.start : bounds.end+1] + } else { + linearFilterFileList(fileList, lastAccessibleFileTime) + } + + return nil +} + +// isInaccessibleFile indicates if the file is past the cloud plan's limit. +func (a *App) isInaccessibleFile(file *model.FileInfo) (int64, *model.AppError) { + if file == nil { + return 0, nil + } + + fl := &model.FileInfoList{ + Order: []string{file.Id}, + FileInfos: map[string]*model.FileInfo{file.Id: file}, + } + + appErr := a.filterInaccessibleFiles(fl, filterFileOptions{assumeSortedCreatedAt: true}) + return fl.FirstInaccessibleFileTime, appErr +} + +// getFilteredAccessibleFiles returns accessible files filtered as per the cloud plan's limit and also indicates if there were any inaccessible files +func (a *App) getFilteredAccessibleFiles(files []*model.FileInfo, options filterFileOptions) ([]*model.FileInfo, int64, *model.AppError) { + if len(files) == 0 { + return files, 0, nil + } + + filteredFiles := []*model.FileInfo{} + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return filteredFiles, 0, model.NewAppError("getFilteredAccessibleFiles", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } else if lastAccessibleFileTime == 0 { + // No need to filter, all files are accessible + return files, 0, nil + } + + if options.assumeSortedCreatedAt { + lenFiles := len(files) + getCreateAt := func(i int) int64 { return files[i].CreateAt } + bounds := getTimeSortedPostAccessibleBounds(lastAccessibleFileTime, lenFiles, getCreateAt) + if bounds.allAccessible(lenFiles) { + return files, 0, nil + } + if bounds.noAccessible() { + var firstInaccessibleFileTime int64 = 0 + if lenFiles > 0 { + firstFileCreatedAt := files[0].CreateAt + lastFileCreatedAt := files[len(files)-1].CreateAt + firstInaccessibleFileTime = max(firstFileCreatedAt, lastFileCreatedAt) + } + return filteredFiles, firstInaccessibleFileTime, nil + } + + startInaccessibleIndex, endInaccessibleIndex := bounds.getInaccessibleRange(len(files)) + firstFileCreatedAt := files[startInaccessibleIndex].CreateAt + lastFileCreatedAt := files[endInaccessibleIndex].CreateAt + firstInaccessibleFileTime := max(firstFileCreatedAt, lastFileCreatedAt) + filteredFiles = files[bounds.start : bounds.end+1] + return filteredFiles, firstInaccessibleFileTime, nil + } + + filteredFiles, firstInaccessibleFileTime := linearFilterFilesSlice(files, lastAccessibleFileTime) + return filteredFiles, firstInaccessibleFileTime, nil +} + +type filterFileOptions struct { + assumeSortedCreatedAt bool +} + +// linearFilterFileList make no assumptions about ordering, go through files one by one +// this is the slower fallback that is still safe +// if we can not assume files are ordered by CreatedAt +func linearFilterFileList(fileList *model.FileInfoList, earliestAccessibleTime int64) { + files := fileList.FileInfos + order := fileList.Order + + n := 0 + for i, fileID := range order { + if createAt := files[fileID].CreateAt; createAt >= earliestAccessibleTime { + order[n] = order[i] + n++ + } else { + if createAt > fileList.FirstInaccessibleFileTime { + fileList.FirstInaccessibleFileTime = createAt + } + delete(files, fileID) + } + } + fileList.Order = order[:n] +} + +// linearFilterFilesSlice make no assumptions about ordering, go through files one by one +// this is the slower fallback that is still safe +// if we can not assume files are ordered by CreatedAt +func linearFilterFilesSlice(files []*model.FileInfo, earliestAccessibleTime int64) ([]*model.FileInfo, int64) { + var firstInaccessibleFileTime int64 = 0 + n := 0 + for i := range files { + if createAt := files[i].CreateAt; createAt >= earliestAccessibleTime { + files[n] = files[i] + n++ + } else { + if createAt > firstInaccessibleFileTime { + firstInaccessibleFileTime = createAt + } + } + } + return files[:n], firstInaccessibleFileTime +} diff --git a/app/file_helper_test.go b/app/file_helper_test.go new file mode 100644 index 0000000000..57842ccdbb --- /dev/null +++ b/app/file_helper_test.go @@ -0,0 +1,206 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "testing" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFilterInaccessibleFiles(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + t.Run("ascending order returns correct files", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_a", "file_b", "file_c", "file_d", "file_e"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_c", + "file_d", + "file_e", + }, fileList.Order) + assert.Equal(t, int64(1), fileList.FirstInaccessibleFileTime) + }) + + t.Run("descending order returns correct files", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_e", "file_d", "file_c", "file_b", "file_a"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_e", + "file_d", + "file_c", + }, fileList.Order) + + assert.Equal(t, int64(1), fileList.FirstInaccessibleFileTime) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_e", "file_b", "file_a", "file_d", "file_c"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_e", + "file_d", + "file_c", + }, fileList.Order) + }) +} + +func TestGetFilteredAccessibleFiles(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + t.Run("ascending order returns correct files", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(0), getFileWithCreateAt(1), getFileWithCreateAt(2), getFileWithCreateAt(3), getFileWithCreateAt(4)} + filteredFiles, firstInaccessibleFileTime, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(2), getFileWithCreateAt(3), getFileWithCreateAt(4)}, filteredFiles) + assert.Equal(t, int64(1), firstInaccessibleFileTime) + }) + + t.Run("descending order returns correct files", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2), getFileWithCreateAt(1), getFileWithCreateAt(0)} + filteredFiles, firstInaccessibleFileTime, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2)}, filteredFiles) + assert.Equal(t, int64(1), firstInaccessibleFileTime) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(1), getFileWithCreateAt(0), getFileWithCreateAt(3), getFileWithCreateAt(2)} + filteredFiles, _, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2)}, filteredFiles) + }) +} + +func TestIsInaccessibleFile(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + file := &model.FileInfo{CreateAt: 3} + firstInaccessibleFileTime, appErr := th.App.isInaccessibleFile(file) + require.Nil(t, appErr) + assert.Equal(t, int64(0), firstInaccessibleFileTime) + + file = &model.FileInfo{CreateAt: 1} + firstInaccessibleFileTime, appErr = th.App.isInaccessibleFile(file) + require.Nil(t, appErr) + assert.Equal(t, int64(1), firstInaccessibleFileTime) +} + +func TestRemoveInaccessibleContentFromFilesSlice(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(1), getFileWithCreateAt(0), getFileWithCreateAt(3), getFileWithCreateAt(2)} + + _, appErr := th.App.removeInaccessibleContentFromFilesSlice(files) + + require.Nil(t, appErr) + assert.Len(t, files, len(files)) + for _, file := range files { + // Inaccessible files are archived + if file.CreateAt < 2 { + assert.True(t, file.Archived) + } else { + assert.False(t, file.Archived) + } + } +} diff --git a/app/file_test.go b/app/file_test.go index 6f84894b8f..c30a2df471 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -16,9 +16,12 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + eMocks "github.com/mattermost/mattermost-server/v6/einterfaces/mocks" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/services/searchengine/mocks" filesStoreMocks "github.com/mattermost/mattermost-server/v6/shared/filestore/mocks" + "github.com/mattermost/mattermost-server/v6/store" + storemocks "github.com/mattermost/mattermost-server/v6/store/storetest/mocks" "github.com/mattermost/mattermost-server/v6/utils/fileutils" ) @@ -553,3 +556,65 @@ func TestExtractContentFromFileInfo(t *testing.T) { // Test that we don't process images. require.NoError(t, app.ExtractContentFromFileInfo(fi)) } + +func TestGetLastAccessibleFileTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + r, err := th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(0), r) + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + + mockSystemStore := storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, store.NewErrNotFound("", "")) + r, err = th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(0), r) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, errors.New("test")) + _, err = th.App.GetLastAccessibleFileTime() + require.NotNil(t, err) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(&model.System{Name: model.SystemLastAccessibleFileTime, Value: "10"}, nil) + r, err = th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(10), r) +} + +func TestComputeLastAccessibleFileTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + cloud := &eMocks.CloudInterface{} + th.App.Srv().Cloud = cloud + + cloud.Mock.On("GetCloudLimits", mock.Anything).Return(&model.ProductLimits{ + Files: &model.FilesLimits{ + TotalStorage: model.NewInt64(1), + }, + }, nil) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + mockFileStore := storemocks.FileInfoStore{} + mockFileStore.On("GetUptoNSizeFileTime", mock.Anything).Return(int64(1), nil) + mockSystemStore := storemocks.SystemStore{} + mockSystemStore.On("SaveOrUpdate", mock.Anything).Return(nil) + mockStore.On("FileInfo").Return(&mockFileStore) + mockStore.On("System").Return(&mockSystemStore) + + err := th.App.ComputeLastAccessibleFileTime() + require.NoError(t, err) + + mockSystemStore.AssertCalled(t, "SaveOrUpdate", mock.Anything) +} diff --git a/app/import_functions.go b/app/import_functions.go index 837a4b875c..240c01ba34 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -1225,7 +1225,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData // Go over existing files in the post and see if there already exists a file with the same name, size and hash. If so - skip it if post.Id != "" { - oldFiles, err := a.GetFileInfosForPost(post.Id, true, false) + oldFiles, err := a.getFileInfosForPostIgnoreCloudLimit(post.Id, true, false) if err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]any{"FilePath": *data.Path}, "", http.StatusBadRequest) } @@ -1235,7 +1235,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData } // check md5 newHash := sha1.Sum(fileData) - oldFileData, err := a.GetFile(oldFile.Id) + oldFileData, err := a.getFileIgnoreCloudLimit(oldFile.Id) if err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]any{"FilePath": *data.Path}, "", http.StatusBadRequest) } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 1869099fb7..30379abfe2 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -1783,6 +1783,28 @@ func (a *OpenTracingAppLayer) CompleteSwitchWithOAuth(service string, userData i return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) ComputeLastAccessibleFileTime() error { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessibleFileTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.ComputeLastAccessibleFileTime() + + if resultVar0 != nil { + span.LogFields(spanlog.Error(resultVar0)) + ext.Error.Set(span, true) + } + + return resultVar0 +} + func (a *OpenTracingAppLayer) ComputeLastAccessiblePostTime() error { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessiblePostTime") @@ -6053,7 +6075,7 @@ func (a *OpenTracingAppLayer) GetFileInfos(page int, perPage int, opt *model.Get return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { +func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetFileInfosForPost") @@ -6065,14 +6087,14 @@ func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool }() defer span.Finish() - resultVar0, resultVar1 := a.app.GetFileInfosForPost(postID, fromMaster, includeDeleted) + resultVar0, resultVar1, resultVar2 := a.app.GetFileInfosForPost(postID, fromMaster, includeDeleted) - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) ext.Error.Set(span, true) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (a *OpenTracingAppLayer) GetFileInfosForPostWithMigration(postID string, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { @@ -6884,6 +6906,28 @@ func (a *OpenTracingAppLayer) GetKnownUsers(userID string) ([]string, *model.App return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetLastAccessibleFileTime() (int64, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessibleFileTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetLastAccessibleFileTime() + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetLastAccessiblePostTime() (int64, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessiblePostTime") diff --git a/app/post.go b/app/post.go index 0a6c300e5c..ea5249ef09 100644 --- a/app/post.go +++ b/app/post.go @@ -1558,12 +1558,12 @@ func (a *App) GetFileInfosForPostWithMigration(postID string, includeDeleted boo close(pchan) }() - infos, err := a.GetFileInfosForPost(postID, false, includeDeleted) + infos, firstInaccessibleFileTime, err := a.GetFileInfosForPost(postID, false, includeDeleted) if err != nil { return nil, err } - if len(infos) == 0 { + if len(infos) == 0 && firstInaccessibleFileTime == 0 { // No FileInfos were returned so check if they need to be created for this post result := <-pchan if result.NErr != nil { @@ -1588,10 +1588,27 @@ func (a *App) GetFileInfosForPostWithMigration(postID string, includeDeleted boo return infos, nil } -func (a *App) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { +// GetFileInfosForPost also returns firstInaccessibleFileTime based on cloud plan's limit. +func (a *App) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) { fileInfos, err := a.Srv().Store.FileInfo().GetForPost(postID, fromMaster, includeDeleted, true) if err != nil { - return nil, model.NewAppError("GetFileInfosForPost", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + return nil, 0, model.NewAppError("GetFileInfosForPost", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + firstInaccessibleFileTime, appErr := a.removeInaccessibleContentFromFilesSlice(fileInfos) + if appErr != nil { + return nil, 0, appErr + } + + a.generateMiniPreviewForInfos(fileInfos) + + return fileInfos, firstInaccessibleFileTime, nil +} + +func (a *App) getFileInfosForPostIgnoreCloudLimit(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { + fileInfos, err := a.Srv().Store.FileInfo().GetForPost(postID, fromMaster, includeDeleted, true) + if err != nil { + return nil, model.NewAppError("getFileInfosForPostIgnoreCloudLimit", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } a.generateMiniPreviewForInfos(fileInfos) diff --git a/app/post_metadata.go b/app/post_metadata.go index f7155e0a7c..b863ac21f4 100644 --- a/app/post_metadata.go +++ b/app/post_metadata.go @@ -123,7 +123,7 @@ func (a *App) PreparePostForClient(originalPost *model.Post, isNewPost, isEditPo } // Files - if fileInfos, err := a.getFileMetadataForPost(post, isNewPost || isEditPost); err != nil { + if fileInfos, _, err := a.getFileMetadataForPost(post, isNewPost || isEditPost); err != nil { mlog.Warn("Failed to get files for a post", mlog.String("post_id", post.Id), mlog.Err(err)) } else { post.Metadata.Files = fileInfos @@ -210,9 +210,9 @@ func (a *App) SanitizePostListMetadataForUser(c request.CTX, postList *model.Pos return clonedPostList, nil } -func (a *App) getFileMetadataForPost(post *model.Post, fromMaster bool) ([]*model.FileInfo, *model.AppError) { +func (a *App) getFileMetadataForPost(post *model.Post, fromMaster bool) ([]*model.FileInfo, int64, *model.AppError) { if len(post.FileIds) == 0 { - return nil, nil + return nil, 0, nil } return a.GetFileInfosForPost(post.Id, fromMaster, false) diff --git a/app/post_test.go b/app/post_test.go index 800511abf9..c51eabb5db 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -219,7 +219,7 @@ func TestAttachFilesToPost(t *testing.T) { appErr := th.App.attachFilesToPost(post) assert.Nil(t, appErr) - infos, appErr := th.App.GetFileInfosForPost(post.Id, false, false) + infos, _, appErr := th.App.GetFileInfosForPost(post.Id, false, false) assert.Nil(t, appErr) assert.Len(t, infos, 2) }) @@ -247,7 +247,7 @@ func TestAttachFilesToPost(t *testing.T) { appErr := th.App.attachFilesToPost(post) assert.Nil(t, appErr) - infos, appErr := th.App.GetFileInfosForPost(post.Id, false, false) + infos, _, appErr := th.App.GetFileInfosForPost(post.Id, false, false) assert.Nil(t, appErr) assert.Len(t, infos, 1) assert.Equal(t, info2.Id, infos[0].Id) diff --git a/app/server.go b/app/server.go index 3e3fd0a18b..997c5a3f64 100644 --- a/app/server.go +++ b/app/server.go @@ -46,6 +46,7 @@ import ( "github.com/mattermost/mattermost-server/v6/jobs/extract_content" "github.com/mattermost/mattermost-server/v6/jobs/import_delete" "github.com/mattermost/mattermost-server/v6/jobs/import_process" + "github.com/mattermost/mattermost-server/v6/jobs/last_accessible_file" "github.com/mattermost/mattermost-server/v6/jobs/last_accessible_post" "github.com/mattermost/mattermost-server/v6/jobs/migrations" "github.com/mattermost/mattermost-server/v6/jobs/notify_admin" @@ -1870,6 +1871,12 @@ func (s *Server) initJobs() { last_accessible_post.MakeScheduler(s.Jobs, s.License()), ) + s.Jobs.RegisterJobType( + model.JobTypeLastAccessibleFile, + last_accessible_file.MakeWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))), + last_accessible_file.MakeScheduler(s.Jobs, s.License()), + ) + s.Jobs.RegisterJobType( model.JobTypeUpgradeNotifyAdmin, notify_admin.MakeUpgradeNotifyWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))), diff --git a/i18n/en.json b/i18n/en.json index eb49526f56..0b4e92e034 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -4987,6 +4987,10 @@ "id": "app.export.zip_create.error", "translation": "Failed to add file to zip archive during export." }, + { + "id": "app.file.cloud.get.app_error", + "translation": "Can not fetch the file as it is past the cloud plan's limit." + }, { "id": "app.file_info.get.app_error", "translation": "Unable to get the file info." @@ -5639,6 +5643,10 @@ "id": "app.job.update.app_error", "translation": "Unable to update the job." }, + { + "id": "app.last_accessible_file.app_error", + "translation": "Error fetching last accessible file" + }, { "id": "app.last_accessible_post.app_error", "translation": "Error fetching last accessible post" diff --git a/jobs/last_accessible_file/scheduler.go b/jobs/last_accessible_file/scheduler.go new file mode 100644 index 0000000000..d351278c20 --- /dev/null +++ b/jobs/last_accessible_file/scheduler.go @@ -0,0 +1,24 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_file + +import ( + "strconv" + "time" + + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/shared/mlog" +) + +const schedFreq = 2 * time.Hour + +func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + enabled := license != nil && *license.Features.Cloud + mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessibleFile)) + return enabled + } + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessibleFile, schedFreq, isEnabled) +} diff --git a/jobs/last_accessible_file/worker.go b/jobs/last_accessible_file/worker.go new file mode 100644 index 0000000000..177c1016b0 --- /dev/null +++ b/jobs/last_accessible_file/worker.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_file + +import ( + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" +) + +const ( + JobName = "LastAccessibleFile" +) + +type AppIface interface { + ComputeLastAccessibleFileTime() error +} + +func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker { + isEnabled := func(_ *model.Config) bool { + return license != nil && *license.Features.Cloud + } + execute := func(_ *model.Job) error { + return app.ComputeLastAccessibleFileTime() + } + worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled) + return worker +} diff --git a/model/client4.go b/model/client4.go index f00852a77e..cfe0f9244e 100644 --- a/model/client4.go +++ b/model/client4.go @@ -35,6 +35,7 @@ const ( HeaderRequestedWith = "X-Requested-With" HeaderRequestedWithXML = "XMLHttpRequest" HeaderFirstInaccessiblePostTime = "First-Inaccessible-Post-Time" + HeaderFirstInaccessibleFileTime = "First-Inaccessible-File-Time" HeaderRange = "Range" STATUS = "status" StatusOk = "OK" diff --git a/model/file_info.go b/model/file_info.go index f6ae41ab36..4aa0a84fa2 100644 --- a/model/file_info.go +++ b/model/file_info.go @@ -188,3 +188,17 @@ func GetEtagForFileInfos(infos []*FileInfo) string { return Etag(infos[0].PostId, maxUpdateAt) } + +func (fi *FileInfo) MakeContentInaccessible() { + if fi == nil { + return + } + + fi.Archived = true + fi.Content = "" + fi.HasPreviewImage = false + fi.MiniPreview = nil + fi.Path = "" + fi.PreviewPath = "" + fi.ThumbnailPath = "" +} diff --git a/model/file_info_list.go b/model/file_info_list.go index 219b24f1dc..e5de2d076b 100644 --- a/model/file_info_list.go +++ b/model/file_info_list.go @@ -12,6 +12,8 @@ type FileInfoList struct { FileInfos map[string]*FileInfo `json:"file_infos"` NextFileInfoId string `json:"next_file_info_id"` PrevFileInfoId string `json:"prev_file_info_id"` + // If there are inaccessible files, FirstInaccessibleFileTime is the time of the latest inaccessible file + FirstInaccessibleFileTime int64 `json:"first_inaccessible_file_time"` } func NewFileInfoList() *FileInfoList { diff --git a/model/job.go b/model/job.go index 0868643557..9b03430718 100644 --- a/model/job.go +++ b/model/job.go @@ -28,6 +28,7 @@ const ( JobTypeResendInvitationEmail = "resend_invitation_email" JobTypeExtractContent = "extract_content" JobTypeLastAccessiblePost = "last_accessible_post" + JobTypeLastAccessibleFile = "last_accessible_file" JobTypeUpgradeNotifyAdmin = "upgrade_notify_admin" JobTypeTrialNotifyAdmin = "trial_notify_admin" @@ -59,6 +60,7 @@ var AllJobTypes = [...]string{ JobTypeCloud, JobTypeExtractContent, JobTypeLastAccessiblePost, + JobTypeLastAccessibleFile, } type Job struct { diff --git a/model/system.go b/model/system.go index 10370d0e08..0ae731f983 100644 --- a/model/system.go +++ b/model/system.go @@ -32,6 +32,7 @@ const ( SystemFirstAdminVisitMarketplace = "FirstAdminVisitMarketplace" SystemFirstAdminSetupComplete = "FirstAdminSetupComplete" SystemLastAccessiblePostTime = "LastAccessiblePostTime" + SystemLastAccessibleFileTime = "LastAccessibleFileTime" AwsMeteringReportInterval = 1 AwsMeteringDimensionUsageHrs = "UsageHrs" ) diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 45b6277728..eb65b1f798 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -3527,6 +3527,24 @@ func (s *OpenTracingLayerFileInfoStore) GetStorageUsage(allowFromCache bool, inc return result, err } +func (s *OpenTracingLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetUptoNSizeFileTime") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetWithOptions") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 6e544138a2..4efa7172ba 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -3957,6 +3957,27 @@ func (s *RetryLayerFileInfoStore) GetStorageUsage(allowFromCache bool, includeDe } +func (s *RetryLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { tries := 0 diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index 4a6eb7e655..dbf47020e2 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -753,3 +753,49 @@ func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) } return size, nil } + +// GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes. +func (fs *SqlFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + if n <= 0 { + return 0, errors.New("n can't be less than 1") + } + + var sizeSubQuery sq.SelectBuilder + // Separate query for MySql, as current min-version 5.x doesn't support window-functions + if fs.DriverName() == model.DatabaseDriverMysql { + sizeSubQuery = sq. + Select("(@runningSum := @runningSum + fi.Size) RunningTotal", "fi.CreateAt"). + From("FileInfo fi"). + Join("(SELECT @runningSum := 0) as tmp"). + Where(sq.Eq{"fi.DeleteAt": 0}). + OrderBy("fi.CreateAt DESC, fi.Id") + } else { + sizeSubQuery = sq. + Select("SUM(fi.Size) OVER(ORDER BY CreateAt DESC, fi.Id) RunningTotal", "fi.CreateAt"). + From("FileInfo fi"). + Where(sq.Eq{"fi.DeleteAt": 0}) + } + + builder := fs.getQueryBuilder(). + Select("fi2.CreateAt"). + FromSelect(sizeSubQuery, "fi2"). + Where(sq.LtOrEq{"fi2.RunningTotal": n}). + OrderBy("fi2.CreateAt"). + Limit(1) + + query, queryArgs, err := builder.ToSql() + if err != nil { + return 0, errors.Wrap(err, "GetUptoNSizeFileTime_tosql") + } + + var createAt int64 + if err := fs.GetReplicaX().Get(&createAt, query, queryArgs...); err != nil { + if err == sql.ErrNoRows { + return 0, store.NewErrNotFound("File", "none") + } + + return 0, errors.Wrapf(err, "failed to get the File for size upto=%d", n) + } + + return createAt, nil +} diff --git a/store/store.go b/store/store.go index df2473b701..4984612a0b 100644 --- a/store/store.go +++ b/store/store.go @@ -698,6 +698,8 @@ type FileInfoStore interface { GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) ClearCaches() GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) + // GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes. + GetUptoNSizeFileTime(n int64) (int64, error) } type UploadSessionStore interface { diff --git a/store/storetest/file_info_store.go b/store/storetest/file_info_store.go index 34cd2da114..1dc72b0536 100644 --- a/store/storetest/file_info_store.go +++ b/store/storetest/file_info_store.go @@ -11,6 +11,7 @@ import ( "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/store" + "github.com/mattermost/mattermost-server/v6/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +31,7 @@ func TestFileInfoStore(t *testing.T, ss store.Store) { t.Run("GetFilesBatchForIndexing", func(t *testing.T) { testFileInfoStoreGetFilesBatchForIndexing(t, ss) }) t.Run("CountAll", func(t *testing.T) { testFileInfoStoreCountAll(t, ss) }) t.Run("GetStorageUsage", func(t *testing.T) { testFileInfoGetStorageUsage(t, ss) }) + t.Run("GetUptoNSizeFileTime", func(t *testing.T) { testGetUptoNSizeFileTime(t, ss) }) } func testFileInfoSaveGet(t *testing.T, ss store.Store) { @@ -772,3 +774,71 @@ func testFileInfoGetStorageUsage(t *testing.T, ss store.Store) { require.NoError(t, err) require.Equal(t, int64(30), usage) } + +func testGetUptoNSizeFileTime(t *testing.T, ss store.Store) { + _, err := ss.FileInfo().GetUptoNSizeFileTime(0) + assert.Error(t, err) + _, err = ss.FileInfo().GetUptoNSizeFileTime(-1) + assert.Error(t, err) + + _, err = ss.FileInfo().PermanentDeleteBatch(model.GetMillis(), 100000) + require.NoError(t, err) + + diff := int64(10000) + now := utils.MillisFromTime(time.Now()) + diff + + f1, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file1.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + f2, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file2.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + f3, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file3.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + _, err = ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file4.txt", + CreateAt: now, + }) + require.NoError(t, err) + + createAt, err := ss.FileInfo().GetUptoNSizeFileTime(20) + require.NoError(t, err) + assert.Equal(t, f3.CreateAt, createAt) + + _, err = ss.FileInfo().GetUptoNSizeFileTime(5) + assert.Error(t, err) + assert.IsType(t, &store.ErrNotFound{}, err) + + createAt, err = ss.FileInfo().GetUptoNSizeFileTime(1000) + require.NoError(t, err) + assert.Equal(t, f1.CreateAt, createAt) + + _, err = ss.FileInfo().DeleteForPost(f3.PostId) + require.NoError(t, err) + + createAt, err = ss.FileInfo().GetUptoNSizeFileTime(20) + require.NoError(t, err) + assert.Equal(t, f2.CreateAt, createAt) +} diff --git a/store/storetest/mocks/FileInfoStore.go b/store/storetest/mocks/FileInfoStore.go index 97432a11f3..c9627a84ad 100644 --- a/store/storetest/mocks/FileInfoStore.go +++ b/store/storetest/mocks/FileInfoStore.go @@ -257,6 +257,27 @@ func (_m *FileInfoStore) GetStorageUsage(allowFromCache bool, includeDeleted boo return r0, r1 } +// GetUptoNSizeFileTime provides a mock function with given fields: n +func (_m *FileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + ret := _m.Called(n) + + var r0 int64 + if rf, ok := ret.Get(0).(func(int64) int64); ok { + r0 = rf(n) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64) error); ok { + r1 = rf(n) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetWithOptions provides a mock function with given fields: page, perPage, opt func (_m *FileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { ret := _m.Called(page, perPage, opt) diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 3a4e40ed39..0d7b338767 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -3222,6 +3222,22 @@ func (s *TimerLayerFileInfoStore) GetStorageUsage(allowFromCache bool, includeDe return result, err } +func (s *TimerLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + start := time.Now() + + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetUptoNSizeFileTime", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { start := time.Now()