MM-28067: Optimize app server startup in tests (#16263)
* MM-28067: Optimize app server startup in tests For every test, we would wipe out the database and then start the server again. This would run through all the migrations and create new rows in Systems and Roles table every time. As a result, this was consuming a lot of setup time for every test. We optimize this by preloading the DB with dummy data in the roles and systems table so that the server code just skips over those migrations. This is completely forward compatible and adding new migrations does not need to generate the sql files again. Only in case of schema changes to the roles or systems table, this would need to be done. It is unlikely the Systems table schema will get changed. The Roles table might change in future but it's a comparatively rare event. Given the reduction in CI runtime we are seeing, it's a worthy optimization. We also apply some more optimizations: - Coalesce multiple UpdateConfig calls into a single one. Each UpdateConfig call has to do a json marshal which would take precious CPU cycles. It's much more efficient to do everything in a single call. - Remove unnecessary debug.FreeOSMemory in reload config. This was an artifact from old days and is no longer required. Numbers: Results show a full **2 minutes** shaved off the test runtime. Earlier, tests would take around 16 minutes. Now they take 14 minutes. ```release-note NONE ``` * fix app package
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
e162c6c787
Коммит
c82c37a36e
@@ -6,6 +6,7 @@ package testlib
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -122,6 +123,42 @@ func (h *MainHelper) setupResources() {
|
||||
}
|
||||
}
|
||||
|
||||
// PreloadMigrations preloads the migrations and roles into the database
|
||||
// so that they are not run again when the migrations happen every time
|
||||
// the server is started.
|
||||
// This change is forward-compatible with new migrations and only new migrations
|
||||
// will get executed.
|
||||
// Only if the schema of either roles or systems table changes, this will break.
|
||||
// In that case, just update the migrations or comment this out for the time being.
|
||||
// In the worst case, only an optimization is lost.
|
||||
//
|
||||
// Re-generate the files with:
|
||||
// pg_dump -a -h localhost -U mmuser -d <> --no-comments --inserts -t roles -t systems
|
||||
// mysqldump -u root -p <> --no-create-info --extended-insert=FALSE Systems Roles
|
||||
// And keep only the permission related rows in the systems table output.
|
||||
func (h *MainHelper) PreloadMigrations() {
|
||||
var buf []byte
|
||||
var err error
|
||||
switch *h.Settings.DriverName {
|
||||
case model.DATABASE_DRIVER_POSTGRES:
|
||||
buf, err = ioutil.ReadFile("mattermost-server/testlib/testdata/postgres_migration_warmup.sql")
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot read file: %v", err))
|
||||
}
|
||||
case model.DATABASE_DRIVER_MYSQL:
|
||||
buf, err = ioutil.ReadFile("mattermost-server/testlib/testdata/mysql_migration_warmup.sql")
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot read file: %v", err))
|
||||
}
|
||||
}
|
||||
handle := h.SQLSupplier.GetMaster()
|
||||
_, err = handle.Exec(string(buf))
|
||||
if err != nil {
|
||||
mlog.Error("Error preloading migrations. Check if you have &multiStatements=true in your DSN if you are using MySQL. Or perhaps the schema changed? If yes, then update the warmup files accordingly.")
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *MainHelper) Close() error {
|
||||
if h.SQLSupplier != nil {
|
||||
h.SQLSupplier.Close()
|
||||
|
||||
Ссылка в новой задаче
Block a user