Update schemas to include new indexes (#18313)
* Update schemas to include new indexes * Increase local usability of gitlab scripts. (#18314) * Increase local usability of gitlab scripts. * Fix loading/saving SQL dumps * Add logging to schema tests. * Fix adding logs dir. Co-authored-by: Claudio Costa <cstcld91@gmail.com> Co-authored-by: Elisabeth Kulzer <elikul@elikul.de>
Этот коммит содержится в:
Claudio Costa
коммит произвёл
GitHub
GitHub
родитель
b6564fec6d
Коммит
9534f9fb62
@@ -1,40 +1,51 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
echo $DOCKER_HOST
|
||||
if [[ "$GITLAB_CI" == "" ]]; then
|
||||
export CI_PROJECT_DIR=$PWD
|
||||
export CI_REGISTRY=registry.internal.mattermost.com
|
||||
export COMPOSE_PROJECT_NAME="1schemamysql"
|
||||
export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7
|
||||
# You need to log in to internal registry to run this script locally
|
||||
fi
|
||||
|
||||
echo "$DOCKER_HOST"
|
||||
docker ps
|
||||
DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
|
||||
DOCKER_NETWORK=$COMPOSE_PROJECT_NAME
|
||||
DOCKER_COMPOSE_FILE="gitlab-dc.schemamysql.yml"
|
||||
CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
|
||||
docker network create ${DOCKER_NETWORK}
|
||||
docker network create $DOCKER_NETWORK
|
||||
ulimit -n 8096
|
||||
cd ${CI_PROJECT_DIR}/build
|
||||
cd "$CI_PROJECT_DIR"/build
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
|
||||
sleep 5
|
||||
docker run --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
||||
docker run --net $DOCKER_NETWORK "$CI_REGISTRY"/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
||||
|
||||
echo "Creating databases"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
|
||||
echo "Importing mysql dump from version 6.0"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -D migrated -uroot -pmostest < ${CI_PROJECT_DIR}/scripts/mattermost-mysql-6.0.sql
|
||||
docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T mysql mysql -D migrated -uroot -pmostest < "$CI_PROJECT_DIR"/scripts/mattermost-mysql-6.0.sql
|
||||
docker run -d -it --rm --name "$CONTAINER_SERVER" --net $DOCKER_NETWORK \
|
||||
--env-file="dotenv/test-schema-validation.env" \
|
||||
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
|
||||
--env MM_SQLSETTINGS_DRIVERNAME=mysql \
|
||||
-v $CI_PROJECT_DIR:/mattermost-server \
|
||||
-v "$CI_PROJECT_DIR":/mattermost-server \
|
||||
-w /mattermost-server \
|
||||
$IMAGE_BUILD_SERVER \
|
||||
bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='version' run-cli"
|
||||
docker logs -f "${CONTAINER_SERVER}"
|
||||
mkdir -p logs
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
docker logs -f $CONTAINER_SERVER
|
||||
tar -czvf logs/docker_logs$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
|
||||
echo "Generating dump"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
|
||||
echo "Removing databases created for db comparison"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
|
||||
|
||||
echo "Generating diff"
|
||||
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
||||
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE down
|
||||
docker network remove ${DOCKER_NETWORK}
|
||||
docker network remove $DOCKER_NETWORK
|
||||
|
||||
@@ -1,38 +1,50 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
if [[ "$GITLAB_CI" == "" ]]; then
|
||||
export CI_PROJECT_DIR=$PWD
|
||||
export CI_REGISTRY=registry.internal.mattermost.com
|
||||
export COMPOSE_PROJECT_NAME="1schemapostgres"
|
||||
export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7
|
||||
# You need to log in to internal registry to run this script locally
|
||||
fi
|
||||
|
||||
echo $DOCKER_HOST
|
||||
docker ps
|
||||
DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
|
||||
DOCKER_NETWORK=$COMPOSE_PROJECT_NAME
|
||||
DOCKER_COMPOSE_FILE="gitlab-dc.schemapostgres.yml"
|
||||
CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
|
||||
docker network create ${DOCKER_NETWORK}
|
||||
docker network create $DOCKER_NETWORK
|
||||
ulimit -n 8096
|
||||
cd ${CI_PROJECT_DIR}/build
|
||||
cd "$CI_PROJECT_DIR"/build
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
|
||||
timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done"
|
||||
|
||||
echo "Creating databases"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
||||
echo "Importing postgres dump from version 6.0"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres psql -U mmuser -d migrated < ${CI_PROJECT_DIR}/scripts/mattermost-postgresql-6.0.sql
|
||||
docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T postgres psql -U mmuser -d migrated < "$CI_PROJECT_DIR"/scripts/mattermost-postgresql-6.0.sql
|
||||
docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \
|
||||
--env-file="dotenv/test-schema-validation.env" \
|
||||
--env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
|
||||
--env MM_SQLSETTINGS_DRIVERNAME=postgres \
|
||||
-v $CI_PROJECT_DIR:/mattermost-server \
|
||||
-v "$CI_PROJECT_DIR":/mattermost-server \
|
||||
-w /mattermost-server \
|
||||
$IMAGE_BUILD_SERVER \
|
||||
bash -c "ulimit -n 8096; make ARGS='version' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='version' run-cli"
|
||||
docker logs -f "${CONTAINER_SERVER}"
|
||||
mkdir -p logs
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
docker logs -f $CONTAINER_SERVER
|
||||
tar -czvf logs/docker_logs$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
|
||||
echo "Generating dump"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
|
||||
echo "Removing databases created for db comparison"
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -d -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
||||
echo "Generating diff"
|
||||
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
||||
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE down
|
||||
docker network remove ${DOCKER_NETWORK}
|
||||
docker stop
|
||||
docker network remove $DOCKER_NETWORK
|
||||
|
||||
@@ -1,36 +1,44 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
echo $DOCKER_HOST
|
||||
if [[ "$GITLAB_CI" == "" ]]; then
|
||||
export CI_PROJECT_DIR=$PWD
|
||||
export CI_REGISTRY=registry.internal.mattermost.com
|
||||
export COMPOSE_PROJECT_NAME="1mysql"
|
||||
export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7
|
||||
# You need to log in to internal registry to run this script locally
|
||||
fi
|
||||
|
||||
echo "$DOCKER_HOST"
|
||||
docker ps
|
||||
DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
|
||||
DOCKER_NETWORK=$COMPOSE_PROJECT_NAME
|
||||
DOCKER_COMPOSE_FILE="gitlab-dc.mysql.yml"
|
||||
CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
|
||||
docker network create ${DOCKER_NETWORK}
|
||||
docker network create $DOCKER_NETWORK
|
||||
ulimit -n 8096
|
||||
cd ${CI_PROJECT_DIR}/build
|
||||
cd "$CI_PROJECT_DIR"/build
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
|
||||
sleep 5
|
||||
cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
||||
docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test';
|
||||
docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_mysql" --net ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
||||
docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
||||
cat ../tests/test-data.ldif | docker-compose exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
||||
docker-compose exec -T minio sh -c 'mkdir -p /data/mattermost-test';
|
||||
docker run --name "${COMPOSE_PROJECT_NAME}_curl_mysql" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
||||
docker run --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
||||
|
||||
docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
|
||||
docker run -d -it --rm --name $CONTAINER_SERVER --net $DOCKER_NETWORK \
|
||||
--env-file="dotenv/test.env" \
|
||||
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true" \
|
||||
--env MM_SQLSETTINGS_DRIVERNAME=mysql \
|
||||
-v ${CI_PROJECT_DIR}:/mattermost-server \
|
||||
-v "$CI_PROJECT_DIR":/mattermost-server \
|
||||
-w /mattermost-server \
|
||||
$IMAGE_BUILD_SERVER \
|
||||
bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \
|
||||
bash -c scripts/diff-email-templates.sh
|
||||
mkdir -p logs
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
|
||||
docker ps -a --no-trunc > logs/docker_ps
|
||||
docker stats -a --no-stream > logs/docker_stats
|
||||
docker logs -f "${CONTAINER_SERVER}"
|
||||
tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
docker ps -a --no-trunc > logs/docker_ps_$COMPOSE_PROJECT_NAME
|
||||
docker stats -a --no-stream > logs/docker_stats_$COMPOSE_PROJECT_NAME
|
||||
docker logs -f $CONTAINER_SERVER
|
||||
tar -czvf logs/docker_logs_$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME logs/docker_ps_$COMPOSE_PROJECT_NAME logs/docker_stats_$COMPOSE_PROJECT_NAME
|
||||
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE down
|
||||
docker network remove ${DOCKER_NETWORK}
|
||||
docker network remove $DOCKER_NETWORK
|
||||
|
||||
34
.gitlab-ci/scripts/test/postgres.sh
Обычный файл → Исполняемый файл
34
.gitlab-ci/scripts/test/postgres.sh
Обычный файл → Исполняемый файл
@@ -1,35 +1,43 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
echo $DOCKER_HOST
|
||||
if [[ "$GITLAB_CI" == "" ]]; then
|
||||
export CI_PROJECT_DIR=$PWD
|
||||
export CI_REGISTRY=registry.internal.mattermost.com
|
||||
export COMPOSE_PROJECT_NAME="1postgres"
|
||||
export IMAGE_BUILD_SERVER=$CI_REGISTRY/mattermost/ci/images/mattermost-build-server:20210810_golang-1.16.7
|
||||
# You need to log in to internal registry to run this script locally
|
||||
fi
|
||||
|
||||
echo "$DOCKER_HOST"
|
||||
docker ps
|
||||
DOCKER_NETWORK="${COMPOSE_PROJECT_NAME}"
|
||||
DOCKER_NETWORK=$COMPOSE_PROJECT_NAME
|
||||
DOCKER_COMPOSE_FILE="gitlab-dc.postgres.yml"
|
||||
CONTAINER_SERVER="${COMPOSE_PROJECT_NAME}_server_1"
|
||||
docker network create ${DOCKER_NETWORK}
|
||||
docker network create $DOCKER_NETWORK
|
||||
ulimit -n 8096
|
||||
cd ${CI_PROJECT_DIR}/build
|
||||
cd "$CI_PROJECT_DIR"/build
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE run -d --rm start_dependencies
|
||||
sleep 5
|
||||
cat ../tests/test-data.ldif | docker-compose exec -d -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
||||
docker-compose exec -d -T minio sh -c 'mkdir -p /data/mattermost-test';
|
||||
timeout 90s bash -c "until docker exec ${COMPOSE_PROJECT_NAME}_postgres_1 pg_isready ; do sleep 5 ; done"
|
||||
docker run -d --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net ${DOCKER_NETWORK} ${CI_REGISTRY}/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
||||
docker run -d -it --rm --name "${CONTAINER_SERVER}" --net ${DOCKER_NETWORK} \
|
||||
docker run --name "${COMPOSE_PROJECT_NAME}_curl_elasticsearch" --net $DOCKER_NETWORK $CI_REGISTRY/mattermost/ci/images/curl:7.59.0-1 sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
||||
docker run -d -it --rm --name "${CONTAINER_SERVER}" --net $DOCKER_NETWORK \
|
||||
--env-file="dotenv/test.env" \
|
||||
--env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
|
||||
--env MM_SQLSETTINGS_DRIVERNAME=postgres \
|
||||
-v $CI_PROJECT_DIR:/mattermost-server \
|
||||
-v "$CI_PROJECT_DIR":/mattermost-server \
|
||||
-w /mattermost-server \
|
||||
$IMAGE_BUILD_SERVER \
|
||||
bash -c "ulimit -n 8096; make test-server$RACE_MODE BUILD_NUMBER=$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA TESTFLAGS= TESTFLAGSEE=" \
|
||||
bash -c scripts/diff-email-templates.sh
|
||||
mkdir -p logs
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
|
||||
docker ps -a --no-trunc > logs/docker_ps
|
||||
docker stats -a --no-stream > logs/docker_stats
|
||||
docker logs -f "${CONTAINER_SERVER}"
|
||||
tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
|
||||
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs_$COMPOSE_PROJECT_NAME
|
||||
docker ps -a --no-trunc > logs/docker_ps_$COMPOSE_PROJECT_NAME
|
||||
docker stats -a --no-stream > logs/docker_stats_$COMPOSE_PROJECT_NAME
|
||||
docker logs -f $CONTAINER_SERVER
|
||||
tar -czvf logs/docker_logs_$COMPOSE_PROJECT_NAME.tar.gz logs/docker-compose_logs_$COMPOSE_PROJECT_NAME logs/docker_ps_$COMPOSE_PROJECT_NAME logs/docker_stats_$COMPOSE_PROJECT_NAME
|
||||
|
||||
docker-compose -f $DOCKER_COMPOSE_FILE down
|
||||
docker network remove ${DOCKER_NETWORK}
|
||||
docker network remove $DOCKER_NETWORK
|
||||
|
||||
Ссылка в новой задаче
Block a user