separate visual tests from main test, update snapshots and dependencies (#31131)

e2e-tests/playwright/CLAUDE.md

@@ -29,9 +29,12 @@ npm run test -- <test-name> --project=chrome
 npm run test -- <test-name> --project=firefox
 npm run test -- <test-name> --project=ipad
 
-# Run all tests
+# Run all tests (including visual tests)
 npm run test
 
+# Run CI tests (excludes visual tests, runs only in Chrome)
+npm run test:ci
+
 # Run tests with UI mode
 npm run playwright-ui
 
@@ -45,7 +48,7 @@ npm run test -- visual
 npm run test:update-snapshots
 
 # Visual testing with Percy
-npm run percy
+npm run percy:docker
 ```
 
 ### Development Commands
@@ -135,6 +138,7 @@ Tests can be configured through environment variables:
 - `PW_SNAPSHOT_ENABLE` - Enable snapshot testing (default: false)
 - `PW_SLOWMO` - Add delay between actions in ms (default: 0)
 - `PW_WORKERS` - Number of parallel workers (default: 1)
+- `PERCY_TOKEN` - Authentication token for Percy visual testing service (required for Percy tests)
 
 ## Server Setup
 
@@ -158,9 +162,15 @@ Before running tests, a Mattermost server must be available. Two options:
 
 2. **Visual Testing**: For visual tests:
 
-    - Run via Docker container for consistency
-    - Use `pw.hideDynamicChannelsContent()` to hide dynamic elements
-    - Update snapshots with `npm run test:update-snapshots`
+    - Place all visual tests in the `specs/visual/` directory
+    - Always include the `@visual` tag in the test tags array
+    - Run via Docker container for consistency to maintain screenshot integrity
+    - Use `pw.hideDynamicChannelsContent()` to hide dynamic elements that could cause flaky tests
+    - Update snapshots with `npm run test:update-snapshots` only from within the Docker container
+    - For Percy-based visual testing:
+        - A valid `PERCY_TOKEN` environment variable must be set
+        - Tests should only be run inside the Playwright Docker container
+    - Follow the visual test documentation format like other tests, with proper JSDoc and comments
 
 3. **Test Title Validation with Claude Code**: When using Claude:
 

e2e-tests/playwright/README.md

@@ -57,8 +57,12 @@ npm run test -- login
 # Run a specific test of a project
 npm run test -- login --project=chrome
 
-# Or run all tests
+# Run all tests (including visual tests)
 npm run test
+
+# Run CI tests (excludes visual tests, runs only in Chrome)
+# Note: visual tests run in a separate workflow
+npm run test:ci
 ```
 
 #### 3. Inspect test results at `/results/output` folder when something fails unexpectedly.
@@ -80,6 +84,68 @@ npm run playwright-ui
 >
 > The "setup" project runs the initial configuration tests in `specs/test_setup.ts` (ensuring plugins are loaded and server deployment is correct). These setup tests are typically run only once before other tests and may be unchecked for subsequent runs, though they can remain checked if needed.
 
+## Visual Testing
+
+All visual tests must be placed in the `specs/visual/` directory and tagged with `@visual` in the test tags array. This organization ensures proper test discovery and execution patterns.
+
+Visual tests are used to verify the UI appearance is consistent across browsers and remains stable across code changes. There are two types of visual tests supported:
+
+1. **Built-in snapshot testing**: Uses Playwright's built-in snapshot comparison
+2. **Percy integration**: Uses the Percy service for more advanced visual testing and reporting
+
+### CI Pipeline for Visual Tests
+
+In CI environments, visual tests run in a separate dedicated pipeline:
+
+- Regular tests run with `npm run test:ci` which excludes all tests with the `@visual` tag
+- Visual tests run in a separate workflow using the Playwright Docker container
+- This separation prevents visual tests from slowing down the main test pipeline
+- It also ensures visual tests always run in a consistent environment
+
+### Writing Visual Tests
+
+When creating visual tests:
+
+1. **Follow the test documentation format** like other tests:
+
+    - Include JSDoc with `@objective` tag
+    - Use action-oriented test title
+    - Add proper comment prefixes (`// #` for actions, `// *` for verifications)
+
+2. **Place in the correct location**:
+
+    - Put visual tests in the `specs/visual/` directory, organized by feature area
+    - Example: `specs/visual/channels/intro_channel.spec.ts`
+
+3. **Add required tags**:
+
+    - Always include `@visual` tag
+    - Add feature-specific tags as needed (e.g., `@login_page`, `@channel_page`)
+
+4. **Manage dynamic content**:
+    - Use `pw.hideDynamicChannelsContent()` to hide elements that could change between runs
+    - Take snapshots only after UI is fully loaded and stable
+
+Example:
+
+```typescript
+/**
+ * @objective Capture visual snapshot of the landing/login page
+ */
+test(
+    'displays landing page with login options',
+    {tag: ['@visual', '@landing_page']},
+    async ({pw, page, browserName, viewport}, testInfo) => {
+        // # Go to landing login page
+        await pw.landingLoginPage.goto();
+        await pw.landingLoginPage.toBeVisible();
+
+        // * Verify landing page appears as expected
+        await pw.matchSnapshot(testInfo, {page, browserName, viewport});
+    },
+);
+```
+
 ## Updating screenshots is done strictly via Playwright's docker container for consistency
 
 #### 1. Run Playwright's docker container
@@ -106,11 +172,15 @@ npm run test -- login --project=chrome
 # Or run all tests
 npm run test
 
-# Run visual tests
-npm run test -- visual
+# Run visual tests (must be run inside Docker for consistency)
+npm run test -- specs/visual
 
-# Update snapshots of visual tests
-npm run test -- visual --update-snapshots
+# Update snapshots of visual tests (must be run inside Docker)
+npm run test -- specs/visual --update-snapshots
+
+# Run Percy visual tests (requires PERCY_TOKEN environment variable)
+export PERCY_TOKEN=<your-percy-token>
+npm run percy:docker
 ```
 
 ## Page/Component Object Model

e2e-tests/playwright/lib/package.json

@@ -45,7 +45,7 @@
     "@axe-core/playwright": "4.10.1",
     "@mattermost/client": "file:../../../webapp/platform/client",
     "@mattermost/types": "file:../../../webapp/platform/types",
-    "@percy/cli": "1.30.10",
+    "@percy/cli": "1.30.11",
     "@percy/playwright": "1.0.8",
     "async-wait-until": "2.0.27",
     "axe-core": "4.10.3",
@@ -57,9 +57,9 @@
   "devDependencies": {
     "@rollup/plugin-typescript": "12.1.2",
     "@types/mime-types": "2.1.4",
-    "@types/node": "22.15.14",
-    "@types/react": "19.1.3",
-    "rollup": "4.40.2",
+    "@types/node": "22.15.21",
+    "@types/react": "19.1.4",
+    "rollup": "4.41.0",
     "rollup-plugin-copy": "3.5.0"
   },
   "peerDependencies": {

e2e-tests/playwright/package.json

@@ -14,10 +14,10 @@
     "prettier:fix": "prettier --write .",
     "check": "npm run lint && npm run prettier && npm run tsc && npm run lint:test-docs",
     "test": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true playwright test",
-    "test:ci": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true playwright test --project=chrome",
-    "test:update-snapshots": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true playwright test --update-snapshots",
+    "test:ci": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true playwright test --grep-invert @visual --project=chrome",
+    "test:update-snapshots": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true playwright test specs/visual --grep @visual --update-snapshots",
     "test:slomo": "npm run build && cross-env PW_SNAPSHOT_ENABLE=true PW_SLOWMO=1000 playwright test",
-    "percy": "npm run build && cross-env PERCY_TOKEN=$PERCY_TOKEN PW_PERCY_ENABLE=true percy exec -- playwright test --project=chrome --project=ipad",
+    "percy:docker": "npm run build && cross-env PW_PERCY_ENABLE=true PERCY_BROWSER_EXECUTABLE='/ms-playwright/chromium-1169/chrome-linux/chrome' percy exec -- playwright test specs/visual --grep @visual --project=chrome --project=ipad",
     "codegen": "npm run build && cross-env playwright codegen $PW_BASE_URL",
     "playwright-ui": "npm run build && cross-env playwright test --ui",
     "show-report": "npm run build && npx playwright show-report results/reporter",
@@ -31,13 +31,13 @@
     "cross-env": "7.0.3",
     "dayjs": "1.11.13",
     "luxon": "3.6.1",
-    "zod": "3.24.4"
+    "zod": "3.25.13"
   },
   "devDependencies": {
     "@types/luxon": "3.6.2",
-    "@typescript-eslint/eslint-plugin": "8.32.0",
-    "eslint": "9.26.0",
-    "eslint-import-resolver-typescript": "4.3.4",
+    "@typescript-eslint/eslint-plugin": "8.32.1",
+    "eslint": "9.27.0",
+    "eslint-import-resolver-typescript": "4.3.5",
     "eslint-plugin-header": "3.1.1",
     "eslint-plugin-import": "2.31.0",
     "glob": "11.0.2",

e2e-tests/playwright/specs/visual/channels/intro_channel.spec.ts

@@ -3,24 +3,31 @@
 
 import {expect, test} from '@mattermost/playwright-lib';
 
-test('Intro to channel as regular user', async ({pw, browserName, viewport}, testInfo) => {
-    // Create and sign in a new user
-    const {user} = await pw.initSetup();
+/**
+ * @objective Capture visual snapshot of the intro channel view for a regular user
+ */
+test(
+    'displays intro to channel view for regular user',
+    {tag: ['@visual', '@channel_page']},
+    async ({pw, browserName, viewport}, testInfo) => {
+        // # Create and sign in a new user
+        const {user} = await pw.initSetup();
 
-    // Log in a user in new browser context
-    const {page, channelsPage} = await pw.testBrowser.login(user);
+        // # Log in a user in new browser context
+        const {page, channelsPage} = await pw.testBrowser.login(user);
 
-    // Visit a default channel page
-    await channelsPage.goto();
-    await channelsPage.toBeVisible();
+        // # Visit a default channel page
+        await channelsPage.goto();
+        await channelsPage.toBeVisible();
 
-    // Wait for Playbooks icon to be loaded in App bar, except in iphone
-    await expect(channelsPage.appBar.playbooksIcon).toBeVisible();
+        // # Wait for Playbooks icon to be loaded in App bar
+        await expect(channelsPage.appBar.playbooksIcon).toBeVisible();
 
-    // Hide dynamic elements of Channels page
-    await pw.hideDynamicChannelsContent(page);
+        // # Hide dynamic elements of Channels page
+        await pw.hideDynamicChannelsContent(page);
 
-    // Match snapshot of channel intro page
-    const testArgs = {page: page, browserName, viewport};
-    await pw.matchSnapshot(testInfo, testArgs);
-});
+        // * Verify channel intro page appears as expected
+        const testArgs = {page: page, browserName, viewport};
+        await pw.matchSnapshot(testInfo, testArgs);
+    },
+);

e2e-tests/playwright/specs/visual/common/landing_page.spec.ts

@@ -3,11 +3,18 @@
 
 import {test} from '@mattermost/playwright-lib';
 
-test('/landing#/login', async ({pw, page, browserName, viewport}, testInfo) => {
-    // Go to landing login page
-    await pw.landingLoginPage.goto();
-    await pw.landingLoginPage.toBeVisible();
+/**
+ * @objective Capture visual snapshot of the landing page
+ */
+test(
+    'landing page visual check',
+    {tag: ['@visual', '@landing_page']},
+    async ({pw, page, browserName, viewport}, testInfo) => {
+        // # Go to landing login page
+        await pw.landingLoginPage.goto();
+        await pw.landingLoginPage.toBeVisible();
 
-    // Match snapshot of landing page
-    await pw.matchSnapshot(testInfo, {page, browserName, viewport});
-});
+        // * Verify landing page appears as expected
+        await pw.matchSnapshot(testInfo, {page, browserName, viewport});
+    },
+);

e2e-tests/playwright/specs/visual/common/login.spec.ts

@@ -3,29 +3,38 @@
 
 import {test} from '@mattermost/playwright-lib';
 
-test('/login', async ({pw, page, browserName, viewport}, testInfo) => {
-    // Set up the page not to redirect to the landing page
-    await pw.hasSeenLandingPage();
+/**
+ * @objective Capture visual snapshots of the login page in normal and error states
+ */
+test(
+    'login page visual check',
+    {tag: ['@visual', '@login_page']},
+    async ({pw, page, browserName, viewport}, testInfo) => {
+        // # Set up the page not to redirect to the landing page
+        await pw.hasSeenLandingPage();
 
-    // Go to login page
-    const {adminClient} = await pw.getAdminClient();
-    await pw.loginPage.goto();
-    await pw.loginPage.toBeVisible();
+        // # Go to login page
+        const {adminClient} = await pw.getAdminClient();
+        await pw.loginPage.goto();
+        await pw.loginPage.toBeVisible();
 
-    // Click to other element to remove focus from email input
-    await pw.loginPage.title.click();
+        // # Click to other element to remove focus from email input
+        await pw.loginPage.title.click();
 
-    // Match snapshot of login page
-    const testArgs = {page, browserName, viewport};
-    const license = await adminClient.getClientLicenseOld();
-    const editionSuffix = license.IsLicensed === 'true' ? '' : 'free edition';
-    await pw.matchSnapshot({...testInfo, title: `${testInfo.title} ${editionSuffix}`}, testArgs);
+        // # Get license information and prepare test args
+        const testArgs = {page, browserName, viewport};
+        const license = await adminClient.getClientLicenseOld();
+        const editionSuffix = license.IsLicensed === 'true' ? '' : 'free edition';
 
-    // Click sign in button without entering user credential
-    await pw.loginPage.signInButton.click();
-    await pw.loginPage.userErrorLabel.waitFor();
-    await pw.waitForAnimationEnd(pw.loginPage.bodyCard);
+        // * Verify login page appears as expected
+        await pw.matchSnapshot({...testInfo, title: `${testInfo.title} ${editionSuffix}`}, testArgs);
 
-    // Match snapshot of login page with error
-    await pw.matchSnapshot({...testInfo, title: `${testInfo.title} error ${editionSuffix}`}, testArgs);
-});
+        // # Click sign in button without entering user credential
+        await pw.loginPage.signInButton.click();
+        await pw.loginPage.userErrorLabel.waitFor();
+        await pw.waitForAnimationEnd(pw.loginPage.bodyCard);
+
+        // * Verify login page with error appears as expected
+        await pw.matchSnapshot({...testInfo, title: `${testInfo.title} error ${editionSuffix}`}, testArgs);
+    },
+);

e2e-tests/playwright/specs/visual/common/signup_email.spec.ts

@@ -4,11 +4,11 @@
 import {test} from '@mattermost/playwright-lib';
 
 /**
- * @objective Verify the appearance of the signup email page in normal and error states
+ * @objective Capture visual snapshots of the signup email page in normal and error states
  */
 test(
-    'signup_email visual verification',
-    {tag: '@visual_signup'},
+    'signup page visual check',
+    {tag: ['@visual', '@signup_email_page']},
     async ({pw, page, browserName, viewport}, testInfo) => {
         // # Set up the page not to redirect to the landing page
         await pw.hasSeenLandingPage();