MM-64282 E2E/Playwright: Test documentation format (#31050)
* initial implementation of test documentation in spec file with AI-assisted prompt from Claude and linter script * update snapshots --------- Co-authored-by: Mattermost Build <build@mattermost.com>
Этот коммит содержится в:
sabril
386
e2e-tests/playwright/script/lint-test-docs.js
Исполняемый файл
386
e2e-tests/playwright/script/lint-test-docs.js
Исполняемый файл
@@ -0,0 +1,386 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/**
|
||||
* Test Documentation Format Linter
|
||||
*
|
||||
* This script verifies that all spec files follow the required documentation format:
|
||||
* - JSDoc with @objective and @precondition
|
||||
* - Proper test title with MM-T ID
|
||||
* - Tag for feature categorization
|
||||
* - Action/Verification comments
|
||||
*/
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-require-imports */
|
||||
/* eslint-disable import/order */
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const glob = require('glob');
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
// Colors for terminal output
|
||||
const colors = {
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
reset: '\x1b[0m',
|
||||
cyan: '\x1b[36m',
|
||||
magenta: '\x1b[35m',
|
||||
white: '\x1b[37m',
|
||||
};
|
||||
|
||||
// Regex patterns
|
||||
const patterns = {
|
||||
// Support both single-line and multi-line test declarations
|
||||
testDeclaration:
|
||||
/test\(\s*['"]([^'"]+)['"]\s*,(?:\s*|\n\s*){.*?tag:\s*['"]@.*?['"].*?}(?:\s*|\n\s*),(?:\s*|\n\s*)async/g,
|
||||
jsdocBlock: /\/\*\*\s*\n(?:.*\n)*?\s*\*\//g,
|
||||
objective: /@objective\s+.*?(?=\n|\*\/)/g,
|
||||
precondition: /@precondition\s+.*?(?=\n|\*\/)/g,
|
||||
tagDeclaration: /{tag:\s*['"]@[\w_]+['"]}|{\s*tag:\s*['"]@[\w_]+['"]\s*}/g,
|
||||
// Match action comments - note the difference between # as a comment marker vs # in strings
|
||||
actionComment: /(?:^|\n)\s*\/\/\s*#\s*.+/g,
|
||||
// Match verification comments - note the difference between * as a comment marker vs * in strings
|
||||
verificationComment: /(?:^|\n)\s*\/\/\s*\*\s*.+/g,
|
||||
};
|
||||
|
||||
// Get all spec files
|
||||
const specFiles = glob.sync(path.join(process.cwd(), 'specs/**/*.spec.ts'));
|
||||
|
||||
// Results
|
||||
const results = {
|
||||
passed: 0,
|
||||
failed: 0,
|
||||
warnings: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
// Process each file
|
||||
specFiles.forEach((filePath) => {
|
||||
const relativeFilePath = path.relative(process.cwd(), filePath);
|
||||
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
||||
const fileErrors = [];
|
||||
|
||||
// Extract all test declarations
|
||||
const testDeclarations = Array.from(fileContent.matchAll(patterns.testDeclaration));
|
||||
const jsdocBlocks = Array.from(fileContent.matchAll(patterns.jsdocBlock));
|
||||
|
||||
// If no test declarations found, warn but don't fail
|
||||
if (testDeclarations.length === 0) {
|
||||
results.warnings++;
|
||||
console.log(
|
||||
`${colors.yellow}WARNING${colors.reset}: No test declarations found in ${colors.cyan}${relativeFilePath}${colors.reset}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check each test for proper format
|
||||
testDeclarations.forEach((testMatch) => {
|
||||
const testDeclaration = testMatch[0];
|
||||
const testName = testMatch[1];
|
||||
|
||||
// Get test function position to find corresponding JSDoc
|
||||
const testPosition = testMatch.index;
|
||||
|
||||
// Get the JSDoc block that immediately precedes this test
|
||||
const precedingJSDoc = jsdocBlocks.find((jsdoc) => {
|
||||
const jsdocEnd = jsdoc.index + jsdoc[0].length;
|
||||
// JSDoc should end right before the test with only whitespace in between
|
||||
const textBetween = fileContent.substring(jsdocEnd, testPosition).trim();
|
||||
return textBetween === '';
|
||||
});
|
||||
|
||||
// Check if test has a test key (MM-T####)
|
||||
if (!testName.match(/^MM-T\d+/)) {
|
||||
// This is a new test without a test key
|
||||
console.log(
|
||||
`${colors.cyan}NEW TEST FOUND: "${testName}"${colors.reset} - Will be registered in the test management system after merge`,
|
||||
);
|
||||
results.newTests = results.newTests || [];
|
||||
results.newTests.push({file: relativeFilePath, testName});
|
||||
} else {
|
||||
// This is an existing test with a test key
|
||||
const baseTestKey = testName.match(/^(MM-T\d+)/)[1];
|
||||
|
||||
// Check if this is a step of a multi-step test case
|
||||
const stepMatch = testName.match(/^MM-T\d+(_\d+)/);
|
||||
const isStep = stepMatch !== null;
|
||||
const stepSuffix = isStep ? stepMatch[1] : '';
|
||||
const testKey = baseTestKey + (stepSuffix || '');
|
||||
|
||||
// Log different message for test steps
|
||||
if (isStep) {
|
||||
// Extract the step number without the underscore
|
||||
const stepNumber = stepSuffix.substring(1);
|
||||
console.log(
|
||||
`${colors.magenta}DOCUMENTATION UPDATE: "${testKey}"${colors.reset} - step ${stepNumber} of ${baseTestKey} - Changes will be mapped to test management system after merge`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.magenta}DOCUMENTATION UPDATE: "${testKey}"${colors.reset} - Changes will be saved to test management system after merge`,
|
||||
);
|
||||
}
|
||||
|
||||
// Check if documentation is present and mark for update
|
||||
if (precedingJSDoc) {
|
||||
results.updatedTests = results.updatedTests || [];
|
||||
results.updatedTests.push({
|
||||
file: relativeFilePath,
|
||||
testKey,
|
||||
baseTestKey,
|
||||
isStep,
|
||||
stepSuffix,
|
||||
testName,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// JSDoc was already retrieved above, no need to get it again
|
||||
|
||||
if (!precedingJSDoc) {
|
||||
fileErrors.push(`Missing JSDoc documentation at "${testName}"`);
|
||||
} else {
|
||||
const jsdocContent = precedingJSDoc[0];
|
||||
|
||||
// Check for @objective
|
||||
if (!jsdocContent.match(patterns.objective)) {
|
||||
fileErrors.push(`Missing @objective in JSDoc at "${testName}"`);
|
||||
}
|
||||
|
||||
// Note: @precondition is optional and should only be included when there are
|
||||
// non-default requirements for the test
|
||||
}
|
||||
|
||||
// Check for tag declaration
|
||||
if (!testDeclaration.match(patterns.tagDeclaration)) {
|
||||
fileErrors.push(`Missing feature tag at "${testName}"`);
|
||||
}
|
||||
|
||||
// Simpler approach to extract test body by using string search and tracking braces to find function boundaries
|
||||
const testNameEscaped = testName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const testDeclarationPattern = new RegExp(`test\\s*\\([\\s\\n]*['"]${testNameEscaped}['"]`);
|
||||
|
||||
// Find the position of the test declaration
|
||||
const testDeclarationMatch = testDeclarationPattern.exec(fileContent);
|
||||
let testFnMatch = null;
|
||||
|
||||
if (testDeclarationMatch) {
|
||||
// Find the position of "async" after the test declaration
|
||||
const startPos = testDeclarationMatch.index;
|
||||
const asyncPattern = /async\s*\([^)]*\)\s*=>\s*{/g;
|
||||
asyncPattern.lastIndex = startPos;
|
||||
|
||||
const asyncMatch = asyncPattern.exec(fileContent);
|
||||
if (asyncMatch) {
|
||||
// Find the position of the opening brace of the function body
|
||||
const openBracePos = asyncMatch.index + asyncMatch[0].length - 1;
|
||||
|
||||
// Find the matching closing brace
|
||||
let braceCount = 1;
|
||||
let currentPos = openBracePos + 1;
|
||||
|
||||
while (braceCount > 0 && currentPos < fileContent.length) {
|
||||
const char = fileContent[currentPos];
|
||||
|
||||
if (char === '{') {
|
||||
braceCount++;
|
||||
} else if (char === '}') {
|
||||
braceCount--;
|
||||
}
|
||||
|
||||
currentPos++;
|
||||
}
|
||||
|
||||
// If we found the matching closing brace, extract everything between them
|
||||
if (braceCount === 0) {
|
||||
const testBody = fileContent.substring(openBracePos + 1, currentPos - 1);
|
||||
testFnMatch = [null, testBody];
|
||||
}
|
||||
}
|
||||
|
||||
// If we couldn't extract using the above method, try a more direct string search approach
|
||||
if (!testFnMatch) {
|
||||
// Find the test function by searching for the test name and extracting all content until the end of the function
|
||||
const testIndex =
|
||||
fileContent.indexOf(`test('${testName}'`) ||
|
||||
fileContent.indexOf(`test("${testName}"`) ||
|
||||
fileContent.indexOf(`test(\n '${testName}'`) ||
|
||||
fileContent.indexOf(`test(\n "${testName}"`);
|
||||
|
||||
if (testIndex !== -1) {
|
||||
// Find the position of "async" which indicates the start of the function body
|
||||
const asyncIndex = fileContent.indexOf('async', testIndex);
|
||||
if (asyncIndex !== -1) {
|
||||
// Find the opening brace after "async"
|
||||
const openBraceIndex = fileContent.indexOf('{', asyncIndex);
|
||||
if (openBraceIndex !== -1) {
|
||||
// Count braces to find the matching closing brace
|
||||
let braceCount = 1;
|
||||
let closePos = openBraceIndex + 1;
|
||||
|
||||
while (braceCount > 0 && closePos < fileContent.length) {
|
||||
const char = fileContent[closePos];
|
||||
|
||||
if (char === '{') {
|
||||
braceCount++;
|
||||
} else if (char === '}') {
|
||||
braceCount--;
|
||||
}
|
||||
|
||||
closePos++;
|
||||
}
|
||||
|
||||
if (braceCount === 0) {
|
||||
const testBody = fileContent.substring(openBraceIndex + 1, closePos - 1);
|
||||
testFnMatch = [null, testBody];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (testFnMatch) {
|
||||
const testBody = testFnMatch[1];
|
||||
|
||||
// Check for action comments (// #) - we need to be more flexible in our detection
|
||||
let hasActionComments = false;
|
||||
const actionPattern = /\/\/\s*#/;
|
||||
|
||||
if (actionPattern.test(testBody)) {
|
||||
hasActionComments = true;
|
||||
}
|
||||
|
||||
if (!hasActionComments) {
|
||||
fileErrors.push(`Missing action comments at "${testName}" (format: "// # Some descriptive action")`);
|
||||
}
|
||||
|
||||
// Check for verification comments (// *) - we need to be more flexible in our detection
|
||||
let hasVerificationComments = false;
|
||||
const verificationPattern = /\/\/\s*\*/;
|
||||
|
||||
if (verificationPattern.test(testBody)) {
|
||||
hasVerificationComments = true;
|
||||
}
|
||||
|
||||
if (!hasVerificationComments) {
|
||||
fileErrors.push(
|
||||
`Missing verification comments at "${testName}" (format: "// * Some descriptive verification")`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
fileErrors.push(`Could not extract test body for "${testName}"`);
|
||||
}
|
||||
});
|
||||
|
||||
if (fileErrors.length > 0) {
|
||||
results.failed++;
|
||||
results.errors.push({
|
||||
file: relativeFilePath,
|
||||
errors: fileErrors,
|
||||
});
|
||||
} else {
|
||||
results.passed++;
|
||||
}
|
||||
});
|
||||
|
||||
// Print results
|
||||
console.log('\n' + '-'.repeat(80));
|
||||
console.log(`${colors.cyan}Test Documentation Format Linter Results${colors.reset}`);
|
||||
console.log(`Files checked: ${specFiles.length}`);
|
||||
console.log(`${colors.green}Passed: ${results.passed}${colors.reset}`);
|
||||
console.log(`${colors.red}Failed: ${results.failed}${colors.reset}`);
|
||||
console.log(`${colors.yellow}Warnings: ${results.warnings}${colors.reset}`);
|
||||
console.log(`${colors.cyan}New tests: ${results.newTests ? results.newTests.length : 0}${colors.reset}`);
|
||||
console.log(`${colors.magenta}Updated tests: ${results.updatedTests ? results.updatedTests.length : 0}${colors.reset}`);
|
||||
|
||||
if (results.errors.length > 0) {
|
||||
console.log('\n' + '-'.repeat(80));
|
||||
console.log(`${colors.red}Errors:${colors.reset}`);
|
||||
|
||||
results.errors.forEach((fileResult) => {
|
||||
console.log(`\n${colors.cyan}${fileResult.file}${colors.reset}:`);
|
||||
fileResult.errors.forEach((error) => {
|
||||
console.log(` ${colors.red}•${colors.reset} ${error}`);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Display new tests summary if any were found
|
||||
if (results.newTests && results.newTests.length > 0) {
|
||||
console.log('\n' + '-'.repeat(80));
|
||||
console.log(`${colors.cyan}New Tests to be Registered:${colors.reset}`);
|
||||
|
||||
results.newTests.forEach((newTest) => {
|
||||
console.log(` ${colors.cyan}•${colors.reset} ${newTest.file}: "${newTest.testName}"`);
|
||||
});
|
||||
}
|
||||
|
||||
// Display updated tests summary if any were found
|
||||
if (results.updatedTests && results.updatedTests.length > 0) {
|
||||
console.log('\n' + '-'.repeat(80));
|
||||
console.log(`${colors.magenta}Tests with Documentation Updates:${colors.reset}`);
|
||||
|
||||
// Group the updated tests by base test key
|
||||
const groupedTests = {};
|
||||
results.updatedTests.forEach((test) => {
|
||||
if (!groupedTests[test.baseTestKey]) {
|
||||
groupedTests[test.baseTestKey] = [];
|
||||
}
|
||||
groupedTests[test.baseTestKey].push(test);
|
||||
});
|
||||
|
||||
// Display the tests grouped by base test key
|
||||
Object.keys(groupedTests).forEach((baseTestKey) => {
|
||||
const tests = groupedTests[baseTestKey];
|
||||
|
||||
// If there's only one test with this base key and it's not a step
|
||||
if (tests.length === 1 && !tests[0].isStep) {
|
||||
const test = tests[0];
|
||||
console.log(
|
||||
` ${colors.magenta}•${colors.reset} ${test.file}: ${test.testKey} "${test.testName.substring(test.testKey.length).trim()}"`,
|
||||
);
|
||||
}
|
||||
// If there are multiple steps of the same test
|
||||
else {
|
||||
console.log(
|
||||
` ${colors.magenta}•${colors.reset} ${tests[0].file}: ${colors.cyan}${baseTestKey}${colors.reset} (with ${tests.length} steps):`,
|
||||
);
|
||||
|
||||
// Sort steps by their step number
|
||||
tests.sort((a, b) => {
|
||||
if (!a.isStep) return -1;
|
||||
if (!b.isStep) return 1;
|
||||
|
||||
const aNum = parseInt(a.stepSuffix.substring(1), 10);
|
||||
const bNum = parseInt(b.stepSuffix.substring(1), 10);
|
||||
return aNum - bNum;
|
||||
});
|
||||
|
||||
// Display each step
|
||||
tests.forEach((test) => {
|
||||
const namePart = test.testName.substring(test.baseTestKey.length).trim();
|
||||
if (test.isStep) {
|
||||
// Extract the step number without the underscore
|
||||
const stepNumber = test.stepSuffix.substring(1);
|
||||
console.log(
|
||||
` ${colors.magenta}◦${colors.reset} step ${stepNumber}: "${namePart.substring(test.stepSuffix.length).trim()}"`,
|
||||
);
|
||||
} else {
|
||||
console.log(` ${colors.magenta}◦${colors.reset} Base test: "${namePart}"`);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log('\n' + '-'.repeat(80));
|
||||
if (results.failed > 0) {
|
||||
console.log(`${colors.red}Linter failed!${colors.reset} Please fix the test documentation issues.`);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(`${colors.green}Linter passed!${colors.reset} All spec files follow the required format.`);
|
||||
process.exit(0);
|
||||
}
|
||||
Ссылка в новой задаче
Block a user