Enabled running both Vitest and Karma tests for packages.

.changelog/20260318101839_i_4307_test_wrapper_v2.md

@@ -0,0 +1,9 @@
+---
+type: Feature
+scope:
+  - ckeditor5-dev-tests
+---
+
+Enabled running mixed Vitest and Karma tests in a single `runAutomatedTests()` invocation. The test runner now partitions test files accordingly, and executes both runners sequentially.
+
+Watch mode is restricted to single-runner selections to avoid interleaved output.

packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js

@@ -4,6 +4,7 @@
  */
 
 import fs from 'node:fs';
+import { spawn } from 'node:child_process';
 import { styleText } from 'node:util';
 import { logger } from '@ckeditor/ckeditor5-dev-utils';
 import getKarmaConfig from '../utils/automated-tests/getkarmaconfig.js';
@@ -24,89 +25,177 @@ const IGNORE_GLOBS = [
 	upath.join( '**', 'tests', '**', '_utils', '**', '*.{js,ts}' )
 ];
 
-// An absolute path to the entry file that will be passed to Karma.
-const ENTRY_FILE_PATH = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' );
-
-export default function runAutomatedTests( options ) {
-	return Promise.resolve().then( () => {
-		if ( !options.production ) {
-			console.warn( styleText(
-				'yellow',
-				'⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' +
-				'to use strictest verification methods.'
-			) );
-		}
+export default async function runAutomatedTests( options ) {
+	if ( !options.production ) {
+		console.warn( styleText(
+			'yellow',
+			'⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' +
+			'to use strictest verification methods.'
+		) );
+	}
 
-		const globPatterns = transformFilesToTestGlob( options.files );
+	const globPatterns = resolveTestGlobs( options.files );
+	const testFiles = collectTestFiles( globPatterns );
+	const { karmaFiles, vitestSelection } = partitionByRunner( testFiles );
 
-		createEntryFile( globPatterns, options.production );
+	if ( !karmaFiles.length && !vitestSelection.length ) {
+		throw new Error( 'No test files found. Specified patterns are invalid.' );
+	}
 
-		const optionsForKarma = Object.assign( {}, options, {
-			entryFile: ENTRY_FILE_PATH,
-			globPatterns
-		} );
+	if ( karmaFiles.length && vitestSelection.length && options.watch ) {
+		throw new Error(
+			'Watch mode cannot be used in a mixed Karma + Vitest run. ' +
+			'Run watch mode separately for Karma and Vitest packages.'
+		);
+	}
 
-		return runKarma( optionsForKarma );
-	} );
+	const errors = [];
+
+	if ( karmaFiles.length ) {
+		try {
+			await runKarmaTests( options, karmaFiles );
+		} catch ( error ) {
+			errors.push( error );
+		}
+	}
+
+	if ( vitestSelection.length ) {
+		try {
+			await spawnVitest( options, vitestSelection );
+		} catch ( error ) {
+			errors.push( error );
+		}
+	}
+
+	if ( errors.length ) {
+		throw aggregateErrors( errors );
+	}
 }
 
-function transformFilesToTestGlob( files ) {
+// -- Glob resolution & file collection -----------------------------------------------------------
+
+function resolveTestGlobs( files ) {
 	if ( !Array.isArray( files ) || files.length === 0 ) {
-		throw new Error( 'Karma requires files to tests. `options.files` has to be non-empty array.' );
+		throw new Error( 'Test runner requires files to test. `options.files` has to be a non-empty array.' );
 	}
 
 	const globMap = {};
 
-	for ( const singleFile of files ) {
-		globMap[ singleFile ] = transformFileOptionToTestGlob( singleFile );
+	for ( const file of files ) {
+		globMap[ file ] = transformFileOptionToTestGlob( file );
 	}
 
 	return globMap;
 }
 
-function createEntryFile( globPatterns, production ) {
-	mkdirp.sync( upath.dirname( ENTRY_FILE_PATH ) );
+function collectTestFiles( globPatterns ) {
 	karmaLogger.setupFromConfig( { logLevel: 'INFO' } );
 
 	const log = karmaLogger.create( 'config' );
 	const allFiles = [];
 
-	for ( const singlePattern of Object.keys( globPatterns ) ) {
+	for ( const [ pattern, resolvedGlobs ] of Object.entries( globPatterns ) ) {
 		let hasFiles = false;
 
-		for ( const resolvedPattern of globPatterns[ singlePattern ] ) {
-			const files = globSync( resolvedPattern ).map( filePath => upath.normalize( filePath ) );
+		for ( const glob of resolvedGlobs ) {
+			const files = globSync( glob ).map( f => upath.normalize( f ) );
 
 			if ( files.length ) {
 				hasFiles = true;
 			}
 
 			allFiles.push(
-				...files.filter( file => !IGNORE_GLOBS.some( globPattern => minimatch( file, globPattern ) ) )
+				...files.filter( file => !IGNORE_GLOBS.some( ignore => minimatch( file, ignore ) ) )
 			);
 		}
 
 		if ( !hasFiles ) {
-			log.warn( 'Pattern "%s" does not match any file.', singlePattern );
+			log.warn( 'Pattern "%s" does not match any file.', pattern );
+		}
+	}
+
+	return allFiles;
+}
+
+// -- Runner partitioning --------------------------------------------------------------------------
+
+function partitionByRunner( testFiles ) {
+	const karmaFiles = [];
+	const vitestSelection = new Map();
+	const runnerCache = new Map();
+
+	for ( const filePath of testFiles ) {
+		const packageRoot = getPackageRoot( filePath );
+
+		if ( !runnerCache.has( packageRoot ) ) {
+			runnerCache.set( packageRoot, detectPackageRunner( packageRoot ) );
+		}
+
+		const { runner, projectName } = runnerCache.get( packageRoot );
+
+		if ( runner === 'vitest' ) {
+			const files = vitestSelection.get( projectName ) || [];
+			files.push( filePath );
+			vitestSelection.set( projectName, files );
+		} else {
+			karmaFiles.push( filePath );
 		}
 	}
 
-	if ( !allFiles.length ) {
-		throw new Error( 'Not found files to tests. Specified patterns are invalid.' );
+	return { karmaFiles, vitestSelection: [ ...vitestSelection.entries() ] };
+}
+
+function getPackageRoot( filePath ) {
+	const normalized = upath.normalize( filePath );
+	const testsIndex = normalized.lastIndexOf( '/tests/' );
+
+	if ( testsIndex === -1 ) {
+		throw new Error( `Cannot determine package root for "${ filePath }".` );
 	}
 
+	return normalized.slice( 0, testsIndex );
+}
+
+function detectPackageRunner( packageRoot ) {
+	const projectName = upath.basename( packageRoot ).replace( /^ckeditor5-/, '' );
+	const packageJson = JSON.parse( fs.readFileSync( upath.join( packageRoot, 'package.json' ), 'utf8' ) );
+	const runner = packageJson.scripts?.test?.includes( 'vitest' ) ? 'vitest' : 'karma';
+
+	return { projectName, runner };
+}
+
+// -- Karma runner ---------------------------------------------------------------------------------
+
+async function runKarmaTests( options, karmaFiles ) {
+	const entryFilePath = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' );
+
+	createKarmaEntryFile( entryFilePath, karmaFiles, options.production );
+
+	// Build globPatterns from karmaFiles only, so the coverage loader instruments
+	// just the Karma packages' source code — not Vitest packages that happen to be
+	// imported transitively.
+	return startKarmaServer( {
+		...options,
+		entryFile: entryFilePath,
+		globPatterns: { karma: karmaFiles }
+	} );
+}
+
+function createKarmaEntryFile( entryFilePath, files, production ) {
+	const utilsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests' );
+	const testImports = [ ...files ];
+
 	// Set global license key in the `before` hook.
-	allFiles.unshift( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'licensekeybefore.js' ) );
+	testImports.unshift( upath.join( utilsDir, 'licensekeybefore.js' ) );
 
 	// Inject the leak detector root hooks. Need to be split into two parts due to #598.
-	allFiles.splice( 0, 0, upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorbefore.js' ) );
-	allFiles.push( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorafter.js' ) );
+	testImports.splice( 0, 0, upath.join( utilsDir, 'leaksdetectorbefore.js' ) );
+	testImports.push( upath.join( utilsDir, 'leaksdetectorafter.js' ) );
 
-	const entryFileContent = allFiles
-		.map( file => 'import "' + file + '";' );
+	const entryLines = testImports.map( file => `import "${ file }";` );
 
 	// Inject the custom chai assertions. See ckeditor/ckeditor5#9668.
-	const assertionsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'assertions' );
+	const assertionsDir = upath.join( utilsDir, 'assertions' );
 	const customAssertions = fs.readdirSync( assertionsDir ).map( assertionFileName => {
 		return [
 			assertionFileName,
@@ -116,27 +205,117 @@ function createEntryFile( globPatterns, production ) {
 
 	// Two loops are needed to achieve correct order in `ckeditor5/build/.automated-tests/entry-point.js`.
 	for ( const [ fileName, functionName ] of customAssertions ) {
-		entryFileContent.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` );
+		entryLines.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` );
 	}
 	for ( const [ , functionName ] of customAssertions ) {
-		entryFileContent.push( `${ functionName }Factory( chai );` );
+		entryLines.push( `${ functionName }Factory( chai );` );
 	}
 
 	if ( production ) {
-		entryFileContent.unshift( assertConsoleUsageToThrowErrors() );
+		entryLines.unshift( assertConsoleUsageToThrowErrors() );
 	}
 
-	fs.writeFileSync( ENTRY_FILE_PATH, entryFileContent.join( '\n' ) + '\n' );
+	mkdirp.sync( upath.dirname( entryFilePath ) );
+	fs.writeFileSync( entryFilePath, entryLines.join( '\n' ) + '\n' );
 
 	// Webpack watcher compiles the file in a loop. It causes to Karma that runs tests multiple times in watch mode.
 	// A ugly hack blocks the loop and tests are executed once.
 	// See: https://github.com/webpack/watchpack/issues/25.
 	const now = Date.now() / 1000;
 	// 10 sec is default value of FS_ACCURENCY (which is hardcoded in Webpack watcher).
 	const then = now - 10;
-	fs.utimesSync( ENTRY_FILE_PATH, then, then );
+	fs.utimesSync( entryFilePath, then, then );
+}
+
+function startKarmaServer( options ) {
+	return new Promise( ( resolve, reject ) => {
+		const KarmaServer = karma.Server;
+		const parseConfig = karma.config.parseConfig;
+
+		const config = getKarmaConfig( options );
+		const parsedConfig = parseConfig( null, config, { throwErrors: true } );
+
+		const server = new KarmaServer( parsedConfig, exitCode => {
+			if ( exitCode === 0 ) {
+				resolve();
+			} else {
+				reject( new Error( `Karma finished with "${ exitCode }" code.` ) );
+			}
+		} );
+
+		if ( options.coverage ) {
+			const coveragePath = upath.join( process.cwd(), 'coverage' );
+
+			server.on( 'run_complete', () => {
+				// Use timeout to not write to the console in the middle of Karma's status.
+				setTimeout( () => {
+					const log = logger();
+
+					log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` );
+				} );
+			} );
+		}
+
+		server.start();
+	} );
+}
+
+// -- Vitest runner --------------------------------------------------------------------------------
+
+async function spawnVitest( options, vitestSelection ) {
+	for ( const [ project, selectedFiles ] of vitestSelection ) {
+		await spawnVitestProject( options, project, selectedFiles );
+	}
+}
+
+function spawnVitestProject( options, project, selectedFiles ) {
+	return new Promise( ( resolve, reject ) => {
+		const args = [ 'vitest' ];
+
+		args.push( options.watch ? '--watch' : '--run' );
+
+		if ( options.coverage ) {
+			const coverageDir = upath.join( process.cwd(), 'coverage-vitest' );
+			args.push( '--coverage', '--coverage.reportsDirectory', coverageDir );
+		}
+
+		args.push( '--project', project );
+
+		for ( const filePath of selectedFiles ) {
+			args.push( upath.relative( process.cwd(), filePath ) );
+		}
+
+		const child = spawn( 'pnpm', args, {
+			stdio: 'inherit',
+			cwd: process.cwd(),
+			shell: process.platform === 'win32'
+		} );
+
+		child.on( 'error', reject );
+
+		child.on( 'close', exitCode => {
+			if ( exitCode === 0 || exitCode === 130 ) {
+				resolve();
+			} else {
+				reject( new Error( `Vitest finished with "${ exitCode }" code.` ) );
+			}
+		} );
+	} );
+}
+
+// -- Error handling -------------------------------------------------------------------------------
+
+function aggregateErrors( errors ) {
+	if ( errors.length === 1 ) {
+		return errors[ 0 ];
+	}
+
+	const details = errors.map( e => `- ${ e.message }` ).join( '\n' );
+	return new Error( `Test execution failed in multiple runners:\n${ details }` );
 }
 
+// -- Console assertion (production mode) ----------------------------------------------------------
+
 function assertConsoleUsageToThrowErrors() {
 	const functionString = makeConsoleUsageToThrowErrors.toString();
 
@@ -186,36 +365,3 @@ function makeConsoleUsageToThrowErrors() {
 			} );
 	} );
 }
-
-function runKarma( options ) {
-	return new Promise( ( resolve, reject ) => {
-		const KarmaServer = karma.Server;
-		const parseConfig = karma.config.parseConfig;
-
-		const config = getKarmaConfig( options );
-		const parsedConfig = parseConfig( null, config, { throwErrors: true } );
-
-		const server = new KarmaServer( parsedConfig, exitCode => {
-			if ( exitCode === 0 ) {
-				resolve();
-			} else {
-				reject( new Error( `Karma finished with "${ exitCode }" code.` ) );
-			}
-		} );
-
-		if ( options.coverage ) {
-			const coveragePath = upath.join( process.cwd(), 'coverage' );
-
-			server.on( 'run_complete', () => {
-				// Use timeout to not write to the console in the middle of Karma's status.
-				setTimeout( () => {
-					const log = logger();
-
-					log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` );
-				} );
-			} );
-		}
-
-		server.start();
-	} );
-}