diff --git a/.changelog/20260318101839_i_4307_test_wrapper_v2.md b/.changelog/20260318101839_i_4307_test_wrapper_v2.md new file mode 100644 index 000000000..0dab99069 --- /dev/null +++ b/.changelog/20260318101839_i_4307_test_wrapper_v2.md @@ -0,0 +1,9 @@ +--- +type: Feature +scope: + - ckeditor5-dev-tests +--- + +Enabled running mixed Vitest and Karma tests in a single `runAutomatedTests()` invocation. The test runner now partitions test files accordingly, and executes both runners sequentially. + +Watch mode is restricted to single-runner selections to avoid interleaved output. diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 0c7f8cb49..0df8f14af 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -4,6 +4,7 @@ */ import fs from 'node:fs'; +import { spawn } from 'node:child_process'; import { styleText } from 'node:util'; import { logger } from '@ckeditor/ckeditor5-dev-utils'; import getKarmaConfig from '../utils/automated-tests/getkarmaconfig.js'; @@ -24,89 +25,184 @@ const IGNORE_GLOBS = [ upath.join( '**', 'tests', '**', '_utils', '**', '*.{js,ts}' ) ]; -// An absolute path to the entry file that will be passed to Karma. -const ENTRY_FILE_PATH = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' ); - -export default function runAutomatedTests( options ) { - return Promise.resolve().then( () => { - if ( !options.production ) { - console.warn( styleText( - 'yellow', - '⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' + - 'to use strictest verification methods.' - ) ); - } +export default async function runAutomatedTests( options ) { + if ( !options.production ) { + console.warn( styleText( + 'yellow', + '⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' + + 'to use strictest verification methods.' + ) ); + } - const globPatterns = transformFilesToTestGlob( options.files ); + const globPatterns = resolveTestGlobs( options.files ); + const testFiles = collectTestFiles( globPatterns ); + const { karmaFiles, vitestSelection } = partitionByRunner( testFiles ); - createEntryFile( globPatterns, options.production ); + if ( !karmaFiles.length && !vitestSelection.length ) { + throw new Error( 'No test files found. Specified patterns are invalid.' ); + } - const optionsForKarma = Object.assign( {}, options, { - entryFile: ENTRY_FILE_PATH, - globPatterns - } ); + if ( karmaFiles.length && vitestSelection.length && ( options.watch || options.server ) ) { + throw new Error( + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' + ); + } - return runKarma( optionsForKarma ); - } ); + if ( options.watch && vitestSelection.length > 1 ) { + throw new Error( + 'Watch mode cannot be used for multiple Vitest projects in one run. ' + + 'Run watch mode separately for each Vitest project.' + ); + } + + const errors = []; + + if ( karmaFiles.length ) { + try { + await runKarmaTests( options, karmaFiles ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( vitestSelection.length ) { + try { + await spawnVitest( options, vitestSelection ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( errors.length ) { + throw aggregateErrors( errors ); + } } -function transformFilesToTestGlob( files ) { +// -- Glob resolution & file collection ----------------------------------------------------------- + +function resolveTestGlobs( files ) { if ( !Array.isArray( files ) || files.length === 0 ) { - throw new Error( 'Karma requires files to tests. `options.files` has to be non-empty array.' ); + throw new Error( 'Test runner requires files to test. `options.files` has to be a non-empty array.' ); } const globMap = {}; - for ( const singleFile of files ) { - globMap[ singleFile ] = transformFileOptionToTestGlob( singleFile ); + for ( const file of files ) { + globMap[ file ] = transformFileOptionToTestGlob( file ); } return globMap; } -function createEntryFile( globPatterns, production ) { - mkdirp.sync( upath.dirname( ENTRY_FILE_PATH ) ); +function collectTestFiles( globPatterns ) { karmaLogger.setupFromConfig( { logLevel: 'INFO' } ); const log = karmaLogger.create( 'config' ); const allFiles = []; - for ( const singlePattern of Object.keys( globPatterns ) ) { + for ( const [ pattern, resolvedGlobs ] of Object.entries( globPatterns ) ) { let hasFiles = false; - for ( const resolvedPattern of globPatterns[ singlePattern ] ) { - const files = globSync( resolvedPattern ).map( filePath => upath.normalize( filePath ) ); + for ( const glob of resolvedGlobs ) { + const files = globSync( glob ).map( f => upath.normalize( f ) ); if ( files.length ) { hasFiles = true; } allFiles.push( - ...files.filter( file => !IGNORE_GLOBS.some( globPattern => minimatch( file, globPattern ) ) ) + ...files.filter( file => !IGNORE_GLOBS.some( ignore => minimatch( file, ignore ) ) ) ); } if ( !hasFiles ) { - log.warn( 'Pattern "%s" does not match any file.', singlePattern ); + log.warn( 'Pattern "%s" does not match any file.', pattern ); } } - if ( !allFiles.length ) { - throw new Error( 'Not found files to tests. Specified patterns are invalid.' ); + return allFiles; +} + +// -- Runner partitioning -------------------------------------------------------------------------- + +function partitionByRunner( testFiles ) { + const karmaFiles = []; + const vitestSelection = new Map(); + const runnerCache = new Map(); + + for ( const filePath of testFiles ) { + const packageRoot = getPackageRoot( filePath ); + + if ( !runnerCache.has( packageRoot ) ) { + runnerCache.set( packageRoot, detectPackageRunner( packageRoot ) ); + } + + const { runner, projectName } = runnerCache.get( packageRoot ); + + if ( runner === 'vitest' ) { + const files = vitestSelection.get( projectName ) || []; + files.push( filePath ); + vitestSelection.set( projectName, files ); + } else { + karmaFiles.push( filePath ); + } + } + + return { karmaFiles, vitestSelection: [ ...vitestSelection.entries() ] }; +} + +function getPackageRoot( filePath ) { + const normalized = upath.normalize( filePath ); + const testsIndex = normalized.lastIndexOf( '/tests/' ); + + if ( testsIndex === -1 ) { + throw new Error( `Cannot determine package root for "${ filePath }".` ); } + return normalized.slice( 0, testsIndex ); +} + +function detectPackageRunner( packageRoot ) { + const projectName = upath.basename( packageRoot ).replace( /^ckeditor5-/, '' ); + const packageJson = JSON.parse( fs.readFileSync( upath.join( packageRoot, 'package.json' ), 'utf8' ) ); + const runner = packageJson.scripts?.test?.includes( 'vitest' ) ? 'vitest' : 'karma'; + + return { projectName, runner }; +} + +// -- Karma runner --------------------------------------------------------------------------------- + +async function runKarmaTests( options, karmaFiles ) { + const entryFilePath = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' ); + + createKarmaEntryFile( entryFilePath, karmaFiles, options.production ); + + // Build globPatterns from karmaFiles only, so the coverage loader instruments + // just the Karma packages' source code — not Vitest packages that happen to be + // imported transitively. + return startKarmaServer( { + ...options, + entryFile: entryFilePath, + globPatterns: { karma: karmaFiles } + } ); +} + +function createKarmaEntryFile( entryFilePath, files, production ) { + const utilsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests' ); + const testImports = [ ...files ]; + // Set global license key in the `before` hook. - allFiles.unshift( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'licensekeybefore.js' ) ); + testImports.unshift( upath.join( utilsDir, 'licensekeybefore.js' ) ); // Inject the leak detector root hooks. Need to be split into two parts due to #598. - allFiles.splice( 0, 0, upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorbefore.js' ) ); - allFiles.push( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorafter.js' ) ); + testImports.splice( 0, 0, upath.join( utilsDir, 'leaksdetectorbefore.js' ) ); + testImports.push( upath.join( utilsDir, 'leaksdetectorafter.js' ) ); - const entryFileContent = allFiles - .map( file => 'import "' + file + '";' ); + const entryLines = testImports.map( file => `import "${ file }";` ); // Inject the custom chai assertions. See ckeditor/ckeditor5#9668. - const assertionsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'assertions' ); + const assertionsDir = upath.join( utilsDir, 'assertions' ); const customAssertions = fs.readdirSync( assertionsDir ).map( assertionFileName => { return [ assertionFileName, @@ -116,17 +212,18 @@ function createEntryFile( globPatterns, production ) { // Two loops are needed to achieve correct order in `ckeditor5/build/.automated-tests/entry-point.js`. for ( const [ fileName, functionName ] of customAssertions ) { - entryFileContent.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` ); + entryLines.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` ); } for ( const [ , functionName ] of customAssertions ) { - entryFileContent.push( `${ functionName }Factory( chai );` ); + entryLines.push( `${ functionName }Factory( chai );` ); } if ( production ) { - entryFileContent.unshift( assertConsoleUsageToThrowErrors() ); + entryLines.unshift( assertConsoleUsageToThrowErrors() ); } - fs.writeFileSync( ENTRY_FILE_PATH, entryFileContent.join( '\n' ) + '\n' ); + mkdirp.sync( upath.dirname( entryFilePath ) ); + fs.writeFileSync( entryFilePath, entryLines.join( '\n' ) + '\n' ); // Webpack watcher compiles the file in a loop. It causes to Karma that runs tests multiple times in watch mode. // A ugly hack blocks the loop and tests are executed once. @@ -134,9 +231,167 @@ function createEntryFile( globPatterns, production ) { const now = Date.now() / 1000; // 10 sec is default value of FS_ACCURENCY (which is hardcoded in Webpack watcher). const then = now - 10; - fs.utimesSync( ENTRY_FILE_PATH, then, then ); + fs.utimesSync( entryFilePath, then, then ); +} + +function startKarmaServer( options ) { + return new Promise( ( resolve, reject ) => { + const KarmaServer = karma.Server; + const parseConfig = karma.config.parseConfig; + + const config = getKarmaConfig( options ); + const parsedConfig = parseConfig( null, config, { throwErrors: true } ); + + const server = new KarmaServer( parsedConfig, exitCode => { + if ( exitCode === 0 ) { + resolve(); + } else { + reject( new Error( `Karma finished with "${ exitCode }" code.` ) ); + } + } ); + + if ( options.coverage ) { + const coveragePath = upath.join( process.cwd(), 'coverage' ); + + server.on( 'run_complete', () => { + // Use timeout to not write to the console in the middle of Karma's status. + setTimeout( () => { + const log = logger(); + + log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` ); + } ); + } ); + } + + server.start(); + } ); +} + +// -- Vitest runner -------------------------------------------------------------------------------- + +async function spawnVitest( options, vitestSelection ) { + const errors = []; + + for ( const [ project, selectedFiles ] of vitestSelection ) { + try { + await spawnVitestProject( options, project, selectedFiles ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( options.coverage ) { + try { + await mergeVitestCoverage( vitestSelection ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( errors.length ) { + if ( errors.length === 1 ) { + throw errors[ 0 ]; + } + + const details = errors.map( e => `- ${ e.message }` ).join( '\n' ); + throw new Error( `Vitest execution failed in multiple projects:\n${ details }` ); + } +} + +function spawnVitestProject( options, project, selectedFiles ) { + return new Promise( ( resolve, reject ) => { + const args = [ 'vitest' ]; + + args.push( options.watch ? '--watch' : '--run' ); + + if ( options.coverage ) { + const coverageDir = upath.join( process.cwd(), 'coverage-vitest', project ); + args.push( '--coverage.enabled', '--coverage.reportsDirectory', coverageDir ); + } + + args.push( '--project', project ); + + for ( const filePath of selectedFiles ) { + args.push( upath.relative( process.cwd(), filePath ) ); + } + + const child = spawn( 'pnpm', args, { + stdio: 'inherit', + cwd: process.cwd(), + shell: process.platform === 'win32' + } ); + + child.on( 'error', reject ); + + child.on( 'close', exitCode => { + if ( exitCode === 0 || exitCode === 130 ) { + resolve(); + } else { + reject( new Error( `Vitest finished with "${ exitCode }" code.` ) ); + } + } ); + } ); +} + +function mergeVitestCoverage( vitestSelection ) { + const cwd = process.cwd(); + const coverageBaseDir = upath.join( cwd, 'coverage-vitest' ); + const nycOutputDir = upath.join( coverageBaseDir, '.nyc_output' ); + + mkdirp.sync( nycOutputDir ); + + // Copy each project's coverage-final.json into .nyc_output/ so nyc can merge them. + for ( const [ project ] of vitestSelection ) { + const sourceFile = upath.join( coverageBaseDir, project, 'coverage-final.json' ); + + if ( fs.existsSync( sourceFile ) ) { + fs.copyFileSync( sourceFile, upath.join( nycOutputDir, `${ project }.json` ) ); + } + } + + const log = logger(); + + return new Promise( ( resolve, reject ) => { + const child = spawn( 'pnpx', [ + 'nyc', 'report', + '--temp-dir', nycOutputDir, + '--report-dir', coverageBaseDir, + '--reporter', 'html', + '--reporter', 'json', + '--reporter', 'lcovonly', + '--reporter', 'text-summary' + ], { + stdio: 'inherit', + cwd, + shell: process.platform === 'win32' + } ); + + child.on( 'error', reject ); + + child.on( 'close', exitCode => { + if ( exitCode === 0 ) { + log.info( `Combined Vitest coverage report saved in '${ styleText( 'cyan', coverageBaseDir ) }'.` ); + resolve(); + } else { + reject( new Error( `nyc report finished with "${ exitCode }" code.` ) ); + } + } ); + } ); } +// -- Error handling ------------------------------------------------------------------------------- + +function aggregateErrors( errors ) { + if ( errors.length === 1 ) { + return errors[ 0 ]; + } + + const details = errors.map( e => `- ${ e.message }` ).join( '\n' ); + return new Error( `Test execution failed in multiple runners:\n${ details }` ); +} + +// -- Console assertion (production mode) ---------------------------------------------------------- + function assertConsoleUsageToThrowErrors() { const functionString = makeConsoleUsageToThrowErrors.toString(); @@ -186,36 +441,3 @@ function makeConsoleUsageToThrowErrors() { } ); } ); } - -function runKarma( options ) { - return new Promise( ( resolve, reject ) => { - const KarmaServer = karma.Server; - const parseConfig = karma.config.parseConfig; - - const config = getKarmaConfig( options ); - const parsedConfig = parseConfig( null, config, { throwErrors: true } ); - - const server = new KarmaServer( parsedConfig, exitCode => { - if ( exitCode === 0 ) { - resolve(); - } else { - reject( new Error( `Karma finished with "${ exitCode }" code.` ) ); - } - } ); - - if ( options.coverage ) { - const coveragePath = upath.join( process.cwd(), 'coverage' ); - - server.on( 'run_complete', () => { - // Use timeout to not write to the console in the middle of Karma's status. - setTimeout( () => { - const log = logger(); - - log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` ); - } ); - } ); - } - - server.start(); - } ); -} diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 395346502..1428c0863 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'node:fs'; +import { spawn } from 'node:child_process'; import { styleText } from 'node:util'; import { globSync } from 'glob'; import { mkdirp } from 'mkdirp'; @@ -26,9 +27,36 @@ const stubs = vi.hoisted( () => ( { on: vi.fn(), start: vi.fn() } + }, + spawn: { + call: vi.fn() + }, + devUtilsLogger: { + info: vi.fn(), + warning: vi.fn(), + error: vi.fn() } } ) ); +vi.mock( 'node:child_process', () => ( { + spawn: vi.fn( ( ...args ) => { + stubs.spawn.call( ...args ); + + const callbacks = {}; + + return { + on: ( eventName, callback ) => { + callbacks[ eventName ] = callback; + }, + emit: ( eventName, ...eventArgs ) => { + if ( callbacks[ eventName ] ) { + callbacks[ eventName ]( ...eventArgs ); + } + } + }; + } ) +} ) ); + vi.mock( 'karma', () => ( { default: { Server: class KarmaServer { @@ -58,6 +86,9 @@ vi.mock( 'node:fs' ); vi.mock( 'mkdirp' ); vi.mock( 'glob' ); vi.mock( 'karma/lib/logger.js' ); +vi.mock( '@ckeditor/ckeditor5-dev-utils', () => ( { + logger: vi.fn( () => stubs.devUtilsLogger ) +} ) ); vi.mock( '../../lib/utils/automated-tests/getkarmaconfig.js' ); vi.mock( '../../lib/utils/transformfileoptiontotestglob.js' ); @@ -66,6 +97,10 @@ describe( 'runAutomatedTests()', () => { beforeEach( async () => { vi.spyOn( process, 'cwd' ).mockReturnValue( '/workspace' ); + stubs.spawn.call.mockReset(); + + // Default: return empty JSON for package.json reads (no scripts → Karma runner). + vi.mocked( fs ).readFileSync.mockReturnValue( '{}' ); vi.mocked( karmaLogger ).create.mockImplementation( name => { expect( name ).to.equal( 'config' ); @@ -76,6 +111,8 @@ describe( 'runAutomatedTests()', () => { runAutomatedTests = ( await import( '../../lib/tasks/runautomatedtests.js' ) ).default; } ); + // -- Karma-only tests ------------------------------------------------------------------------- + it( 'should create an entry file before tests execution', async () => { const options = { files: [ @@ -125,7 +162,7 @@ describe( 'runAutomatedTests()', () => { it( 'throws when files are not specified', async () => { await expect( runAutomatedTests( { production: true } ) ) - .rejects.toThrow( 'Karma requires files to tests. `options.files` has to be non-empty array.' ); + .rejects.toThrow( 'Test runner requires files to test. `options.files` has to be a non-empty array.' ); } ); it( 'throws when specified files are invalid', async () => { @@ -152,7 +189,7 @@ describe( 'runAutomatedTests()', () => { vi.mocked( globSync ).mockReturnValue( [] ); await expect( runAutomatedTests( options ) ) - .rejects.toThrow( 'Not found files to tests. Specified patterns are invalid.' ); + .rejects.toThrow( 'No test files found. Specified patterns are invalid.' ); expect( stubs.log.warn ).toHaveBeenCalledTimes( 2 ); expect( stubs.log.warn ).toHaveBeenCalledWith( 'Pattern "%s" does not match any file.', 'basic-foo' ); @@ -473,4 +510,755 @@ describe( 'runAutomatedTests()', () => { ].join( '\n' ) ) ); } ); + + // -- Vitest-only tests ------------------------------------------------------------------------ + + it( 'should run only Vitest when all selected packages use Vitest', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.karma.server.constructor ).not.toHaveBeenCalled(); + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ + 'vitest', + '--run', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } + ); + expect( vi.mocked( fs ).writeFileSync ).not.toHaveBeenCalledWith( + '/workspace/build/.automated-tests/entry-point.js', + expect.any( String ) + ); + } ); + + it( 'should pass --watch flag to Vitest when watch mode is enabled', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: true, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ + 'vitest', + '--watch', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], + expect.any( Object ) + ); + } ); + + it( 'should pass coverage flags to Vitest and merge coverage with nyc', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( true ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + // First spawn: vitest project run. + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + // Second spawn: nyc report merge. + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 0 ); + + await promise; + + // Vitest was called with per-project coverage directory. + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 1, + 'pnpm', + [ + 'vitest', + '--run', + '--coverage.enabled', + '--coverage.reportsDirectory', + '/workspace/coverage-vitest/engine', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], + expect.any( Object ) + ); + + // coverage-final.json was copied into .nyc_output. + expect( vi.mocked( fs ).existsSync ).toHaveBeenCalledWith( + '/workspace/coverage-vitest/engine/coverage-final.json' + ); + expect( vi.mocked( fs ).copyFileSync ).toHaveBeenCalledWith( + '/workspace/coverage-vitest/engine/coverage-final.json', + '/workspace/coverage-vitest/.nyc_output/engine.json' + ); + + // nyc report was called with correct reporters. + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 2, + 'pnpx', + [ + 'nyc', 'report', + '--temp-dir', '/workspace/coverage-vitest/.nyc_output', + '--report-dir', '/workspace/coverage-vitest', + '--reporter', 'html', + '--reporter', 'json', + '--reporter', 'lcovonly', + '--reporter', 'text-summary' + ], + expect.objectContaining( { stdio: 'inherit', cwd: '/workspace' } ) + ); + + // Log message was printed. + expect( stubs.devUtilsLogger.info ).toHaveBeenCalled(); + } ); + + it( 'should reject when Vitest process exits with non-zero code', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 1 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + } ); + + // -- Mixed Karma + Vitest tests --------------------------------------------------------------- + + it( 'should route mixed package selection to Karma and Vitest', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: false + }; + + vi.mocked( fs ).readdirSync.mockReturnValue( [] ); + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + + setTimeout( () => { + const [ firstCall ] = stubs.karma.server.constructor.mock.calls; + const [ , exitCallback ] = firstCall; + + exitCallback( 0 ); + + setTimeout( () => { + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + + subprocess.emit( 'close', 0 ); + } ); + } ); + + await promise; + + expect( stubs.karma.server.constructor ).toHaveBeenCalledOnce(); + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--run', '--project', 'utils', 'packages/ckeditor5-utils/tests/first.js' ], + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } + ); + } ); + + it( 'should throw when watch mode is used with mixed Karma + Vitest packages', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' + ); + } ); + + it( 'should throw when server mode is used with mixed Karma + Vitest packages', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + server: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' + ); + } ); + + it( 'should aggregate errors when both runners fail', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: false + }; + + vi.mocked( fs ).readdirSync.mockReturnValue( [] ); + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: {} } ); + } + + return '{}'; + } ); + + vi.mocked( karma ).config.parseConfig.mockImplementation( () => { + throw new Error( 'Karma finished with "1" code.' ); + } ); + + const promise = runAutomatedTests( options ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 2 ); + + await expect( promise ).rejects.toThrow( /Test execution failed in multiple runners/ ); + } ); + + // -- Multiple Vitest projects test ------------------------------------------------------------ + + it( 'should run each Vitest project in a separate process with selected files', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 1, + 'pnpm', + [ + 'vitest', + '--run', + '--project', + 'utils', + 'external/ckeditor5/packages/ckeditor5-utils/tests/first.js' + ], + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } + ); + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 2, + 'pnpm', + [ + 'vitest', + '--run', + '--project', + 'engine', + 'external/ckeditor5/packages/ckeditor5-engine/tests/model.js' + ], + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } + ); + } ); + + it( 'should throw when watch mode is used with multiple Vitest projects', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: true, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch mode cannot be used for multiple Vitest projects in one run. ' + + 'Run watch mode separately for each Vitest project.' + ); + + expect( stubs.spawn.call ).not.toHaveBeenCalled(); + } ); + + it( 'should continue running remaining Vitest projects after a project failure', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 1 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + expect( stubs.spawn.call ).toHaveBeenCalledTimes( 2 ); + } ); + + it( 'should merge Vitest coverage even when a project fails', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 1 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , , nycProcess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + nycProcess.emit( 'close', 0 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + expect( stubs.spawn.call ).toHaveBeenCalledTimes( 3 ); + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 3, + 'pnpx', + expect.arrayContaining( [ 'nyc', 'report' ] ), + expect.objectContaining( { cwd: '/workspace' } ) + ); + } ); + + // -- Edge cases ------------------------------------------------------------------------------- + + it( 'should resolve when Vitest exits with code 130 (SIGINT)', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 130 ); + + await promise; + } ); + + it( 'should reject when spawn emits an error event', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'error', new Error( 'spawn ENOENT' ) ); + + await expect( promise ).rejects.toThrow( 'spawn ENOENT' ); + } ); + + it( 'should skip copying coverage-final.json when the file does not exist', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 0 ); + + await promise; + + expect( vi.mocked( fs ).copyFileSync ).not.toHaveBeenCalled(); + } ); + + it( 'should reject when nyc report fails', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 1 ); + + await expect( promise ).rejects.toThrow( 'nyc report finished with "1" code.' ); + } ); + + it( 'should reject when nyc spawn emits an error', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'error', new Error( 'nyc ENOENT' ) ); + + await expect( promise ).rejects.toThrow( 'nyc ENOENT' ); + } ); + + it( 'should throw when a test file path does not contain /tests/ segment', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/src/model.js' + ] ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Cannot determine package root for "/workspace/packages/ckeditor5-engine/src/model.js".' + ); + } ); } );