diff --git a/examples/sleeping.js b/examples/sleeping.js index 8d78648..bdb6a86 100644 --- a/examples/sleeping.js +++ b/examples/sleeping.js @@ -61,12 +61,15 @@ Example.sleeping = function() { Composite.add(world, stack); + /* + // sleep events for (var i = 0; i < stack.bodies.length; i++) { Events.on(stack.bodies[i], 'sleepStart sleepEnd', function(event) { var body = this; console.log('body id', body.id, 'sleeping:', body.isSleeping); }); } + */ // add mouse control var mouse = Mouse.create(render.canvas), diff --git a/test/Examples.spec.js b/test/Examples.spec.js index 00a85dc..a40bb42 100644 --- a/test/Examples.spec.js +++ b/test/Examples.spec.js @@ -32,21 +32,67 @@ const examples = Object.keys(Example).filter(key => { }); const captureExamples = async useDev => { - const worker = new Worker(require.resolve('./ExampleWorker'), { + const multiThreadWorker = new Worker(require.resolve('./ExampleWorker'), { + enableWorkerThreads: true + }); + + const overlapRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({ + name, + useDev, + updates: 1, + stableSort: true, + jitter: excludeJitter.includes(name) ? 0 : 1e-10 + }))); + + const behaviourRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({ + name, + useDev, + updates: 2, + stableSort: true, + jitter: excludeJitter.includes(name) ? 0 : 1e-10 + }))); + + const similarityRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({ + name, + useDev, + updates: 2, + stableSort: false, + jitter: excludeJitter.includes(name) ? 0 : 1e-10 + }))); + + await multiThreadWorker.end(); + + const singleThreadWorker = new Worker(require.resolve('./ExampleWorker'), { enableWorkerThreads: true, numWorkers: 1 }); - const result = await Promise.all(examples.map(name => worker.runExample({ + const completeRuns = await Promise.all(examples.map(name => singleThreadWorker.runExample({ name, useDev, - totalUpdates: 120, + updates: 150, + stableSort: false, jitter: excludeJitter.includes(name) ? 0 : 1e-10 }))); - await worker.end(); + await singleThreadWorker.end(); - return result.reduce((out, capture) => (out[capture.name] = capture, out), {}); + const capture = {}; + + for (const completeRun of completeRuns) { + const behaviourRun = behaviourRuns.find(({ name }) => name === completeRun.name); + const similarityRun = similarityRuns.find(({ name }) => name === completeRun.name); + const overlapRun = overlapRuns.find(({ name }) => name === completeRun.name); + + capture[overlapRun.name] = { + ...completeRun, + behaviourExtrinsic: behaviourRun.extrinsic, + similarityExtrinsic: similarityRun.extrinsic, + overlap: overlapRun.overlap + }; + } + + return capture; }; const capturesDev = captureExamples(true); diff --git a/test/TestTools.js b/test/TestTools.js index 7e41739..f0da95c 100644 --- a/test/TestTools.js +++ b/test/TestTools.js @@ -9,23 +9,92 @@ const compareCommand = 'open http://localhost:8000/?compare'; const diffSaveCommand = 'npm run test-save'; const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json'; const equalityThreshold = 0.99999; - const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 }; -const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text; -const toPercent = val => (100 * val).toFixed(3); -const toPercentRound = val => Math.round(100 * val); -const requireUncached = path => { - delete require.cache[require.resolve(path)]; - const module = require(path); - delete require.cache[require.resolve(path)]; - return module; -}; +const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => { + const performanceDev = capturePerformanceTotals(capturesDev); + const performanceBuild = capturePerformanceTotals(capturesBuild); -const noiseThreshold = (val, threshold) => { - const sign = val < 0 ? -1 : 1; - const magnitude = Math.abs(val); - return sign * Math.max(0, magnitude - threshold) / (1 - threshold); + const perfChange = noiseThreshold(1 - (performanceDev.duration / performanceBuild.duration), 0.01); + const memoryChange = noiseThreshold((performanceDev.memory / performanceBuild.memory) - 1, 0.01); + const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1; + const filesizeChange = (devSize / buildSize) - 1; + + const behaviourSimilaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'behaviourExtrinsic'); + const behaviourSimilarityAverage = extrinsicSimilarityAverage(behaviourSimilaritys); + const behaviourSimilarityEntries = Object.entries(behaviourSimilaritys); + behaviourSimilarityEntries.sort((a, b) => a[1] - b[1]); + + const similaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'similarityExtrinsic'); + const similarityAverage = extrinsicSimilarityAverage(similaritys); + + const devIntrinsicsChanged = {}; + const buildIntrinsicsChanged = {}; + let intrinsicChangeCount = 0; + + const captureSummary = Object.entries(capturesDev) + .map(([name]) => { + const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic); + if (changedIntrinsics) { + capturesDev[name].changedIntrinsics = true; + if (intrinsicChangeCount < 2) { + devIntrinsicsChanged[name] = capturesDev[name].intrinsic; + buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic; + intrinsicChangeCount += 1; + } + } + + return { name }; + }) + .sort((a, b) => a.name.localeCompare(b.name)); + + const report = (breakEvery, format) => [ + [`Output comparison of ${behaviourSimilarityEntries.length}`, + `examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}` + ].join(' '), + + `\n\n${format('Behaviour ', colors.White)}`, + `${format(formatPercent(behaviourSimilarityAverage), behaviourSimilarityAverage === 1 ? colors.Green : colors.Yellow)}%`, + + ` ${format('Similarity', colors.White)}`, + `${format(formatPercent(similarityAverage), similarityAverage === 1 ? colors.Green : colors.Yellow)}%`, + + ` ${format('Overlap', colors.White)}`, + ` ${format((overlapChange >= 0 ? '+' : '-') + formatPercent(overlapChange, true), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`, + + `\n${format('Performance', colors.White)}`, + `${format((perfChange >= 0 ? '+' : '-') + formatPercent(perfChange, true), perfChange >= 0 ? colors.Green : colors.Yellow)}%`, + + ` ${format('Memory', colors.White)}`, + ` ${format((memoryChange >= 0 ? '+' : '-') + formatPercent(memoryChange, true), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`, + + ` ${format('Filesize', colors.White)}`, + `${format((filesizeChange >= 0 ? '+' : '-') + formatPercent(filesizeChange, true), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`, + `${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`, + + captureSummary.reduce((output, p, i) => { + output += `${p.name} `; + output += `${similarityRatings(behaviourSimilaritys[p.name])} `; + output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `; + if (i > 0 && i < captureSummary.length && breakEvery > 0 && i % breakEvery === 0) { + output += '\n'; + } + return output; + }, '\n\n'), + + `\n\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`, + + behaviourSimilarityAverage < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 150 + '#' + behaviourSimilarityEntries[0][0], colors.BrightCyan)}` : '', + intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : '' + ].join(' '); + + if (save) { + writeResult('examples-dev', devIntrinsicsChanged); + writeResult('examples-build', buildIntrinsicsChanged); + writeResult('examples-report', report(5, s => s)); + } + + return report(5, color); }; const similarity = (a, b) => { @@ -35,18 +104,56 @@ const similarity = (a, b) => { return 1 / (1 + (distance / a.length)); }; -const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => { +const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·'; +const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·'; +const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text; +const formatPercent = (val, abs) => (100 * (abs ? Math.abs(val) : val)).toFixed(2); + +const noiseThreshold = (val, threshold) => { + const sign = val < 0 ? -1 : 1; + const magnitude = Math.abs(val); + return sign * Math.max(0, magnitude - threshold) / (1 - threshold); +}; + +const equals = (a, b) => { + try { + expect(a).toEqual(b); + } catch (e) { + return false; + } + return true; +}; + +const capturePerformanceTotals = (captures) => { + const totals = { + duration: 0, + overlap: 0, + memory: 0 + }; + + for (const [ name ] of Object.entries(captures)) { + totals.duration += captures[name].duration; + totals.overlap += captures[name].overlap; + totals.memory += captures[name].memory; + }; + + return totals; +}; + +const extrinsicSimilarity = (currentCaptures, referenceCaptures, key) => { const result = {}; Object.entries(currentCaptures).forEach(([name, current]) => { const reference = referenceCaptures[name]; const worldVector = []; const worldVectorRef = []; + const currentExtrinsic = current[key]; + const referenceExtrinsic = reference[key]; - Object.keys(current.extrinsic).forEach(objectType => { - Object.keys(current.extrinsic[objectType]).forEach(objectId => { - worldVector.push(...current.extrinsic[objectType][objectId]); - worldVectorRef.push(...reference.extrinsic[objectType][objectId]); + Object.keys(currentExtrinsic).forEach(objectType => { + Object.keys(currentExtrinsic[objectType]).forEach(objectId => { + worldVector.push(...currentExtrinsic[objectType][objectId]); + worldVectorRef.push(...referenceExtrinsic[objectType][objectId]); }); }); @@ -56,6 +163,15 @@ const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => { return result; }; +const extrinsicSimilarityAverage = (similaritys) => { + const entries = Object.entries(similaritys); + let average = 0; + + entries.forEach(([_, similarity]) => average += similarity); + + return average /= entries.length; +}; + const writeResult = (name, obj) => { try { fs.mkdirSync(comparePath, { recursive: true }); @@ -70,9 +186,35 @@ const writeResult = (name, obj) => { } }; +const logReport = (captures, version) => { + let report = ''; + + for (const capture of Object.values(captures)) { + if (!capture.logs.length) { + continue; + } + + report += ` ${capture.name}\n`; + + for (const log of capture.logs) { + report += ` ${log}\n`; + } + } + + return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n` + + (report ? report : ' None\n'); +}; + +const requireUncached = path => { + delete require.cache[require.resolve(path)]; + const module = require(path); + delete require.cache[require.resolve(path)]; + return module; +}; + const toMatchExtrinsics = { toMatchExtrinsics(received, value) { - const similaritys = captureSimilarityExtrinsic(received, value); + const similaritys = extrinsicSimilarity(received, value, 'extrinsic'); const pass = Object.values(similaritys).every(similarity => similarity >= equalityThreshold); return { @@ -101,126 +243,6 @@ const toMatchIntrinsics = { } }; -const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·'; -const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·'; - -const equals = (a, b) => { - try { - expect(a).toEqual(b); - } catch (e) { - return false; - } - return true; -}; - -const logReport = (captures, version) => { - let report = ''; - - for (const capture of Object.values(captures)) { - if (!capture.logs.length) { - continue; - } - - report += ` ${capture.name}\n`; - - for (const log of capture.logs) { - report += ` ${log}\n`; - } - } - - return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n` - + (report ? report : ' None\n'); -}; - -const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => { - const similaritys = captureSimilarityExtrinsic(capturesDev, capturesBuild); - const similarityEntries = Object.entries(similaritys); - const devIntrinsicsChanged = {}; - const buildIntrinsicsChanged = {}; - let intrinsicChangeCount = 0; - let totalTimeBuild = 0; - let totalTimeDev = 0; - let totalOverlapBuild = 0; - let totalOverlapDev = 0; - let totalMemoryBuild = 0; - let totalMemoryDev = 0; - - const capturePerformance = Object.entries(capturesDev).map(([name]) => { - totalTimeBuild += capturesBuild[name].duration; - totalTimeDev += capturesDev[name].duration; - - totalOverlapBuild += capturesBuild[name].overlap; - totalOverlapDev += capturesDev[name].overlap; - - totalMemoryBuild += capturesBuild[name].memory; - totalMemoryDev += capturesDev[name].memory; - - const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic); - if (changedIntrinsics) { - capturesDev[name].changedIntrinsics = true; - if (intrinsicChangeCount < 2) { - devIntrinsicsChanged[name] = capturesDev[name].intrinsic; - buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic; - intrinsicChangeCount += 1; - } - } - - return { name }; - }); - - capturePerformance.sort((a, b) => a.name.localeCompare(b.name)); - similarityEntries.sort((a, b) => a[1] - b[1]); - - const perfChange = noiseThreshold(1 - (totalTimeDev / totalTimeBuild), 0.01); - const memoryChange = noiseThreshold((totalMemoryDev / totalMemoryBuild) - 1, 0.01); - const overlapChange = (totalOverlapDev / (totalOverlapBuild || 1)) - 1; - const filesizeChange = (devSize / buildSize) - 1; - - let similarityAvg = 0; - similarityEntries.forEach(([_, similarity]) => { - similarityAvg += similarity; - }); - - similarityAvg /= similarityEntries.length; - - const report = (breakEvery, format) => [ - [`Output comparison of ${similarityEntries.length}`, - `examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}` - ].join(' '), - `\n\n${format('Similarity', colors.White)}`, - `${format(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`, - `${format('Overlap', colors.White)}`, - `${format((overlapChange >= 0 ? '+' : '-') + toPercent(Math.abs(overlapChange)), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`, - `${format('Performance ~', colors.White)}`, - `${format((perfChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(perfChange)), perfChange >= 0 ? colors.Green : colors.Yellow)}%`, - `${format('Memory ~', colors.White)}`, - `${format((memoryChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(memoryChange)), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`, - capturePerformance.reduce((output, p, i) => { - output += `${p.name} `; - output += `${similarityRatings(similaritys[p.name])} `; - output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `; - if (i > 0 && i < capturePerformance.length && breakEvery > 0 && i % breakEvery === 0) { - output += '\n'; - } - return output; - }, '\n\n'), - `\n\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`, - similarityAvg < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 120 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '', - intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : '', - `\n\n${format('Filesize', colors.White)}`, - `${format((filesizeChange >= 0 ? '+' : '-') + toPercent(Math.abs(filesizeChange)), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`, - `${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`, - ].join(' '); - - if (save) { - writeResult('examples-dev', devIntrinsicsChanged); - writeResult('examples-build', buildIntrinsicsChanged); - writeResult('examples-report', report(5, s => s)); - } - - return report(5, color); -}; - module.exports = { requireUncached, comparisonReport, logReport, toMatchExtrinsics, toMatchIntrinsics