mirror of
https://github.com/liabru/matter-js.git
synced 2024-11-23 09:26:51 -05:00
added behaviour metric to tests and refactor tests
This commit is contained in:
liabru
parent
81dd2fb695
commit
81259663fd
3 changed files with 216 additions and 145 deletions
|
|
@ -61,12 +61,15 @@ Example.sleeping = function() {
|
|||
|
||||
Composite.add(world, stack);
|
||||
|
||||
/*
|
||||
// sleep events
|
||||
for (var i = 0; i < stack.bodies.length; i++) {
|
||||
Events.on(stack.bodies[i], 'sleepStart sleepEnd', function(event) {
|
||||
var body = this;
|
||||
console.log('body id', body.id, 'sleeping:', body.isSleeping);
|
||||
});
|
||||
}
|
||||
*/
|
||||
|
||||
// add mouse control
|
||||
var mouse = Mouse.create(render.canvas),
|
||||
|
|
|
|||
|
|
@ -32,21 +32,67 @@ const examples = Object.keys(Example).filter(key => {
|
|||
});
|
||||
|
||||
const captureExamples = async useDev => {
|
||||
const worker = new Worker(require.resolve('./ExampleWorker'), {
|
||||
const multiThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
|
||||
enableWorkerThreads: true
|
||||
});
|
||||
|
||||
const overlapRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
|
||||
name,
|
||||
useDev,
|
||||
updates: 1,
|
||||
stableSort: true,
|
||||
jitter: excludeJitter.includes(name) ? 0 : 1e-10
|
||||
})));
|
||||
|
||||
const behaviourRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
|
||||
name,
|
||||
useDev,
|
||||
updates: 2,
|
||||
stableSort: true,
|
||||
jitter: excludeJitter.includes(name) ? 0 : 1e-10
|
||||
})));
|
||||
|
||||
const similarityRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
|
||||
name,
|
||||
useDev,
|
||||
updates: 2,
|
||||
stableSort: false,
|
||||
jitter: excludeJitter.includes(name) ? 0 : 1e-10
|
||||
})));
|
||||
|
||||
await multiThreadWorker.end();
|
||||
|
||||
const singleThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
|
||||
enableWorkerThreads: true,
|
||||
numWorkers: 1
|
||||
});
|
||||
|
||||
const result = await Promise.all(examples.map(name => worker.runExample({
|
||||
const completeRuns = await Promise.all(examples.map(name => singleThreadWorker.runExample({
|
||||
name,
|
||||
useDev,
|
||||
totalUpdates: 120,
|
||||
updates: 150,
|
||||
stableSort: false,
|
||||
jitter: excludeJitter.includes(name) ? 0 : 1e-10
|
||||
})));
|
||||
|
||||
await worker.end();
|
||||
await singleThreadWorker.end();
|
||||
|
||||
return result.reduce((out, capture) => (out[capture.name] = capture, out), {});
|
||||
const capture = {};
|
||||
|
||||
for (const completeRun of completeRuns) {
|
||||
const behaviourRun = behaviourRuns.find(({ name }) => name === completeRun.name);
|
||||
const similarityRun = similarityRuns.find(({ name }) => name === completeRun.name);
|
||||
const overlapRun = overlapRuns.find(({ name }) => name === completeRun.name);
|
||||
|
||||
capture[overlapRun.name] = {
|
||||
...completeRun,
|
||||
behaviourExtrinsic: behaviourRun.extrinsic,
|
||||
similarityExtrinsic: similarityRun.extrinsic,
|
||||
overlap: overlapRun.overlap
|
||||
};
|
||||
}
|
||||
|
||||
return capture;
|
||||
};
|
||||
|
||||
const capturesDev = captureExamples(true);
|
||||
|
|
|
|||
|
|
@ -9,23 +9,92 @@ const compareCommand = 'open http://localhost:8000/?compare';
|
|||
const diffSaveCommand = 'npm run test-save';
|
||||
const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json';
|
||||
const equalityThreshold = 0.99999;
|
||||
|
||||
const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };
|
||||
const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
|
||||
const toPercent = val => (100 * val).toFixed(3);
|
||||
const toPercentRound = val => Math.round(100 * val);
|
||||
|
||||
const requireUncached = path => {
|
||||
delete require.cache[require.resolve(path)];
|
||||
const module = require(path);
|
||||
delete require.cache[require.resolve(path)];
|
||||
return module;
|
||||
};
|
||||
const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => {
|
||||
const performanceDev = capturePerformanceTotals(capturesDev);
|
||||
const performanceBuild = capturePerformanceTotals(capturesBuild);
|
||||
|
||||
const noiseThreshold = (val, threshold) => {
|
||||
const sign = val < 0 ? -1 : 1;
|
||||
const magnitude = Math.abs(val);
|
||||
return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
|
||||
const perfChange = noiseThreshold(1 - (performanceDev.duration / performanceBuild.duration), 0.01);
|
||||
const memoryChange = noiseThreshold((performanceDev.memory / performanceBuild.memory) - 1, 0.01);
|
||||
const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1;
|
||||
const filesizeChange = (devSize / buildSize) - 1;
|
||||
|
||||
const behaviourSimilaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'behaviourExtrinsic');
|
||||
const behaviourSimilarityAverage = extrinsicSimilarityAverage(behaviourSimilaritys);
|
||||
const behaviourSimilarityEntries = Object.entries(behaviourSimilaritys);
|
||||
behaviourSimilarityEntries.sort((a, b) => a[1] - b[1]);
|
||||
|
||||
const similaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'similarityExtrinsic');
|
||||
const similarityAverage = extrinsicSimilarityAverage(similaritys);
|
||||
|
||||
const devIntrinsicsChanged = {};
|
||||
const buildIntrinsicsChanged = {};
|
||||
let intrinsicChangeCount = 0;
|
||||
|
||||
const captureSummary = Object.entries(capturesDev)
|
||||
.map(([name]) => {
|
||||
const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
|
||||
if (changedIntrinsics) {
|
||||
capturesDev[name].changedIntrinsics = true;
|
||||
if (intrinsicChangeCount < 2) {
|
||||
devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
|
||||
buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
|
||||
intrinsicChangeCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { name };
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
const report = (breakEvery, format) => [
|
||||
[`Output comparison of ${behaviourSimilarityEntries.length}`,
|
||||
`examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}`
|
||||
].join(' '),
|
||||
|
||||
`\n\n${format('Behaviour ', colors.White)}`,
|
||||
`${format(formatPercent(behaviourSimilarityAverage), behaviourSimilarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
|
||||
|
||||
` ${format('Similarity', colors.White)}`,
|
||||
`${format(formatPercent(similarityAverage), similarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
|
||||
|
||||
` ${format('Overlap', colors.White)}`,
|
||||
` ${format((overlapChange >= 0 ? '+' : '-') + formatPercent(overlapChange, true), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
|
||||
`\n${format('Performance', colors.White)}`,
|
||||
`${format((perfChange >= 0 ? '+' : '-') + formatPercent(perfChange, true), perfChange >= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
|
||||
` ${format('Memory', colors.White)}`,
|
||||
` ${format((memoryChange >= 0 ? '+' : '-') + formatPercent(memoryChange, true), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
|
||||
` ${format('Filesize', colors.White)}`,
|
||||
`${format((filesizeChange >= 0 ? '+' : '-') + formatPercent(filesizeChange, true), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
`${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`,
|
||||
|
||||
captureSummary.reduce((output, p, i) => {
|
||||
output += `${p.name} `;
|
||||
output += `${similarityRatings(behaviourSimilaritys[p.name])} `;
|
||||
output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
|
||||
if (i > 0 && i < captureSummary.length && breakEvery > 0 && i % breakEvery === 0) {
|
||||
output += '\n';
|
||||
}
|
||||
return output;
|
||||
}, '\n\n'),
|
||||
|
||||
`\n\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`,
|
||||
|
||||
behaviourSimilarityAverage < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 150 + '#' + behaviourSimilarityEntries[0][0], colors.BrightCyan)}` : '',
|
||||
intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : ''
|
||||
].join(' ');
|
||||
|
||||
if (save) {
|
||||
writeResult('examples-dev', devIntrinsicsChanged);
|
||||
writeResult('examples-build', buildIntrinsicsChanged);
|
||||
writeResult('examples-report', report(5, s => s));
|
||||
}
|
||||
|
||||
return report(5, color);
|
||||
};
|
||||
|
||||
const similarity = (a, b) => {
|
||||
|
|
@ -35,18 +104,56 @@ const similarity = (a, b) => {
|
|||
return 1 / (1 + (distance / a.length));
|
||||
};
|
||||
|
||||
const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
|
||||
const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
|
||||
const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
|
||||
const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
|
||||
const formatPercent = (val, abs) => (100 * (abs ? Math.abs(val) : val)).toFixed(2);
|
||||
|
||||
const noiseThreshold = (val, threshold) => {
|
||||
const sign = val < 0 ? -1 : 1;
|
||||
const magnitude = Math.abs(val);
|
||||
return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
|
||||
};
|
||||
|
||||
const equals = (a, b) => {
|
||||
try {
|
||||
expect(a).toEqual(b);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const capturePerformanceTotals = (captures) => {
|
||||
const totals = {
|
||||
duration: 0,
|
||||
overlap: 0,
|
||||
memory: 0
|
||||
};
|
||||
|
||||
for (const [ name ] of Object.entries(captures)) {
|
||||
totals.duration += captures[name].duration;
|
||||
totals.overlap += captures[name].overlap;
|
||||
totals.memory += captures[name].memory;
|
||||
};
|
||||
|
||||
return totals;
|
||||
};
|
||||
|
||||
const extrinsicSimilarity = (currentCaptures, referenceCaptures, key) => {
|
||||
const result = {};
|
||||
|
||||
Object.entries(currentCaptures).forEach(([name, current]) => {
|
||||
const reference = referenceCaptures[name];
|
||||
const worldVector = [];
|
||||
const worldVectorRef = [];
|
||||
const currentExtrinsic = current[key];
|
||||
const referenceExtrinsic = reference[key];
|
||||
|
||||
Object.keys(current.extrinsic).forEach(objectType => {
|
||||
Object.keys(current.extrinsic[objectType]).forEach(objectId => {
|
||||
worldVector.push(...current.extrinsic[objectType][objectId]);
|
||||
worldVectorRef.push(...reference.extrinsic[objectType][objectId]);
|
||||
Object.keys(currentExtrinsic).forEach(objectType => {
|
||||
Object.keys(currentExtrinsic[objectType]).forEach(objectId => {
|
||||
worldVector.push(...currentExtrinsic[objectType][objectId]);
|
||||
worldVectorRef.push(...referenceExtrinsic[objectType][objectId]);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -56,6 +163,15 @@ const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const extrinsicSimilarityAverage = (similaritys) => {
|
||||
const entries = Object.entries(similaritys);
|
||||
let average = 0;
|
||||
|
||||
entries.forEach(([_, similarity]) => average += similarity);
|
||||
|
||||
return average /= entries.length;
|
||||
};
|
||||
|
||||
const writeResult = (name, obj) => {
|
||||
try {
|
||||
fs.mkdirSync(comparePath, { recursive: true });
|
||||
|
|
@ -70,9 +186,35 @@ const writeResult = (name, obj) => {
|
|||
}
|
||||
};
|
||||
|
||||
const logReport = (captures, version) => {
|
||||
let report = '';
|
||||
|
||||
for (const capture of Object.values(captures)) {
|
||||
if (!capture.logs.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
report += ` ${capture.name}\n`;
|
||||
|
||||
for (const log of capture.logs) {
|
||||
report += ` ${log}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n`
|
||||
+ (report ? report : ' None\n');
|
||||
};
|
||||
|
||||
const requireUncached = path => {
|
||||
delete require.cache[require.resolve(path)];
|
||||
const module = require(path);
|
||||
delete require.cache[require.resolve(path)];
|
||||
return module;
|
||||
};
|
||||
|
||||
const toMatchExtrinsics = {
|
||||
toMatchExtrinsics(received, value) {
|
||||
const similaritys = captureSimilarityExtrinsic(received, value);
|
||||
const similaritys = extrinsicSimilarity(received, value, 'extrinsic');
|
||||
const pass = Object.values(similaritys).every(similarity => similarity >= equalityThreshold);
|
||||
|
||||
return {
|
||||
|
|
@ -101,126 +243,6 @@ const toMatchIntrinsics = {
|
|||
}
|
||||
};
|
||||
|
||||
const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
|
||||
const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
|
||||
|
||||
const equals = (a, b) => {
|
||||
try {
|
||||
expect(a).toEqual(b);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const logReport = (captures, version) => {
|
||||
let report = '';
|
||||
|
||||
for (const capture of Object.values(captures)) {
|
||||
if (!capture.logs.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
report += ` ${capture.name}\n`;
|
||||
|
||||
for (const log of capture.logs) {
|
||||
report += ` ${log}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n`
|
||||
+ (report ? report : ' None\n');
|
||||
};
|
||||
|
||||
const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => {
|
||||
const similaritys = captureSimilarityExtrinsic(capturesDev, capturesBuild);
|
||||
const similarityEntries = Object.entries(similaritys);
|
||||
const devIntrinsicsChanged = {};
|
||||
const buildIntrinsicsChanged = {};
|
||||
let intrinsicChangeCount = 0;
|
||||
let totalTimeBuild = 0;
|
||||
let totalTimeDev = 0;
|
||||
let totalOverlapBuild = 0;
|
||||
let totalOverlapDev = 0;
|
||||
let totalMemoryBuild = 0;
|
||||
let totalMemoryDev = 0;
|
||||
|
||||
const capturePerformance = Object.entries(capturesDev).map(([name]) => {
|
||||
totalTimeBuild += capturesBuild[name].duration;
|
||||
totalTimeDev += capturesDev[name].duration;
|
||||
|
||||
totalOverlapBuild += capturesBuild[name].overlap;
|
||||
totalOverlapDev += capturesDev[name].overlap;
|
||||
|
||||
totalMemoryBuild += capturesBuild[name].memory;
|
||||
totalMemoryDev += capturesDev[name].memory;
|
||||
|
||||
const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
|
||||
if (changedIntrinsics) {
|
||||
capturesDev[name].changedIntrinsics = true;
|
||||
if (intrinsicChangeCount < 2) {
|
||||
devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
|
||||
buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
|
||||
intrinsicChangeCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { name };
|
||||
});
|
||||
|
||||
capturePerformance.sort((a, b) => a.name.localeCompare(b.name));
|
||||
similarityEntries.sort((a, b) => a[1] - b[1]);
|
||||
|
||||
const perfChange = noiseThreshold(1 - (totalTimeDev / totalTimeBuild), 0.01);
|
||||
const memoryChange = noiseThreshold((totalMemoryDev / totalMemoryBuild) - 1, 0.01);
|
||||
const overlapChange = (totalOverlapDev / (totalOverlapBuild || 1)) - 1;
|
||||
const filesizeChange = (devSize / buildSize) - 1;
|
||||
|
||||
let similarityAvg = 0;
|
||||
similarityEntries.forEach(([_, similarity]) => {
|
||||
similarityAvg += similarity;
|
||||
});
|
||||
|
||||
similarityAvg /= similarityEntries.length;
|
||||
|
||||
const report = (breakEvery, format) => [
|
||||
[`Output comparison of ${similarityEntries.length}`,
|
||||
`examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}`
|
||||
].join(' '),
|
||||
`\n\n${format('Similarity', colors.White)}`,
|
||||
`${format(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`,
|
||||
`${format('Overlap', colors.White)}`,
|
||||
`${format((overlapChange >= 0 ? '+' : '-') + toPercent(Math.abs(overlapChange)), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
`${format('Performance ~', colors.White)}`,
|
||||
`${format((perfChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(perfChange)), perfChange >= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
`${format('Memory ~', colors.White)}`,
|
||||
`${format((memoryChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(memoryChange)), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
capturePerformance.reduce((output, p, i) => {
|
||||
output += `${p.name} `;
|
||||
output += `${similarityRatings(similaritys[p.name])} `;
|
||||
output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
|
||||
if (i > 0 && i < capturePerformance.length && breakEvery > 0 && i % breakEvery === 0) {
|
||||
output += '\n';
|
||||
}
|
||||
return output;
|
||||
}, '\n\n'),
|
||||
`\n\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`,
|
||||
similarityAvg < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 120 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '',
|
||||
intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : '',
|
||||
`\n\n${format('Filesize', colors.White)}`,
|
||||
`${format((filesizeChange >= 0 ? '+' : '-') + toPercent(Math.abs(filesizeChange)), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`,
|
||||
`${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`,
|
||||
].join(' ');
|
||||
|
||||
if (save) {
|
||||
writeResult('examples-dev', devIntrinsicsChanged);
|
||||
writeResult('examples-build', buildIntrinsicsChanged);
|
||||
writeResult('examples-report', report(5, s => s));
|
||||
}
|
||||
|
||||
return report(5, color);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
requireUncached, comparisonReport, logReport,
|
||||
toMatchExtrinsics, toMatchIntrinsics
|
||||
|
|
|
|||
Loading…
Reference in a new issue