0
0
Fork 0
mirror of https://github.com/liabru/matter-js.git synced 2024-11-23 09:26:51 -05:00

improve test comparison report

This commit is contained in:
liabru 2024-03-03 21:01:45 +00:00
parent a3e801acfd
commit 2cc1c1c4e5
5 changed files with 3298 additions and 4768 deletions

7729
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -23,8 +23,8 @@
"conventional-changelog-cli": "^2.1.1",
"eslint": "^6.8.0",
"html-webpack-plugin": "^4.5.1",
"jest": "^25.1.0",
"jest-worker": "^24.9.0",
"jest": "^29.7.0",
"jest-worker": "^29.7.0",
"json-stringify-pretty-compact": "^2.0.0",
"matter-tools": "^0.14.0",
"matter-wrap": "^0.2.0",
@ -50,7 +50,7 @@
"lint": "eslint 'src/**/*.js' 'demo/src/**/*.js' 'examples/*.js' 'webpack.*.js'",
"doc": "yuidoc --config yuidoc.json --project-version $npm_package_version",
"doc-watch": "nodemon --delay 3 --watch 'matter-doc-theme' --watch src -e 'js,html,css,handlebars' --exec 'npm run doc'",
"benchmark": "npm run test-node -- --examples=stress3,stress4 --updates=300 --repeats=3",
"benchmark": "npm run test-node -- --examples=stress3,stress4 --benchmark=true --updates=300 --repeats=3",
"test": "npm run test-node",
"test-node": "npm run build-dev && node --expose-gc node_modules/.bin/jest --force-exit --no-cache --runInBand ./test/Examples.spec.js",
"test-browser": "node --expose-gc node_modules/.bin/jest --force-exit --no-cache --runInBand ./test/Browser.spec.js",

View file

@ -3,8 +3,9 @@
"use strict";
const mock = require('mock-require');
const { requireUncached, serialize } = require('./TestTools');
const { requireUncached, serialize, smoothExp } = require('./TestTools');
const consoleOriginal = global.console;
const DateOriginal = global.Date;
const runExample = options => {
const {
@ -13,8 +14,8 @@ const runExample = options => {
frameCallbacks
} = prepareEnvironment(options);
let totalMemory = 0;
let totalDuration = 0;
let memoryDeltaAverage = 0;
let timeDeltaAverage = 0;
let overlapTotal = 0;
let overlapCount = 0;
let i;
@ -24,6 +25,12 @@ const runExample = options => {
let runner;
let engine;
let render;
let extrinsicCapture;
const bodyOverlap = (bodyA, bodyB) => {
const collision = Matter.Collision.collides(bodyA, bodyB);
return collision ? collision.depth : 0;
};
for (i = 0; i < options.repeats; i += 1) {
if (global.gc) {
@ -36,35 +43,47 @@ const runExample = options => {
runner = example.runner;
engine = example.engine;
render = example.render;
for (j = 0; j < options.updates; j += 1) {
const time = j * runner.delta;
const callbackCount = frameCallbacks.length;
global.timeNow = time;
for (let p = 0; p < callbackCount; p += 1) {
totalMemory += process.memoryUsage().heapUsed;
const callback = frameCallbacks.shift();
const startTime = process.hrtime();
const frameCallback = frameCallbacks.shift();
const memoryBefore = process.memoryUsage().heapUsed;
const timeBefore = process.hrtime();
callback(time);
frameCallback(time);
const duration = process.hrtime(startTime);
totalMemory += process.memoryUsage().heapUsed;
totalDuration += duration[0] * 1e9 + duration[1];
const timeDuration = process.hrtime(timeBefore);
const timeDelta = timeDuration[0] * 1e9 + timeDuration[1];
const memoryAfter = process.memoryUsage().heapUsed;
const memoryDelta = Math.max(memoryAfter - memoryBefore, 0);
memoryDeltaAverage = smoothExp(memoryDeltaAverage, memoryDelta);
timeDeltaAverage = smoothExp(timeDeltaAverage, timeDelta);
}
const pairsList = engine.pairs.list;
const pairsListLength = engine.pairs.list.length;
if (j === 1) {
const pairsList = engine.pairs.list;
const pairsListLength = engine.pairs.list.length;
for (let p = 0; p < pairsListLength; p += 1) {
const pair = pairsList[p];
const separation = pair.separation - pair.slop;
for (let p = 0; p < pairsListLength; p += 1) {
const pair = pairsList[p];
if (pair.isActive && !pair.isSensor) {
overlapTotal += separation > 0 ? separation : 0;
overlapCount += 1;
if (pair.isActive && !pair.isSensor) {
overlapTotal += bodyOverlap(pair.bodyA, pair.bodyB);
overlapCount += 1;
}
}
}
if (!extrinsicCapture && engine.timing.timestamp >= 1000) {
extrinsicCapture = captureExtrinsics(engine, Matter);
extrinsicCapture.updates = j;
}
}
}
@ -72,13 +91,13 @@ const runExample = options => {
return {
name: options.name,
duration: totalDuration,
duration: timeDeltaAverage,
memory: memoryDeltaAverage,
overlap: overlapTotal / (overlapCount || 1),
memory: totalMemory,
logs: logs,
extrinsic: captureExtrinsics(engine, Matter),
extrinsic: extrinsicCapture,
intrinsic: captureIntrinsics(engine, Matter),
state: captureState(engine, runner, render)
state: captureState(engine, runner, render),
logs
};
} catch (err) {
@ -144,6 +163,7 @@ const prepareEnvironment = options => {
const frameCallbacks = [];
global.document = global.window = {
performance: {},
addEventListener: () => {},
requestAnimationFrame: callback => {
frameCallbacks.push(callback);
@ -174,6 +194,21 @@ const prepareEnvironment = options => {
}
};
global.Math.random = () => {
throw new Error("Math.random was called during tests, output can not be compared.");
};
global.timeNow = 0;
global.window.performance.now = () => global.timeNow;
global.Date = function() {
this.toString = () => global.timeNow.toString();
this.valueOf = () => global.timeNow;
};
global.Date.now = () => global.timeNow;
const Matter = prepareMatter(options);
mock('matter-js', Matter);
global.Matter = Matter;
@ -187,6 +222,7 @@ const prepareEnvironment = options => {
const resetEnvironment = () => {
global.console = consoleOriginal;
global.Date = DateOriginal;
global.window = undefined;
global.document = undefined;
global.Matter = undefined;
@ -195,46 +231,22 @@ const resetEnvironment = () => {
const captureExtrinsics = ({ world }, Matter) => ({
bodies: Matter.Composite.allBodies(world).reduce((bodies, body) => {
bodies[body.id] = [
body.position.x,
body.position.y,
body.positionPrev.x,
body.positionPrev.y,
body.angle,
body.anglePrev,
...body.vertices.reduce((flat, vertex) => (flat.push(vertex.x, vertex.y), flat), [])
];
bodies[body.id] = {
position: { x: body.position.x, y: body.position.y },
vertices: body.vertices.map(vertex => ({ x: vertex.x, y: vertex.y }))
};
return bodies;
}, {}),
constraints: Matter.Composite.allConstraints(world).reduce((constraints, constraint) => {
let positionA;
let positionB;
try {
positionA = Matter.Constraint.pointAWorld(constraint);
} catch (err) {
positionA = { x: 0, y: 0 };
}
try {
positionB = Matter.Constraint.pointBWorld(constraint);
} catch (err) {
positionB = { x: 0, y: 0 };
}
constraints[constraint.id] = [
positionA.x,
positionA.y,
positionB.x,
positionB.y
];
return constraints;
}, {})
});
const captureIntrinsics = ({ world }, Matter) => serialize({
const captureIntrinsics = ({ world, timing }, Matter) => serialize({
engine: {
timing: {
timeScale: timing.timeScale,
timestamp: timing.timestamp
}
},
bodies: Matter.Composite.allBodies(world).reduce((bodies, body) => {
bodies[body.id] = body;
return bodies;
@ -289,6 +301,9 @@ const extrinsicProperties = [
'velocity',
'position',
'positionPrev',
'motion',
'sleepCounter',
'positionImpulse'
];
const excludeStateProperties = [

View file

@ -17,13 +17,14 @@ const {
const Example = requireUncached('../examples/index');
const MatterBuild = requireUncached('../build/matter');
const { versionSatisfies } = requireUncached('../src/core/Plugin');
const Worker = require('jest-worker').default;
const Worker = require('jest-worker').Worker;
const testComparison = getArg('compare', null) === 'true';
const saveComparison = getArg('save', null) === 'true';
const specificExamples = getArg('examples', null, (val) => val.split(','));
const repeats = getArg('repeats', 1, parseFloat);
const updates = getArg('updates', 150, parseFloat);
const benchmark = getArg('benchmark', null) === 'true';
const excludeExamples = ['svg', 'terrain'];
const excludeJitter = ['stack', 'circleStack', 'restitution', 'staticFriction', 'friction', 'newtonsCradle', 'catapult'];
@ -37,68 +38,26 @@ const examples = (specificExamples || Object.keys(Example)).filter(key => {
});
const captureExamples = async useDev => {
const multiThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
enableWorkerThreads: true
});
const overlapRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
name,
useDev,
updates: 2,
repeats: 1,
stableSort: true,
jitter: excludeJitter.includes(name) ? 0 : 1e-10
})));
const behaviourRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
name,
useDev,
updates: 2,
repeats: 1,
stableSort: true,
jitter: excludeJitter.includes(name) ? 0 : 1e-10
})));
const similarityRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
name,
useDev,
updates: 2,
repeats: 1,
stableSort: false,
jitter: excludeJitter.includes(name) ? 0 : 1e-10
})));
await multiThreadWorker.end();
const singleThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
const worker = new Worker(require.resolve('./ExampleWorker'), {
enableWorkerThreads: true,
numWorkers: 1
numWorkers: benchmark ? 1 : undefined
});
const completeRuns = await Promise.all(examples.map(name => singleThreadWorker.runExample({
const completeRuns = await Promise.all(examples.map(name => worker.runExample({
name,
useDev,
updates: updates,
repeats: repeats,
repeats: benchmark ? Math.max(repeats, 3) : repeats,
stableSort: false,
jitter: excludeJitter.includes(name) ? 0 : 1e-10
})));
await singleThreadWorker.end();
await worker.end();
const capture = {};
for (const completeRun of completeRuns) {
const behaviourRun = behaviourRuns.find(({ name }) => name === completeRun.name);
const similarityRun = similarityRuns.find(({ name }) => name === completeRun.name);
const overlapRun = overlapRuns.find(({ name }) => name === completeRun.name);
capture[overlapRun.name] = {
...completeRun,
behaviourExtrinsic: behaviourRun.extrinsic,
similarityExtrinsic: similarityRun.extrinsic,
overlap: overlapRun.overlap
};
capture[completeRun.name] = completeRun;
}
return capture;
@ -119,7 +78,7 @@ afterAll(async () => {
'Examples ran against previous release and current build\n\n'
+ logReport(build, `release`) + '\n'
+ logReport(dev, `current`) + '\n'
+ comparisonReport(dev, build, devSize, buildSize, MatterBuild.version, saveComparison)
+ comparisonReport(dev, build, devSize, buildSize, MatterBuild.version, saveComparison, benchmark)
);
});

View file

@ -8,10 +8,10 @@ const comparePath = './test/__compare__';
const compareCommand = 'open http://localhost:8000/?compare';
const diffSaveCommand = 'npm run test-save';
const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json';
const equalityThreshold = 0.99999;
const equalityThreshold = 1;
const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };
const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => {
const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save, benchmark) => {
const performanceDev = capturePerformanceTotals(capturesDev);
const performanceBuild = capturePerformanceTotals(capturesBuild);
@ -20,13 +20,15 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1;
const filesizeChange = (devSize / buildSize) - 1;
const behaviourSimilaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'behaviourExtrinsic');
const behaviourSimilarityAverage = extrinsicSimilarityAverage(behaviourSimilaritys);
const behaviourSimilarityEntries = Object.entries(behaviourSimilaritys);
behaviourSimilarityEntries.sort((a, b) => a[1] - b[1]);
const similaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'similarityExtrinsic');
const similaritys = extrinsicSimilarity(capturesDev, capturesBuild);
const similarityAverage = extrinsicSimilarityAverage(similaritys);
const similarityEntries = Object.entries(similaritys);
similarityEntries.sort((a, b) => a[1] - b[1]);
const firstCapture = Object.entries(capturesDev)[0][1];
const updates = firstCapture.extrinsic.updates;
const similarityAveragePerUpdate = Math.pow(1, -1 / updates) * Math.pow(similarityAverage, 1 / updates);
const devIntrinsicsChanged = {};
const buildIntrinsicsChanged = {};
@ -49,33 +51,34 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
})
.sort((a, b) => a.name.localeCompare(b.name));
const formatColor = green => green ? colors.Green : colors.Yellow;
const report = (breakEvery, format) => [
[`Output comparison of ${behaviourSimilarityEntries.length}`,
`examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}`
[`Output sample comparison estimates of ${similarityEntries.length} examples`,
`against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}:`
].join(' '),
`\n\n${format('Behaviour ', colors.White)}`,
`${format(formatPercent(behaviourSimilarityAverage), behaviourSimilarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
` ${format('Similarity', colors.White)}`,
`${format(formatPercent(similarityAverage), similarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
`\n\n${format(`Similarity`, colors.White)} `,
`${format(formatPercent(similarityAveragePerUpdate, false, true), formatColor(similarityAveragePerUpdate === 1))}% `,
` ${format('Overlap', colors.White)}`,
` ${format((overlapChange >= 0 ? '+' : '-') + formatPercent(overlapChange, true), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`,
`\n${format('Performance', colors.White)}`,
`${format((perfChange >= 0 ? '+' : '-') + formatPercent(perfChange, true), perfChange >= 0 ? colors.Green : colors.Yellow)}%`,
` ${format('Memory', colors.White)}`,
` ${format((memoryChange >= 0 ? '+' : '-') + formatPercent(memoryChange, true), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`,
` ${format(formatPercent(overlapChange), formatColor(overlapChange <= 0))}%`,
` ${format('Filesize', colors.White)}`,
`${format((filesizeChange >= 0 ? '+' : '-') + formatPercent(filesizeChange, true), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`,
`${format(formatPercent(filesizeChange), formatColor(filesizeChange <= 0))}%`,
`${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`,
...(benchmark ? [
`\n${format('Performance', colors.White)}`,
` ${format(formatPercent(perfChange), formatColor(perfChange >= 0))}%`,
` ${format('Memory', colors.White)} `,
` ${format(formatPercent(memoryChange), formatColor(memoryChange <= 0))}%`,
] : []),
captureSummary.reduce((output, p, i) => {
output += `${p.name} `;
output += `${similarityRatings(behaviourSimilaritys[p.name])} `;
output += `${similarityRatings(similaritys[p.name])} `;
output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
if (i > 0 && i < captureSummary.length && breakEvery > 0 && i % breakEvery === 0) {
output += '\n';
@ -83,9 +86,9 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
return output;
}, '\n\n'),
`\n\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`,
`\n\nwhere for the sample · no change detected ● extrinsics changed ◆ intrinsics changed\n`,
behaviourSimilarityAverage < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 150 + '#' + behaviourSimilarityEntries[0][0], colors.BrightCyan)}` : '',
similarityAverage < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 150 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '',
intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : ''
].join(' ');
@ -98,17 +101,25 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
return report(5, color);
};
const similarity = (a, b) => {
const distance = Math.sqrt(a.reduce(
(sum, _val, i) => sum + Math.pow((a[i] || 0) - (b[i] || 0), 2), 0)
);
return 1 / (1 + (distance / a.length));
};
const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
const formatPercent = (val, abs) => (100 * (abs ? Math.abs(val) : val)).toFixed(2);
const formatPercent = (val, showSign=true, showFractional=false, padStart=6) => {
let fractionalSign = '';
if (showFractional && val > 0.9999 && val < 1) {
val = 0.9999;
fractionalSign = '>';
} else if (showFractional && val > 0 && val < 0.0001) {
val = 0.0001;
fractionalSign = '<';
}
const percentFixed = Math.abs(100 * val).toFixed(2);
const sign = parseFloat((100 * val).toFixed(2)) >= 0 ? '+' : '-';
return ((showFractional ? fractionalSign : '') + (showSign ? sign : '') + percentFixed).padStart(padStart, ' ');
};
const noiseThreshold = (val, threshold) => {
const sign = val < 0 ? -1 : 1;
@ -116,6 +127,13 @@ const noiseThreshold = (val, threshold) => {
return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
};
const smoothExp = (last, current) => {
const delta = current - last;
const sign = delta < 0 ? -1 : 1;
const magnitude = Math.abs(delta);
return last + Math.sqrt(magnitude) * sign;
};
const equals = (a, b) => {
try {
expect(a).toEqual(b);
@ -141,25 +159,46 @@ const capturePerformanceTotals = (captures) => {
return totals;
};
const extrinsicSimilarity = (currentCaptures, referenceCaptures, key) => {
const extrinsicSimilarity = (currentCaptures, referenceCaptures, key='extrinsic') => {
const result = {};
const zeroVector = { x: 0, y: 0 };
Object.entries(currentCaptures).forEach(([name, current]) => {
const reference = referenceCaptures[name];
const worldVector = [];
const worldVectorRef = [];
const currentExtrinsic = current[key];
const referenceExtrinsic = reference[key];
for (const name in currentCaptures) {
const currentExtrinsic = currentCaptures[name][key];
const referenceExtrinsic = referenceCaptures[name][key];
Object.keys(currentExtrinsic).forEach(objectType => {
Object.keys(currentExtrinsic[objectType]).forEach(objectId => {
worldVector.push(...currentExtrinsic[objectType][objectId]);
worldVectorRef.push(...referenceExtrinsic[objectType][objectId]);
});
});
let totalCount = 0;
let totalSimilarity = 0;
result[name] = similarity(worldVector, worldVectorRef);
});
for (const objectType in currentExtrinsic) {
for (const objectId in currentExtrinsic[objectType]) {
const currentObject = currentExtrinsic[objectType][objectId];
const referenceObject = referenceExtrinsic[objectType][objectId];
for (let i = 0; i < currentObject.vertices.length; i += 1) {
const currentPosition = currentObject.position;
const currentVertex = currentObject.vertices[i];
const referenceVertex = referenceObject.vertices[i] ? referenceObject.vertices[i] : zeroVector;
const radius = Math.sqrt(
Math.pow(currentVertex.x - currentPosition.x, 2)
+ Math.pow(currentVertex.y - currentPosition.y, 2)
);
const distance = Math.sqrt(
Math.pow(currentVertex.x - referenceVertex.x, 2)
+ Math.pow(currentVertex.y - referenceVertex.y, 2)
);
totalSimilarity += Math.min(1, distance / (2 * radius)) / currentObject.vertices.length;
}
totalCount += 1;
}
}
result[name] = 1 - (totalSimilarity / totalCount);
}
return result;
};
@ -291,6 +330,6 @@ const toMatchIntrinsics = {
};
module.exports = {
requireUncached, comparisonReport, logReport, getArg,
requireUncached, comparisonReport, logReport, getArg, smoothExp,
serialize, toMatchExtrinsics, toMatchIntrinsics
};