foster/liabru-matter-js
0
0
Fork 0
mirror of https://github.com/liabru/matter-js.git synced 2024-11-23 09:26:51 -05:00
Code Issues Wiki Activity

implemented threaded comparison testing

Browse source
This commit is contained in:
liabru 2019-11-07 22:34:59 +00:00 committed by liabru
parent 82bb41535b
commit 285d70df34
7 changed files with 239 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
examples/avalanche.js
View file

@ -1,10 +1,10 @@
var Example = Example || {};
Matter.use(
'matter-wrap'
);
Example.avalanche = function() {
Matter.use(
'matter-wrap'
);
var Engine = Matter.Engine,
Render = Matter.Render,
Runner = Matter.Runner,

8
examples/ballPool.js
View file

@ -1,10 +1,10 @@
var Example = Example || {};
Matter.use(
'matter-wrap'
);
Example.ballPool = function() {
Matter.use(
'matter-wrap'
);
var Engine = Matter.Engine,
Render = Matter.Render,
Runner = Matter.Runner,

6
package.json
View file

@ -27,6 +27,7 @@
"gulp-tag-version": "^1.3.0",
"gulp-util": "^3.0.8",
"jest": "^24.9.0",
"jest-worker": "^24.9.0",
"json-stringify-pretty-compact": "^2.0.0",
"run-sequence": "^1.1.4",
"webpack": "^4.39.3",
@ -38,10 +39,11 @@
"build": "webpack --mode=production & webpack --mode=production --env.MINIMIZE",
"build-alpha": "webpack --mode=production --env.EDGE & webpack --mode=production --env.MINIMIZE --env.EDGE",
"build-examples": "webpack --config webpack.examples.config.js --mode=production --env.EDGE & webpack --config webpack.examples.config.js --mode=production --env.MINIMIZE --env.EDGE",
"lint": "eslint 'src/**/*.js' 'demo/js/Demo.js' 'demo/js/Compare.js' 'examples/*.js' 'test/*.spec.js' 'webpack.*.js' 'Gulpfile.js'",
"lint": "eslint 'src/**/*.js' 'demo/js/Demo.js' 'demo/js/Compare.js' 'examples/*.js' 'webpack.*.js' 'Gulpfile.js'",
"doc": "gulp doc",
"test": "jest",
"compare": "COMPARE=true jest"
"test-save": "SAVE=true jest",
"test-watch": "jest --watch"
},
"dependencies": {},
"files": [

2
src/factory/Bodies.js
View file

@ -196,7 +196,7 @@ var Vector = require('../geometry/Vector');
* @return {body}
*/
Bodies.fromVertices = function(x, y, vertexSets, options, flagInternal, removeCollinear, minimumArea) {
var decomp = typeof decomp !== 'undefined' ? decomp : require('poly-decomp'),
var decomp = global.decomp || require('poly-decomp'),
body,
parts,
isConvex,

61
test/ExampleWorker.js Normal file
View file

@ -0,0 +1,61 @@
/* eslint-env es6 */
/* eslint no-global-assign: 0 */
"use strict";
const stubBrowserFeatures = M => {
const noop = () => ({ collisionFilter: {}, mouse: {} });
M.Render.create = () => ({ options: {}, bounds: { min: { x: 0, y: 0 }, max: { x: 800, y: 600 }}});
M.Render.run = M.Render.lookAt = noop;
M.Runner.create = M.Runner.run = noop;
M.MouseConstraint.create = M.Mouse.create = noop;
M.Common.log = M.Common.info = M.Common.warn = noop;
return M;
};
const reset = M => {
M.Common._nextId = M.Common._seed = 0;
M.Body._nextCollidingGroupId = 1;
M.Body._nextNonCollidingGroupId = -1;
M.Body._nextCategory = 0x0001;
};
const { engineCapture } = require('./TestTools');
const MatterDev = stubBrowserFeatures(require('../src/module/main'));
const MatterBuild = stubBrowserFeatures(require('../build/Matter'));
const Example = require('../examples/index');
const decomp = require('../demo/lib/decomp');
const runExample = options => {
const Matter = options.useDev ? MatterDev : MatterBuild;
const consoleOriginal = global.console;
global.console = { log: () => {} };
global.document = {};
global.decomp = decomp;
global.Matter = Matter;
reset(Matter);
const example = Example[options.name]();
const engine = example.engine;
const startTime = process.hrtime();
for (let i = 0; i < options.totalUpdates; i += 1) {
Matter.Engine.update(engine, 1000 / 60);
}
const duration = process.hrtime(startTime);
global.console = consoleOriginal;
global.document = undefined;
global.decomp = undefined;
global.Matter = undefined;
return {
name: options.name,
duration: duration[0] * 1e9 + duration[1],
...engineCapture(engine)
};
};
module.exports = { runExample };

95
test/Examples.spec.js
View file

@ -1,73 +1,72 @@
/* eslint-env es6 */
/* eslint no-global-assign: 0 */
"use strict";
const {
stubBrowserFeatures, engineSnapshot, toMatchExtrinsics, toMatchIntrinsics
} = require('./TestTools');
jest.setTimeout(30 * 1000);
const totalUpdates = 120;
const isCompare = process.env.COMPARE === 'true';
const excludeExamples = ['stress', 'stress2', 'svg', 'terrain'];
const { comparisonReport, toMatchExtrinsics, toMatchIntrinsics } = require('./TestTools');
const Example = require('../examples/index');
const MatterBuild = require('../build/matter');
const MatterDev = require('../src/module/main');
const Worker = require('jest-worker').default;
jest.mock('matter-wrap', () => require('../demo/lib/matter-wrap'), { virtual: true });
jest.mock('poly-decomp', () => require('../demo/lib/decomp'), { virtual: true });
const testComparison = process.env.COMPARE === 'true';
const saveComparison = process.env.SAVE === 'true';
const excludeExamples = [ 'svg', 'terrain' ];
const examples = Object.keys(Example).filter(key => !excludeExamples.includes(key));
const runExamples = (matter) => {
let snapshots = {};
matter = stubBrowserFeatures(matter);
global.Matter = matter;
matter.use(require('matter-wrap'));
const runExamples = async useDev => {
const worker = new Worker(require.resolve('./ExampleWorker'), {
enableWorkerThreads: true
});
const Example = require('../examples/index');
const examples = Object.keys(Example).filter(key => !excludeExamples.includes(key));
const result = await Promise.all(examples.map(name => worker.runExample({
name,
useDev,
totalUpdates: 120
})));
const consoleOriginal = global.console;
global.console = { log: () => {} };
await worker.end();
for (name of examples) {
matter.Common._nextId = matter.Common._seed = 0;
const example = Example[name]();
const engine = example.engine;
for (let i = 0; i < totalUpdates; i += 1) {
matter.Engine.update(engine, 1000 / 60);
}
snapshots[name] = isCompare ? engineSnapshot(engine) : {};
}
global.console = consoleOriginal;
global.Matter = undefined;
return snapshots;
return result.reduce((out, capture) => (out[capture.name] = capture, out), {});
};
const snapshotsDev = runExamples(MatterDev);
const snapshotsBuild = runExamples(MatterBuild);
const examples = Object.keys(snapshotsDev);
const capturesDev = runExamples(true);
const capturesBuild = runExamples(false);
describe(`Integration tests (${examples.length})`, () => {
test(`Examples run without throwing`, () => {
expect(Object.keys(snapshotsDev)).toEqual(examples);
expect(Object.keys(snapshotsBuild)).toEqual(examples);
afterAll(async () => {
// Report experimental capture comparison.
const dev = await capturesDev;
const build = await capturesBuild;
console.log(comparisonReport(dev, build, MatterBuild.version, saveComparison));
});
describe(`Integration checks (${examples.length})`, () => {
test(`Examples run without throwing`, async () => {
const dev = await capturesDev;
const build = await capturesBuild;
expect(Object.keys(dev)).toEqual(examples);
expect(Object.keys(build)).toEqual(examples);
});
});
if (isCompare) {
describe(`Regression tests (${examples.length})`, () => {
// Experimental regression comparison checks.
if (testComparison) {
describe(`Regression checks (${examples.length})`, () => {
expect.extend(toMatchExtrinsics);
expect.extend(toMatchIntrinsics);
test(`Examples match properties with release build`, () => {
expect(snapshotsDev).toMatchIntrinsics(snapshotsBuild, totalUpdates);
test(`Examples match intrinsic properties with release build`, async () => {
const dev = await capturesDev;
const build = await capturesBuild;
// compare mass, inertia, friction etc.
expect(dev).toMatchIntrinsics(build);
});
test(`Examples match positions and velocities with release build`, () => {
expect(snapshotsDev).toMatchExtrinsics(snapshotsBuild, totalUpdates);
test(`Examples match extrinsic positions and velocities with release build`, async () => {
const dev = await capturesDev;
const build = await capturesBuild;
// compare position, linear and angular velocity
expect(dev).toMatchExtrinsics(build);
});
});
}

223
test/TestTools.js
View file

@ -6,8 +6,10 @@ const compactStringify = require('json-stringify-pretty-compact');
const { Composite, Constraint } = require('../src/module/main');
const comparePath = './test/__compare__';
const compareCommand = 'open http://localhost:8000/?compare'
const diffCommand = 'code -n -d test/__compare__/examples-dev.json test/__compare__/examples-build.json';
const compareCommand = 'open http://localhost:8000/?compare';
const diffSaveCommand = 'npm run test-save';
const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json';
const equalityThreshold = 0.99999;
const intrinsicProps = [
// Common
@ -26,27 +28,18 @@ const intrinsicProps = [
'bodies', 'constraints', 'composites'
];
const stubBrowserFeatures = M => {
const noop = () => ({ collisionFilter: {}, mouse: {} });
M.Render.create = () => ({ options: {}, bounds: { min: { x: 0, y: 0 }, max: { x: 800, y: 600 }}});
M.Render.run = M.Render.lookAt = noop;
M.Runner.create = M.Runner.run = noop;
M.MouseConstraint.create = M.Mouse.create = noop;
M.Common.log = M.Common.info = M.Common.warn = noop;
return M;
};
const colors = { White: 37, BrightWhite: 90, BrightCyan: 36 };
const color = (text, number) => `\x1b[${number}m${text}\x1b[0m`;
const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };
const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
const limit = (val, precision=3) => parseFloat(val.toPrecision(precision));
const toPercent = val => (100 * val).toPrecision(3);
const engineSnapshot = (engine) => ({
const engineCapture = (engine) => ({
timestamp: limit(engine.timing.timestamp),
world: worldSnapshotExtrinsic(engine.world),
worldIntrinsic: worldSnapshotIntrinsic(engine.world)
extrinsic: worldCaptureExtrinsic(engine.world),
intrinsic: worldCaptureIntrinsic(engine.world)
});
const worldSnapshotExtrinsic = world => ({
const worldCaptureExtrinsic = world => ({
bodies: Composite.allBodies(world).reduce((bodies, body) => {
bodies[body.id] = [
body.position.x,
@ -75,7 +68,7 @@ const worldSnapshotExtrinsic = world => ({
}, {})
});
const worldSnapshotIntrinsic = world => worldSnapshotIntrinsicBase({
const worldCaptureIntrinsic = world => worldCaptureIntrinsicBase({
bodies: Composite.allBodies(world).reduce((bodies, body) => {
bodies[body.id] = body;
return bodies;
@ -94,13 +87,13 @@ const worldSnapshotIntrinsic = world => worldSnapshotIntrinsicBase({
}, {})
});
const worldSnapshotIntrinsicBase = (obj, depth=0) => {
const worldCaptureIntrinsicBase = (obj, depth=0) => {
if (obj === Infinity) {
return 'Infinity';
} else if (typeof obj === 'number') {
return limit(obj);
} else if (Array.isArray(obj)) {
return obj.map(item => worldSnapshotIntrinsicBase(item, depth + 1));
return obj.map(item => worldCaptureIntrinsicBase(item, depth + 1));
} else if (typeof obj !== 'object') {
return obj;
}
@ -116,7 +109,7 @@ const worldSnapshotIntrinsicBase = (obj, depth=0) => {
val = `[${val.length}]`;
}
cleaned[key] = worldSnapshotIntrinsicBase(val, depth + 1);
cleaned[key] = worldCaptureIntrinsicBase(val, depth + 1);
return cleaned;
}, {});
@ -129,18 +122,18 @@ const similarity = (a, b) => {
return 1 / (1 + (distance / a.length));
};
const snapshotSimilarityExtrinsic = (currentSnapshots, referenceSnapshots) => {
const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
const result = {};
Object.entries(currentSnapshots).forEach(([name, current]) => {
const reference = referenceSnapshots[name];
Object.entries(currentCaptures).forEach(([name, current]) => {
const reference = referenceCaptures[name];
const worldVector = [];
const worldVectorRef = [];
Object.keys(current.world).forEach(objectType => {
Object.keys(current.world[objectType]).forEach(objectId => {
worldVector.push(...current.world[objectType][objectId]);
worldVectorRef.push(...reference.world[objectType][objectId]);
Object.keys(current.extrinsic).forEach(objectType => {
Object.keys(current.extrinsic[objectType]).forEach(objectId => {
worldVector.push(...current.extrinsic[objectType][objectId]);
worldVectorRef.push(...reference.extrinsic[objectType][objectId]);
});
});
@ -150,7 +143,7 @@ const snapshotSimilarityExtrinsic = (currentSnapshots, referenceSnapshots) => {
return result;
};
const writeSnapshots = (name, obj) => {
const writeCaptures = (name, obj) => {
try {
fs.mkdirSync(comparePath, { recursive: true });
} catch (err) {
@ -160,98 +153,118 @@ const writeSnapshots = (name, obj) => {
};
const toMatchExtrinsics = {
toMatchExtrinsics(received, value, ticks) {
const changed = [];
const borderline = [];
const equal = [];
const similaritys = snapshotSimilarityExtrinsic(received, value);
const entries = Object.entries(similaritys);
entries.sort(([_nameA, similarityA], [_nameB, similarityB]) => similarityA - similarityB);
entries.forEach(([name, similarity], i) => {
const percentSimilar = similarity * 100;
if (percentSimilar < 99.99) {
const col = i < 5 ? colors.White : colors.BrightWhite;
changed.push(color(`◇ ${name}`, col) + ` ${percentSimilar.toFixed(2)}%`);
} else if (percentSimilar !== 100) {
borderline.push(`~ ${name}`);
} else {
equal.push(`✓ ${name}`);
}
});
const pass = equal.length === entries.length && changed.length === 0 && borderline.length === 0;
toMatchExtrinsics(received, value) {
const similaritys = captureSimilarityExtrinsic(received, value);
const pass = Object.values(similaritys).every(similarity => similarity >= equalityThreshold);
return {
message: () => `Expected positions and velocities to match between builds.
${color('▶', colors.White)} Debug using ${color(compareCommand + '=' + ticks + '#' + entries[0][0], colors.BrightCyan)}
(${changed.length}) Changed
${changed.join(' ')}
(${borderline.length}) Borderline (> 99.99%)
${borderline.join(' ').slice(0, 80)}...
(${equal.length}) Equal
${equal.join(' ').slice(0, 80)}...`,
message: () => 'Expected positions and velocities to match between builds.',
pass
};
}
};
const toMatchIntrinsics = {
toMatchIntrinsics(currentSnapshots, referenceSnapshots) {
const changed = [];
const equal = [];
const currentChanged = {};
const referenceChanged = {};
const entries = Object.entries(currentSnapshots);
toMatchIntrinsics(currentCaptures, referenceCaptures) {
const entries = Object.entries(currentCaptures);
let changed = false;
entries.forEach(([name, current]) => {
const reference = referenceSnapshots[name];
const endWorld = current.worldIntrinsic;
const endWorldRef = reference.worldIntrinsic;
if (this.equals(endWorld, endWorldRef)) {
equal.push(`✓ ${name}`);
} else {
changed.push(color(`◇ ${name}`, changed.length < 5 ? colors.White : colors.BrightWhite));
if (changed.length < 2) {
currentChanged[name] = endWorld;
referenceChanged[name] = endWorldRef;
}
const reference = referenceCaptures[name];
if (!this.equals(current.intrinsic, reference.intrinsic)) {
changed = true;
}
});
const pass = equal.length === entries.length && changed.length === 0;
writeSnapshots('examples-dev', currentChanged);
writeSnapshots('examples-build', referenceChanged);
return {
message: () => `Expected intrinsic properties to match between builds.
(${changed.length}) Changed
${changed.join(' ')}
(${equal.length}) Equal
${equal.join(' ').slice(0, 80)}...
${color('▶', colors.White)} Inspect using ${color(diffCommand, colors.BrightCyan)}`,
pass
message: () => 'Expected intrinsic properties to match between builds.',
pass: !changed
};
}
};
const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
const equals = (a, b) => {
try {
expect(a).toEqual(b);
} catch (e) {
return false;
}
return true;
};
const comparisonReport = (capturesDev, capturesBuild, buildVersion, save) => {
const similaritys = captureSimilarityExtrinsic(capturesDev, capturesBuild);
const similarityEntries = Object.entries(similaritys);
const devIntrinsicsChanged = {};
const buildIntrinsicsChanged = {};
let intrinsicChangeCount = 0;
let totalTimeBuild = 0;
let totalTimeDev = 0;
const capturePerformance = Object.entries(capturesDev).map(([name]) => {
const buildDuration = capturesBuild[name].duration;
const devDuration = capturesDev[name].duration;
totalTimeBuild += buildDuration;
totalTimeDev += devDuration;
const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
if (changedIntrinsics) {
capturesDev[name].changedIntrinsics = true;
if (intrinsicChangeCount < 2) {
devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
intrinsicChangeCount += 1;
}
}
return { name };
});
capturePerformance.sort((a, b) => a.name.localeCompare(b.name));
similarityEntries.sort((a, b) => a[1] - b[1]);
let similarityAvg = 0;
let perfChange = 1 - (totalTimeDev / totalTimeBuild);
perfChange = perfChange < -0.05 || perfChange > 0.05 ? perfChange : 0;
similarityEntries.forEach(([_, similarity]) => {
similarityAvg += similarity;
});
similarityAvg /= similarityEntries.length;
if (save) {
writeCaptures('examples-dev', devIntrinsicsChanged);
writeCaptures('examples-build', buildIntrinsicsChanged);
}
return [
[`Output comparison of ${similarityEntries.length}`,
`examples against ${color('matter-js@' + buildVersion, colors.Yellow)} build on last run`
].join(' '),
`\n\n${color('Similarity', colors.White)}`,
`${color(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`,
`${color('Performance', colors.White)}`,
`${color((perfChange >= 0 ? '+' : '') + toPercent(perfChange), perfChange >= 0 ? colors.Green : colors.Red)}%`,
capturePerformance.reduce((output, p, i) => {
output += `${p.name} `;
output += `${similarityRatings(similaritys[p.name])} `;
output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
if (i > 0 && i < capturePerformance.length && i % 5 === 0) {
output += '\n';
}
return output;
}, '\n\n'),
`\nwhere · no change ● extrinsics changed ◆ intrinsics changed\n`,
similarityAvg < 1 ? `\n${color('▶', colors.White)} ${color(compareCommand + '=' + 120 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '',
intrinsicChangeCount > 0 ? `\n${color('▶', colors.White)} ${color((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : ''
].join(' ');
};
module.exports = {
stubBrowserFeatures, engineSnapshot, toMatchExtrinsics, toMatchIntrinsics
engineCapture, comparisonReport,
toMatchExtrinsics, toMatchIntrinsics
};