implemented threaded comparison testing

examples/avalanche.js

@@ -1,10 +1,10 @@
 var Example = Example || {};
 
+Example.avalanche = function() {
     Matter.use(
         'matter-wrap'
     );
     
-Example.avalanche = function() {
     var Engine = Matter.Engine,
         Render = Matter.Render,
         Runner = Matter.Runner,

examples/ballPool.js

@@ -1,10 +1,10 @@
 var Example = Example || {};
 
+Example.ballPool = function() {
     Matter.use(
         'matter-wrap'
     );
     
-Example.ballPool = function() {
     var Engine = Matter.Engine,
         Render = Matter.Render,
         Runner = Matter.Runner,

package.json

@@ -27,6 +27,7 @@
     "gulp-tag-version": "^1.3.0",
     "gulp-util": "^3.0.8",
     "jest": "^24.9.0",
+    "jest-worker": "^24.9.0",
     "json-stringify-pretty-compact": "^2.0.0",
     "run-sequence": "^1.1.4",
     "webpack": "^4.39.3",
@@ -38,10 +39,11 @@
     "build": "webpack --mode=production & webpack --mode=production --env.MINIMIZE",
     "build-alpha": "webpack --mode=production --env.EDGE & webpack --mode=production --env.MINIMIZE --env.EDGE",
     "build-examples": "webpack --config webpack.examples.config.js --mode=production --env.EDGE & webpack --config webpack.examples.config.js --mode=production --env.MINIMIZE --env.EDGE",
-    "lint": "eslint 'src/**/*.js' 'demo/js/Demo.js' 'demo/js/Compare.js' 'examples/*.js' 'test/*.spec.js' 'webpack.*.js' 'Gulpfile.js'",
+    "lint": "eslint 'src/**/*.js' 'demo/js/Demo.js' 'demo/js/Compare.js' 'examples/*.js' 'webpack.*.js' 'Gulpfile.js'",
     "doc": "gulp doc",
     "test": "jest",
-    "compare": "COMPARE=true jest"
+    "test-save": "SAVE=true jest",
+    "test-watch": "jest --watch"
   },
   "dependencies": {},
   "files": [

src/factory/Bodies.js

@@ -196,7 +196,7 @@ var Vector = require('../geometry/Vector');
      * @return {body}
      */
     Bodies.fromVertices = function(x, y, vertexSets, options, flagInternal, removeCollinear, minimumArea) {
-        var decomp = typeof decomp !== 'undefined' ? decomp : require('poly-decomp'),
+        var decomp = global.decomp || require('poly-decomp'),
             body,
             parts,
             isConvex,

test/ExampleWorker.js

@@ -0,0 +1,61 @@
+/* eslint-env es6 */
+/* eslint no-global-assign: 0 */
+"use strict";
+
+const stubBrowserFeatures = M => {
+  const noop = () => ({ collisionFilter: {}, mouse: {} });
+  M.Render.create = () => ({ options: {}, bounds: { min: { x: 0, y: 0 }, max: { x: 800, y: 600 }}});
+  M.Render.run = M.Render.lookAt = noop;
+  M.Runner.create = M.Runner.run = noop;
+  M.MouseConstraint.create = M.Mouse.create = noop;
+  M.Common.log = M.Common.info = M.Common.warn = noop;
+  return M;
+};
+
+const reset = M => {
+  M.Common._nextId = M.Common._seed = 0;
+  M.Body._nextCollidingGroupId = 1;
+  M.Body._nextNonCollidingGroupId = -1;
+  M.Body._nextCategory = 0x0001;
+};
+
+const { engineCapture } = require('./TestTools');
+const MatterDev = stubBrowserFeatures(require('../src/module/main'));
+const MatterBuild = stubBrowserFeatures(require('../build/Matter'));
+const Example = require('../examples/index');
+const decomp = require('../demo/lib/decomp');
+
+const runExample = options => {
+  const Matter = options.useDev ? MatterDev : MatterBuild;
+  const consoleOriginal = global.console;
+
+  global.console = { log: () => {} };
+  global.document = {};
+  global.decomp = decomp;
+  global.Matter = Matter;
+
+  reset(Matter);
+
+  const example = Example[options.name]();
+  const engine = example.engine;
+  const startTime = process.hrtime();
+
+  for (let i = 0; i < options.totalUpdates; i += 1) {
+      Matter.Engine.update(engine, 1000 / 60);
+  }
+
+  const duration = process.hrtime(startTime);
+
+  global.console = consoleOriginal;
+  global.document = undefined;
+  global.decomp = undefined;
+  global.Matter = undefined;
+
+  return {
+    name: options.name,
+    duration: duration[0] * 1e9 + duration[1],
+    ...engineCapture(engine)
+  };
+};
+
+module.exports = { runExample };

test/Examples.spec.js

@@ -1,73 +1,72 @@
 /* eslint-env es6 */
-/* eslint no-global-assign: 0 */
 "use strict";
 
-const { 
+jest.setTimeout(30 * 1000);
-    stubBrowserFeatures, engineSnapshot, toMatchExtrinsics, toMatchIntrinsics 
-} = require('./TestTools');
 
-const totalUpdates = 120;
+const { comparisonReport, toMatchExtrinsics, toMatchIntrinsics } = require('./TestTools');
-const isCompare = process.env.COMPARE === 'true';
-const excludeExamples = ['stress', 'stress2', 'svg', 'terrain'];
-
-const MatterBuild = require('../build/matter');
-const MatterDev = require('../src/module/main');
-
-jest.mock('matter-wrap', () => require('../demo/lib/matter-wrap'), { virtual: true });
-jest.mock('poly-decomp', () => require('../demo/lib/decomp'), { virtual: true });
-
-const runExamples = (matter) => {
-    let snapshots = {};
-    matter = stubBrowserFeatures(matter);
-    global.Matter = matter;
-    matter.use(require('matter-wrap'));
 
 const Example = require('../examples/index');
+const MatterBuild = require('../build/matter');
+const Worker = require('jest-worker').default;
+
+const testComparison = process.env.COMPARE === 'true';
+const saveComparison = process.env.SAVE === 'true';
+const excludeExamples = [ 'svg', 'terrain' ];
 const examples = Object.keys(Example).filter(key => !excludeExamples.includes(key));
 
-    const consoleOriginal = global.console;
+const runExamples = async useDev => {
-    global.console = { log: () => {} };
+    const worker = new Worker(require.resolve('./ExampleWorker'), {
+        enableWorkerThreads: true
+    });
 
-    for (name of examples) {
+    const result = await Promise.all(examples.map(name => worker.runExample({
-        matter.Common._nextId = matter.Common._seed = 0;
+        name,
+        useDev,
+        totalUpdates: 120
+    })));
 
-        const example = Example[name]();
+    await worker.end();
-        const engine = example.engine;
 
-        for (let i = 0; i < totalUpdates; i += 1) {
+    return result.reduce((out, capture) => (out[capture.name] = capture, out), {});
-            matter.Engine.update(engine, 1000 / 60);
-        }
-    
-        snapshots[name] = isCompare ? engineSnapshot(engine) : {};
-    }
-
-    global.console = consoleOriginal;
-    global.Matter = undefined;
-    return snapshots;
 };
 
-const snapshotsDev = runExamples(MatterDev);
+const capturesDev = runExamples(true);
-const snapshotsBuild = runExamples(MatterBuild);
+const capturesBuild = runExamples(false);
-const examples = Object.keys(snapshotsDev);
 
-describe(`Integration tests (${examples.length})`, () => {
+afterAll(async () => {
-    test(`Examples run without throwing`, () => {
+    // Report experimental capture comparison.
-        expect(Object.keys(snapshotsDev)).toEqual(examples);
+    const dev = await capturesDev;
-        expect(Object.keys(snapshotsBuild)).toEqual(examples);
+    const build = await capturesBuild;
+    console.log(comparisonReport(dev, build, MatterBuild.version, saveComparison));
+});
+
+describe(`Integration checks (${examples.length})`, () => {
+    test(`Examples run without throwing`, async () => {
+        const dev = await capturesDev;
+        const build = await capturesBuild;
+        expect(Object.keys(dev)).toEqual(examples);
+        expect(Object.keys(build)).toEqual(examples);
     });
 });
 
-if (isCompare) {
+// Experimental regression comparison checks.
-    describe(`Regression tests (${examples.length})`, () => {
+if (testComparison) {
+    describe(`Regression checks (${examples.length})`, () => {
         expect.extend(toMatchExtrinsics);
         expect.extend(toMatchIntrinsics);
 
-        test(`Examples match properties with release build`, () => {
+        test(`Examples match intrinsic properties with release build`, async () => {
-            expect(snapshotsDev).toMatchIntrinsics(snapshotsBuild, totalUpdates);
+            const dev = await capturesDev;
+            const build = await capturesBuild;
+            // compare mass, inertia, friction etc.
+            expect(dev).toMatchIntrinsics(build);
         });
 
-        test(`Examples match positions and velocities with release build`, () => {
+        test(`Examples match extrinsic positions and velocities with release build`, async () => {
-            expect(snapshotsDev).toMatchExtrinsics(snapshotsBuild, totalUpdates);
+            const dev = await capturesDev;
+            const build = await capturesBuild;
+            // compare position, linear and angular velocity
+            expect(dev).toMatchExtrinsics(build);
         });
     });
 }

test/TestTools.js

@@ -6,8 +6,10 @@ const compactStringify = require('json-stringify-pretty-compact');
 const { Composite, Constraint } = require('../src/module/main');
 
 const comparePath = './test/__compare__';
-const compareCommand = 'open http://localhost:8000/?compare'
+const compareCommand = 'open http://localhost:8000/?compare';
-const diffCommand = 'code -n -d test/__compare__/examples-dev.json test/__compare__/examples-build.json';
+const diffSaveCommand = 'npm run test-save';
+const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json';
+const equalityThreshold = 0.99999;
 
 const intrinsicProps = [
     // Common
@@ -26,27 +28,18 @@ const intrinsicProps = [
     'bodies', 'constraints', 'composites'
 ];
 
-const stubBrowserFeatures = M => {
+const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };
-    const noop = () => ({ collisionFilter: {}, mouse: {} });
+const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
-    M.Render.create = () => ({ options: {}, bounds: { min: { x: 0, y: 0 }, max: { x: 800, y: 600 }}});
-    M.Render.run = M.Render.lookAt = noop;
-    M.Runner.create = M.Runner.run = noop;
-    M.MouseConstraint.create = M.Mouse.create = noop;
-    M.Common.log = M.Common.info = M.Common.warn = noop;
-    return M;
-};
-
-const colors = { White: 37, BrightWhite: 90, BrightCyan: 36 };
-const color = (text, number) => `\x1b[${number}m${text}\x1b[0m`;
 const limit = (val, precision=3) => parseFloat(val.toPrecision(precision));
+const toPercent = val => (100 * val).toPrecision(3);
 
-const engineSnapshot = (engine) => ({
+const engineCapture = (engine) => ({
     timestamp: limit(engine.timing.timestamp),
-    world: worldSnapshotExtrinsic(engine.world),
+    extrinsic: worldCaptureExtrinsic(engine.world),
-    worldIntrinsic: worldSnapshotIntrinsic(engine.world)
+    intrinsic: worldCaptureIntrinsic(engine.world)
 });
 
-const worldSnapshotExtrinsic = world => ({
+const worldCaptureExtrinsic = world => ({
     bodies: Composite.allBodies(world).reduce((bodies, body) => {
         bodies[body.id] = [
             body.position.x,
@@ -75,7 +68,7 @@ const worldSnapshotExtrinsic = world => ({
     }, {})
 });
 
-const worldSnapshotIntrinsic = world => worldSnapshotIntrinsicBase({
+const worldCaptureIntrinsic = world => worldCaptureIntrinsicBase({
     bodies: Composite.allBodies(world).reduce((bodies, body) => {
         bodies[body.id] = body;
         return bodies;
@@ -94,13 +87,13 @@ const worldSnapshotIntrinsic = world => worldSnapshotIntrinsicBase({
     }, {})
 });
 
-const worldSnapshotIntrinsicBase = (obj, depth=0) => {
+const worldCaptureIntrinsicBase = (obj, depth=0) => {
     if (obj === Infinity) {
         return 'Infinity';
     } else if (typeof obj === 'number') {
         return limit(obj);
     } else if (Array.isArray(obj)) {
-        return obj.map(item => worldSnapshotIntrinsicBase(item, depth + 1));
+        return obj.map(item => worldCaptureIntrinsicBase(item, depth + 1));
     } else if (typeof obj !== 'object') {
         return obj;
     }
@@ -116,7 +109,7 @@ const worldSnapshotIntrinsicBase = (obj, depth=0) => {
                 val = `[${val.length}]`;
             }
 
-            cleaned[key] = worldSnapshotIntrinsicBase(val, depth + 1);
+            cleaned[key] = worldCaptureIntrinsicBase(val, depth + 1);
             return cleaned;
         }, {});
 
@@ -129,18 +122,18 @@ const similarity = (a, b) => {
     return 1 / (1 + (distance / a.length));
 };
 
-const snapshotSimilarityExtrinsic = (currentSnapshots, referenceSnapshots) => {
+const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
     const result = {};
 
-    Object.entries(currentSnapshots).forEach(([name, current]) => {
+    Object.entries(currentCaptures).forEach(([name, current]) => {
-        const reference = referenceSnapshots[name];
+        const reference = referenceCaptures[name];
         const worldVector = [];
         const worldVectorRef = [];
 
-        Object.keys(current.world).forEach(objectType => {
+        Object.keys(current.extrinsic).forEach(objectType => {
-            Object.keys(current.world[objectType]).forEach(objectId => {
+            Object.keys(current.extrinsic[objectType]).forEach(objectId => {
-                worldVector.push(...current.world[objectType][objectId]);
+                worldVector.push(...current.extrinsic[objectType][objectId]);
-                worldVectorRef.push(...reference.world[objectType][objectId]);
+                worldVectorRef.push(...reference.extrinsic[objectType][objectId]);
             });
         });
 
@@ -150,7 +143,7 @@ const snapshotSimilarityExtrinsic = (currentSnapshots, referenceSnapshots) => {
     return result;
 };
 
-const writeSnapshots = (name, obj) => {
+const writeCaptures = (name, obj) => {
     try {
         fs.mkdirSync(comparePath, { recursive: true });
     } catch (err) {
@@ -160,98 +153,118 @@ const writeSnapshots = (name, obj) => {
 };
 
 const toMatchExtrinsics = {
-    toMatchExtrinsics(received, value, ticks) {
+    toMatchExtrinsics(received, value) {
-        const changed = [];
+        const similaritys = captureSimilarityExtrinsic(received, value);
-        const borderline = [];
+        const pass = Object.values(similaritys).every(similarity => similarity >= equalityThreshold);
-        const equal = [];
-        const similaritys = snapshotSimilarityExtrinsic(received, value);
-        const entries = Object.entries(similaritys);
-
-        entries.sort(([_nameA, similarityA], [_nameB, similarityB]) => similarityA - similarityB);
-
-        entries.forEach(([name, similarity], i) => {
-            const percentSimilar = similarity * 100;
-
-            if (percentSimilar < 99.99) {
-                const col = i < 5 ? colors.White : colors.BrightWhite;
-                changed.push(color(`◇ ${name}`, col) + ` ${percentSimilar.toFixed(2)}%`);
-            } else if (percentSimilar !== 100) {
-                borderline.push(`~ ${name}`);
-            } else {
-                equal.push(`✓ ${name}`);
-            }
-        });
-
-        const pass = equal.length === entries.length && changed.length === 0 && borderline.length === 0;
 
         return {
-            message: () => `Expected positions and velocities to match between builds.
+            message: () => 'Expected positions and velocities to match between builds.',
-
-${color('▶', colors.White)} Debug using ${color(compareCommand + '=' + ticks + '#' + entries[0][0], colors.BrightCyan)}
-
-(${changed.length}) Changed
-
-    ${changed.join(' ')}
-
-(${borderline.length}) Borderline (> 99.99%)
-
-    ${borderline.join(' ').slice(0, 80)}...
-
-(${equal.length}) Equal
-
-    ${equal.join(' ').slice(0, 80)}...`,
             pass
         };
     }
 };
 
 const toMatchIntrinsics = {
-    toMatchIntrinsics(currentSnapshots, referenceSnapshots) {
+    toMatchIntrinsics(currentCaptures, referenceCaptures) {
-        const changed = [];
+        const entries = Object.entries(currentCaptures);
-        const equal = [];
+        let changed = false;
-        const currentChanged = {};
-        const referenceChanged = {};
-        const entries = Object.entries(currentSnapshots);
 
         entries.forEach(([name, current]) => {
-            const reference = referenceSnapshots[name];
+            const reference = referenceCaptures[name];
-            const endWorld = current.worldIntrinsic;
+            if (!this.equals(current.intrinsic, reference.intrinsic)) {
-            const endWorldRef = reference.worldIntrinsic;
+                changed = true;
-
-            if (this.equals(endWorld, endWorldRef)) {
-                equal.push(`✓ ${name}`);
-            } else {
-                changed.push(color(`◇ ${name}`, changed.length < 5 ? colors.White : colors.BrightWhite));
-
-                if (changed.length < 2) {
-                    currentChanged[name] = endWorld;
-                    referenceChanged[name] = endWorldRef;
-                }
             }
         });
 
-        const pass = equal.length === entries.length && changed.length === 0;
-
-        writeSnapshots('examples-dev', currentChanged);
-        writeSnapshots('examples-build', referenceChanged);
-
         return {
-            message: () => `Expected intrinsic properties to match between builds.
+            message: () => 'Expected intrinsic properties to match between builds.',
-
+            pass: !changed
-(${changed.length}) Changed
-
-${changed.join(' ')}
-
-(${equal.length}) Equal
-
-${equal.join(' ').slice(0, 80)}...
-
-${color('▶', colors.White)} Inspect using ${color(diffCommand, colors.BrightCyan)}`,
-            pass
         };
     }
 };
 
-module.exports = {
+const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
-    stubBrowserFeatures, engineSnapshot, toMatchExtrinsics, toMatchIntrinsics
+const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
+
+const equals = (a, b) => {
+    try {
+        expect(a).toEqual(b);
+    } catch (e) {
+        return false;
+    }
+    return true;
+};
+
+const comparisonReport = (capturesDev, capturesBuild, buildVersion, save) => {
+    const similaritys = captureSimilarityExtrinsic(capturesDev, capturesBuild);
+    const similarityEntries = Object.entries(similaritys);
+    const devIntrinsicsChanged = {};
+    const buildIntrinsicsChanged = {};
+    let intrinsicChangeCount = 0;
+    let totalTimeBuild = 0;
+    let totalTimeDev = 0;
+
+    const capturePerformance = Object.entries(capturesDev).map(([name]) => {
+        const buildDuration = capturesBuild[name].duration;
+        const devDuration = capturesDev[name].duration;
+        totalTimeBuild += buildDuration;
+        totalTimeDev += devDuration;
+
+        const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
+        if (changedIntrinsics) {
+            capturesDev[name].changedIntrinsics = true;
+            if (intrinsicChangeCount < 2) {
+                devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
+                buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
+                intrinsicChangeCount += 1;
+            }
+        }
+        
+        return { name };
+    });
+
+    capturePerformance.sort((a, b) => a.name.localeCompare(b.name));
+    similarityEntries.sort((a, b) => a[1] - b[1]);
+
+    let similarityAvg = 0;
+    let perfChange = 1 - (totalTimeDev / totalTimeBuild);
+    perfChange = perfChange < -0.05 || perfChange > 0.05 ? perfChange : 0;
+
+    similarityEntries.forEach(([_, similarity]) => {
+        similarityAvg += similarity;
+    });
+
+    similarityAvg /= similarityEntries.length;
+
+    if (save) {
+        writeCaptures('examples-dev', devIntrinsicsChanged);
+        writeCaptures('examples-build', buildIntrinsicsChanged);
+    }
+
+    return [
+        [`Output comparison of ${similarityEntries.length}`,
+         `examples against ${color('matter-js@' + buildVersion, colors.Yellow)} build on last run`
+        ].join(' '),
+        `\n\n${color('Similarity', colors.White)}`,
+        `${color(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`,
+        `${color('Performance', colors.White)}`,
+        `${color((perfChange >= 0 ? '+' : '') + toPercent(perfChange), perfChange >= 0 ? colors.Green : colors.Red)}%`,
+        capturePerformance.reduce((output, p, i) => {
+            output += `${p.name} `;
+            output += `${similarityRatings(similaritys[p.name])} `;
+            output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
+            if (i > 0 && i < capturePerformance.length && i % 5 === 0) {
+                output += '\n';
+            }
+            return output;
+        }, '\n\n'),
+        `\nwhere  · no change  ● extrinsics changed  ◆ intrinsics changed\n`,
+        similarityAvg < 1 ? `\n${color('▶', colors.White)} ${color(compareCommand + '=' + 120 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '',
+        intrinsicChangeCount > 0 ? `\n${color('▶', colors.White)} ${color((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : ''
+    ].join('  ');
+};
+
+module.exports = {
+    engineCapture, comparisonReport,
+    toMatchExtrinsics, toMatchIntrinsics
 };