added behaviour metric to tests and refactor tests

examples/sleeping.js

@@ -61,12 +61,15 @@ Example.sleeping = function() {
     
     Composite.add(world, stack);
 
+    /*
+    // sleep events
     for (var i = 0; i < stack.bodies.length; i++) {
         Events.on(stack.bodies[i], 'sleepStart sleepEnd', function(event) {
             var body = this;
             console.log('body id', body.id, 'sleeping:', body.isSleeping);
         });
     }
+    */
 
     // add mouse control
     var mouse = Mouse.create(render.canvas),

test/Examples.spec.js

@@ -32,21 +32,67 @@ const examples = Object.keys(Example).filter(key => {
 });
 
 const captureExamples = async useDev => {
-    const worker = new Worker(require.resolve('./ExampleWorker'), {
+    const multiThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
+        enableWorkerThreads: true
+    });
+
+    const overlapRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
+        name,
+        useDev,
+        updates: 1,
+        stableSort: true,
+        jitter: excludeJitter.includes(name) ? 0 : 1e-10
+    })));
+
+    const behaviourRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
+        name,
+        useDev,
+        updates: 2,
+        stableSort: true,
+        jitter: excludeJitter.includes(name) ? 0 : 1e-10
+    })));
+
+    const similarityRuns = await Promise.all(examples.map(name => multiThreadWorker.runExample({
+        name,
+        useDev,
+        updates: 2,
+        stableSort: false,
+        jitter: excludeJitter.includes(name) ? 0 : 1e-10
+    })));
+
+    await multiThreadWorker.end();
+
+    const singleThreadWorker = new Worker(require.resolve('./ExampleWorker'), {
         enableWorkerThreads: true,
         numWorkers: 1
     });
 
-    const result = await Promise.all(examples.map(name => worker.runExample({
+    const completeRuns = await Promise.all(examples.map(name => singleThreadWorker.runExample({
         name,
         useDev,
-        totalUpdates: 120,
+        updates: 150,
+        stableSort: false,
         jitter: excludeJitter.includes(name) ? 0 : 1e-10
     })));
 
-    await worker.end();
+    await singleThreadWorker.end();
 
-    return result.reduce((out, capture) => (out[capture.name] = capture, out), {});
+    const capture = {};
+
+    for (const completeRun of completeRuns) {
+        const behaviourRun = behaviourRuns.find(({ name }) => name === completeRun.name);
+        const similarityRun = similarityRuns.find(({ name }) => name === completeRun.name);
+        const overlapRun = overlapRuns.find(({ name }) => name === completeRun.name);
+
+        capture[overlapRun.name] = {
+            ...completeRun,
+            behaviourExtrinsic: behaviourRun.extrinsic,
+            similarityExtrinsic: similarityRun.extrinsic,
+            overlap: overlapRun.overlap
+        };
+    }
+
+    return capture;
 };
 
 const capturesDev = captureExamples(true);

test/TestTools.js

@@ -9,23 +9,92 @@ const compareCommand = 'open http://localhost:8000/?compare';
 const diffSaveCommand = 'npm run test-save';
 const diffCommand = 'code -n -d test/__compare__/examples-build.json test/__compare__/examples-dev.json';
 const equalityThreshold = 0.99999;
-
 const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };
-const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
-const toPercent = val => (100 * val).toFixed(3);
-const toPercentRound = val => Math.round(100 * val);
 
-const requireUncached = path => {
+const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => {
-    delete require.cache[require.resolve(path)];
+    const performanceDev = capturePerformanceTotals(capturesDev);
-    const module = require(path);
+    const performanceBuild = capturePerformanceTotals(capturesBuild);
-    delete require.cache[require.resolve(path)];
-    return module;
-};
 
-const noiseThreshold = (val, threshold) => {
+    const perfChange = noiseThreshold(1 - (performanceDev.duration / performanceBuild.duration), 0.01);
-    const sign = val < 0 ? -1 : 1;
+    const memoryChange = noiseThreshold((performanceDev.memory / performanceBuild.memory) - 1, 0.01);
-    const magnitude = Math.abs(val);
+    const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1;
-    return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
+    const filesizeChange = (devSize / buildSize) - 1;
+
+    const behaviourSimilaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'behaviourExtrinsic');
+    const behaviourSimilarityAverage = extrinsicSimilarityAverage(behaviourSimilaritys);
+    const behaviourSimilarityEntries = Object.entries(behaviourSimilaritys);
+    behaviourSimilarityEntries.sort((a, b) => a[1] - b[1]);
+
+    const similaritys = extrinsicSimilarity(capturesDev, capturesBuild, 'similarityExtrinsic');
+    const similarityAverage = extrinsicSimilarityAverage(similaritys);
+
+    const devIntrinsicsChanged = {};
+    const buildIntrinsicsChanged = {};
+    let intrinsicChangeCount = 0;
+
+    const captureSummary = Object.entries(capturesDev)
+        .map(([name]) => {
+            const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
+            if (changedIntrinsics) {
+                capturesDev[name].changedIntrinsics = true;
+                if (intrinsicChangeCount < 2) {
+                    devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
+                    buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
+                    intrinsicChangeCount += 1;
+                }
+            }
+            
+            return { name };
+        })
+        .sort((a, b) => a.name.localeCompare(b.name));
+
+    const report = (breakEvery, format) => [
+        [`Output comparison of ${behaviourSimilarityEntries.length}`,
+         `examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}`
+        ].join(' '),
+
+        `\n\n${format('Behaviour  ', colors.White)}`,
+        `${format(formatPercent(behaviourSimilarityAverage), behaviourSimilarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
+
+        ` ${format('Similarity', colors.White)}`,
+        `${format(formatPercent(similarityAverage), similarityAverage === 1 ? colors.Green : colors.Yellow)}%`,
+
+        ` ${format('Overlap', colors.White)}`,
+        ` ${format((overlapChange >= 0 ? '+' : '-') + formatPercent(overlapChange, true), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`,
+
+        `\n${format('Performance', colors.White)}`,
+        `${format((perfChange >= 0 ? '+' : '-') + formatPercent(perfChange, true), perfChange >= 0 ? colors.Green : colors.Yellow)}%`,
+
+        ` ${format('Memory', colors.White)}`,
+        `    ${format((memoryChange >= 0 ? '+' : '-') + formatPercent(memoryChange, true), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`,
+
+        ` ${format('Filesize', colors.White)}`,
+        `${format((filesizeChange >= 0 ? '+' : '-') + formatPercent(filesizeChange, true), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`,
+        `${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`,
+
+        captureSummary.reduce((output, p, i) => {
+            output += `${p.name} `;
+            output += `${similarityRatings(behaviourSimilaritys[p.name])} `;
+            output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
+            if (i > 0 && i < captureSummary.length && breakEvery > 0 && i % breakEvery === 0) {
+                output += '\n';
+            }
+            return output;
+        }, '\n\n'),
+
+        `\n\nwhere  · no change  ● extrinsics changed  ◆ intrinsics changed\n`,
+
+        behaviourSimilarityAverage < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 150 + '#' + behaviourSimilarityEntries[0][0], colors.BrightCyan)}` : '',
+        intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : ''
+    ].join('  ');
+
+    if (save) {
+        writeResult('examples-dev', devIntrinsicsChanged);
+        writeResult('examples-build', buildIntrinsicsChanged);
+        writeResult('examples-report', report(5, s => s));
+    }
+
+    return report(5, color);
 };
 
 const similarity = (a, b) => {
@@ -35,18 +104,56 @@ const similarity = (a, b) => {
     return 1 / (1 + (distance / a.length));
 };
 
-const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
+const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
+const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
+const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
+const formatPercent = (val, abs) => (100 * (abs ? Math.abs(val) : val)).toFixed(2);
+
+const noiseThreshold = (val, threshold) => {
+    const sign = val < 0 ? -1 : 1;
+    const magnitude = Math.abs(val);
+    return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
+};
+
+const equals = (a, b) => {
+    try {
+        expect(a).toEqual(b);
+    } catch (e) {
+        return false;
+    }
+    return true;
+};
+
+const capturePerformanceTotals = (captures) => {
+    const totals = {
+        duration: 0,
+        overlap: 0,
+        memory: 0
+    };
+
+    for (const [ name ] of Object.entries(captures)) {
+        totals.duration += captures[name].duration;
+        totals.overlap += captures[name].overlap;
+        totals.memory += captures[name].memory;
+    };
+
+    return totals;
+};
+
+const extrinsicSimilarity = (currentCaptures, referenceCaptures, key) => {
     const result = {};
 
     Object.entries(currentCaptures).forEach(([name, current]) => {
         const reference = referenceCaptures[name];
         const worldVector = [];
         const worldVectorRef = [];
+        const currentExtrinsic = current[key];
+        const referenceExtrinsic = reference[key];
 
-        Object.keys(current.extrinsic).forEach(objectType => {
+        Object.keys(currentExtrinsic).forEach(objectType => {
-            Object.keys(current.extrinsic[objectType]).forEach(objectId => {
+            Object.keys(currentExtrinsic[objectType]).forEach(objectId => {
-                worldVector.push(...current.extrinsic[objectType][objectId]);
+                worldVector.push(...currentExtrinsic[objectType][objectId]);
-                worldVectorRef.push(...reference.extrinsic[objectType][objectId]);
+                worldVectorRef.push(...referenceExtrinsic[objectType][objectId]);
             });
         });
 
@@ -56,6 +163,15 @@ const captureSimilarityExtrinsic = (currentCaptures, referenceCaptures) => {
     return result;
 };
 
+const extrinsicSimilarityAverage = (similaritys) => {
+    const entries = Object.entries(similaritys);
+    let average = 0;
+
+    entries.forEach(([_, similarity]) => average += similarity);
+
+    return average /= entries.length;
+};
+
 const writeResult = (name, obj) => {
     try {
         fs.mkdirSync(comparePath, { recursive: true });
@@ -70,9 +186,35 @@ const writeResult = (name, obj) => {
     }
 };
 
+const logReport = (captures, version) => {
+    let report = '';
+
+    for (const capture of Object.values(captures)) {
+        if (!capture.logs.length) {
+            continue;
+        }
+
+        report += `  ${capture.name}\n`;
+
+        for (const log of capture.logs) {
+            report += `    ${log}\n`;
+        }
+    }
+
+    return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n` 
+        + (report ? report : '  None\n');
+};
+
+const requireUncached = path => {
+    delete require.cache[require.resolve(path)];
+    const module = require(path);
+    delete require.cache[require.resolve(path)];
+    return module;
+};
+
 const toMatchExtrinsics = {
     toMatchExtrinsics(received, value) {
-        const similaritys = captureSimilarityExtrinsic(received, value);
+        const similaritys = extrinsicSimilarity(received, value, 'extrinsic');
         const pass = Object.values(similaritys).every(similarity => similarity >= equalityThreshold);
 
         return {
@@ -101,126 +243,6 @@ const toMatchIntrinsics = {
     }
 };
 
-const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';
-const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';
-
-const equals = (a, b) => {
-    try {
-        expect(a).toEqual(b);
-    } catch (e) {
-        return false;
-    }
-    return true;
-};
-
-const logReport = (captures, version) => {
-    let report = '';
-
-    for (const capture of Object.values(captures)) {
-        if (!capture.logs.length) {
-            continue;
-        }
-
-        report += `  ${capture.name}\n`;
-
-        for (const log of capture.logs) {
-            report += `    ${log}\n`;
-        }
-    }
-
-    return `Output logs from ${color(version, colors.Yellow)} build on last run\n\n` 
-        + (report ? report : '  None\n');
-};
-
-const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save) => {
-    const similaritys = captureSimilarityExtrinsic(capturesDev, capturesBuild);
-    const similarityEntries = Object.entries(similaritys);
-    const devIntrinsicsChanged = {};
-    const buildIntrinsicsChanged = {};
-    let intrinsicChangeCount = 0;
-    let totalTimeBuild = 0;
-    let totalTimeDev = 0;
-    let totalOverlapBuild = 0;
-    let totalOverlapDev = 0;
-    let totalMemoryBuild = 0;
-    let totalMemoryDev = 0;
-
-    const capturePerformance = Object.entries(capturesDev).map(([name]) => {
-        totalTimeBuild += capturesBuild[name].duration;
-        totalTimeDev += capturesDev[name].duration;
-
-        totalOverlapBuild += capturesBuild[name].overlap;
-        totalOverlapDev += capturesDev[name].overlap;
-
-        totalMemoryBuild += capturesBuild[name].memory;
-        totalMemoryDev += capturesDev[name].memory;
-
-        const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
-        if (changedIntrinsics) {
-            capturesDev[name].changedIntrinsics = true;
-            if (intrinsicChangeCount < 2) {
-                devIntrinsicsChanged[name] = capturesDev[name].intrinsic;
-                buildIntrinsicsChanged[name] = capturesBuild[name].intrinsic;
-                intrinsicChangeCount += 1;
-            }
-        }
-        
-        return { name };
-    });
-
-    capturePerformance.sort((a, b) => a.name.localeCompare(b.name));
-    similarityEntries.sort((a, b) => a[1] - b[1]);
-
-    const perfChange = noiseThreshold(1 - (totalTimeDev / totalTimeBuild), 0.01);
-    const memoryChange = noiseThreshold((totalMemoryDev / totalMemoryBuild) - 1, 0.01);
-    const overlapChange = (totalOverlapDev / (totalOverlapBuild || 1)) - 1;
-    const filesizeChange = (devSize / buildSize) - 1;
-
-    let similarityAvg = 0;
-    similarityEntries.forEach(([_, similarity]) => {
-        similarityAvg += similarity;
-    });
-
-    similarityAvg /= similarityEntries.length;
-
-    const report = (breakEvery, format) => [
-        [`Output comparison of ${similarityEntries.length}`,
-         `examples against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}`
-        ].join(' '),
-        `\n\n${format('Similarity', colors.White)}`,
-        `${format(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`,
-        `${format('Overlap', colors.White)}`,
-        `${format((overlapChange >= 0 ? '+' : '-') + toPercent(Math.abs(overlapChange)), overlapChange <= 0 ? colors.Green : colors.Yellow)}%`,
-        `${format('Performance ~', colors.White)}`,
-        `${format((perfChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(perfChange)), perfChange >= 0 ? colors.Green : colors.Yellow)}%`,
-        `${format('Memory ~', colors.White)}`,
-        `${format((memoryChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(memoryChange)), memoryChange <= 0 ? colors.Green : colors.Yellow)}%`,
-        capturePerformance.reduce((output, p, i) => {
-            output += `${p.name} `;
-            output += `${similarityRatings(similaritys[p.name])} `;
-            output += `${changeRatings(capturesDev[p.name].changedIntrinsics)} `;
-            if (i > 0 && i < capturePerformance.length && breakEvery > 0 && i % breakEvery === 0) {
-                output += '\n';
-            }
-            return output;
-        }, '\n\n'),
-        `\n\nwhere  · no change  ● extrinsics changed  ◆ intrinsics changed\n`,
-        similarityAvg < 1 ? `\n${format('▶', colors.White)} ${format(compareCommand + '=' + 120 + '#' + similarityEntries[0][0], colors.BrightCyan)}` : '',
-        intrinsicChangeCount > 0 ? `\n${format('▶', colors.White)} ${format((save ? diffCommand : diffSaveCommand), colors.BrightCyan)}` : '',
-        `\n\n${format('Filesize', colors.White)}`,
-        `${format((filesizeChange >= 0 ? '+' : '-') + toPercent(Math.abs(filesizeChange)), filesizeChange <= 0 ? colors.Green : colors.Yellow)}%`,
-        `${format(`${(devSize / 1024).toPrecision(4)} KB`, colors.White)}`,
-    ].join('  ');
-
-    if (save) {
-        writeResult('examples-dev', devIntrinsicsChanged);
-        writeResult('examples-build', buildIntrinsicsChanged);
-        writeResult('examples-report', report(5, s => s));
-    }
-
-    return report(5, color);
-};
-
 module.exports = {
     requireUncached, comparisonReport, logReport,
     toMatchExtrinsics, toMatchIntrinsics