foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-22 07:14:47 -05:00
Code Issues Packages 1 Wiki Activity

Improve tools/unit_tests.py (#958)

Browse source 
Checks the output more carefully. The first line of output from
js/unit_tests.ts should be something like "running 96 tests"
And the last line should be something like
"test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36
filtered out"
This parses those strings and make sure they align.

This will catch silent death bugs.
This commit is contained in:
Ryan Dahl 2018-10-11 16:56:50 -04:00 committed by GitHub
parent c814d5a914
commit d4f72e18be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 675 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

238
tools/testdata/unit_test_output1.txt vendored Normal file
View file

@ -0,0 +1,238 @@
running 96 tests
test permSerialization_permW0N0E0
... ok
test permFromStringThrows_permW0N0E0
... ok
test compilerInstance_permW0N0E0
... ok
test compilerRun_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRunMultiModule_permW0N0E0
... ok
test compilerRunCircularDependency_permW0N0E0
Compiling modA
Compiling modB
... ok
test compilerResolveModule_permW0N0E0
... ok
test compilerGetModuleDependencies_permW0N0E0
... ok
test compilerGetCompilationSettings_permW0N0E0
... ok
test compilerGetNewLine_permW0N0E0
... ok
test compilerGetScriptFileNames_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRecompileFlag_permW0N0E0
Compiling /root/project/foo/bar.ts
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptKind_permW0N0E0
... ok
test compilerGetScriptVersion_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptVersionUnknown_permW0N0E0
... ok
test compilerGetScriptSnapshot_permW0N0E0
... ok
test compilerGetCurrentDirectory_permW0N0E0
... ok
test compilerGetDefaultLibFileName_permW0N0E0
... ok
test compilerUseCaseSensitiveFileNames_permW0N0E0
... ok
test compilerReadFile_permW0N0E0
... ok
test compilerFileExists_permW0N0E0
... ok
test compilerResolveModuleNames_permW0N0E0
... ok
test consoleTestAssert_permW0N0E0
... ok
test consoleTestStringifyComplexObjects_permW0N0E0
... ok
test consoleTestStringifyCircular_permW0N0E0
... ok
test consoleTestStringifyWithDepth_permW0N0E0
... ok
test consoleTestError_permW0N0E0
... ok
test consoleDetachedLog_permW0N0E0
Hello world
Hello world
Hello world
Hello world
Hello world
Hello world
... ok
test fetchPerm_permW0N0E0
... ok
test headersAppend_permW0N0E0
... ok
test newHeaderTest_permW0N0E0
... ok
test newHeaderWithSequence_permW0N0E0
... ok
test newHeaderWithRecord_permW0N0E0
... ok
test newHeaderWithHeadersInstance_permW0N0E0
... ok
test headerAppendSuccess_permW0N0E0
... ok
test headerSetSuccess_permW0N0E0
... ok
test headerHasSuccess_permW0N0E0
... ok
test headerDeleteSuccess_permW0N0E0
... ok
test headerGetSuccess_permW0N0E0
... ok
test headerForEachSuccess_permW0N0E0
... ok
test envFailure_permW0N0E0
... ok
test filesStdioFileDescriptors_permW0N0E0
... ok
test filesCopyToStdout_permW0N0E0
{
"name": "deno",
"devDependencies": {
"@types/base64-js": "^1.2.5",
"@types/flatbuffers": "^1.9.0",
"@types/source-map-support": "^0.4.1",
"@types/text-encoding": "0.0.33",
"base64-js": "^1.3.0",
"flatbuffers": "^1.9.0",
"magic-string": "^0.22.5",
"prettier": "^1.14.0",
"rollup": "^0.63.2",
"rollup-plugin-alias": "^1.4.0",
"rollup-plugin-analyzer": "^2.1.0",
"rollup-plugin-commonjs": "^9.1.3",
"rollup-plugin-node-globals": "^1.2.1",
"rollup-plugin-node-resolve": "^3.3.0",
"rollup-plugin-string": "^2.0.2",
"rollup-plugin-typescript2": "^0.16.1",
"rollup-pluginutils": "^2.3.0",
"source-map-support": "^0.5.6",
"text-encoding": "0.6.4",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.3.1",
"tslint-no-circular-imports": "^0.5.0",
"typescript": "3.0.3"
}
}
bytes written 860
... ok
test readFileSyncSuccess_permW0N0E0
... ok
test readFileSyncNotFound_permW0N0E0
... ok
test readFileSuccess_permW0N0E0
... ok
test readDirSyncNotDir_permW0N0E0
... ok
test readDirSyncNotFound_permW0N0E0
... ok
test writeFileSyncPerm_permW0N0E0
... ok
test writeFilePerm_permW0N0E0
... ok
test copyFileSyncPerm_permW0N0E0
... ok
test copyFilePerm_permW0N0E0
... ok
test mkdirSyncPerm_permW0N0E0
... ok
test makeTempDirSyncPerm_permW0N0E0
... ok
test statSyncSuccess_permW0N0E0
... ok
test statSyncNotFound_permW0N0E0
... ok
test lstatSyncSuccess_permW0N0E0
... ok
test lstatSyncNotFound_permW0N0E0
... ok
test statSuccess_permW0N0E0
... ok
test statNotFound_permW0N0E0
... ok
test lstatSuccess_permW0N0E0
... ok
test lstatNotFound_permW0N0E0
... ok
test renameSyncPerm_permW0N0E0
... ok
test readlinkSyncNotFound_permW0N0E0
... ok
test blobString_permW0N0E0
... ok
test blobBuffer_permW0N0E0
... ok
test blobSlice_permW0N0E0
... ok
test timeoutSuccess_permW0N0E0
... ok
test timeoutArgs_permW0N0E0
... ok
test timeoutCancelSuccess_permW0N0E0
... ok
test timeoutCancelMultiple_permW0N0E0
... ok
test timeoutCancelInvalidSilentFail_permW0N0E0
... ok
test intervalSuccess_permW0N0E0
... ok
test intervalCancelSuccess_permW0N0E0
... ok
test intervalOrdering_permW0N0E0
... ok
test intervalCancelInvalidSilentFail_permW0N0E0
... ok
test symlinkSyncPerm_permW0N0E0
... ok
test platformTransform_permW0N0E0
... ok
test atobSuccess_permW0N0E0
... ok
test btoaSuccess_permW0N0E0
... ok
test btoaFailed_permW0N0E0
... ok
test truncateSyncPerm_permW0N0E0
... ok
test truncatePerm_permW0N0E0
... ok
test evalErrorFormatted_permW0N0E0
... ok
test createExecTimeColumnsRegularData_permW0N0E0
... ok
test createExecTimeColumnsIrregularData_permW0N0E0
... ok
test createBinarySizeColumnsRegularData_permW0N0E0
... ok
test createBinarySizeColumnsIrregularData_permW0N0E0
... ok
test createThreadCountColumnsRegularData_permW0N0E0
... ok
test createThreadCountColumnsIrregularData_permW0N0E0
... ok
test createSyscallCountColumnsRegularData_permW0N0E0
... ok
test createSyscallCountColumnsIrregularData_permW0N0E0
... ok
test createSha1ListRegularData_permW0N0E0
... ok
test formatBytesPatterns_permW0N0E0
... ok
test formatSecondsPatterns_permW0N0E0
... ok
test getTravisDataSuccess_permW0N0E0
... ok
test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out

71
tools/testdata/unit_test_output2.txt vendored Normal file
View file

@ -0,0 +1,71 @@
running 96 tests
test permSerialization_permW0N0E0
... ok
test permFromStringThrows_permW0N0E0
... ok
test compilerInstance_permW0N0E0
... ok
test compilerRun_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRunMultiModule_permW0N0E0
... ok
test compilerRunCircularDependency_permW0N0E0
Compiling modA
Compiling modB
... ok
test compilerResolveModule_permW0N0E0
... ok
test compilerGetModuleDependencies_permW0N0E0
... ok
test compilerGetCompilationSettings_permW0N0E0
... ok
test compilerGetNewLine_permW0N0E0
... ok
test compilerGetScriptFileNames_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRecompileFlag_permW0N0E0
Compiling /root/project/foo/bar.ts
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptKind_permW0N0E0
... ok
test compilerGetScriptVersion_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptVersionUnknown_permW0N0E0
... ok
test compilerGetScriptSnapshot_permW0N0E0
... ok
test compilerGetCurrentDirectory_permW0N0E0
... ok
test compilerGetDefaultLibFileName_permW0N0E0
... ok
test compilerUseCaseSensitiveFileNames_permW0N0E0
... ok
test compilerReadFile_permW0N0E0
... ok
test compilerFileExists_permW0N0E0
... ok
test compilerResolveModuleNames_permW0N0E0
... ok
test consoleTestAssert_permW0N0E0
... ok
test consoleTestStringifyComplexObjects_permW0N0E0
... ok
test consoleTestStringifyCircular_permW0N0E0
... ok
test consoleTestStringifyWithDepth_permW0N0E0
... ok
test consoleTestError_permW0N0E0
... ok
test consoleDetachedLog_permW0N0E0
Hello world
Hello world
Hello world
Hello world
Hello world
Hello world
... ok
test fetchPerm_permW0N0E0

268
tools/testdata/unit_test_output3.txt vendored Normal file
View file

@ -0,0 +1,268 @@
Compiling /Users/rld/src/deno/js/unit_tests.ts
Compiling /Users/rld/src/deno/js/compiler_test.ts
Compiling /Users/rld/src/deno/js/test_util.ts
Compiling /Users/rld/src/deno/js/testing/testing.ts
Compiling /Users/rld/src/deno/js/testing/util.ts
Compiling /Users/rld/src/deno/js/console_test.ts
Compiling /Users/rld/src/deno/js/console.ts
Compiling /Users/rld/src/deno/js/fetch_test.ts
Compiling /Users/rld/src/deno/js/os_test.ts
Compiling /Users/rld/src/deno/js/files_test.ts
Compiling /Users/rld/src/deno/js/read_file_test.ts
Compiling /Users/rld/src/deno/js/read_dir_test.ts
Compiling /Users/rld/src/deno/js/write_file_test.ts
Compiling /Users/rld/src/deno/js/copy_file_test.ts
Compiling /Users/rld/src/deno/js/mkdir_test.ts
Compiling /Users/rld/src/deno/js/make_temp_dir_test.ts
Compiling /Users/rld/src/deno/js/stat_test.ts
Compiling /Users/rld/src/deno/js/rename_test.ts
Compiling /Users/rld/src/deno/js/read_link_test.ts
Compiling /Users/rld/src/deno/js/blob_test.ts
Compiling /Users/rld/src/deno/js/timers_test.ts
Compiling /Users/rld/src/deno/js/symlink_test.ts
Compiling /Users/rld/src/deno/js/platform_test.ts
Compiling /Users/rld/src/deno/js/text_encoding_test.ts
Compiling /Users/rld/src/deno/js/net_test.ts
Compiling /Users/rld/src/deno/js/trace_test.ts
Compiling /Users/rld/src/deno/js/truncate_test.ts
Compiling /Users/rld/src/deno/js/v8_source_maps_test.ts
Compiling /Users/rld/src/deno/website/app_test.js
Compiling /Users/rld/src/deno/website/app.js
running 96 tests
test permSerialization_permW0N0E0
... ok
test permFromStringThrows_permW0N0E0
... ok
test compilerInstance_permW0N0E0
... ok
test compilerRun_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRunMultiModule_permW0N0E0
... ok
test compilerRunCircularDependency_permW0N0E0
Compiling modA
Compiling modB
... ok
test compilerResolveModule_permW0N0E0
... ok
test compilerGetModuleDependencies_permW0N0E0
... ok
test compilerGetCompilationSettings_permW0N0E0
... ok
test compilerGetNewLine_permW0N0E0
... ok
test compilerGetScriptFileNames_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerRecompileFlag_permW0N0E0
Compiling /root/project/foo/bar.ts
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptKind_permW0N0E0
... ok
test compilerGetScriptVersion_permW0N0E0
Compiling /root/project/foo/bar.ts
... ok
test compilerGetScriptVersionUnknown_permW0N0E0
... ok
test compilerGetScriptSnapshot_permW0N0E0
... ok
test compilerGetCurrentDirectory_permW0N0E0
... ok
test compilerGetDefaultLibFileName_permW0N0E0
... ok
test compilerUseCaseSensitiveFileNames_permW0N0E0
... ok
test compilerReadFile_permW0N0E0
... ok
test compilerFileExists_permW0N0E0
... ok
test compilerResolveModuleNames_permW0N0E0
... ok
test consoleTestAssert_permW0N0E0
... ok
test consoleTestStringifyComplexObjects_permW0N0E0
... ok
test consoleTestStringifyCircular_permW0N0E0
... ok
test consoleTestStringifyWithDepth_permW0N0E0
... ok
test consoleTestError_permW0N0E0
... ok
test consoleDetachedLog_permW0N0E0
Hello world
Hello world
Hello world
Hello world
Hello world
Hello world
... ok
test fetchPerm_permW0N0E0
... ok
test headersAppend_permW0N0E0
... ok
test newHeaderTest_permW0N0E0
... ok
test newHeaderWithSequence_permW0N0E0
... ok
test newHeaderWithRecord_permW0N0E0
... ok
test newHeaderWithHeadersInstance_permW0N0E0
... ok
test headerAppendSuccess_permW0N0E0
... ok
test headerSetSuccess_permW0N0E0
... ok
test headerHasSuccess_permW0N0E0
... ok
test headerDeleteSuccess_permW0N0E0
... ok
test headerGetSuccess_permW0N0E0
... ok
test headerForEachSuccess_permW0N0E0
... ok
test envFailure_permW0N0E0
... ok
test filesStdioFileDescriptors_permW0N0E0
... ok
test filesCopyToStdout_permW0N0E0
{
"name": "deno",
"devDependencies": {
"@types/base64-js": "^1.2.5",
"@types/flatbuffers": "^1.9.0",
"@types/source-map-support": "^0.4.1",
"@types/text-encoding": "0.0.33",
"base64-js": "^1.3.0",
"flatbuffers": "^1.9.0",
"magic-string": "^0.22.5",
"prettier": "^1.14.0",
"rollup": "^0.63.2",
"rollup-plugin-alias": "^1.4.0",
"rollup-plugin-analyzer": "^2.1.0",
"rollup-plugin-commonjs": "^9.1.3",
"rollup-plugin-node-globals": "^1.2.1",
"rollup-plugin-node-resolve": "^3.3.0",
"rollup-plugin-string": "^2.0.2",
"rollup-plugin-typescript2": "^0.16.1",
"rollup-pluginutils": "^2.3.0",
"source-map-support": "^0.5.6",
"text-encoding": "0.6.4",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.3.1",
"tslint-no-circular-imports": "^0.5.0",
"typescript": "3.0.3"
}
}
bytes written 860
... ok
test readFileSyncSuccess_permW0N0E0
... ok
test readFileSyncNotFound_permW0N0E0
... ok
test readFileSuccess_permW0N0E0
... ok
test readDirSyncNotDir_permW0N0E0
... ok
test readDirSyncNotFound_permW0N0E0
... ok
test writeFileSyncPerm_permW0N0E0
... ok
test writeFilePerm_permW0N0E0
... ok
test copyFileSyncPerm_permW0N0E0
... ok
test copyFilePerm_permW0N0E0
... ok
test mkdirSyncPerm_permW0N0E0
... ok
test makeTempDirSyncPerm_permW0N0E0
... ok
test statSyncSuccess_permW0N0E0
... ok
test statSyncNotFound_permW0N0E0
... ok
test lstatSyncSuccess_permW0N0E0
... ok
test lstatSyncNotFound_permW0N0E0
... ok
test statSuccess_permW0N0E0
... ok
test statNotFound_permW0N0E0
... ok
test lstatSuccess_permW0N0E0
... ok
test lstatNotFound_permW0N0E0
... ok
test renameSyncPerm_permW0N0E0
... ok
test readlinkSyncNotFound_permW0N0E0
... ok
test blobString_permW0N0E0
... ok
test blobBuffer_permW0N0E0
... ok
test blobSlice_permW0N0E0
... ok
test timeoutSuccess_permW0N0E0
... ok
test timeoutArgs_permW0N0E0
... ok
test timeoutCancelSuccess_permW0N0E0
... ok
test timeoutCancelMultiple_permW0N0E0
... ok
test timeoutCancelInvalidSilentFail_permW0N0E0
... ok
test intervalSuccess_permW0N0E0
... ok
test intervalCancelSuccess_permW0N0E0
... ok
test intervalOrdering_permW0N0E0
... ok
test intervalCancelInvalidSilentFail_permW0N0E0
... ok
test symlinkSyncPerm_permW0N0E0
... ok
test platformTransform_permW0N0E0
... ok
test atobSuccess_permW0N0E0
... ok
test btoaSuccess_permW0N0E0
... ok
test btoaFailed_permW0N0E0
... ok
test truncateSyncPerm_permW0N0E0
... ok
test truncatePerm_permW0N0E0
... ok
test evalErrorFormatted_permW0N0E0
... ok
test createExecTimeColumnsRegularData_permW0N0E0
... ok
test createExecTimeColumnsIrregularData_permW0N0E0
... ok
test createBinarySizeColumnsRegularData_permW0N0E0
... ok
test createBinarySizeColumnsIrregularData_permW0N0E0
... ok
test createThreadCountColumnsRegularData_permW0N0E0
... ok
test createThreadCountColumnsIrregularData_permW0N0E0
... ok
test createSyscallCountColumnsRegularData_permW0N0E0
... ok
test createSyscallCountColumnsIrregularData_permW0N0E0
... ok
test createSha1ListRegularData_permW0N0E0
... ok
test formatBytesPatterns_permW0N0E0
... ok
test formatSecondsPatterns_permW0N0E0
... ok
test getTravisDataSuccess_permW0N0E0
... ok
test result: ok. 96 passed; 0 failed; 0 ignored; 0 measured; 36 filtered out

53
tools/unit_tests.py
View file

@ -1,6 +1,32 @@
#!/usr/bin/env python
from util import run
import util
import sys
import subprocess
import re
def run_unit_test(deno_exe, permStr, flags=[]):
cmd = [deno_exe, "--reload", "js/unit_tests.ts", permStr] + flags
process = subprocess.Popen(
cmd,
bufsize=1,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
(actual, expected) = util.parse_unit_test_output(process.stdout, True)
process.wait()
errcode = process.returncode
if errcode != 0:
sys.exit(errcode)
if actual == None and expected == None:
raise AssertionError("Bad js/unit_test.ts output")
if expected != actual:
print "expected", expected, "actual", actual
raise AssertionError("expected tests did not equal actual")
process.wait()
errcode = process.returncode
if errcode != 0:
sys.exit(errcode)
# We want to test many ops in deno which have different behavior depending on
@ -10,25 +36,12 @@ import sys
# tests by the special string. permW0N0 means allow-write but not allow-net.
# See js/test_util.ts for more details.
def unit_tests(deno_exe):
run([deno_exe, "--reload", "js/unit_tests.ts", "permW0N0E0"])
run([
deno_exe, "--reload", "js/unit_tests.ts", "permW1N0E0", "--allow-write"
])
run([
deno_exe, "--reload", "js/unit_tests.ts", "permW0N1E0", "--allow-net"
])
run([
deno_exe, "--reload", "js/unit_tests.ts", "permW0N0E1", "--allow-env"
])
run([
deno_exe,
"--reload",
"js/unit_tests.ts",
"permW1N1E1",
"--allow-write",
"--allow-net",
"--allow-env",
])
run_unit_test(deno_exe, "permW0N0E0")
run_unit_test(deno_exe, "permW1N0E0", ["--allow-write"])
run_unit_test(deno_exe, "permW0N1E0", ["--allow-net"])
run_unit_test(deno_exe, "permW0N0E1", ["--allow-env"])
# TODO We might accidentally miss some. We should be smarter about which we
# run. Maybe we can use the "filtered out" number to check this.
if __name__ == '__main__':

30
tools/util.py
View file

@ -294,3 +294,33 @@ def enable_ansi_colors_win10():
CloseHandle(conout)
return True
def parse_unit_test_output(output, print_to_stdout):
first = True
expected = None
actual = None
result = None
for line in iter(output.readline, ''):
if expected is None:
# expect "running 30 tests"
expected = extract_number(r'running (\d+) tests', line)
elif "test result:" in line:
result = line
if print_to_stdout:
sys.stdout.write(line)
sys.stdout.flush()
# Check that the number of expected tests equals what was reported at the
# bottom.
if result:
# result should be a string like this:
# "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..."
actual = extract_number(r'(\d+) passed', result)
return (actual, expected)
def extract_number(pattern, string):
matches = re.findall(pattern, string)
if len(matches) != 1:
return None
return int(matches[0])

35
tools/util_test.py
View file

@ -1,5 +1,8 @@
# Copyright 2018 the Deno authors. All rights reserved. MIT license.
from util import pattern_match, parse_exit_code, shell_quote_win
import util
import os
import sys
def pattern_match_test():
@ -44,10 +47,42 @@ def shell_quote_win_test():
'a"b""c\\d\\"e\\\\')
def parse_unit_test_output_test():
print "Testing util.parse_unit_test_output()..."
# This is an example of a successful unit test output.
output = open(
os.path.join(util.root_path, "tools/testdata/unit_test_output1.txt"))
(actual, expected) = util.parse_unit_test_output(output, False)
assert actual == 96
assert expected == 96
# This is an example of a silently dying unit test.
output = open(
os.path.join(util.root_path, "tools/testdata/unit_test_output2.txt"))
(actual, expected) = util.parse_unit_test_output(output, False)
assert actual == None
assert expected == 96
# This is an example of compiling before successful unit tests.
output = open(
os.path.join(util.root_path, "tools/testdata/unit_test_output3.txt"))
(actual, expected) = util.parse_unit_test_output(output, False)
assert actual == 96
assert expected == 96
# Check what happens on empty output.
from StringIO import StringIO
output = StringIO("\n\n\n")
(actual, expected) = util.parse_unit_test_output(output, False)
assert actual == None
assert expected == None
def util_test():
pattern_match_test()
parse_exit_code_test()
shell_quote_win_test()
parse_unit_test_output_test()
if __name__ == '__main__':