Refactor test infrastructure (#2432)

* use subclass of unittest.TestCase for all test cases

* allow to run single test file (eg. python tools/integration_tests.py)

* test filtering (via --pattern/-p CLI flag)

* use common CLI parser for all tests:
  usage: test.py [-h] [--failfast] [--verbose] [--executable EXECUTABLE]
               [--release] [--pattern PATTERN] [--build-dir BUILD_DIR]

  optional arguments:
  -h, --help            show this help message and exit
  --failfast, -f        Stop on first failure
  --verbose, -v         Verbose output
  --executable EXECUTABLE
                        Use external executable of Deno
  --release             Test against release executable
  --pattern PATTERN, -p PATTERN
                        Run tests that match provided pattern
  --build-dir BUILD_DIR
                        Deno build directory

* respect NO_COLOR variable

.appveyor.yml

@@ -196,7 +196,7 @@ build_script:
 test_script:
   - python tools\lint.py
   - python tools\test_format.py
-  - ps: Exec { & python tools\test.py -v $env:DENO_BUILD_PATH }
+  - ps: Exec { & python tools\test.py -v --build-dir $env:DENO_BUILD_PATH }
 
 after_test:
   # Delete the the rollup cache, which is unreliable, so that it doesn't get

tools/benchmark_test.py

@@ -4,7 +4,7 @@ import sys
 import os
 import benchmark
 import unittest
-from util import DenoTestCase, test_main
+from test_util import DenoTestCase, run_tests
 
 
 class TestBenchmark(DenoTestCase):
@@ -56,4 +56,4 @@ class TestBenchmark(DenoTestCase):
 if __name__ == '__main__':
     # FIME this doesn't appear to be the case.
     # This test assumes tools/http_server.py is running in the background.
-    test_main()
+    run_tests()

tools/complex_permissions_test.py

@@ -2,13 +2,11 @@
 # -*- coding: utf-8 -*-
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
-import subprocess
-import sys
-import time
 import unittest
 
-from http_server import spawn
-from util import DenoTestCase, root_path, test_main, tty_capture
+import http_server
+from test_util import DenoTestCase, run_tests
+from util import root_path, tty_capture
 
 PERMISSIONS_PROMPT_TEST_TS = "tools/complex_permissions_test.ts"
 
@@ -25,8 +23,8 @@ class BaseComplexPermissionTest(DenoTestCase):
         return tty_capture(cmd, b'')
 
 
-class TestReadPermissions(BaseComplexPermissionTest):
-    test_type = "read"
+class BaseReadWritePermissionsTest(object):
+    test_type = None
 
     def test_inside_project_dir(self):
         code, _stdout, stderr = self._run_deno(
@@ -97,7 +95,13 @@ class TestReadPermissions(BaseComplexPermissionTest):
         os.chdir(saved_curdir)
 
 
-class TestWritePermissions(TestReadPermissions):
+class TestReadPermissions(BaseReadWritePermissionsTest,
+                          BaseComplexPermissionTest):
+    test_type = "read"
+
+
+class TestWritePermissions(BaseReadWritePermissionsTest,
+                           BaseComplexPermissionTest):
     test_type = "write"
 
 
@@ -211,5 +215,5 @@ def complex_permissions_tests():
 
 
 if __name__ == "__main__":
-    with spawn():
-        test_main()
+    with http_server.spawn():
+        run_tests()

tools/deno_dir_test.py

@@ -3,10 +3,9 @@
 # Check deno dir is created properly
 # Usage: deno_dir_test.py [path to deno dir]
 import os
-import subprocess
-import sys
 
-from util import DenoTestCase, mkdtemp, rmtree, run, test_main
+from test_util import DenoTestCase, run_tests
+from util import mkdtemp, rmtree, run
 
 
 class TestDenoDir(DenoTestCase):
@@ -43,4 +42,4 @@ class TestDenoDir(DenoTestCase):
 
 
 if __name__ == '__main__':
-    test_main()
+    run_tests()

tools/fetch_test.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
-import sys
 import shutil
 
-from http_server import spawn
-from util import DenoTestCase, mkdtemp, tests_path, run_output, test_main
+import http_server
+from test_util import DenoTestCase, run_tests
+from util import mkdtemp, tests_path, run_output
 
 
 class FetchTest(DenoTestCase):
@@ -26,5 +26,5 @@ class FetchTest(DenoTestCase):
 
 
 if __name__ == "__main__":
-    with spawn():
-        test_main()
+    with http_server.spawn():
+        run_tests()

tools/fmt_test.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import json
 import os
 import shutil
-import sys
 
-from util import (DenoTestCase, mkdtemp, root_path, tests_path, run, test_main)
+from test_util import DenoTestCase, run_tests
+from util import mkdtemp, root_path, tests_path, run
 
 
 class FmtTest(DenoTestCase):
@@ -43,4 +42,4 @@ class FmtTest(DenoTestCase):
 
 
 if __name__ == "__main__":
-    test_main()
+    run_tests()

tools/integration_tests.py

@@ -7,16 +7,13 @@
 # exit code can be specified.
 #
 # Usage: integration_tests.py [path to deno executable]
-import argparse
 import os
 import re
-import sys
 import subprocess
-import unittest
 
-from http_server import spawn
-from util import (DenoTestCase, ColorTextTestRunner, root_path, tests_path,
-                  pattern_match, rmtree, test_main)
+import http_server
+from test_util import DenoTestCase, run_tests
+from util import root_path, tests_path, pattern_match, rmtree
 
 
 def strip_ansi_codes(s):
@@ -107,36 +104,6 @@ for fn in sorted(
     tn = t.__name__ = "test_" + fn.split(".")[0]
     setattr(TestIntegrations, tn, t)
 
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--filter", help="Run specific tests")
-    parser.add_argument(
-        "--release", help="Use release build of Deno", action="store_true")
-    parser.add_argument("--executable", help="Use external executable of Deno")
-    args = parser.parse_args()
-
-    target = "release" if args.release else "debug"
-    build_dir = os.environ.get("DENO_BUILD_PATH",
-                               os.path.join(root_path, "target", target))
-
-    deno_dir = os.path.join(build_dir, ".deno_test")
-    if os.path.isdir(deno_dir):
-        rmtree(deno_dir)
-    os.environ["DENO_DIR"] = deno_dir
-
-    test_names = [
-        test_name for test_name in unittest.TestLoader().getTestCaseNames(
-            TestIntegrations) if not args.filter or args.filter in test_name
-    ]
-    suite = unittest.TestLoader().loadTestsFromNames(
-        test_names, module=TestIntegrations)
-
-    with spawn():
-        result = ColorTextTestRunner(verbosity=2).run(suite)
-        if not result.wasSuccessful():
-            sys.exit(1)
-
-
 if __name__ == "__main__":
-    main()
+    with http_server.spawn():
+        run_tests()

tools/is_tty_test.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
-import subprocess
 import unittest
 from sys import stdin
 
-from util import DenoTestCase, test_main, tty_capture
+from test_util import DenoTestCase, run_tests
+from util import tty_capture
 
 IS_TTY_TEST_TS = "tests/is_tty.ts"
 
@@ -20,4 +20,4 @@ class TestIsTty(DenoTestCase):
 
 
 if __name__ == "__main__":
-    test_main()
+    run_tests()

tools/permission_prompt_test.py

@@ -2,12 +2,10 @@
 # -*- coding: utf-8 -*-
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
-import subprocess
-import sys
-import time
 import unittest
 
-from util import DenoTestCase, test_main, tty_capture
+from test_util import DenoTestCase, run_tests
+from util import tty_capture
 
 PERMISSIONS_PROMPT_TEST_TS = "tools/permission_prompt_test.ts"
 
@@ -143,4 +141,4 @@ def permission_prompt_tests():
 
 
 if __name__ == "__main__":
-    test_main()
+    run_tests()

tools/repl_test.py

@@ -4,18 +4,10 @@ from subprocess import CalledProcessError, PIPE, Popen
 import sys
 import time
 
-from util import DenoTestCase, test_main
+from test_util import DenoTestCase, run_tests
 
 
 class TestRepl(DenoTestCase):
-    def __init__(self, *args, **kwargs):
-        super(TestRepl, self).__init__(*args, **kwargs)
-        self._warm_up()
-
-    def _warm_up(self):
-        # This may output an error message about the history file (ignore it).
-        self.input("")
-
     def input(self, *lines, **kwargs):
         exit_ = kwargs.pop("exit", True)
         sleep_ = kwargs.pop("sleep", 0)
@@ -141,4 +133,4 @@ class TestRepl(DenoTestCase):
 
 
 if __name__ == "__main__":
-    test_main()
+    run_tests()

tools/setup_test.py

@@ -6,7 +6,7 @@ from setup import gn_string, read_gn_args, write_gn_args
 from shutil import rmtree
 from tempfile import mktemp
 
-from util import DenoTestCase, test_main
+from test_util import DenoTestCase, run_tests
 
 
 class TestSetup(DenoTestCase):
@@ -63,4 +63,4 @@ class TestSetup(DenoTestCase):
 
 
 if __name__ == '__main__':
-    test_main()
+    run_tests()

tools/target_test.py

@@ -0,0 +1,60 @@
+import os
+import sys
+
+from test_util import DenoTestCase, run_tests
+from util import executable_suffix, run, tests_path, run_output
+
+
+class TestTarget(DenoTestCase):
+    @staticmethod
+    def check_exists(filename):
+        if not os.path.exists(filename):
+            print "Required target doesn't exist:", filename
+            print "Run ./tools/build.py"
+            sys.exit(1)
+
+    def test_executable_exists(self):
+        self.check_exists(self.deno_exe)
+
+    def _test(self, executable):
+        "Test executable runs and exits with code 0."
+        bin_file = os.path.join(self.build_dir, executable + executable_suffix)
+        self.check_exists(bin_file)
+        run([bin_file])
+
+    def test_libdeno(self):
+        self._test("libdeno_test")
+
+    def test_cli(self):
+        self._test("cli_test")
+
+    def test_core(self):
+        self._test("deno_core_test")
+
+    def test_core_http_benchmark(self):
+        self._test("deno_core_http_bench_test")
+
+    def test_ts_library_builder(self):
+        run([
+            "node", "./node_modules/.bin/ts-node", "--project",
+            "tools/ts_library_builder/tsconfig.json",
+            "tools/ts_library_builder/test.ts"
+        ])
+
+    def test_no_color(self):
+        t = os.path.join(tests_path, "no_color.js")
+        output = run_output([self.deno_exe, "run", t],
+                            merge_env={"NO_COLOR": "1"})
+        assert output.strip() == "noColor true"
+        t = os.path.join(tests_path, "no_color.js")
+        output = run_output([self.deno_exe, "run", t])
+        assert output.strip() == "noColor false"
+
+    def test_exec_path(self):
+        cmd = [self.deno_exe, "run", "tests/exec_path.ts"]
+        output = run_output(cmd)
+        assert self.deno_exe in output.strip()
+
+
+if __name__ == "main":
+    run_tests()

tools/test.py

@@ -3,9 +3,6 @@
 # Runs the full test suite.
 # Usage: ./tools/test.py out/Debug
 import os
-import subprocess
-import sys
-import unittest
 
 from benchmark_test import TestBenchmark
 from deno_dir_test import TestDenoDir
@@ -14,73 +11,22 @@ from fmt_test import FmtTest
 from integration_tests import TestIntegrations
 from repl_test import TestRepl
 from setup_test import TestSetup
+from target_test import TestTarget
 from unit_tests import JsUnitTests
 from util_test import TestUtil
-
-from is_tty_test import TestIsTty
 # NOTE: These tests are skipped on Windows
+from is_tty_test import TestIsTty
 from permission_prompt_test import permission_prompt_tests
 from complex_permissions_test import complex_permissions_tests
 
-from http_server import spawn
-from util import (DenoTestCase, ColorTextTestRunner, enable_ansi_colors,
-                  executable_suffix, run, run_output, rmtree, tests_path,
-                  test_args)
+import http_server
+from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
+                  executable_suffix, run, run_output, rmtree, tests_path)
+from test_util import parse_test_args, run_tests
 
 
-class TestTarget(DenoTestCase):
-    @staticmethod
-    def check_exists(filename):
-        if not os.path.exists(filename):
-            print "Required target doesn't exist:", filename
-            print "Run ./tools/build.py"
-            sys.exit(1)
-
-    def test_executable_exists(self):
-        self.check_exists(self.deno_exe)
-
-    def _test(self, executable):
-        "Test executable runs and exits with code 0."
-        bin_file = os.path.join(self.build_dir, executable + executable_suffix)
-        self.check_exists(bin_file)
-        run([bin_file])
-
-    def test_libdeno(self):
-        self._test("libdeno_test")
-
-    def test_cli(self):
-        self._test("cli_test")
-
-    def test_core(self):
-        self._test("deno_core_test")
-
-    def test_core_http_benchmark(self):
-        self._test("deno_core_http_bench_test")
-
-    def test_ts_library_builder(self):
-        run([
-            "node", "./node_modules/.bin/ts-node", "--project",
-            "tools/ts_library_builder/tsconfig.json",
-            "tools/ts_library_builder/test.ts"
-        ])
-
-    def test_no_color(self):
-        t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t],
-                            merge_env={"NO_COLOR": "1"})
-        assert output.strip() == "noColor true"
-        t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t])
-        assert output.strip() == "noColor false"
-
-    def test_exec_path(self):
-        cmd = [self.deno_exe, "run", "tests/exec_path.ts"]
-        output = run_output(cmd)
-        assert self.deno_exe in output.strip()
-
-
-def main(argv):
-    args = test_args(argv)
+def main():
+    args = parse_test_args()
 
     deno_dir = os.path.join(args.build_dir, ".deno_test")
     if os.path.isdir(deno_dir):
@@ -89,7 +35,6 @@ def main(argv):
 
     enable_ansi_colors()
 
-    with spawn():
     test_cases = [
         TestSetup,
         TestUtil,
@@ -101,24 +46,14 @@ def main(argv):
         TestRepl,
         TestDenoDir,
         TestBenchmark,
+        TestIsTty,
     ]
-        # These tests are skipped, but to make the test output less noisy
-        # we'll avoid triggering them.
-        if os.name != 'nt':
-            test_cases.append(TestIsTty)
     test_cases += permission_prompt_tests()
     test_cases += complex_permissions_tests()
 
-        suite = unittest.TestSuite([
-            unittest.TestLoader().loadTestsFromTestCase(tc)
-            for tc in test_cases
-        ])
-
-        result = ColorTextTestRunner(
-            verbosity=args.verbosity + 1, failfast=args.failfast).run(suite)
-        if not result.wasSuccessful():
-            sys.exit(1)
+    with http_server.spawn():
+        run_tests(test_cases)
 
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main()

tools/test_util.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+# Runs the full test suite.
+# Usage: ./tools/test.py out/Debug
+import argparse
+import os
+import sys
+import unittest
+
+from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
+                  executable_suffix, run, run_output, rmtree, tests_path)
+
+
+class DenoTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        args = parse_test_args()
+
+        cls.build_dir = args.build_dir
+        cls.deno_exe = args.executable
+
+
+# overload the test result class
+class ColorTextTestResult(unittest.TextTestResult):
+    def getDescription(self, test):
+        name = str(test)
+        if name.startswith("test_"):
+            name = name[5:]
+        return name
+
+    def addSuccess(self, test):
+        if self.showAll:
+            self.stream.write(FG_GREEN)
+        super(ColorTextTestResult, self).addSuccess(test)
+        if self.showAll:
+            self.stream.write(RESET)
+
+    def addError(self, test, err):
+        if self.showAll:
+            self.stream.write(FG_RED)
+        super(ColorTextTestResult, self).addError(test, err)
+        if self.showAll:
+            self.stream.write(RESET)
+
+    def addFailure(self, test, err):
+        if self.showAll:
+            self.stream.write(FG_RED)
+        super(ColorTextTestResult, self).addFailure(test, err)
+        if self.showAll:
+            self.stream.write(RESET)
+
+
+class ColorTextTestRunner(unittest.TextTestRunner):
+    resultclass = ColorTextTestResult
+
+
+def create_test_arg_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--failfast', '-f', action='store_true', help='Stop on first failure')
+    parser.add_argument(
+        '--verbose', '-v', action='store_true', help='Verbose output')
+    parser.add_argument("--executable", help="Use external executable of Deno")
+    parser.add_argument(
+        '--release',
+        action='store_true',
+        help='Test against release executable')
+    parser.add_argument(
+        '--pattern', '-p', help='Run tests that match provided pattern')
+    parser.add_argument(
+        '--build-dir', dest="build_dir", help='Deno build directory')
+    return parser
+
+
+TestArgParser = create_test_arg_parser()
+
+
+def parse_test_args(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = TestArgParser.parse_args(argv)
+
+    if args.executable and args.release:
+        raise argparse.ArgumentError(
+            None, "Path to executable is inferred from "
+            "--release, cannot provide both.")
+
+    if not args.build_dir:
+        args.build_dir = build_path()
+
+    if not args.executable:
+        args.executable = os.path.join(args.build_dir,
+                                       "deno" + executable_suffix)
+
+    if not os.path.isfile(args.executable):
+        raise argparse.ArgumentError(
+            None, "deno executable not found at {}".format(args.executable))
+
+    return args
+
+
+def filter_test_suite(suite, pattern):
+    filtered_tests = []
+
+    for test_case in suite:
+        if isinstance(test_case, unittest.TestSuite):
+            filtered_tests += filter_test_suite(test_case, pattern)
+        else:
+            if pattern in str(test_case):
+                filtered_tests.append(test_case)
+
+    return filtered_tests
+
+
+def run_tests(test_cases=None):
+    args = parse_test_args()
+
+    loader = unittest.TestLoader()
+
+    # if suite was not explicitly passed load test
+    # cases from calling module
+    if test_cases is None:
+        import __main__
+        suite = loader.loadTestsFromModule(__main__)
+    else:
+        suite = unittest.TestSuite()
+        for test_case in test_cases:
+            suite.addTests(loader.loadTestsFromTestCase(test_case))
+
+    if args.pattern:
+        filtered_tests = filter_test_suite(suite, args.pattern)
+        suite = unittest.TestSuite(filtered_tests)
+
+    runner = ColorTextTestRunner(
+        verbosity=args.verbose + 1, failfast=args.failfast)
+
+    result = runner.run(suite)
+    if not result.wasSuccessful():
+        sys.exit(1)

tools/unit_tests.py

@@ -3,8 +3,8 @@
 import sys
 import subprocess
 
-from http_server import spawn
-from util import DenoTestCase, test_main
+import http_server
+from test_util import DenoTestCase, run_tests
 
 
 class JsUnitTests(DenoTestCase):
@@ -24,5 +24,5 @@ class JsUnitTests(DenoTestCase):
 
 
 if __name__ == '__main__':
-    with spawn():
-        test_main()
+    with http_server.spawn():
+        run_tests()

tools/util.py

@@ -1,5 +1,4 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import argparse
 import os
 import re
 import shutil
@@ -9,12 +8,13 @@ import sys
 import subprocess
 import tempfile
 import time
-import unittest
 
-# FIXME support nocolor (use "" if passed?)
-RESET = "\x1b[0m"
-FG_RED = "\x1b[31m"
-FG_GREEN = "\x1b[32m"
+if os.environ.get("NO_COLOR", None):
+    RESET = FG_READ = FG_GREEN = ""
+else:
+    RESET = "\x1b[0m"
+    FG_RED = "\x1b[31m"
+    FG_GREEN = "\x1b[32m"
 
 executable_suffix = ".exe" if os.name == "nt" else ""
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -373,88 +373,6 @@ def mkdtemp():
     return tempfile.mkdtemp(dir=temp_dir)
 
 
-class DenoTestCase(unittest.TestCase):
-    @property
-    def build_dir(self):
-        args = test_args()
-        return args.build_dir
-
-    @property
-    def deno_exe(self):
-        return os.path.join(self.build_dir, "deno" + executable_suffix)
-
-
-# overload the test result class
-class ColorTextTestResult(unittest.TextTestResult):
-    def getDescription(self, test):
-        name = str(test)
-        if name.startswith("test_"):
-            name = name[5:]
-        return name
-
-    def addSuccess(self, test):
-        if self.showAll:
-            self.stream.write(FG_GREEN)
-        super(ColorTextTestResult, self).addSuccess(test)
-        if self.showAll:
-            self.stream.write(RESET)
-
-    def addError(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
-        super(ColorTextTestResult, self).addError(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
-
-    def addFailure(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
-        super(ColorTextTestResult, self).addFailure(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
-
-
-class ColorTextTestRunner(unittest.TextTestRunner):
-    resultclass = ColorTextTestResult
-
-
-def test_main():
-    args = test_args()
-    # FIXME(hayd) support more of the unittest.main API.
-    return unittest.main(
-        verbosity=args.verbosity + 1,
-        testRunner=ColorTextTestRunner,
-        failfast=args.failfast,
-        argv=[''])
-
-
-def test_args(argv=None):
-    if argv is None:
-        argv = sys.argv[1:]
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--failfast', '-f', action='store_true', help='Stop on first failure')
-    parser.add_argument(
-        '--verbosity', '-v', action='store_true', help='Verbose output')
-    parser.add_argument(
-        '--release',
-        action='store_true',
-        help='Test against release deno_executable')
-    parser.add_argument('build_dir', nargs='?', help='Deno build directory')
-    args = parser.parse_args(argv)
-    if args.build_dir and args.release:
-        raise argparse.ArgumentError(
-            None, "build_dir is inferred from --release, cannot provide both")
-    if not args.build_dir:
-        args.build_dir = build_path()
-
-    if not os.path.isfile(
-            os.path.join(args.build_dir, "deno" + executable_suffix)):
-        raise argparse.ArgumentError(None,
-                                     "deno executable not found in build_dir")
-    return args
-
-
 # This function is copied from:
 # https://gist.github.com/hayd/4f46a68fc697ba8888a7b517a414583e
 # https://stackoverflow.com/q/52954248/1240268

tools/util_test.py

@@ -1,16 +1,9 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
-import sys
 
-from util import (
-    DenoTestCase,
-    pattern_match,
-    parse_exit_code,
-    shell_quote_win,
-    parse_wrk_output,
-    root_path,
-    test_main,
-)
+from test_util import DenoTestCase, run_tests
+from util import (pattern_match, parse_exit_code, shell_quote_win,
+                  parse_wrk_output, root_path)
 
 
 class TestUtil(DenoTestCase):
@@ -70,4 +63,4 @@ class TestUtil(DenoTestCase):
 
 
 if __name__ == '__main__':
-    test_main()
+    run_tests()