make tests quieter (#2468)

Don't mix every http request in with the tests output.
Don't print that the file servers are starting unless
-vv flag is passed.

Capture the output of run with run_output which returns
stdout, stderr and exit_code. Test against this rather
than relying on sys.exit.

.appveyor.yml

@@ -196,7 +196,7 @@ build_script:
 test_script:
   - python tools\lint.py
   - python tools\test_format.py
-  - ps: Exec { & python tools\test.py -v --build-dir $env:DENO_BUILD_PATH }
+  - ps: Exec { & python tools\test.py --build-dir $env:DENO_BUILD_PATH }
 
 after_test:
   # Delete the the rollup cache, which is unreliable, so that it doesn't get

.travis.yml

@@ -73,7 +73,7 @@ script:
 - ./tools/lint.py
 - ./tools/test_format.py
 - ./tools/build.py -C target/release
-- DENO_BUILD_MODE=release ./tools/test.py -v
+- DENO_BUILD_MODE=release ./tools/test.py
 
 jobs:
   fast_finish: true

tools/benchmark.py

@@ -11,7 +11,7 @@ import sys
 import json
 import time
 import shutil
-from util import run, run_output, root_path, build_path, executable_suffix
+from util import root_path, run, run_output, build_path, executable_suffix
 import tempfile
 import http_server
 import throughput_benchmark
@@ -212,7 +212,8 @@ def main(argv):
         print "Usage: tools/benchmark.py [build_dir]"
         sys.exit(1)
 
-    sha1 = run_output(["git", "rev-parse", "HEAD"]).strip()
+    sha1 = run_output(["git", "rev-parse", "HEAD"],
+                      exit_on_fail=True).out.strip()
     http_server.spawn()
 
     deno_exe = os.path.join(build_dir, "deno")

tools/deno_dir_test.py

@@ -5,7 +5,7 @@
 import os
 
 from test_util import DenoTestCase, run_tests
-from util import mkdtemp, rmtree, run
+from util import mkdtemp, rmtree, run_output
 
 
 class TestDenoDir(DenoTestCase):
@@ -38,7 +38,8 @@ class TestDenoDir(DenoTestCase):
     def run_deno(self, deno_dir=None):
         cmd = [self.deno_exe, "run", "tests/002_hello.ts"]
         deno_dir_env = {"DENO_DIR": deno_dir} if deno_dir is not None else None
-        run(cmd, quiet=True, env=deno_dir_env)
+        res = run_output(cmd, quiet=True, env=deno_dir_env)
+        self.assertEqual(res.code, 0)
 
 
 if __name__ == '__main__':

tools/fetch_test.py

@@ -2,20 +2,23 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import os
 import shutil
+import sys
 
 import http_server
 from test_util import DenoTestCase, run_tests
 from util import mkdtemp, tests_path, run_output
 
 
-class FetchTest(DenoTestCase):
+class TestFetch(DenoTestCase):
     def test_fetch(self):
         deno_dir = mkdtemp()
         try:
             t = os.path.join(tests_path, "006_url_imports.ts")
-            output = run_output([self.deno_exe, "fetch", t],
+            result = run_output([self.deno_exe, "fetch", t],
+                                quiet=True,
                                 merge_env={"DENO_DIR": deno_dir})
-            assert output == ""
+            self.assertEqual(result.out, "")
+            self.assertEqual(result.code, 0)
             # Check that we actually did the prefetch.
             os.path.exists(
                 os.path.join(

tools/fmt_test.py

@@ -4,10 +4,10 @@ import os
 import shutil
 
 from test_util import DenoTestCase, run_tests
-from util import mkdtemp, root_path, tests_path, run
+from util import mkdtemp, root_path, tests_path, run_output
 
 
-class FmtTest(DenoTestCase):
+class TestFmt(DenoTestCase):
     def test_fmt(self):
         d = mkdtemp()
         try:
@@ -26,12 +26,15 @@ class FmtTest(DenoTestCase):
             # TODO(kt3k) Below can be run([deno_exe, "fmt", dst], ...)
             # once the following issue is addressed:
             # https://github.com/denoland/deno_std/issues/330
-            run([
+            result = run_output([
                 os.path.join(root_path, self.deno_exe), "fmt",
                 "badly_formatted.js"
             ],
                                 cwd=d,
-                merge_env={"DENO_DIR": deno_dir})
+                                merge_env={"DENO_DIR": deno_dir},
+                                exit_on_fail=True,
+                                quiet=True)
+            self.assertEqual(result.code, 0)
             with open(fixed_filename) as f:
                 expected = f.read()
             with open(dst) as f:

tools/http_server.py

@@ -17,8 +17,17 @@ REDIRECT_PORT = 4546
 ANOTHER_REDIRECT_PORT = 4547
 DOUBLE_REDIRECTS_PORT = 4548
 
+QUIET = '-v' not in sys.argv and '--verbose' not in sys.argv
 
-class ContentTypeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+
+class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    def log_request(self, code='-', size='-'):
+        if not QUIET:
+            SimpleHTTPServer.SimpleHTTPRequestHandler.log_request(
+                self, code, size)
+
+
+class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
     def do_GET(self):
         if "multipart_form_data.txt" in self.path:
             self.protocol_version = 'HTTP/1.1'
@@ -102,6 +111,7 @@ def server():
     })
     SocketServer.TCPServer.allow_reuse_address = True
     s = SocketServer.TCPServer(("", PORT), Handler)
+    if not QUIET:
         print "Deno test server http://localhost:%d/" % PORT
     return RunningServer(s, start(s))
 
@@ -110,7 +120,7 @@ def base_redirect_server(host_port, target_port, extra_path_segment=""):
     os.chdir(root_path)
     target_host = "http://localhost:%d" % target_port
 
-    class RedirectHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    class RedirectHandler(QuietSimpleHTTPRequestHandler):
         def do_GET(self):
             self.send_response(301)
             self.send_header('Location',
@@ -120,6 +130,7 @@ def base_redirect_server(host_port, target_port, extra_path_segment=""):
     Handler = RedirectHandler
     SocketServer.TCPServer.allow_reuse_address = True
     s = SocketServer.TCPServer(("", host_port), Handler)
+    if not QUIET:
         print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
             host_port, target_port)
     return RunningServer(s, start(s))
@@ -153,7 +164,8 @@ def start(s):
 def spawn():
     servers = (server(), redirect_server(), another_redirect_server(),
                double_redirects_server())
-    sleep(1)  # TODO I'm too lazy to figure out how to do this properly.
+    while any(not s.thread.is_alive() for s in servers):
+        sleep(0.01)
     try:
         yield
     finally:

tools/setup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import third_party
-from util import build_mode, build_path, enable_ansi_colors, root_path, run
-from util import shell_quote, run_output
+from util import (build_mode, build_path, enable_ansi_colors, root_path, run,
+                  shell_quote)
 import os
 import re
 import sys

tools/target_test.py

@@ -2,7 +2,7 @@ import os
 import sys
 
 from test_util import DenoTestCase, run_tests
-from util import executable_suffix, run, tests_path, run_output
+from util import executable_suffix, tests_path, run, run_output
 
 
 class TestTarget(DenoTestCase):
@@ -20,7 +20,7 @@ class TestTarget(DenoTestCase):
         "Test executable runs and exits with code 0."
         bin_file = os.path.join(self.build_dir, executable + executable_suffix)
         self.check_exists(bin_file)
-        run([bin_file])
+        run([bin_file], quiet=True)
 
     def test_libdeno(self):
         self._test("libdeno_test")
@@ -35,26 +35,31 @@ class TestTarget(DenoTestCase):
         self._test("deno_core_http_bench_test")
 
     def test_ts_library_builder(self):
-        run([
+        result = run_output([
             "node", "./node_modules/.bin/ts-node", "--project",
             "tools/ts_library_builder/tsconfig.json",
             "tools/ts_library_builder/test.ts"
-        ])
+        ],
+                            quiet=True)
+        self.assertEqual(result.code, 0)
+        assert "ts_library_builder ok" in result.out
 
     def test_no_color(self):
         t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t],
-                            merge_env={"NO_COLOR": "1"})
-        assert output.strip() == "noColor true"
+        result = run_output([self.deno_exe, "run", t],
+                            merge_env={"NO_COLOR": "1"},
+                            quiet=True)
+        assert result.out.strip() == "noColor true"
         t = os.path.join(tests_path, "no_color.js")
-        output = run_output([self.deno_exe, "run", t])
-        assert output.strip() == "noColor false"
+        result = run_output([self.deno_exe, "run", t], quiet=True)
+        assert result.out.strip() == "noColor false"
 
     def test_exec_path(self):
         cmd = [self.deno_exe, "run", "tests/exec_path.ts"]
-        output = run_output(cmd)
-        assert self.deno_exe in output.strip()
+        result = run_output(cmd, quiet=True)
+        assert self.deno_exe in result.out.strip()
+        self.assertEqual(result.code, 0)
 
 
-if __name__ == "main":
+if __name__ == "__main__":
     run_tests()

tools/test.py

@@ -6,8 +6,8 @@ import os
 
 from benchmark_test import TestBenchmark
 from deno_dir_test import TestDenoDir
-from fetch_test import FetchTest
-from fmt_test import FmtTest
+from fetch_test import TestFetch
+from fmt_test import TestFmt
 from integration_tests import TestIntegrations
 from repl_test import TestRepl
 from setup_test import TestSetup
@@ -21,7 +21,7 @@ from complex_permissions_test import complex_permissions_tests
 
 import http_server
 from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
-                  executable_suffix, run, run_output, rmtree, tests_path)
+                  executable_suffix, rmtree, tests_path)
 from test_util import parse_test_args, run_tests
 
 
@@ -40,8 +40,8 @@ def main():
         TestUtil,
         TestTarget,
         JsUnitTests,
-        FetchTest,
-        FmtTest,
+        TestFetch,
+        TestFmt,
         TestIntegrations,
         TestRepl,
         TestDenoDir,

tools/test_format.py

@@ -9,11 +9,12 @@ import subprocess
 
 def main():
     util.run([sys.executable, "tools/format.py"])
-    output = util.run_output(
-        ["git", "status", "-uno", "--porcelain", "--ignore-submodules"])
-    if len(output) > 0:
+    result = util.run_output(
+        ["git", "status", "-uno", "--porcelain", "--ignore-submodules"],
+        exit_on_fail=True)
+    if result.out:
         print "Run tools/format.py "
-        print output
+        print result.out
         sys.exit(1)
 
 

tools/test_util.py

@@ -3,12 +3,13 @@
 # Runs the full test suite.
 # Usage: ./tools/test.py out/Debug
 import argparse
+import contextlib
 import os
 import sys
 import unittest
 
 from util import (enable_ansi_colors, build_path, RESET, FG_RED, FG_GREEN,
-                  executable_suffix, run, run_output, rmtree, tests_path)
+                  executable_suffix, rmtree, tests_path)
 
 
 class DenoTestCase(unittest.TestCase):
@@ -22,6 +23,14 @@ class DenoTestCase(unittest.TestCase):
 
 # overload the test result class
 class ColorTextTestResult(unittest.TextTestResult):
+    @contextlib.contextmanager
+    def color(self, code):
+        self.stream.write(code)
+        try:
+            yield
+        finally:
+            self.stream.write(RESET)
+
     def getDescription(self, test):
         name = str(test)
         if name.startswith("test_"):
@@ -29,25 +38,16 @@ class ColorTextTestResult(unittest.TextTestResult):
         return name
 
     def addSuccess(self, test):
-        if self.showAll:
-            self.stream.write(FG_GREEN)
+        with self.color(FG_GREEN):
             super(ColorTextTestResult, self).addSuccess(test)
-        if self.showAll:
-            self.stream.write(RESET)
 
     def addError(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
+        with self.color(FG_RED):
             super(ColorTextTestResult, self).addError(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
 
     def addFailure(self, test, err):
-        if self.showAll:
-            self.stream.write(FG_RED)
+        with self.color(FG_RED):
             super(ColorTextTestResult, self).addFailure(test, err)
-        if self.showAll:
-            self.stream.write(RESET)
 
 
 class ColorTextTestRunner(unittest.TextTestRunner):
@@ -133,7 +133,7 @@ def run_tests(test_cases=None):
         suite = unittest.TestSuite(filtered_tests)
 
     runner = ColorTextTestRunner(
-        verbosity=args.verbose + 1, failfast=args.failfast)
+        verbosity=args.verbose + 2, failfast=args.failfast)
 
     result = runner.run(suite)
     if not result.wasSuccessful():

tools/util.py

@@ -1,4 +1,5 @@
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+import collections
 import os
 import re
 import shutil
@@ -59,7 +60,15 @@ def run(args, quiet=False, cwd=None, env=None, merge_env=None):
         sys.exit(rc)
 
 
-def run_output(args, quiet=False, cwd=None, env=None, merge_env=None):
+CmdResult = collections.namedtuple('CmdResult', ['out', 'err', 'code'])
+
+
+def run_output(args,
+               quiet=False,
+               cwd=None,
+               env=None,
+               merge_env=None,
+               exit_on_fail=False):
     if merge_env is None:
         merge_env = {}
     args[0] = os.path.normpath(args[0])
@@ -67,7 +76,25 @@ def run_output(args, quiet=False, cwd=None, env=None, merge_env=None):
         print " ".join(args)
     env = make_env(env=env, merge_env=merge_env)
     shell = os.name == "nt"  # Run through shell to make .bat/.cmd files work.
-    return subprocess.check_output(args, cwd=cwd, env=env, shell=shell)
+    p = subprocess.Popen(
+        args,
+        cwd=cwd,
+        env=env,
+        shell=shell,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    try:
+        out, err = p.communicate()
+    except subprocess.CalledProcessError as e:
+        p.kill()
+        p.wait()
+        raise e
+    retcode = p.poll()
+    if retcode and exit_on_fail:
+        sys.exit(retcode)
+    # Ignore Windows CRLF (\r\n).
+    return CmdResult(
+        out.replace('\r\n', '\n'), err.replace('\r\n', '\n'), retcode)
 
 
 def shell_quote_win(arg):

tools/write_gn_args.py

@@ -11,7 +11,8 @@ args_list = run_output([
     build_path(), "--list", "--short", "--overrides-only"
 ],
                        quiet=True,
-                       env=third_party.google_env())
+                       env=third_party.google_env(),
+                       exit_on_fail=True).out
 
 with open(out_filename, "w") as f:
     f.write(args_list)