From fbfd895bc7439c16d8cdbf209ed9ef28d3985703 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Sat, 14 Sep 2019 16:25:49 +0200 Subject: [PATCH] tools: refactor lint.py and format.py (#2950) --- .eslintignore | 1 - .gitignore | 1 - tools/format.py | 118 +++++++++++++++++++++++------------------------- tools/lint.py | 76 +++++++++++++++++++++++-------- tools/util.py | 26 +++++++++++ 5 files changed, 139 insertions(+), 83 deletions(-) diff --git a/.eslintignore b/.eslintignore index de3425c583..a76bd68241 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,2 @@ /js/deps/ -/js/gen/ /tests/error_syntax.js diff --git a/.gitignore b/.gitignore index 5c16b784d8..4118680a3a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,3 @@ node_modules /website/data.json /website/recent.json /website/*.bundle.js -/js/gen diff --git a/tools/format.py b/tools/format.py index 5c9fcfcef0..2628928f1e 100755 --- a/tools/format.py +++ b/tools/format.py @@ -1,102 +1,96 @@ #!/usr/bin/env python # Copyright 2018 the Deno authors. All rights reserved. MIT license. -from glob import glob import os import sys import argparse from third_party import google_env, python_env -from util import root_path, run, find_exts, platform, third_party_path - -parser = argparse.ArgumentParser() -parser.add_argument("--js", help="only run prettier", action="store_true") -parser.add_argument("--rs", help="only run rustfmt", action="store_true") -parser.add_argument("--py", help="only run yapf", action="store_true") -parser.add_argument("--gn", help="only run gn format", action="store_true") -parser.add_argument("--cc", help="only run clang format", action="store_true") - -clang_format_path = os.path.join(third_party_path, "depot_tools", - "clang-format") -prettier_path = os.path.join(third_party_path, "node_modules", "prettier", - "bin-prettier.js") -tools_path = os.path.join(root_path, "tools") -rustfmt_config = os.path.join(root_path, ".rustfmt.toml") +from util import git_ls_files, third_party_path, root_path, run def main(): os.chdir(root_path) + + parser = argparse.ArgumentParser() + parser.add_argument("--cc", help="run clang-format", action="store_true") + parser.add_argument("--gn", help="run gn format", action="store_true") + parser.add_argument("--js", help="run prettier", action="store_true") + parser.add_argument("--py", help="run yapf", action="store_true") + parser.add_argument("--rs", help="run rustfmt", action="store_true") args = parser.parse_args() + did_fmt = False - if args.rs: - rustfmt() - did_fmt = True if args.cc: clang_format() did_fmt = True if args.gn: gn_format() did_fmt = True - if args.py: - yapf() - did_fmt = True if args.js: prettier() did_fmt = True - if not did_fmt: + if args.py: + yapf() + did_fmt = True + if args.rs: rustfmt() + did_fmt = True + + if not did_fmt: clang_format() gn_format() - yapf() prettier() - - -def qrun(cmd, env=None): - run(cmd, quiet=True, env=env) + yapf() + rustfmt() def clang_format(): print "clang_format" - qrun([clang_format_path, "-i", "-style", "Google"] + - find_exts(["core"], [".cc", ".h"])) - - -def rustfmt(): - print "rustfmt" - qrun([ - "rustfmt", - "--config-path", - rustfmt_config, - ] + find_exts(["cli", "core", "tools", "deno_typescript", "cli_snapshots"], - [".rs"])) + exe = os.path.join(third_party_path, "depot_tools", "clang-format") + source_files = git_ls_files(root_path, ["*.cc", "*.h"]) + run([exe, "-i", "-style", "Google", "--"] + source_files, + env=google_env(), + quiet=True) def gn_format(): print "gn format" - for fn in ["BUILD.gn", ".gn"] + find_exts( - ["build_extra", "cli", "core", "deno_typescript", "cli_snapshots"], - [".gn", ".gni"]): - qrun(["third_party/depot_tools/gn", "format", fn], env=google_env()) - - -def yapf(): - print "yapf" - qrun( - [sys.executable, "third_party/python_packages/bin/yapf", "-i"] + - find_exts(["tools", "build_extra", "deno_typescript", "cli_snapshots"], - [".py"], - skip=["tools/clang"]), - env=python_env()) + exe = os.path.join(third_party_path, "depot_tools", "gn") + source_files = git_ls_files(root_path, ["*.gn", "*.gni"]) + run([exe, "format", "--"] + source_files, env=google_env(), quiet=True) def prettier(): print "prettier" - files = find_exts([ - ".github", "js", "tests", "tools", "website", "core", - "deno_typescript", "cli_snapshots" - ], [".js", ".json", ".ts", ".md"], - skip=["tools/clang", "js/deps", "js/gen"]) - qrun(["node", prettier_path, "--write", "--loglevel=error"] + - ["rollup.config.js"] + files) + script = os.path.join(third_party_path, "node_modules", "prettier", + "bin-prettier.js") + source_files = git_ls_files(root_path, ["*.js", "*.json", "*.ts", "*.md"]) + run(["node", script, "--write", "--loglevel=error", "--"] + source_files, + shell=False, + quiet=True) -if __name__ == '__main__': +def yapf(): + print "yapf" + script = os.path.join(third_party_path, "python_packages", "bin", "yapf") + source_files = git_ls_files(root_path, ["*.py"]) + run([sys.executable, script, "-i", "--"] + source_files, + env=python_env(), + shell=False, + quiet=True) + + +def rustfmt(): + print "rustfmt" + config_file = os.path.join(root_path, ".rustfmt.toml") + source_files = git_ls_files(root_path, ["*.rs"]) + run([ + "rustfmt", + "--config-path=" + config_file, + "--", + ] + source_files, + shell=False, + quiet=True) + + +if __name__ == "__main__": sys.exit(main()) diff --git a/tools/lint.py b/tools/lint.py index 46a3c94ca8..29ed86fa99 100755 --- a/tools/lint.py +++ b/tools/lint.py @@ -4,28 +4,66 @@ import os import sys -from util import enable_ansi_colors, find_exts, root_path, run, third_party_path +from util import enable_ansi_colors, git_ls_files, root_path, run +from util import third_party_path from third_party import python_env -enable_ansi_colors() -cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py") -eslint = os.path.join(third_party_path, "node_modules", "eslint", "bin", - "eslint") +def main(): + enable_ansi_colors() + os.chdir(root_path) + cpplint() + eslint() + pylint() -os.chdir(root_path) -run([ - sys.executable, cpplint, "--filter=-build/include_subdir", - "--repository=core/libdeno", "--extensions=cc,h", "--recursive", "core" -]) -run([ - "node", eslint, "--max-warnings=0", "./js/**/*.{ts,js}", - "./core/**/*.{ts,js}", "./tests/**/*.{ts,js}" -]) +def cpplint(): + print "cpplint" + script = os.path.join(third_party_path, "cpplint", "cpplint.py") + libdeno_dir = os.path.join(root_path, "core", "libdeno") + source_files = git_ls_files(libdeno_dir, ["*.cc", "*.h"]) + run([ + sys.executable, + script, + "--quiet", + "--filter=-build/include_subdir", + "--repository=" + libdeno_dir, + "--", + ] + source_files, + env=python_env(), + shell=False, + quiet=True) -run([ - sys.executable, "third_party/python_packages/pylint", - "--rcfile=third_party/depot_tools/pylintrc" -] + find_exts(["tools", "build_extra"], [".py"], skip=["tools/clang"]), - env=python_env()) + +def eslint(): + print "eslint" + script = os.path.join(third_party_path, "node_modules", "eslint", "bin", + "eslint") + # TODO: Files in 'deno_typescript', 'tools' and 'website' directories are + # currently not linted, but they should. + source_files = git_ls_files( + root_path, + ["*.js", "*.ts", ":!:deno_typescript/", ":!:tools/", ":!:website/"]) + # Find all *directories* in the main repo that contain .ts/.js files. + source_dirs = set([os.path.dirname(f) for f in source_files]) + # Within the source dirs, eslint does its own globbing, taking into account + # the exclusion rules listed in '.eslintignore'. + source_globs = ["%s/*.{js,ts}" % d for d in source_dirs] + run(["node", script, "--max-warnings=0", "--"] + source_globs, + shell=False, + quiet=True) + + +def pylint(): + print "pylint" + script = os.path.join(third_party_path, "python_packages", "pylint") + rcfile = os.path.join(third_party_path, "depot_tools", "pylintrc") + source_files = git_ls_files(root_path, ["*.py", ":!:gclient_config.py"]) + run([sys.executable, script, "--rcfile=" + rcfile, "--"] + source_files, + env=python_env(), + shell=False, + quiet=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/util.py b/tools/util.py index ce4c835734..7f5b95ea2d 100644 --- a/tools/util.py +++ b/tools/util.py @@ -166,6 +166,32 @@ def touch(fname): open(fname, 'a').close() +# Recursively list all files in (a subdirectory of) a git worktree. +# * Optionally, glob patterns may be specified to e.g. only list files with a +# certain extension. +# * Untracked files are included, unless they're listed in .gitignore. +# * Directory names themselves are not listed (but the files inside are). +# * Submodules and their contents are ignored entirely. +# * This function fails if the query matches no files. +def git_ls_files(base_dir, patterns=None): + base_dir = os.path.abspath(base_dir) + args = [ + "git", "-C", base_dir, "ls-files", "-z", "--exclude-standard", + "--cached", "--modified", "--others" + ] + if patterns: + args += ["--"] + patterns + output = subprocess.check_output(args) + files = [ + os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0") + if f != "" + ] + if not files: + raise RuntimeError("git_ls_files: no files in '%s'" % base_dir + + (" matching %s" % patterns if patterns else "")) + return files + + # Recursive search for files of certain extensions. # * Recursive glob doesn't exist in python 2.7. # * On windows, `os.walk()` unconditionally follows symlinks.