foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-22 07:14:47 -05:00
Code Issues Packages 1 Wiki Activity

build: rewrite tools/ scripts to deno (#8247)

Browse source 
This commit rewrites scripts in "tools/" directory
to use Deno instead of Python. In return it allows 
to remove huge number of Python packages in "third_party/".
This commit is contained in:
Bartek Iwańczuk 2020-11-05 15:53:21 +01:00 committed by GitHub
parent e7cfd90b0f
commit 791119d4af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 343 additions and 1140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -17,6 +17,6 @@ https://github.com/denoland/deno/blob/master/docs/contributing.md
2. Ensure there is a related issue and it is referenced in the PR text.
3. Ensure there are tests that cover the changes.
4. Ensure `cargo test` passes.
5. Ensure `./tools/format.py` passes without changing files.
6. Ensure `./tools/lint.py` passes.
5. Ensure `./tools/format.js` passes without changing files.
6. Ensure `./tools/lint.js` passes.
-->

24
.github/workflows/ci.yml vendored
View file

@ -73,6 +73,19 @@ jobs:
rustup component add clippy
rustup component add rustfmt
- name: Install Deno
if: |
!startsWith(matrix.os, 'windows')
run: |-
curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.5.1
echo "$HOME/.deno/bin" >> $GITHUB_PATH
- name: Install Deno (Windows)
if: startsWith(matrix.os, 'windows')
run: |-
curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.5.1
echo "$HOME/.deno/bin" >> $env:GITHUB_PATH
- name: Install Python
uses: actions/setup-python@v1
with:
@ -93,14 +106,15 @@ jobs:
python --version
rustc --version
cargo --version
deno --version
- name: lint.py
- name: lint.js
if: matrix.kind == 'lint'
run: python ./tools/lint.py
run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js
- name: test_format.py
- name: test_format.js
if: matrix.kind == 'lint'
run: python ./tools/test_format.py
run: deno run --unstable --allow-write --allow-read --allow-run ./tools/format.js --check
- name: Build release
if: |
@ -133,7 +147,7 @@ jobs:
DENOBOT_PAT: ${{ secrets.DENOBOT_PAT }}
run: |
git clone --depth 1 -b gh-pages https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git gh-pages
python ./tools/build_benchmark_jsons.py --release
deno run --unstable -A ./tools/build_benchmark_jsons.js --release
cd gh-pages
git config user.email "propelml@gmail.com"
git config user.name "denobot"

5
cli/tests/integration_tests.rs
View file

@ -1728,11 +1728,6 @@ fn deno_test_no_color() {
assert!(out.contains("test result: FAILED. 1 passed; 1 failed; 1 ignored; 0 measured; 0 filtered out"));
}
#[test]
fn util_test() {
util::run_python_script("tools/util_test.py")
}
macro_rules! itest(
($name:ident {$( $key:ident: $value:expr,)*}) => {
#[test]

4
docs/contributing.md
View file

@ -42,8 +42,8 @@ Examples of bad PR title:
2. Ensure there is a related issue and it is referenced in the PR text.
3. Ensure there are tests that cover the changes.
4. Ensure `cargo test` passes.
5. Ensure `./tools/format.py` passes without changing files.
6. Ensure `./tools/lint.py` passes.
5. Ensure `./tools/format.js` passes without changing files.
6. Ensure `./tools/lint.js` passes.
## Changes to `third_party`

4
docs/contributing/development_tools.md
View file

@ -23,13 +23,13 @@ cargo test std_tests
Lint the code:
```shell
./tools/lint.py
deno run -A --unstable ./tools/lint.js
```
Format the code:
```shell
./tools/format.py
deno run -A --unstable ./tools/format.js
```
### Profiling

20
test_util/src/lib.rs
View file

@ -776,26 +776,6 @@ pub fn deno_cmd() -> Command {
c
}
pub fn run_python_script(script: &str) {
let deno_dir = new_deno_dir();
let output = Command::new("python")
.env("DENO_DIR", deno_dir.path())
.current_dir(root_path())
.arg(script)
.arg(format!("--build-dir={}", target_dir().display()))
.arg(format!("--executable={}", deno_exe_path().display()))
.output()
.expect("failed to spawn script");
if !output.status.success() {
let stdout = String::from_utf8(output.stdout).unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
panic!(
"{} executed with failing error code\n{}{}",
script, stdout, stderr
);
}
}
pub fn run_powershell_script_file(
script_file_path: &str,
args: Vec<&str>,

2
third_party

@ -1 +1 @@
Subproject commit 64944a3a65e15a02d117a641f40de50179358259
Subproject commit 76b283c420be9302309e94394d1b3f1ebb425f29

14
tools/README.md
View file

@ -2,24 +2,24 @@
Documentation for various tooling in support of Deno development.
## format.py
## format.js
This script will format the code (currently using dprint, yapf and rustfmt). It
is a prerequisite to run this before code check in.
This script will format the code (currently using dprint, rustfmt). It is a
prerequisite to run this before code check in.
To run formatting:
```sh
./tools/format.py
deno run -A --unstable ./tools/format.js
```
## lint.py
This script will lint the code base (currently using eslint, pylint and clippy).
It is a prerequisite to run this before code check in.
This script will lint the code base (currently using dlint, clippy). It is a
prerequisite to run this before code check in.
To run linting:
```sh
./tools/lint.py
deno run -A --unstable ./tools/lint.js
```

31
tools/build_benchmark_jsons.js Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { buildPath, existsSync, join } from "./util.js";
const currentDataFile = join(buildPath(), "bench.json");
const allDataFile = "gh-pages/data.json"; // Includes all benchmark data.
const recentDataFile = "gh-pages/recent.json"; // Includes recent 20 benchmark data.
function readJson(filename) {
return JSON.parse(Deno.readTextFileSync(filename));
}
function writeJson(filename, data) {
return Deno.writeTextFileSync(filename, JSON.stringify(data));
}
if (!existsSync(currentDataFile)) {
throw new Error(`${currentDataFile} doesn't exist`);
}
if (!existsSync(allDataFile)) {
throw new Error(`${allDataFile} doesn't exist`);
}
const newData = readJson(currentDataFile);
const allData = readJson(allDataFile);
allData.push(newData);
const allDataLen = allData.length;
const recentData = allData.slice(allDataLen - 20);
writeJson(allDataFile, allData);
writeJson(recentDataFile, recentData);

30
tools/build_benchmark_jsons.py
View file

@ -1,30 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import os
import json
from util import build_path
def read_json(filename):
with open(filename) as json_file:
return json.load(json_file)
def write_json(filename, data):
with open(filename, 'w') as outfile:
json.dump(data, outfile)
current_data_file = os.path.join(build_path(), "bench.json")
all_data_file = "gh-pages/data.json" # Includes all benchmark data.
recent_data_file = "gh-pages/recent.json" # Includes recent 20 benchmark data.
assert os.path.exists(current_data_file)
assert os.path.exists(all_data_file)
new_data = read_json(current_data_file)
all_data = read_json(all_data_file)
all_data.append(new_data)
write_json(all_data_file, all_data)
write_json(recent_data_file, all_data[-20:])

63
tools/format.js Executable file
View file

@ -0,0 +1,63 @@
#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-run
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import { getPrebuiltToolPath, getSources, join, ROOT_PATH } from "./util.js";
async function dprint() {
const execPath = getPrebuiltToolPath("dprint");
console.log("dprint");
const p = Deno.run({
cmd: [execPath, "fmt"],
});
const { success } = await p.status();
if (!success) {
throw new Error("dprint failed");
}
p.close();
}
async function rustfmt() {
const configFile = join(ROOT_PATH, ".rustfmt.toml");
const sourceFiles = await getSources(ROOT_PATH, ["*.rs"]);
if (!sourceFiles.length) {
return;
}
console.log(`rustfmt ${sourceFiles.length} file(s)`);
const p = Deno.run({
cmd: ["rustfmt", "--config-path=" + configFile, "--", ...sourceFiles],
});
const { success } = await p.status();
if (!success) {
throw new Error("rustfmt failed");
}
p.close();
}
async function main() {
await Deno.chdir(ROOT_PATH);
await dprint();
await rustfmt();
if (Deno.args.includes("--check")) {
const git = Deno.run({
cmd: ["git", "status", "-uno", "--porcelain", "--ignore-submodules"],
stdout: "piped",
});
const { success } = await git.status();
if (!success) {
throw new Error("git status failed");
}
const out = new TextDecoder().decode(await git.output());
git.close();
if (out) {
console.log("run tools/format.js");
console.log(out);
Deno.exit(1);
}
}
}
await main();

89
tools/format.py
View file

@ -1,89 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import os
import sys
import argparse
from third_party import python_env, get_prebuilt_tool_path
from util import git_ls_files, git_staged, third_party_path, root_path
from util import print_command, run
cmd_args = None
def get_cmd_args():
global cmd_args
if cmd_args:
return cmd_args
parser = argparse.ArgumentParser()
parser.add_argument("--js", help="run dprint", action="store_true")
parser.add_argument("--py", help="run yapf", action="store_true")
parser.add_argument("--rs", help="run rustfmt", action="store_true")
parser.add_argument(
"--staged", help="run only on staged files", action="store_true")
cmd_args = parser.parse_args()
return cmd_args
def get_sources(*args):
getter = git_staged if get_cmd_args().staged else git_ls_files
return getter(*args)
def main():
os.chdir(root_path)
args = get_cmd_args()
did_fmt = False
if args.js:
dprint()
did_fmt = True
if args.py:
yapf()
did_fmt = True
if args.rs:
rustfmt()
did_fmt = True
if not did_fmt:
dprint()
yapf()
rustfmt()
def dprint():
executable_path = get_prebuilt_tool_path("dprint")
command = [executable_path, "fmt"]
run(command, shell=False, quiet=True)
def yapf():
script = os.path.join(third_party_path, "python_packages", "bin", "yapf")
source_files = get_sources(root_path, ["*.py"])
if source_files:
print_command("yapf", source_files)
run([sys.executable, script, "-i", "--style=pep8", "--"] +
source_files,
env=python_env(),
shell=False,
quiet=True)
def rustfmt():
config_file = os.path.join(root_path, ".rustfmt.toml")
source_files = get_sources(root_path, ["*.rs"])
if source_files:
print_command("rustfmt", source_files)
run([
"rustfmt",
"--config-path=" + config_file,
"--",
] + source_files,
shell=False,
quiet=True)
if __name__ == "__main__":
sys.exit(main())

102
tools/lint.js Executable file
View file

@ -0,0 +1,102 @@
#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-run
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import {
buildMode,
getPrebuiltToolPath,
getSources,
ROOT_PATH,
} from "./util.js";
async function dlint() {
const execPath = getPrebuiltToolPath("dlint");
console.log("dlint");
const sourceFiles = await getSources(ROOT_PATH, [
"*.js",
"*.ts",
":!:cli/tests/swc_syntax_error.ts",
":!:cli/tests/038_checkjs.js",
":!:cli/tests/error_008_checkjs.js",
":!:std/**/testdata/*",
":!:std/**/node_modules/*",
":!:cli/bench/node*.js",
":!:cli/compilers/wasm_wrap.js",
":!:cli/dts/**",
":!:cli/tests/encoding/**",
":!:cli/tests/error_syntax.js",
":!:cli/tests/lint/**",
":!:cli/tests/tsc/**",
":!:cli/tsc/*typescript.js",
]);
if (!sourceFiles.length) {
return;
}
const MAX_COMMAND_LEN = 30000;
const preCommand = [execPath, "run"];
const chunks = [[]];
let cmdLen = preCommand.join(" ").length;
for (const f of sourceFiles) {
if (cmdLen + f.length > MAX_COMMAND_LEN) {
chunks.push([f]);
cmdLen = preCommand.join(" ").length;
} else {
chunks[chunks.length - 1].push(f);
cmdLen = preCommand.join(" ").length;
}
}
for (const chunk of chunks) {
const p = Deno.run({
cmd: [execPath, "run", ...chunk],
});
const { success } = await p.status();
if (!success) {
throw new Error("dlint failed");
}
p.close();
}
}
async function clippy() {
console.log("clippy");
const currentBuildMode = buildMode();
const cmd = ["cargo", "clippy", "--all-targets", "--locked"];
if (currentBuildMode != "debug") {
cmd.push("--release");
}
const p = Deno.run({
cmd: [...cmd, "--", "-D", "clippy::all"],
});
const { success } = await p.status();
if (!success) {
throw new Error("clippy failed");
}
p.close();
}
async function main() {
await Deno.chdir(ROOT_PATH);
let didLint = false;
if (Deno.args.includes("--js")) {
await dlint();
didLint = true;
}
if (Deno.args.includes("--rs")) {
await clippy();
didLint = true;
}
if (!didLint) {
await dlint();
await clippy();
}
}
await main();

123
tools/lint.py
View file

@ -1,123 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
# Does google-lint on c++ files and ts-lint on typescript files
import os
import sys
import argparse
from util import enable_ansi_colors, git_ls_files, git_staged, root_path, run
from util import third_party_path, build_mode, print_command
from third_party import python_env, get_prebuilt_tool_path
cmd_args = None
def get_cmd_args():
global cmd_args
if cmd_args:
return cmd_args
parser = argparse.ArgumentParser()
parser.add_argument("--js", help="run dlint", action="store_true")
parser.add_argument("--py", help="run pylint", action="store_true")
parser.add_argument("--rs", help="run clippy", action="store_true")
parser.add_argument(
"--staged", help="run only on staged files", action="store_true")
cmd_args = parser.parse_args()
return cmd_args
def get_sources(*args):
getter = git_staged if get_cmd_args().staged else git_ls_files
return getter(*args)
def main():
enable_ansi_colors()
os.chdir(root_path)
args = get_cmd_args()
did_fmt = False
if args.js:
dlint()
did_fmt = True
if args.py:
pylint()
did_fmt = True
if args.rs:
clippy()
did_fmt = True
if not did_fmt:
dlint()
pylint()
clippy()
def dlint():
executable_path = get_prebuilt_tool_path("dlint")
# Find all *directories* in the main repo that contain .ts/.js files.
source_files = get_sources(root_path, [
"*.js",
"*.ts",
":!:cli/tests/swc_syntax_error.ts",
":!:cli/tests/038_checkjs.js",
":!:cli/tests/error_008_checkjs.js",
":!:std/**/testdata/*",
":!:std/**/node_modules/*",
":!:cli/bench/node*.js",
":!:cli/compilers/wasm_wrap.js",
":!:cli/dts/**",
":!:cli/tests/encoding/**",
":!:cli/tests/error_syntax.js",
":!:cli/tests/lint/**",
":!:cli/tests/tsc/**",
":!:cli/tsc/*typescript.js",
])
if source_files:
max_command_len = 30000
pre_command = [executable_path, "run"]
chunks = [[]]
cmd_len = len(" ".join(pre_command))
for f in source_files:
if cmd_len + len(f) > max_command_len:
chunks.append([f])
cmd_len = len(" ".join(pre_command))
else:
chunks[-1].append(f)
cmd_len = cmd_len + len(f) + 1
for c in chunks:
print_command("dlint", c)
run(pre_command + c, shell=False, quiet=True)
def pylint():
script = os.path.join(third_party_path, "python_packages", "pylint")
rcfile = os.path.join(root_path, "tools", "pylintrc")
msg_template = "{path}({line}:{column}) {category}: {msg} ({symbol})"
source_files = get_sources(root_path, ["*.py"])
if source_files:
print_command("pylint", source_files)
run([
sys.executable, script, "--rcfile=" + rcfile,
"--msg-template=" + msg_template, "--"
] + source_files,
env=python_env(),
shell=False,
quiet=True)
def clippy():
print("clippy")
current_build_mode = build_mode()
args = ["cargo", "clippy", "--all-targets", "--locked"]
if current_build_mode != "debug":
args += ["--release"]
run(args + ["--", "-D", "clippy::all"], shell=False, quiet=True)
if __name__ == "__main__":
sys.exit(main())

337
tools/pylintrc
View file

@ -1,337 +0,0 @@
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Pickle collected data for later comparisons.
persistent=yes
[MESSAGES CONTROL]
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once).
#
# These should get enabled, but the codebase has too many violations currently:
# bad-continuation
# anomalous-backslash-in-string
# bad-context-manager
# bad-indentation
# bad-str-strip-call
# bad-whitespace
# cell-var-from-loop
# deprecated-lambda
# eval-used
# function-redefined
# import-error
# locally-enabled
# missing-final-newline
# no-init
# no-name-in-module
# no-self-use
# not-callable
# old-style-class
# protected-access
# superfluous-parens
# super-on-old-class
# too-many-function-args
# trailing-whitespace
# unnecessary-semicolon
# unpacking-non-sequence
# unused-import
# useless-else-on-loop
#
# CHANGED:
disable=
invalid-name,
missing-docstring,
too-many-lines,
bad-inline-option,
locally-disabled,
duplicate-code,
too-many-ancestors,
too-many-instance-attributes,
too-few-public-methods,
too-many-public-methods,
too-many-return-statements,
too-many-branches,
too-many-arguments,
too-many-locals,
too-many-statements,
abstract-class-not-used,
abstract-class-little-used,
exec-used,
bad-builtin,
star-args,
deprecated-module,
reimported,
fixme,
global-statement,
broad-except,
logging-not-lazy,
bad-continuation,
anomalous-backslash-in-string,
assigning-non-slot,
bad-context-manager,
bad-indentation,
bad-str-strip-call,
bad-super-call,
bad-whitespace,
cell-var-from-loop,
consider-using-enumerate,
deprecated-lambda,
deprecated-method,
eval-used,
function-redefined,
import-error,
invalid-docstring-quote,
invalid-string-quote,
invalid-triple-quote,
locally-enabled,
misplaced-comparison-constant,
misplaced-bare-raise,
missing-final-newline,
multiple-imports,
no-init,
no-name-in-module,
no-self-argument,
no-self-use,
not-an-iterable,
not-callable,
old-style-class,
protected-access,
redefined-variable-type,
simplifiable-if-statement,
singleton-comparison,
superfluous-parens,
super-on-old-class,
too-many-boolean-expressions,
too-many-function-args,
too-many-nested-blocks,
trailing-whitespace,
undefined-variable,
ungrouped-imports,
unnecessary-semicolon,
unneeded-not,
unpacking-non-sequence,
unsubscriptable-object,
unsupported-membership-test,
unused-import,
useless-else-on-loop,
using-constant-test,
wrong-import-order,
wrong-import-position,
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
# CHANGED:
reports=no
# Activate the evaluation score.
score=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the beginning of the name of dummy variables
# (i.e. not used).
dummy-variables-rgx=_|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=80
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
# CHANGED:
indent-string=' '
[BASIC]
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input
# Regular expression which should only match correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression which should only match correct module level names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression which should only match correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Regular expression which should only match correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct instance attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match correct list comprehension /
# generator expression variable names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Regular expression which should only match functions or classes name which do
# not require a docstring
no-docstring-rgx=__.*__
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branchs=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception

22
tools/test_format.py
View file

@ -1,22 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
# This program fails if ./tools/format.py changes any files.
import sys
import subprocess
import util
def main():
util.run([sys.executable, "tools/format.py"])
result = util.run_output(
["git", "status", "-uno", "--porcelain", "--ignore-submodules"],
exit_on_fail=True)
if result.out:
print("Run tools/format.py ")
print(result.out)
sys.exit(1)
if __name__ == '__main__':
main()

139
tools/test_util.py
View file

@ -1,139 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
# Runs the full test suite.
# Usage: ./tools/test.py out/Debug
import argparse
import contextlib
import os
import sys
import unittest
from util import (build_path, RESET, FG_RED, FG_GREEN, executable_suffix)
class DenoTestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
args = parse_test_args()
cls.build_dir = args.build_dir
cls.deno_exe = args.executable
# overload the test result class
class ColorTextTestResult(unittest.TextTestResult):
@contextlib.contextmanager
def color(self, code):
self.stream.write(code)
try:
yield
finally:
self.stream.write(RESET)
def getDescription(self, test):
name = str(test)
if name.startswith("test_"):
name = name[5:]
return name
def addSuccess(self, test):
with self.color(FG_GREEN):
super(ColorTextTestResult, self).addSuccess(test)
def addError(self, test, err):
with self.color(FG_RED):
super(ColorTextTestResult, self).addError(test, err)
def addFailure(self, test, err):
with self.color(FG_RED):
super(ColorTextTestResult, self).addFailure(test, err)
class ColorTextTestRunner(unittest.TextTestRunner):
resultclass = ColorTextTestResult
def create_test_arg_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
'--failfast', '-f', action='store_true', help='Stop on first failure')
parser.add_argument(
'--verbose', '-v', action='store_true', help='Verbose output')
parser.add_argument("--executable", help="Use external executable of Deno")
parser.add_argument(
'--release',
action='store_true',
help='Test against release executable')
parser.add_argument(
'--pattern', '-p', help='Run tests that match provided pattern')
parser.add_argument(
'--build-dir', dest="build_dir", help='Deno build directory')
return parser
TestArgParser = create_test_arg_parser()
def parse_test_args(argv=None):
if argv is None:
argv = sys.argv[1:]
args = TestArgParser.parse_args(argv)
if args.executable and args.release:
raise argparse.ArgumentError(
None, "Path to executable is inferred from "
"--release, cannot provide both.")
if not args.build_dir:
args.build_dir = build_path()
if not args.executable:
args.executable = os.path.join(args.build_dir,
"deno" + executable_suffix)
if not os.path.isfile(args.executable):
raise argparse.ArgumentError(
None, "deno executable not found at {}".format(args.executable))
return args
def filter_test_suite(suite, pattern):
filtered_tests = []
for test_case in suite:
if isinstance(test_case, unittest.TestSuite):
filtered_tests += filter_test_suite(test_case, pattern)
else:
if pattern in str(test_case):
filtered_tests.append(test_case)
return filtered_tests
def run_tests(test_cases=None):
args = parse_test_args()
loader = unittest.TestLoader()
# if suite was not explicitly passed load test
# cases from calling module
if test_cases is None:
import __main__
suite = loader.loadTestsFromModule(__main__)
else:
suite = unittest.TestSuite()
for test_case in test_cases:
suite.addTests(loader.loadTestsFromTestCase(test_case))
if args.pattern:
filtered_tests = filter_test_suite(suite, args.pattern)
suite = unittest.TestSuite(filtered_tests)
runner = ColorTextTestRunner(
verbosity=args.verbose + 2, failfast=args.failfast)
result = runner.run(suite)
if not result.wasSuccessful():
sys.exit(1)

55
tools/third_party.py
View file

@ -1,55 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
# This script contains helper functions to work with the third_party subrepo.
import os
import re
import site
import sys
from util import add_env_path, executable_suffix, make_env, third_party_path
prebuilt_path = os.path.join(third_party_path, "prebuilt")
python_packages_path = os.path.join(third_party_path, "python_packages")
python_site_env = None
# Creates/modifies an environment so python can find packages that are bundled
# in the 'third_party' directory.
def python_env(env=None, merge_env=None):
if merge_env is None:
merge_env = {}
global python_site_env
# Use site.addsitedir() to determine which search paths would be considered
# if 'third_party/python_packages' was a site-packages directory.
# PATH is also updated, so windows can find the DLLs that ship with pywin32.
if python_site_env is None:
python_site_env = {}
temp = os.environ["PATH"], sys.path
os.environ["PATH"], sys.path = "", []
site.addsitedir(python_packages_path) # Modifies PATH and sys.path.
python_site_env = {"PATH": os.environ["PATH"], "PYTHONPATH": sys.path}
os.environ["PATH"], sys.path = temp
# Make a new environment object.
env = make_env(env=env, merge_env=merge_env)
# Apply PATH and PYTHONPATH from the site-packages environment.
add_env_path(python_site_env["PATH"], env=env, key="PATH")
add_env_path(python_site_env["PYTHONPATH"], env=env, key="PYTHONPATH")
return env
def get_platform_dir_name():
if sys.platform == "win32":
return "win"
elif sys.platform == "darwin":
return "mac"
elif sys.platform.startswith("linux"):
return "linux64"
def get_prebuilt_tool_path(tool):
return os.path.join(prebuilt_path, get_platform_dir_name(),
tool + executable_suffix)

114
tools/util.js Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import {
dirname,
fromFileUrl,
join,
} from "https://deno.land/std@0.76.0/path/mod.ts";
export { dirname, join };
export { existsSync } from "https://deno.land/std@0.76.0/fs/mod.ts";
export const ROOT_PATH = dirname(dirname(fromFileUrl(import.meta.url)));
async function getFilesFromGit(baseDir, cmd) {
const p = Deno.run({
cmd,
stdout: "piped",
});
const { success } = await p.status();
if (!success) {
throw new Error("gitLsFiles failed");
}
const output = new TextDecoder().decode(await p.output());
p.close();
const files = output.split("\0").filter((line) => line.length > 0).map(
(filePath) => {
return Deno.realPathSync(join(baseDir, filePath));
},
);
return files;
}
async function gitLsFiles(baseDir, patterns) {
baseDir = Deno.realPathSync(baseDir);
const cmd = [
"git",
"-C",
baseDir,
"ls-files",
"-z",
"--exclude-standard",
"--cached",
"--modified",
"--others",
"--",
...patterns,
];
return getFilesFromGit(baseDir, cmd);
}
/** List all files staged for commit */
async function gitStaged(baseDir, patterns) {
baseDir = Deno.realPathSync(baseDir);
const cmd = [
"git",
"-C",
baseDir,
"diff",
"--staged",
"--diff-filter=ACMR",
"--name-only",
"-z",
"--",
...patterns,
];
return getFilesFromGit(baseDir, cmd);
}
/**
* Recursively list all files in (a subdirectory of) a git worktree.
* * Optionally, glob patterns may be specified to e.g. only list files with a
* certain extension.
* * Untracked files are included, unless they're listed in .gitignore.
* * Directory names themselves are not listed (but the files inside are).
* * Submodules and their contents are ignored entirely.
* * This function fails if the query matches no files.
*
* If --staged argument was provided when program is run
* only staged sources will be returned.
*/
export async function getSources(baseDir, patterns) {
const stagedOnly = Deno.args.includes("--staged");
if (stagedOnly) {
return await gitStaged(baseDir, patterns);
} else {
return await gitLsFiles(baseDir, patterns);
}
}
export function buildMode() {
if (Deno.args.includes("--release")) {
return "release";
}
return "debug";
}
export function buildPath() {
return join(ROOT_PATH, "target", buildMode());
}
export function getPrebuiltToolPath(toolName) {
const PREBUILT_PATH = join(ROOT_PATH, "third_party", "prebuilt");
const platformDirName = {
"windows": "win",
"darwin": "mac",
"linux": "linux64",
}[Deno.build.os];
const executableSuffix = Deno.build.os === "windows" ? ".exe" : "";
return join(PREBUILT_PATH, platformDirName, toolName + executableSuffix);
}

278
tools/util.py
View file

@ -1,278 +0,0 @@
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import collections
import os
import re
import shutil
import select
import stat
import sys
import subprocess
import tempfile
import time
if os.environ.get("NO_COLOR", None):
RESET = FG_READ = FG_GREEN = ""
else:
RESET = "\x1b[0m"
FG_RED = "\x1b[31m"
FG_GREEN = "\x1b[32m"
executable_suffix = ".exe" if os.name == "nt" else ""
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
third_party_path = os.path.join(root_path, "third_party")
def make_env(merge_env=None, env=None):
if env is None:
env = os.environ
env = env.copy()
if merge_env is None:
merge_env = {}
for key in merge_env.keys():
env[key] = merge_env[key]
return env
def add_env_path(add, env, key="PATH", prepend=False):
dirs_left = env[key].split(os.pathsep) if key in env else []
dirs_right = add.split(os.pathsep) if isinstance(add, str) else add
if prepend:
dirs_left, dirs_right = dirs_right, dirs_left
for d in dirs_right:
if not d in dirs_left:
dirs_left += [d]
env[key] = os.pathsep.join(dirs_left)
def run(args, quiet=False, cwd=None, env=None, merge_env=None, shell=None):
args[0] = os.path.normpath(args[0])
env = make_env(env=env, merge_env=merge_env)
if shell is None:
# Use the default value for 'shell' parameter.
# - Posix: do not use shell.
# - Windows: use shell; this makes .bat/.cmd files work.
shell = os.name == "nt"
if not quiet:
print(" ".join([shell_quote(arg) for arg in args]))
rc = subprocess.call(args, cwd=cwd, env=env, shell=shell)
if rc != 0:
sys.exit(rc)
CmdResult = collections.namedtuple('CmdResult', ['out', 'err', 'code'])
def run_output(args,
quiet=False,
cwd=None,
env=None,
merge_env=None,
exit_on_fail=False):
if merge_env is None:
merge_env = {}
args[0] = os.path.normpath(args[0])
if not quiet:
print(" ".join(args))
env = make_env(env=env, merge_env=merge_env)
shell = os.name == "nt" # Run through shell to make .bat/.cmd files work.
p = subprocess.Popen(
args,
cwd=cwd,
env=env,
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
out, err = p.communicate()
except subprocess.CalledProcessError as e:
p.kill()
p.wait()
raise e
retcode = p.poll()
if retcode and exit_on_fail:
sys.exit(retcode)
# Ignore Windows CRLF (\r\n).
return CmdResult(
out.replace('\r\n', '\n'), err.replace('\r\n', '\n'), retcode)
def shell_quote_win(arg):
if re.search(r'[\x00-\x20"^%~!@&?*<>|()=]', arg):
# Double all " quote characters.
arg = arg.replace('"', '""')
# Wrap the entire string in " quotes.
arg = '"' + arg + '"'
# Double any N backslashes that are immediately followed by a " quote.
arg = re.sub(r'(\\+)(?=")', r'\1\1', arg)
return arg
def shell_quote(arg):
if os.name == "nt":
return shell_quote_win(arg)
else:
# Python 2 has posix shell quoting built in, albeit in a weird place.
from pipes import quote
return quote(arg)
# Recursively list all files in (a subdirectory of) a git worktree.
# * Optionally, glob patterns may be specified to e.g. only list files with a
# certain extension.
# * Untracked files are included, unless they're listed in .gitignore.
# * Directory names themselves are not listed (but the files inside are).
# * Submodules and their contents are ignored entirely.
# * This function fails if the query matches no files.
def git_ls_files(base_dir, patterns=None):
base_dir = os.path.abspath(base_dir)
args = [
"git", "-C", base_dir, "ls-files", "-z", "--exclude-standard",
"--cached", "--modified", "--others"
]
if patterns:
args += ["--"] + patterns
output = subprocess.check_output(args)
files = [
os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0")
if f != ""
]
if not files:
raise RuntimeError("git_ls_files: no files in '%s'" % base_dir +
(" matching %s" % patterns if patterns else ""))
return files
# list all files staged for commit
def git_staged(base_dir, patterns=None):
base_dir = os.path.abspath(base_dir)
args = [
"git", "-C", base_dir, "diff", "--staged", "--diff-filter=ACMR",
"--name-only", "-z"
]
if patterns:
args += ["--"] + patterns
output = subprocess.check_output(args)
files = [
os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0")
if f != ""
]
return files
def build_mode():
if "--release" in sys.argv:
return "release"
else:
return "debug"
# E.G. "target/debug"
def build_path():
return os.path.join(root_path, "target", build_mode())
# Attempts to enable ANSI escape code support.
# Returns True if successful, False if not supported.
def enable_ansi_colors():
if os.name != 'nt':
return True # On non-windows platforms this just works.
return enable_ansi_colors_win10()
# The windows 10 implementation of enable_ansi_colors.
def enable_ansi_colors_win10():
import ctypes
# Function factory for errcheck callbacks that raise WinError on failure.
def raise_if(error_result):
def check(result, _func, args):
if result == error_result:
raise ctypes.WinError(ctypes.get_last_error())
return args
return check
# Windows API types.
from ctypes.wintypes import BOOL, DWORD, HANDLE, LPCWSTR, LPVOID
LPDWORD = ctypes.POINTER(DWORD)
# Generic constants.
NULL = ctypes.c_void_p(0).value
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
ERROR_INVALID_PARAMETER = 87
# CreateFile flags.
# yapf: disable
GENERIC_READ = 0x80000000
GENERIC_WRITE = 0x40000000
FILE_SHARE_READ = 0x01
FILE_SHARE_WRITE = 0x02
OPEN_EXISTING = 3
# yapf: enable
# Get/SetConsoleMode flags.
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
# HANDLE CreateFileW(...)
CreateFileW = kernel32.CreateFileW
CreateFileW.restype = HANDLE
CreateFileW.errcheck = raise_if(INVALID_HANDLE_VALUE)
# yapf: disable
CreateFileW.argtypes = (LPCWSTR, # lpFileName
DWORD, # dwDesiredAccess
DWORD, # dwShareMode
LPVOID, # lpSecurityAttributes
DWORD, # dwCreationDisposition
DWORD, # dwFlagsAndAttributes
HANDLE) # hTemplateFile
# yapf: enable
# BOOL CloseHandle(HANDLE hObject)
CloseHandle = kernel32.CloseHandle
CloseHandle.restype = BOOL
CloseHandle.errcheck = raise_if(False)
CloseHandle.argtypes = (HANDLE, )
# BOOL GetConsoleMode(HANDLE hConsoleHandle, LPDWORD lpMode)
GetConsoleMode = kernel32.GetConsoleMode
GetConsoleMode.restype = BOOL
GetConsoleMode.errcheck = raise_if(False)
GetConsoleMode.argtypes = (HANDLE, LPDWORD)
# BOOL SetConsoleMode(HANDLE hConsoleHandle, DWORD dwMode)
SetConsoleMode = kernel32.SetConsoleMode
SetConsoleMode.restype = BOOL
SetConsoleMode.errcheck = raise_if(False)
SetConsoleMode.argtypes = (HANDLE, DWORD)
# Open the console output device.
conout = CreateFileW("CONOUT$", GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
OPEN_EXISTING, 0, 0)
# Get the current mode.
mode = DWORD()
GetConsoleMode(conout, ctypes.byref(mode))
# Try to set the flag that controls ANSI escape code support.
try:
SetConsoleMode(conout, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
except WindowsError as e: # pylint:disable=undefined-variable
if e.winerror == ERROR_INVALID_PARAMETER:
return False # Not supported, likely an older version of Windows.
raise
finally:
CloseHandle(conout)
return True
def print_command(cmd, files):
noun = "file" if len(files) == 1 else "files"
print("%s (%d %s)" % (cmd, len(files), noun))

23
tools/util_test.py
View file

@ -1,23 +0,0 @@
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
import os
from test_util import DenoTestCase, run_tests
from util import (shell_quote_win, root_path)
class TestUtil(DenoTestCase):
def test_shell_quote_win(self):
assert shell_quote_win('simple') == 'simple'
assert shell_quote_win(
'roof/\\isoprojection') == 'roof/\\isoprojection'
assert shell_quote_win('with space') == '"with space"'
assert shell_quote_win('embedded"quote') == '"embedded""quote"'
assert shell_quote_win(
'a"b""c\\d\\"e\\\\') == '"a""b""""c\\d\\\\""e\\\\\\\\"'
def test_executable_exists(self):
assert os.path.exists(self.deno_exe)
if __name__ == '__main__':
run_tests()