From c131b8f3b6664dfa69d80c2643b3261540b58fd7 Mon Sep 17 00:00:00 2001 From: Vincent LE GOFF Date: Sat, 2 Mar 2019 20:56:19 +0100 Subject: [PATCH] Glob integration for the FS walker (denoland/deno_std#219) Original: https://github.com/denoland/deno_std/commit/0c3ba838fa7e74a859d2a6dbfec3941a521c7988 --- fs/glob.ts | 6 + fs/glob_test.ts | 134 +++++++ fs/globrex.ts | 315 +++++++++++++++ fs/globrex_test.ts | 935 +++++++++++++++++++++++++++++++++++++++++++++ fs/walk.ts | 15 +- fs/walk_test.ts | 2 +- test.ts | 2 + 7 files changed, 1406 insertions(+), 3 deletions(-) create mode 100644 fs/glob.ts create mode 100644 fs/glob_test.ts create mode 100644 fs/globrex.ts create mode 100644 fs/globrex_test.ts diff --git a/fs/glob.ts b/fs/glob.ts new file mode 100644 index 0000000000..1031bd75d5 --- /dev/null +++ b/fs/glob.ts @@ -0,0 +1,6 @@ +import { FileInfo } from "deno"; +import { globrex, GlobOptions } from "./globrex.ts"; + +export function glob(glob: string, options: GlobOptions = {}): RegExp { + return globrex(glob, options).regex; +} diff --git a/fs/glob_test.ts b/fs/glob_test.ts new file mode 100644 index 0000000000..50e6abef89 --- /dev/null +++ b/fs/glob_test.ts @@ -0,0 +1,134 @@ +const { mkdir, open } = Deno; +import { FileInfo } from "deno"; +import { test, assert } from "../testing/mod.ts"; +import { glob } from "./glob.ts"; +import { join } from "./path.ts"; +import { testWalk } from "./walk_test.ts"; +import { walk, walkSync, WalkOptions } from "./walk.ts"; + +async function touch(path: string): Promise { + await open(path, "w"); +} + +async function walkArray( + dirname: string = ".", + options: WalkOptions = {} +): Promise> { + const arr: string[] = []; + for await (const f of walk(dirname, { ...options })) { + arr.push(f.path.replace(/\\/g, "/")); + } + arr.sort(); + const arr_sync = Array.from(walkSync(dirname, options), (f: FileInfo) => + f.path.replace(/\\/g, "/") + ).sort(); + assert.equal(arr, arr_sync); + return arr; +} + +test({ + name: "glob: glob to regex", + fn() { + assert.equal(glob("unicorn.*") instanceof RegExp, true); + assert.equal(glob("unicorn.*").test("poney.ts"), false); + assert.equal(glob("unicorn.*").test("unicorn.py"), true); + assert.equal(glob("*.ts").test("poney.ts"), true); + assert.equal(glob("*.ts").test("unicorn.js"), false); + assert.equal( + glob(join("unicorn", "**", "cathedral.ts")).test( + join("unicorn", "in", "the", "cathedral.ts") + ), + true + ); + assert.equal( + glob(join("unicorn", "**", "cathedral.ts")).test( + join("unicorn", "in", "the", "kitchen.ts") + ), + false + ); + assert.equal( + glob(join("unicorn", "**", "bathroom.*")).test( + join("unicorn", "sleeping", "in", "bathroom.py") + ), + true + ); + assert.equal( + glob(join("unicorn", "!(sleeping)", "bathroom.ts"), { + extended: true + }).test(join("unicorn", "flying", "bathroom.ts")), + true + ); + assert.equal( + glob(join("unicorn", "(!sleeping)", "bathroom.ts"), { + extended: true + }).test(join("unicorn", "sleeping", "bathroom.ts")), + false + ); + } +}); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await touch(d + "/a/x.ts"); + }, + async function globInWalk() { + const arr = await walkArray(".", { match: [glob("*.ts")] }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./a/x.ts"); + } +); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await mkdir(d + "/b"); + await touch(d + "/a/x.ts"); + await touch(d + "/b/z.ts"); + await touch(d + "/b/z.js"); + }, + async function globInWalkWildcardFiles() { + const arr = await walkArray(".", { match: [glob("*.ts")] }); + assert.equal(arr.length, 2); + assert.equal(arr[0], "./a/x.ts"); + assert.equal(arr[1], "./b/z.ts"); + } +); + +testWalk( + async (d: string) => { + await mkdir(d + "/a"); + await mkdir(d + "/a/yo"); + await touch(d + "/a/yo/x.ts"); + }, + async function globInWalkFolderWildcard() { + const arr = await walkArray(".", { + match: [ + glob(join("a", "**", "*.ts"), { + flags: "g", + extended: true, + globstar: true + }) + ] + }); + assert.equal(arr.length, 1); + assert.equal(arr[0], "./a/yo/x.ts"); + } +); + +testWalk( + async (d: string) => { + await touch(d + "/x.ts"); + await touch(d + "/x.js"); + await touch(d + "/b.js"); + }, + async function globInWalkWildcardExtension() { + const arr = await walkArray(".", { + match: [glob("x.*", { flags: "g", extended: true, globstar: true })] + }); + console.log(arr); + assert.equal(arr.length, 2); + assert.equal(arr[0], "./x.js"); + assert.equal(arr[1], "./x.ts"); + } +); diff --git a/fs/globrex.ts b/fs/globrex.ts new file mode 100644 index 0000000000..06a6b79bf3 --- /dev/null +++ b/fs/globrex.ts @@ -0,0 +1,315 @@ +// This file is ported from globrex@0.1.2 +// MIT License +// Copyright (c) 2018 Terkel Gjervig Nielsen + +import * as deno from "deno"; + +const isWin = deno.platform.os === "win"; +const SEP = isWin ? `\\\\+` : `\\/`; +const SEP_ESC = isWin ? `\\\\` : `/`; +const GLOBSTAR = `((?:[^/]*(?:/|$))*)`; +const WILDCARD = `([^/]*)`; +const GLOBSTAR_SEGMENT = `((?:[^${SEP_ESC}]*(?:${SEP_ESC}|$))*)`; +const WILDCARD_SEGMENT = `([^${SEP_ESC}]*)`; + +export interface GlobOptions { + extended?: boolean; + globstar?: boolean; + strict?: boolean; + filepath?: boolean; + flags?: string; +} + +/** + * Convert any glob pattern to a JavaScript Regexp object + * @param {String} glob Glob pattern to convert + * @param {Object} opts Configuration object + * @param {Boolean} [opts.extended=false] Support advanced ext globbing + * @param {Boolean} [opts.globstar=false] Support globstar + * @param {Boolean} [opts.strict=true] be laissez faire about mutiple slashes + * @param {Boolean} [opts.filepath=''] Parse as filepath for extra path related features + * @param {String} [opts.flags=''] RegExp globs + * @returns {Object} converted object with string, segments and RegExp object + */ +export function globrex( + glob: string, + { + extended = false, + globstar = false, + strict = false, + filepath = false, + flags = "" + }: GlobOptions = {} +) { + let regex = ""; + let segment = ""; + let path: { + regex: string | RegExp; + segments: Array; + globstar?: RegExp; + } = { regex: "", segments: [] }; + + // If we are doing extended matching, this boolean is true when we are inside + // a group (eg {*.html,*.js}), and false otherwise. + let inGroup = false; + let inRange = false; + + // extglob stack. Keep track of scope + const ext = []; + + interface AddOptions { + split?: boolean; + last?: boolean; + only?: string; + } + + // Helper function to build string and segments + function add( + str, + options: AddOptions = { split: false, last: false, only: "" } + ) { + const { split, last, only } = options; + if (only !== "path") regex += str; + if (filepath && only !== "regex") { + path.regex += str === "\\/" ? SEP : str; + if (split) { + if (last) segment += str; + if (segment !== "") { + if (!flags.includes("g")) segment = `^${segment}$`; // change it 'includes' + path.segments.push(new RegExp(segment, flags)); + } + segment = ""; + } else { + segment += str; + } + } + } + + let c, n; + for (let i = 0; i < glob.length; i++) { + c = glob[i]; + n = glob[i + 1]; + + if (["\\", "$", "^", ".", "="].includes(c)) { + add(`\\${c}`); + continue; + } + + if (c === "/") { + add(`\\${c}`, { split: true }); + if (n === "/" && !strict) regex += "?"; + continue; + } + + if (c === "(") { + if (ext.length) { + add(c); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === ")") { + if (ext.length) { + add(c); + let type = ext.pop(); + if (type === "@") { + add("{1}"); + } else if (type === "!") { + add("([^/]*)"); + } else { + add(type); + } + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "|") { + if (ext.length) { + add(c); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "+") { + if (n === "(" && extended) { + ext.push(c); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "@" && extended) { + if (n === "(") { + ext.push(c); + continue; + } + } + + if (c === "!") { + if (extended) { + if (inRange) { + add("^"); + continue; + } + if (n === "(") { + ext.push(c); + add("(?!"); + i++; + continue; + } + add(`\\${c}`); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "?") { + if (extended) { + if (n === "(") { + ext.push(c); + } else { + add("."); + } + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "[") { + if (inRange && n === ":") { + i++; // skip [ + let value = ""; + while (glob[++i] !== ":") value += glob[i]; + if (value === "alnum") add("(\\w|\\d)"); + else if (value === "space") add("\\s"); + else if (value === "digit") add("\\d"); + i++; // skip last ] + continue; + } + if (extended) { + inRange = true; + add(c); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "]") { + if (extended) { + inRange = false; + add(c); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "{") { + if (extended) { + inGroup = true; + add("("); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "}") { + if (extended) { + inGroup = false; + add(")"); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === ",") { + if (inGroup) { + add("|"); + continue; + } + add(`\\${c}`); + continue; + } + + if (c === "*") { + if (n === "(" && extended) { + ext.push(c); + continue; + } + // Move over all consecutive "*"'s. + // Also store the previous and next characters + let prevChar = glob[i - 1]; + let starCount = 1; + while (glob[i + 1] === "*") { + starCount++; + i++; + } + let nextChar = glob[i + 1]; + if (!globstar) { + // globstar is disabled, so treat any number of "*" as one + add(".*"); + } else { + // globstar is enabled, so determine if this is a globstar segment + let isGlobstar = + starCount > 1 && // multiple "*"'s + (prevChar === "/" || prevChar === undefined) && // from the start of the segment + (nextChar === "/" || nextChar === undefined); // to the end of the segment + if (isGlobstar) { + // it's a globstar, so match zero or more path segments + add(GLOBSTAR, { only: "regex" }); + add(GLOBSTAR_SEGMENT, { only: "path", last: true, split: true }); + i++; // move over the "/" + } else { + // it's not a globstar, so only match one path segment + add(WILDCARD, { only: "regex" }); + add(WILDCARD_SEGMENT, { only: "path" }); + } + } + continue; + } + + add(c); + } + + // When regexp 'g' flag is specified don't + // constrain the regular expression with ^ & $ + if (!flags.includes("g")) { + regex = `^${regex}$`; + segment = `^${segment}$`; + if (filepath) path.regex = `^${path.regex}$`; + } + + const result: { + regex: RegExp; + path?: { + regex: string | RegExp; + segments: Array; + globstar?: RegExp; + }; + } = { regex: new RegExp(regex, flags) }; + + // Push the last segment + if (filepath) { + path.segments.push(new RegExp(segment, flags)); + path.regex = new RegExp(path.regex.toString(), flags); + path.globstar = new RegExp( + !flags.includes("g") ? `^${GLOBSTAR_SEGMENT}$` : GLOBSTAR_SEGMENT, + flags + ); + result.path = path; + } + + return result; +} diff --git a/fs/globrex_test.ts b/fs/globrex_test.ts new file mode 100644 index 0000000000..8f97f2897e --- /dev/null +++ b/fs/globrex_test.ts @@ -0,0 +1,935 @@ +// This file is ported from globrex@0.1.2 +// MIT License +// Copyright (c) 2018 Terkel Gjervig Nielsen + +import * as deno from "deno"; +import { test, assert } from "../testing/mod.ts"; +import { globrex } from "./globrex.ts"; + +const isWin = deno.platform.os === "win"; +const t = { equal: assert.equal, is: assert.equal }; + +function match(glob, strUnix, strWin?, opts = {}) { + if (typeof strWin === "object") { + opts = strWin; + strWin = false; + } + let res = globrex(glob, opts); + return res.regex.test(isWin && strWin ? strWin : strUnix); +} + +function matchRegex(t, pattern, ifUnix, ifWin, opts) { + const res = globrex(pattern, opts); + const { regex } = opts.filepath ? res.path : res; + t.is(regex.toString(), isWin ? ifWin : ifUnix, "~> regex matches expectant"); + return res; +} + +function matchSegments(t, pattern, ifUnix, ifWin, opts) { + const res = globrex(pattern, { filepath: true, ...opts }); + const str = res.path.segments.join(" "); + const exp = (isWin ? ifWin : ifUnix).join(" "); + t.is(str, exp); + return res; +} + +test({ + name: "globrex: standard", + fn() { + let res = globrex("*.js"); + t.equal(typeof globrex, "function", "constructor is a typeof function"); + t.equal(res instanceof Object, true, "returns object"); + t.equal(res.regex.toString(), "/^.*\\.js$/", "returns regex object"); + } +}); + +test({ + name: "globrex: Standard * matching", + fn() { + t.equal(match("*", "foo"), true, "match everything"); + t.equal(match("*", "foo", { flags: "g" }), true, "match everything"); + t.equal(match("f*", "foo"), true, "match the end"); + t.equal(match("f*", "foo", { flags: "g" }), true, "match the end"); + t.equal(match("*o", "foo"), true, "match the start"); + t.equal(match("*o", "foo", { flags: "g" }), true, "match the start"); + t.equal(match("u*orn", "unicorn"), true, "match the middle"); + t.equal( + match("u*orn", "unicorn", { flags: "g" }), + true, + "match the middle" + ); + t.equal(match("ico", "unicorn"), false, "do not match without g"); + t.equal( + match("ico", "unicorn", { flags: "g" }), + true, + 'match anywhere with RegExp "g"' + ); + t.equal(match("u*nicorn", "unicorn"), true, "match zero characters"); + t.equal( + match("u*nicorn", "unicorn", { flags: "g" }), + true, + "match zero characters" + ); + } +}); + +test({ + name: "globrex: advance * matching", + fn() { + t.equal( + match("*.min.js", "http://example.com/jquery.min.js", { + globstar: false + }), + true, + "complex match" + ); + t.equal( + match("*.min.*", "http://example.com/jquery.min.js", { globstar: false }), + true, + "complex match" + ); + t.equal( + match("*/js/*.js", "http://example.com/js/jquery.min.js", { + globstar: false + }), + true, + "complex match" + ); + t.equal( + match("*.min.*", "http://example.com/jquery.min.js", { flags: "g" }), + true, + "complex match global" + ); + t.equal( + match("*.min.js", "http://example.com/jquery.min.js", { flags: "g" }), + true, + "complex match global" + ); + t.equal( + match("*/js/*.js", "http://example.com/js/jquery.min.js", { flags: "g" }), + true, + "complex match global" + ); + + const str = "\\/$^+?.()=!|{},[].*"; + t.equal(match(str, str), true, "battle test complex string - strict"); + t.equal( + match(str, str, { flags: "g" }), + true, + "battle test complex string - strict" + ); + + t.equal( + match(".min.", "http://example.com/jquery.min.js"), + false, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("*.min.*", "http://example.com/jquery.min.js"), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match(".min.", "http://example.com/jquery.min.js", { flags: "g" }), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("http:", "http://example.com/jquery.min.js"), + false, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("http:*", "http://example.com/jquery.min.js"), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("http:", "http://example.com/jquery.min.js", { flags: "g" }), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("min.js", "http://example.com/jquery.min.js"), + false, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("*.min.js", "http://example.com/jquery.min.js"), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("min.js", "http://example.com/jquery.min.js", { flags: "g" }), + true, + 'matches without/with using RegExp "g"' + ); + t.equal( + match("min", "http://example.com/jquery.min.js", { flags: "g" }), + true, + 'match anywhere (globally) using RegExp "g"' + ); + t.equal( + match("/js/", "http://example.com/js/jquery.min.js", { flags: "g" }), + true, + 'match anywhere (globally) using RegExp "g"' + ); + t.equal(match("/js*jq*.js", "http://example.com/js/jquery.min.js"), false); + t.equal( + match("/js*jq*.js", "http://example.com/js/jquery.min.js", { + flags: "g" + }), + true + ); + } +}); + +test({ + name: "globrex: ? match one character, no more and no less", + fn() { + t.equal(match("f?o", "foo", { extended: true }), true); + t.equal(match("f?o", "fooo", { extended: true }), false); + t.equal(match("f?oo", "foo", { extended: true }), false); + + const tester = globstar => { + t.equal( + match("f?o", "foo", { extended: true, globstar, flags: "g" }), + true + ); + t.equal( + match("f?o", "fooo", { extended: true, globstar, flags: "g" }), + true + ); + t.equal( + match("f?o?", "fooo", { extended: true, globstar, flags: "g" }), + true + ); + + t.equal( + match("?fo", "fooo", { extended: true, globstar, flags: "g" }), + false + ); + t.equal( + match("f?oo", "foo", { extended: true, globstar, flags: "g" }), + false + ); + t.equal( + match("foo?", "foo", { extended: true, globstar, flags: "g" }), + false + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: [] match a character range", + fn() { + t.equal(match("fo[oz]", "foo", { extended: true }), true); + t.equal(match("fo[oz]", "foz", { extended: true }), true); + t.equal(match("fo[oz]", "fog", { extended: true }), false); + t.equal(match("fo[a-z]", "fob", { extended: true }), true); + t.equal(match("fo[a-d]", "fot", { extended: true }), false); + t.equal(match("fo[!tz]", "fot", { extended: true }), false); + t.equal(match("fo[!tz]", "fob", { extended: true }), true); + + const tester = globstar => { + t.equal( + match("fo[oz]", "foo", { extended: true, globstar, flags: "g" }), + true + ); + t.equal( + match("fo[oz]", "foz", { extended: true, globstar, flags: "g" }), + true + ); + t.equal( + match("fo[oz]", "fog", { extended: true, globstar, flags: "g" }), + false + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: [] extended character ranges", + fn() { + t.equal( + match("[[:alnum:]]/bar.txt", "a/bar.txt", { extended: true }), + true + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "11/bar.txt", { extended: true }), + true + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "a/bar.txt", { extended: true }), + true + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "b/bar.txt", { extended: true }), + true + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "c/bar.txt", { extended: true }), + true + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "abc/bar.txt", { extended: true }), + false + ); + t.equal( + match("@([[:alnum:]abc]|11)/bar.txt", "3/bar.txt", { extended: true }), + true + ); + t.equal( + match("[[:digit:]]/bar.txt", "1/bar.txt", { extended: true }), + true + ); + t.equal( + match("[[:digit:]b]/bar.txt", "b/bar.txt", { extended: true }), + true + ); + t.equal( + match("[![:digit:]b]/bar.txt", "a/bar.txt", { extended: true }), + true + ); + t.equal( + match("[[:alnum:]]/bar.txt", "!/bar.txt", { extended: true }), + false + ); + t.equal( + match("[[:digit:]]/bar.txt", "a/bar.txt", { extended: true }), + false + ); + t.equal( + match("[[:digit:]b]/bar.txt", "a/bar.txt", { extended: true }), + false + ); + } +}); + +test({ + name: "globrex: {} match a choice of different substrings", + fn() { + t.equal(match("foo{bar,baaz}", "foobaaz", { extended: true }), true); + t.equal(match("foo{bar,baaz}", "foobar", { extended: true }), true); + t.equal(match("foo{bar,baaz}", "foobuzz", { extended: true }), false); + t.equal(match("foo{bar,b*z}", "foobuzz", { extended: true }), true); + + const tester = globstar => { + t.equal( + match("foo{bar,baaz}", "foobaaz", { + extended: true, + globstar, + flag: "g" + }), + true + ); + t.equal( + match("foo{bar,baaz}", "foobar", { + extended: true, + globstar, + flag: "g" + }), + true + ); + t.equal( + match("foo{bar,baaz}", "foobuzz", { + extended: true, + globstar, + flag: "g" + }), + false + ); + t.equal( + match("foo{bar,b*z}", "foobuzz", { + extended: true, + globstar, + flag: "g" + }), + true + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: complex extended matches", + fn() { + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://foo.baaz.com/jquery.min.js", + { extended: true } + ), + true + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.buzz.com/index.html", + { extended: true } + ), + true + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.buzz.com/index.htm", + { extended: true } + ), + false + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.bar.com/index.html", + { extended: true } + ), + false + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://flozz.buzz.com/index.html", + { extended: true } + ), + false + ); + + const tester = globstar => { + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://foo.baaz.com/jquery.min.js", + { extended: true, globstar, flags: "g" } + ), + true + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.buzz.com/index.html", + { extended: true, globstar, flags: "g" } + ), + true + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.buzz.com/index.htm", + { extended: true, globstar, flags: "g" } + ), + false + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://moz.bar.com/index.html", + { extended: true, globstar, flags: "g" } + ), + false + ); + t.equal( + match( + "http://?o[oz].b*z.com/{*.js,*.html}", + "http://flozz.buzz.com/index.html", + { extended: true, globstar, flags: "g" } + ), + false + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: standard globstar", + fn() { + const tester = globstar => { + t.equal( + match( + "http://foo.com/**/{*.js,*.html}", + "http://foo.com/bar/jquery.min.js", + { extended: true, globstar, flags: "g" } + ), + true + ); + t.equal( + match( + "http://foo.com/**/{*.js,*.html}", + "http://foo.com/bar/baz/jquery.min.js", + { extended: true, globstar, flags: "g" } + ), + true + ); + t.equal( + match("http://foo.com/**", "http://foo.com/bar/baz/jquery.min.js", { + extended: true, + globstar, + flags: "g" + }), + true + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: remaining chars should match themself", + fn() { + const tester = globstar => { + const testExtStr = "\\/$^+.()=!|,.*"; + t.equal(match(testExtStr, testExtStr, { extended: true }), true); + t.equal( + match(testExtStr, testExtStr, { extended: true, globstar, flags: "g" }), + true + ); + }; + + tester(true); + tester(false); + } +}); + +test({ + name: "globrex: globstar advance testing", + fn() { + t.equal(match("/foo/*", "/foo/bar.txt", { globstar: true }), true); + t.equal(match("/foo/**", "/foo/bar.txt", { globstar: true }), true); + t.equal(match("/foo/**", "/foo/bar/baz.txt", { globstar: true }), true); + t.equal(match("/foo/**", "/foo/bar/baz.txt", { globstar: true }), true); + t.equal( + match("/foo/*/*.txt", "/foo/bar/baz.txt", { globstar: true }), + true + ); + t.equal( + match("/foo/**/*.txt", "/foo/bar/baz.txt", { globstar: true }), + true + ); + t.equal( + match("/foo/**/*.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + true + ); + t.equal(match("/foo/**/bar.txt", "/foo/bar.txt", { globstar: true }), true); + t.equal( + match("/foo/**/**/bar.txt", "/foo/bar.txt", { globstar: true }), + true + ); + t.equal( + match("/foo/**/*/baz.txt", "/foo/bar/baz.txt", { globstar: true }), + true + ); + t.equal(match("/foo/**/*.txt", "/foo/bar.txt", { globstar: true }), true); + t.equal( + match("/foo/**/**/*.txt", "/foo/bar.txt", { globstar: true }), + true + ); + t.equal( + match("/foo/**/*/*.txt", "/foo/bar/baz.txt", { globstar: true }), + true + ); + t.equal( + match("**/*.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + true + ); + t.equal(match("**/foo.txt", "foo.txt", { globstar: true }), true); + t.equal(match("**/*.txt", "foo.txt", { globstar: true }), true); + t.equal(match("/foo/*", "/foo/bar/baz.txt", { globstar: true }), false); + t.equal(match("/foo/*.txt", "/foo/bar/baz.txt", { globstar: true }), false); + t.equal( + match("/foo/*/*.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + false + ); + t.equal(match("/foo/*/bar.txt", "/foo/bar.txt", { globstar: true }), false); + t.equal( + match("/foo/*/*/baz.txt", "/foo/bar/baz.txt", { globstar: true }), + false + ); + t.equal( + match("/foo/**.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + false + ); + t.equal( + match("/foo/bar**/*.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + false + ); + t.equal(match("/foo/bar**", "/foo/bar/baz.txt", { globstar: true }), false); + t.equal( + match("**/.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + false + ); + t.equal( + match("*/*.txt", "/foo/bar/baz/qux.txt", { globstar: true }), + false + ); + t.equal(match("*/*.txt", "foo.txt", { globstar: true }), false); + t.equal( + match("http://foo.com/*", "http://foo.com/bar/baz/jquery.min.js", { + extended: true, + globstar: true + }), + false + ); + t.equal( + match("http://foo.com/*", "http://foo.com/bar/baz/jquery.min.js", { + globstar: true + }), + false + ); + t.equal( + match("http://foo.com/*", "http://foo.com/bar/baz/jquery.min.js", { + globstar: false + }), + true + ); + t.equal( + match("http://foo.com/**", "http://foo.com/bar/baz/jquery.min.js", { + globstar: true + }), + true + ); + t.equal( + match( + "http://foo.com/*/*/jquery.min.js", + "http://foo.com/bar/baz/jquery.min.js", + { globstar: true } + ), + true + ); + t.equal( + match( + "http://foo.com/**/jquery.min.js", + "http://foo.com/bar/baz/jquery.min.js", + { globstar: true } + ), + true + ); + t.equal( + match( + "http://foo.com/*/*/jquery.min.js", + "http://foo.com/bar/baz/jquery.min.js", + { globstar: false } + ), + true + ); + t.equal( + match( + "http://foo.com/*/jquery.min.js", + "http://foo.com/bar/baz/jquery.min.js", + { globstar: false } + ), + true + ); + t.equal( + match( + "http://foo.com/*/jquery.min.js", + "http://foo.com/bar/baz/jquery.min.js", + { globstar: true } + ), + false + ); + } +}); + +test({ + name: "globrex: extended extglob ?", + fn() { + t.equal(match("(foo).txt", "(foo).txt", { extended: true }), true); + t.equal(match("?(foo).txt", "foo.txt", { extended: true }), true); + t.equal(match("?(foo).txt", ".txt", { extended: true }), true); + t.equal(match("?(foo|bar)baz.txt", "foobaz.txt", { extended: true }), true); + t.equal( + match("?(ba[zr]|qux)baz.txt", "bazbaz.txt", { extended: true }), + true + ); + t.equal( + match("?(ba[zr]|qux)baz.txt", "barbaz.txt", { extended: true }), + true + ); + t.equal( + match("?(ba[zr]|qux)baz.txt", "quxbaz.txt", { extended: true }), + true + ); + t.equal( + match("?(ba[!zr]|qux)baz.txt", "batbaz.txt", { extended: true }), + true + ); + t.equal(match("?(ba*|qux)baz.txt", "batbaz.txt", { extended: true }), true); + t.equal( + match("?(ba*|qux)baz.txt", "batttbaz.txt", { extended: true }), + true + ); + t.equal(match("?(ba*|qux)baz.txt", "quxbaz.txt", { extended: true }), true); + t.equal( + match("?(ba?(z|r)|qux)baz.txt", "bazbaz.txt", { extended: true }), + true + ); + t.equal( + match("?(ba?(z|?(r))|qux)baz.txt", "bazbaz.txt", { extended: true }), + true + ); + t.equal(match("?(foo).txt", "foo.txt", { extended: false }), false); + t.equal( + match("?(foo|bar)baz.txt", "foobarbaz.txt", { extended: true }), + false + ); + t.equal( + match("?(ba[zr]|qux)baz.txt", "bazquxbaz.txt", { extended: true }), + false + ); + t.equal( + match("?(ba[!zr]|qux)baz.txt", "bazbaz.txt", { extended: true }), + false + ); + } +}); + +test({ + name: "globrex: extended extglob *", + fn() { + t.equal(match("*(foo).txt", "foo.txt", { extended: true }), true); + t.equal(match("*foo.txt", "bofoo.txt", { extended: true }), true); + t.equal(match("*(foo).txt", "foofoo.txt", { extended: true }), true); + t.equal(match("*(foo).txt", ".txt", { extended: true }), true); + t.equal(match("*(fooo).txt", ".txt", { extended: true }), true); + t.equal(match("*(fooo).txt", "foo.txt", { extended: true }), false); + t.equal(match("*(foo|bar).txt", "foobar.txt", { extended: true }), true); + t.equal(match("*(foo|bar).txt", "barbar.txt", { extended: true }), true); + t.equal(match("*(foo|bar).txt", "barfoobar.txt", { extended: true }), true); + t.equal(match("*(foo|bar).txt", ".txt", { extended: true }), true); + t.equal(match("*(foo|ba[rt]).txt", "bat.txt", { extended: true }), true); + t.equal(match("*(foo|b*[rt]).txt", "blat.txt", { extended: true }), true); + t.equal(match("*(foo|b*[rt]).txt", "tlat.txt", { extended: true }), false); + t.equal( + match("*(*).txt", "whatever.txt", { extended: true, globstar: true }), + true + ); + t.equal( + match("*(foo|bar)/**/*.txt", "foo/hello/world/bar.txt", { + extended: true, + globstar: true + }), + true + ); + t.equal( + match("*(foo|bar)/**/*.txt", "foo/world/bar.txt", { + extended: true, + globstar: true + }), + true + ); + } +}); + +test({ + name: "globrex: extended extglob +", + fn() { + t.equal(match("+(foo).txt", "foo.txt", { extended: true }), true); + t.equal(match("+foo.txt", "+foo.txt", { extended: true }), true); + t.equal(match("+(foo).txt", ".txt", { extended: true }), false); + t.equal(match("+(foo|bar).txt", "foobar.txt", { extended: true }), true); + } +}); + +test({ + name: "globrex: extended extglob @", + fn() { + t.equal(match("@(foo).txt", "foo.txt", { extended: true }), true); + t.equal(match("@foo.txt", "@foo.txt", { extended: true }), true); + t.equal(match("@(foo|baz)bar.txt", "foobar.txt", { extended: true }), true); + t.equal( + match("@(foo|baz)bar.txt", "foobazbar.txt", { extended: true }), + false + ); + t.equal( + match("@(foo|baz)bar.txt", "foofoobar.txt", { extended: true }), + false + ); + t.equal( + match("@(foo|baz)bar.txt", "toofoobar.txt", { extended: true }), + false + ); + } +}); + +test({ + name: "globrex: extended extglob !", + fn() { + t.equal(match("!(boo).txt", "foo.txt", { extended: true }), true); + t.equal(match("!(foo|baz)bar.txt", "buzbar.txt", { extended: true }), true); + t.equal(match("!bar.txt", "!bar.txt", { extended: true }), true); + t.equal( + match("!({foo,bar})baz.txt", "notbaz.txt", { extended: true }), + true + ); + t.equal( + match("!({foo,bar})baz.txt", "foobaz.txt", { extended: true }), + false + ); + } +}); + +test({ + name: "globrex: strict", + fn() { + t.equal(match("foo//bar.txt", "foo/bar.txt"), true); + t.equal(match("foo///bar.txt", "foo/bar.txt"), true); + t.equal(match("foo///bar.txt", "foo/bar.txt", { strict: true }), false); + } +}); + +test({ + name: "globrex: filepath path-regex", + fn() { + let opts = { extended: true, filepath: true, globstar: false }, + res, + pattern; + + res = globrex("", opts); + t.is(res.hasOwnProperty("path"), true); + t.is(res.path.hasOwnProperty("regex"), true); + t.is(res.path.hasOwnProperty("segments"), true); + t.is(Array.isArray(res.path.segments), true); + + pattern = "foo/bar/baz.js"; + res = matchRegex( + t, + pattern, + "/^foo\\/bar\\/baz\\.js$/", + "/^foo\\\\+bar\\\\+baz\\.js$/", + opts + ); + t.is(res.path.segments.length, 3); + + res = matchRegex( + t, + "../foo/bar.js", + "/^\\.\\.\\/foo\\/bar\\.js$/", + "/^\\.\\.\\\\+foo\\\\+bar\\.js$/", + opts + ); + t.is(res.path.segments.length, 3); + + res = matchRegex( + t, + "*/bar.js", + "/^.*\\/bar\\.js$/", + "/^.*\\\\+bar\\.js$/", + opts + ); + t.is(res.path.segments.length, 2); + + opts.globstar = true; + res = matchRegex( + t, + "**/bar.js", + "/^((?:[^\\/]*(?:\\/|$))*)bar\\.js$/", + "/^((?:[^\\\\]*(?:\\\\|$))*)bar\\.js$/", + opts + ); + t.is(res.path.segments.length, 2); + } +}); + +test({ + name: "globrex: filepath path segments", + fn() { + let opts = { extended: true }, + res, + win, + unix; + + unix = [/^foo$/, /^bar$/, /^([^\/]*)$/, /^baz\.(md|js|txt)$/]; + win = [/^foo$/, /^bar$/, /^([^\\]*)$/, /^baz\.(md|js|txt)$/]; + matchSegments(t, "foo/bar/*/baz.{md,js,txt}", unix, win, { + ...opts, + globstar: true + }); + + unix = [/^foo$/, /^.*$/, /^baz\.md$/]; + win = [/^foo$/, /^.*$/, /^baz\.md$/]; + matchSegments(t, "foo/*/baz.md", unix, win, opts); + + unix = [/^foo$/, /^.*$/, /^baz\.md$/]; + win = [/^foo$/, /^.*$/, /^baz\.md$/]; + matchSegments(t, "foo/**/baz.md", unix, win, opts); + + unix = [/^foo$/, /^((?:[^\/]*(?:\/|$))*)$/, /^baz\.md$/]; + win = [/^foo$/, /^((?:[^\\]*(?:\\|$))*)$/, /^baz\.md$/]; + matchSegments(t, "foo/**/baz.md", unix, win, { ...opts, globstar: true }); + + unix = [/^foo$/, /^.*$/, /^.*\.md$/]; + win = [/^foo$/, /^.*$/, /^.*\.md$/]; + matchSegments(t, "foo/**/*.md", unix, win, opts); + + unix = [/^foo$/, /^((?:[^\/]*(?:\/|$))*)$/, /^([^\/]*)\.md$/]; + win = [/^foo$/, /^((?:[^\\]*(?:\\|$))*)$/, /^([^\\]*)\.md$/]; + matchSegments(t, "foo/**/*.md", unix, win, { ...opts, globstar: true }); + + unix = [/^foo$/, /^:$/, /^b:az$/]; + win = [/^foo$/, /^:$/, /^b:az$/]; + matchSegments(t, "foo/:/b:az", unix, win, opts); + + unix = [/^foo$/, /^baz\.md$/]; + win = [/^foo$/, /^baz\.md$/]; + matchSegments(t, "foo///baz.md", unix, win, { ...opts, strict: true }); + + unix = [/^foo$/, /^baz\.md$/]; + win = [/^foo$/, /^baz\.md$/]; + matchSegments(t, "foo///baz.md", unix, win, { ...opts, strict: false }); + } +}); + +test({ + name: "globrex: stress testing", + fn() { + t.equal( + match("**/*/?yfile.{md,js,txt}", "foo/bar/baz/myfile.md", { + extended: true + }), + true + ); + t.equal( + match("**/*/?yfile.{md,js,txt}", "foo/baz/myfile.md", { extended: true }), + true + ); + t.equal( + match("**/*/?yfile.{md,js,txt}", "foo/baz/tyfile.js", { extended: true }), + true + ); + t.equal( + match("[[:digit:]_.]/file.js", "1/file.js", { extended: true }), + true + ); + t.equal( + match("[[:digit:]_.]/file.js", "2/file.js", { extended: true }), + true + ); + t.equal( + match("[[:digit:]_.]/file.js", "_/file.js", { extended: true }), + true + ); + t.equal( + match("[[:digit:]_.]/file.js", "./file.js", { extended: true }), + true + ); + t.equal( + match("[[:digit:]_.]/file.js", "z/file.js", { extended: true }), + false + ); + } +}); diff --git a/fs/walk.ts b/fs/walk.ts index 10393ae0be..f2a9b6c57a 100644 --- a/fs/walk.ts +++ b/fs/walk.ts @@ -95,15 +95,26 @@ function include(f: FileInfo, options: WalkOptions): Boolean { if (options.exts && !options.exts.some(ext => f.path.endsWith(ext))) { return false; } - if (options.match && !options.match.some(pattern => pattern.test(f.path))) { + if (options.match && !patternTest(options.match, f.path)) { return false; } - if (options.skip && options.skip.some(pattern => pattern.test(f.path))) { + if (options.skip && patternTest(options.skip, f.path)) { return false; } return true; } +function patternTest(patterns: RegExp[], path: string) { + // Forced to reset last index on regex while iterating for have + // consistent results. + // See: https://stackoverflow.com/a/1520853 + return patterns.some(pattern => { + let r = pattern.test(path); + pattern.lastIndex = 0; + return r; + }); +} + async function resolve(f: FileInfo): Promise { // This is the full path, unfortunately if we were to make it relative // it could resolve to a symlink and cause an infinite loop. diff --git a/fs/walk_test.ts b/fs/walk_test.ts index d542739950..f78765b1d4 100644 --- a/fs/walk_test.ts +++ b/fs/walk_test.ts @@ -14,7 +14,7 @@ import { test, assert, TestFunction } from "../testing/mod.ts"; const isWindows = platform.os === "win"; -async function testWalk( +export async function testWalk( setup: (string) => void | Promise, t: TestFunction ): Promise { diff --git a/test.ts b/test.ts index f740d497b4..09ce823dec 100755 --- a/test.ts +++ b/test.ts @@ -12,6 +12,8 @@ import "./io/writers_test.ts"; import "./io/readers_test.ts"; import "./fs/path/test.ts"; import "./fs/walk_test.ts"; +import "./fs/globrex_test.ts"; +import "./fs/glob_test.ts"; import "./io/test.ts"; import "./http/server_test.ts"; import "./http/file_server_test.ts";