diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b2941c3c2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +.idea +tsconfig.json +deno.d.ts \ No newline at end of file diff --git a/bytes/bytes.ts b/bytes/bytes.ts new file mode 100644 index 0000000000..ef333288e7 --- /dev/null +++ b/bytes/bytes.ts @@ -0,0 +1,60 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +/** Find first index of binary pattern from a. If not found, then return -1 **/ +export function bytesFindIndex(a: Uint8Array, pat: Uint8Array): number { + const s = pat[0]; + for (let i = 0; i < a.length; i++) { + if (a[i] !== s) continue; + const pin = i; + let matched = 1; + while (matched < pat.length) { + i++; + if (a[i] !== pat[i - pin]) { + break; + } + matched++; + } + if (matched === pat.length) { + return pin; + } + } + return -1; +} + +/** Find last index of binary pattern from a. If not found, then return -1 **/ +export function bytesFindLastIndex(a: Uint8Array, pat: Uint8Array) { + const e = pat[pat.length - 1]; + for (let i = a.length - 1; i >= 0; i--) { + if (a[i] !== e) continue; + const pin = i; + let matched = 1; + while (matched < pat.length) { + i--; + if (a[i] !== pat[pat.length - 1 - (pin - i)]) { + break; + } + matched++; + } + if (matched === pat.length) { + return pin - pat.length + 1; + } + } + return -1; +} + +/** Check whether binary arrays are equal to each other **/ +export function bytesEqual(a: Uint8Array, match: Uint8Array): boolean { + if (a.length !== match.length) return false; + for (let i = 0; i < match.length; i++) { + if (a[i] !== match[i]) return false; + } + return true; +} + +/** Check whether binary array has binary prefix **/ +export function bytesHasPrefix(a: Uint8Array, prefix: Uint8Array): boolean { + for (let i = 0, max = prefix.length; i < max; i++) { + if (a[i] !== prefix[i]) return false; + } + return true; +} diff --git a/bytes/bytes_test.ts b/bytes/bytes_test.ts new file mode 100644 index 0000000000..3d87497fe5 --- /dev/null +++ b/bytes/bytes_test.ts @@ -0,0 +1,36 @@ +import { + bytesFindIndex, + bytesFindLastIndex, + bytesEqual, + bytesHasPrefix +} from "./bytes.ts"; +import { assertEqual, test } from "./deps.ts"; + +test(function bytesBytesFindIndex() { + const i = bytesFindIndex( + new Uint8Array([1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 3]), + new Uint8Array([0, 1, 2]) + ); + assertEqual(i, 2); +}); + +test(function bytesBytesFindLastIndex1() { + const i = bytesFindLastIndex( + new Uint8Array([0, 1, 2, 0, 1, 2, 0, 1, 3]), + new Uint8Array([0, 1, 2]) + ); + assertEqual(i, 3); +}); + +test(function bytesBytesBytesEqual() { + const v = bytesEqual( + new Uint8Array([0, 1, 2, 3]), + new Uint8Array([0, 1, 2, 3]) + ); + assertEqual(v, true); +}); + +test(function bytesBytesHasPrefix() { + const v = bytesHasPrefix(new Uint8Array([0, 1, 2]), new Uint8Array([0, 1])); + assertEqual(v, true); +}); diff --git a/io/bufio_test.ts b/io/bufio_test.ts index fa8f4b73bd..e63f1c5c97 100644 --- a/io/bufio_test.ts +++ b/io/bufio_test.ts @@ -30,7 +30,7 @@ test(async function bufioReaderSimple() { const data = "hello world"; const b = new BufReader(stringsReader(data)); const s = await readBytes(b); - assertEqual(s, data); + assert.equal(s, data); }); type ReadMaker = { name: string; fn: (r: Reader) => Reader }; diff --git a/io/ioutil.ts b/io/ioutil.ts index 68d6e51905..6590c0f663 100644 --- a/io/ioutil.ts +++ b/io/ioutil.ts @@ -1,27 +1,55 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import { BufReader } from "./bufio.ts"; +import { Reader, Writer } from "deno"; +import { assert } from "../testing/mod.ts"; -/* Read big endian 16bit short from BufReader */ +/** copy N size at the most. If read size is lesser than N, then returns nread */ +export async function copyN( + dest: Writer, + r: Reader, + size: number +): Promise { + let bytesRead = 0; + let buf = new Uint8Array(1024); + while (bytesRead < size) { + if (size - bytesRead < 1024) { + buf = new Uint8Array(size - bytesRead); + } + const { nread, eof } = await r.read(buf); + bytesRead += nread; + if (nread > 0) { + const n = await dest.write(buf.slice(0, nread)); + assert.assert(n === nread, "could not write"); + } + if (eof) { + break; + } + } + return bytesRead; +} + +/** Read big endian 16bit short from BufReader */ export async function readShort(buf: BufReader): Promise { const [high, low] = [await buf.readByte(), await buf.readByte()]; return (high << 8) | low; } -/* Read big endian 32bit integer from BufReader */ +/** Read big endian 32bit integer from BufReader */ export async function readInt(buf: BufReader): Promise { const [high, low] = [await readShort(buf), await readShort(buf)]; return (high << 16) | low; } const BIT32 = 0xffffffff; -/* Read big endian 64bit long from BufReader */ + +/** Read big endian 64bit long from BufReader */ export async function readLong(buf: BufReader): Promise { const [high, low] = [await readInt(buf), await readInt(buf)]; // ECMAScript doesn't support 64bit bit ops. return high ? high * (BIT32 + 1) + low : low; } -/* Slice number into 64bit big endian byte array */ +/** Slice number into 64bit big endian byte array */ export function sliceLongToBytes(d: number, dest = new Array(8)): number[] { let mask = 0xff; let low = (d << 32) >>> 32; diff --git a/io/ioutil_test.ts b/io/ioutil_test.ts index 4ff69352a7..2c78b25620 100644 --- a/io/ioutil_test.ts +++ b/io/ioutil_test.ts @@ -1,8 +1,15 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -import { Reader, ReadResult } from "deno"; -import { assertEqual, test } from "../testing/mod.ts"; -import { readInt, readLong, readShort, sliceLongToBytes } from "./ioutil.ts"; +import { Buffer, Reader, ReadResult } from "deno"; +import { assert, assertEqual, runTests, test } from "../testing/mod.ts"; +import { + copyN, + readInt, + readLong, + readShort, + sliceLongToBytes +} from "./ioutil.ts"; import { BufReader } from "./bufio.ts"; +import { stringsReader } from "./util.ts"; class BinaryReader implements Reader { index = 0; @@ -61,3 +68,19 @@ test(async function testSliceLongToBytes2() { const arr = sliceLongToBytes(0x12345678); assertEqual(arr, [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78]); }); + +test(async function testCopyN1() { + const w = new Buffer(); + const r = stringsReader("abcdefghij"); + const n = await copyN(w, r, 3); + assert.equal(n, 3); + assert.equal(w.toString(), "abc"); +}); + +test(async function testCopyN2() { + const w = new Buffer(); + const r = stringsReader("abcdefghij"); + const n = await copyN(w, r, 11); + assert.equal(n, 10); + assert.equal(w.toString(), "abcdefghij"); +}); diff --git a/io/readers.ts b/io/readers.ts new file mode 100644 index 0000000000..df02993560 --- /dev/null +++ b/io/readers.ts @@ -0,0 +1,38 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import { Reader, ReadResult } from "deno"; +import { encode } from "../strings/strings.ts"; + +/** Reader utility for strings */ +export class StringReader implements Reader { + private offs = 0; + private buf = new Uint8Array(encode(this.s)); + + constructor(private readonly s: string) {} + + async read(p: Uint8Array): Promise { + const n = Math.min(p.byteLength, this.buf.byteLength - this.offs); + p.set(this.buf.slice(this.offs, this.offs + n)); + this.offs += n; + return { nread: n, eof: this.offs === this.buf.byteLength }; + } +} + +/** Reader utility for combining multiple readers */ +export class MultiReader implements Reader { + private readonly readers: Reader[]; + private currentIndex = 0; + + constructor(...readers: Reader[]) { + this.readers = readers; + } + + async read(p: Uint8Array): Promise { + const r = this.readers[this.currentIndex]; + if (!r) return { nread: 0, eof: true }; + const { nread, eof } = await r.read(p); + if (eof) { + this.currentIndex++; + } + return { nread, eof: false }; + } +} diff --git a/io/readers_test.ts b/io/readers_test.ts new file mode 100644 index 0000000000..0bc8ca36a0 --- /dev/null +++ b/io/readers_test.ts @@ -0,0 +1,36 @@ +import { assert, test } from "../testing/mod.ts"; +import { MultiReader, StringReader } from "./readers.ts"; +import { StringWriter } from "./writers.ts"; +import { copy } from "deno"; +import { copyN } from "./ioutil.ts"; +import { decode } from "../strings/strings.ts"; + +test(async function ioStringReader() { + const r = new StringReader("abcdef"); + const { nread, eof } = await r.read(new Uint8Array(6)); + assert.equal(nread, 6); + assert.equal(eof, true); +}); + +test(async function ioStringReader() { + const r = new StringReader("abcdef"); + const buf = new Uint8Array(3); + let res1 = await r.read(buf); + assert.equal(res1.nread, 3); + assert.equal(res1.eof, false); + assert.equal(decode(buf), "abc"); + let res2 = await r.read(buf); + assert.equal(res2.nread, 3); + assert.equal(res2.eof, true); + assert.equal(decode(buf), "def"); +}); + +test(async function ioMultiReader() { + const r = new MultiReader(new StringReader("abc"), new StringReader("def")); + const w = new StringWriter(); + const n = await copyN(w, r, 4); + assert.equal(n, 4); + assert.equal(w.toString(), "abcd"); + await copy(w, r); + assert.equal(w.toString(), "abcdef"); +}); diff --git a/io/util.ts b/io/util.ts index 8726a18879..954808c6cd 100644 --- a/io/util.ts +++ b/io/util.ts @@ -1,6 +1,7 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -import { Buffer, Reader } from "deno"; - +import { Buffer, File, mkdir, open, Reader } from "deno"; +import { encode } from "../strings/strings.ts"; +import * as path from "../fs/path.ts"; // `off` is the offset into `dst` where it will at which to begin writing values // from `src`. // Returns the number of bytes copied. @@ -18,8 +19,23 @@ export function charCode(s: string): number { return s.charCodeAt(0); } -const encoder = new TextEncoder(); export function stringsReader(s: string): Reader { - const ui8 = encoder.encode(s); - return new Buffer(ui8.buffer as ArrayBuffer); + return new Buffer(encode(s).buffer); +} + +/** Create or open a temporal file at specified directory with prefix and postfix */ +export async function tempFile( + dir: string, + opts: { + prefix?: string; + postfix?: string; + } = { prefix: "", postfix: "" } +): Promise<{ file: File; filepath: string }> { + const r = Math.floor(Math.random() * 1000000); + const filepath = path.resolve( + `${dir}/${opts.prefix || ""}${r}${opts.postfix || ""}` + ); + await mkdir(path.dirname(filepath), true); + const file = await open(filepath, "a"); + return { file, filepath }; } diff --git a/io/util_test.ts b/io/util_test.ts index 90ae5d4c1b..c3f134616f 100644 --- a/io/util_test.ts +++ b/io/util_test.ts @@ -1,6 +1,8 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import { test, assert } from "../testing/mod.ts"; -import { copyBytes } from "./util.ts"; +import { copyBytes, tempFile } from "./util.ts"; +import { remove } from "deno"; +import * as path from "../fs/path.ts"; test(function testCopyBytes() { let dst = new Uint8Array(4); @@ -35,3 +37,14 @@ test(function testCopyBytes() { assert(len === 2); assert.equal(dst, Uint8Array.of(3, 4, 0, 0)); }); + +test(async function ioTempfile() { + const f = await tempFile(".", { + prefix: "prefix-", + postfix: "-postfix" + }); + console.log(f.file, f.filepath); + const base = path.basename(f.filepath); + assert.assert(!!base.match(/^prefix-.+?-postfix$/)); + await remove(f.filepath); +}); diff --git a/io/writers.ts b/io/writers.ts new file mode 100644 index 0000000000..15c2628aca --- /dev/null +++ b/io/writers.ts @@ -0,0 +1,38 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import { Writer } from "deno"; +import { decode, encode } from "../strings/strings.ts"; + +/** Writer utility for buffering string chunks */ +export class StringWriter implements Writer { + private chunks: Uint8Array[] = []; + private byteLength: number = 0; + + constructor(private base: string = "") { + const c = encode(base); + this.chunks.push(c); + this.byteLength += c.byteLength; + } + + async write(p: Uint8Array): Promise { + this.chunks.push(p); + this.byteLength += p.byteLength; + this.cache = null; + return p.byteLength; + } + + private cache: string; + + toString(): string { + if (this.cache) { + return this.cache; + } + const buf = new Uint8Array(this.byteLength); + let offs = 0; + for (const chunk of this.chunks) { + buf.set(chunk, offs); + offs += chunk.byteLength; + } + this.cache = decode(buf); + return this.cache; + } +} diff --git a/io/writers_test.ts b/io/writers_test.ts new file mode 100644 index 0000000000..01388497cd --- /dev/null +++ b/io/writers_test.ts @@ -0,0 +1,14 @@ +import { assert, test } from "../testing/mod.ts"; +import { StringWriter } from "./writers.ts"; +import { StringReader } from "./readers.ts"; +import { copyN } from "./ioutil.ts"; +import { copy } from "deno"; + +test(async function ioStringWriter() { + const w = new StringWriter("base"); + const r = new StringReader("0123456789"); + const n = await copyN(w, r, 4); + assert.equal(w.toString(), "base0123"); + await copy(w, r); + assert.equal(w.toString(), "base0123456789"); +}); diff --git a/multipart/fixtures/sample.txt b/multipart/fixtures/sample.txt new file mode 100644 index 0000000000..97e9bf5531 --- /dev/null +++ b/multipart/fixtures/sample.txt @@ -0,0 +1,27 @@ +----------------------------434049563556637648550474 +content-disposition: form-data; name="foo" +content-type: application/octet-stream + +foo +----------------------------434049563556637648550474 +content-disposition: form-data; name="bar" +content-type: application/octet-stream + +bar +----------------------------434049563556637648550474 +content-disposition: form-data; name="file"; filename="tsconfig.json" +content-type: application/octet-stream + +{ + "compilerOptions": { + "target": "es2018", + "baseUrl": ".", + "paths": { + "deno": ["./deno.d.ts"], + "https://*": ["../../.deno/deps/https/*"], + "http://*": ["../../.deno/deps/http/*"] + } + } +} + +----------------------------434049563556637648550474-- diff --git a/multipart/formfile.ts b/multipart/formfile.ts new file mode 100644 index 0000000000..b1b63eb157 --- /dev/null +++ b/multipart/formfile.ts @@ -0,0 +1,24 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +/** FormFile object */ +export type FormFile = { + /** filename */ + filename: string; + /** content-type header value of file */ + type: string; + /** byte size of file */ + size: number; + /** in-memory content of file. Either content or tempfile is set */ + content?: Uint8Array; + /** temporal file path. Set if file size is bigger than specified max-memory size at reading form */ + tempfile?: string; +}; + +/** Type guard for FormFile */ +export function isFormFile(x): x is FormFile { + return ( + typeof x === "object" && + x.hasOwnProperty("filename") && + x.hasOwnProperty("type") + ); +} diff --git a/multipart/formfile_test.ts b/multipart/formfile_test.ts new file mode 100644 index 0000000000..e6f73b826d --- /dev/null +++ b/multipart/formfile_test.ts @@ -0,0 +1,19 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import { assert, test } from "../testing/mod.ts"; +import { isFormFile } from "./formfile.ts"; + +test(function multipartIsFormFile() { + assert.equal( + isFormFile({ + filename: "foo", + type: "application/json" + }), + true + ); + assert.equal( + isFormFile({ + filename: "foo" + }), + false + ); +}); diff --git a/multipart/multipart.ts b/multipart/multipart.ts new file mode 100644 index 0000000000..f0caa2160e --- /dev/null +++ b/multipart/multipart.ts @@ -0,0 +1,492 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { Buffer, Closer, copy, Reader, ReadResult, remove, Writer } from "deno"; + +import { FormFile } from "./formfile.ts"; +import { + bytesFindIndex, + bytesFindLastIndex, + bytesHasPrefix, + bytesEqual +} from "../bytes/bytes.ts"; +import { copyN } from "../io/ioutil.ts"; +import { MultiReader } from "../io/readers.ts"; +import { tempFile } from "../io/util.ts"; +import { BufReader, BufState, BufWriter } from "../io/bufio.ts"; +import { TextProtoReader } from "../textproto/mod.ts"; +import { encoder } from "../strings/strings.ts"; +import * as path from "../fs/path.ts"; + +function randomBoundary() { + let boundary = "--------------------------"; + for (let i = 0; i < 24; i++) { + boundary += Math.floor(Math.random() * 10).toString(16); + } + return boundary; +} + +/** Reader for parsing multipart/form-data */ +export class MultipartReader { + readonly newLine = encoder.encode("\r\n"); + readonly newLineDashBoundary = encoder.encode(`\r\n--${this.boundary}`); + readonly dashBoundaryDash = encoder.encode(`--${this.boundary}--`); + readonly dashBoundary = encoder.encode(`--${this.boundary}`); + readonly bufReader: BufReader; + + constructor(private reader: Reader, private boundary: string) { + this.bufReader = new BufReader(reader); + } + + /** Read all form data from stream. + * If total size of stored data in memory exceed maxMemory, + * overflowed file data will be written to temporal files. + * String field values are never written to files */ + async readForm( + maxMemory: number + ): Promise<{ [key: string]: string | FormFile }> { + const result = Object.create(null); + let maxValueBytes = maxMemory + (10 << 20); + const buf = new Buffer(new Uint8Array(maxValueBytes)); + for (;;) { + const p = await this.nextPart(); + if (!p) { + break; + } + if (p.formName === "") { + continue; + } + buf.reset(); + if (!p.fileName) { + // value + const n = await copyN(buf, p, maxValueBytes); + maxValueBytes -= n; + if (maxValueBytes < 0) { + throw new RangeError("message too large"); + } + const value = buf.toString(); + result[p.formName] = value; + continue; + } + // file + let formFile: FormFile; + const n = await copy(buf, p); + if (n > maxMemory) { + // too big, write to disk and flush buffer + const ext = path.extname(p.fileName); + const { file, filepath } = await tempFile(".", { + prefix: "multipart-", + postfix: ext + }); + try { + const size = await copyN( + file, + new MultiReader(buf, p), + maxValueBytes + ); + file.close(); + formFile = { + filename: p.fileName, + type: p.headers.get("content-type"), + tempfile: filepath, + size + }; + } catch (e) { + await remove(filepath); + } + } else { + formFile = { + filename: p.fileName, + type: p.headers.get("content-type"), + content: buf.bytes(), + size: buf.bytes().byteLength + }; + maxMemory -= n; + maxValueBytes -= n; + } + result[p.formName] = formFile; + } + return result; + } + + private currentPart: PartReader; + private partsRead: number; + + private async nextPart(): Promise { + if (this.currentPart) { + this.currentPart.close(); + } + if (bytesEqual(this.dashBoundary, encoder.encode("--"))) { + throw new Error("boundary is empty"); + } + let expectNewPart = false; + for (;;) { + const [line, state] = await this.bufReader.readSlice("\n".charCodeAt(0)); + if (state === "EOF" && this.isFinalBoundary(line)) { + break; + } + if (state) { + throw new Error("aa" + state.toString()); + } + if (this.isBoundaryDelimiterLine(line)) { + this.partsRead++; + const r = new TextProtoReader(this.bufReader); + const [headers, state] = await r.readMIMEHeader(); + if (state) { + throw state; + } + const np = new PartReader(this, headers); + this.currentPart = np; + return np; + } + if (this.isFinalBoundary(line)) { + break; + } + if (expectNewPart) { + throw new Error(`expecting a new Part; got line ${line}`); + } + if (this.partsRead === 0) { + continue; + } + if (bytesEqual(line, this.newLine)) { + expectNewPart = true; + continue; + } + throw new Error(`unexpected line in next(): ${line}`); + } + } + + private isFinalBoundary(line: Uint8Array) { + if (!bytesHasPrefix(line, this.dashBoundaryDash)) { + return false; + } + let rest = line.slice(this.dashBoundaryDash.length, line.length); + return rest.length === 0 || bytesEqual(skipLWSPChar(rest), this.newLine); + } + + private isBoundaryDelimiterLine(line: Uint8Array) { + if (!bytesHasPrefix(line, this.dashBoundary)) { + return false; + } + const rest = line.slice(this.dashBoundary.length); + return bytesEqual(skipLWSPChar(rest), this.newLine); + } +} + +function skipLWSPChar(u: Uint8Array): Uint8Array { + const ret = new Uint8Array(u.length); + const sp = " ".charCodeAt(0); + const ht = "\t".charCodeAt(0); + let j = 0; + for (let i = 0; i < u.length; i++) { + if (u[i] === sp || u[i] === ht) continue; + ret[j++] = u[i]; + } + return ret.slice(0, j); +} + +let i = 0; + +class PartReader implements Reader, Closer { + n: number = 0; + total: number = 0; + bufState: BufState = null; + index = i++; + + constructor(private mr: MultipartReader, public readonly headers: Headers) {} + + async read(p: Uint8Array): Promise { + const br = this.mr.bufReader; + const returnResult = (nread: number, bufState: BufState): ReadResult => { + if (bufState && bufState !== "EOF") { + throw bufState; + } + return { nread, eof: bufState === "EOF" }; + }; + if (this.n === 0 && !this.bufState) { + const [peek] = await br.peek(br.buffered()); + const [n, state] = scanUntilBoundary( + peek, + this.mr.dashBoundary, + this.mr.newLineDashBoundary, + this.total, + this.bufState + ); + this.n = n; + this.bufState = state; + if (this.n === 0 && !this.bufState) { + const [_, state] = await br.peek(peek.length + 1); + this.bufState = state; + if (this.bufState === "EOF") { + this.bufState = new RangeError("unexpected eof"); + } + } + } + if (this.n === 0) { + return returnResult(0, this.bufState); + } + + let n = 0; + if (p.byteLength > this.n) { + n = this.n; + } + const buf = p.slice(0, n); + const [nread] = await this.mr.bufReader.readFull(buf); + p.set(buf); + this.total += nread; + this.n -= nread; + if (this.n === 0) { + return returnResult(n, this.bufState); + } + return returnResult(n, null); + } + + close(): void {} + + private contentDisposition: string; + private contentDispositionParams: { [key: string]: string }; + + private getContentDispositionParams() { + if (this.contentDispositionParams) return this.contentDispositionParams; + const cd = this.headers.get("content-disposition"); + const params = {}; + const comps = cd.split(";"); + this.contentDisposition = comps[0]; + comps + .slice(1) + .map(v => v.trim()) + .map(kv => { + const [k, v] = kv.split("="); + if (v) { + const s = v.charAt(0); + const e = v.charAt(v.length - 1); + if ((s === e && s === '"') || s === "'") { + params[k] = v.substr(1, v.length - 2); + } else { + params[k] = v; + } + } + }); + return (this.contentDispositionParams = params); + } + + get fileName(): string { + return this.getContentDispositionParams()["filename"]; + } + + get formName(): string { + const p = this.getContentDispositionParams(); + if (this.contentDisposition === "form-data") { + return p["name"]; + } + return ""; + } +} + +export function scanUntilBoundary( + buf: Uint8Array, + dashBoundary: Uint8Array, + newLineDashBoundary: Uint8Array, + total: number, + state: BufState +): [number, BufState] { + if (total === 0) { + if (bytesHasPrefix(buf, dashBoundary)) { + switch (matchAfterPrefix(buf, dashBoundary, state)) { + case -1: + return [dashBoundary.length, null]; + case 0: + return [0, null]; + case 1: + return [0, "EOF"]; + } + if (bytesHasPrefix(dashBoundary, buf)) { + return [0, state]; + } + } + } + const i = bytesFindIndex(buf, newLineDashBoundary); + if (i >= 0) { + switch (matchAfterPrefix(buf.slice(i), newLineDashBoundary, state)) { + case -1: + return [i + newLineDashBoundary.length, null]; + case 0: + return [i, null]; + case 1: + return [i, "EOF"]; + } + } + if (bytesHasPrefix(newLineDashBoundary, buf)) { + return [0, state]; + } + const j = bytesFindLastIndex(buf, newLineDashBoundary.slice(0, 1)); + if (j >= 0 && bytesHasPrefix(newLineDashBoundary, buf.slice(j))) { + return [j, null]; + } + return [buf.length, state]; +} + +export function matchAfterPrefix( + a: Uint8Array, + prefix: Uint8Array, + bufState: BufState +): number { + if (a.length === prefix.length) { + if (bufState) { + return 1; + } + return 0; + } + const c = a[prefix.length]; + if ( + c === " ".charCodeAt(0) || + c === "\t".charCodeAt(0) || + c === "\r".charCodeAt(0) || + c === "\n".charCodeAt(0) || + c === "-".charCodeAt(0) + ) { + return 1; + } + return -1; +} + +class PartWriter implements Writer { + closed = false; + private readonly partHeader: string; + private headersWritten: boolean = false; + + constructor( + private writer: Writer, + readonly boundary: string, + public headers: Headers, + isFirstBoundary: boolean + ) { + let buf = ""; + if (isFirstBoundary) { + buf += `--${boundary}\r\n`; + } else { + buf += `\r\n--${boundary}\r\n`; + } + for (const [key, value] of headers.entries()) { + buf += `${key}: ${value}\r\n`; + } + buf += `\r\n`; + this.partHeader = buf; + } + + close(): void { + this.closed = true; + } + + async write(p: Uint8Array): Promise { + if (this.closed) { + throw new Error("part is closed"); + } + if (!this.headersWritten) { + await this.writer.write(encoder.encode(this.partHeader)); + this.headersWritten = true; + } + return this.writer.write(p); + } +} + +function checkBoundary(b: string) { + if (b.length < 1 || b.length > 70) { + throw new Error("invalid boundary length: " + b.length); + } + const end = b.length - 1; + for (let i = 0; i < end; i++) { + const c = b.charAt(i); + if (!c.match(/[a-zA-Z0-9'()+_,\-./:=?]/) || (c === " " && i !== end)) { + throw new Error("invalid boundary character: " + c); + } + } + return b; +} + +/** Writer for creating multipart/form-data */ +export class MultipartWriter { + private readonly _boundary: string; + + get boundary() { + return this._boundary; + } + + private lastPart: PartWriter; + private bufWriter: BufWriter; + private isClosed: boolean = false; + + constructor(private readonly writer: Writer, boundary?: string) { + if (boundary !== void 0) { + this._boundary = checkBoundary(boundary); + } else { + this._boundary = randomBoundary(); + } + this.bufWriter = new BufWriter(writer); + } + + formDataContentType(): string { + return `multipart/form-data; boundary=${this.boundary}`; + } + + private createPart(headers: Headers): Writer { + if (this.isClosed) { + throw new Error("multipart: writer is closed"); + } + if (this.lastPart) { + this.lastPart.close(); + } + const part = new PartWriter( + this.writer, + this.boundary, + headers, + !this.lastPart + ); + this.lastPart = part; + return part; + } + + createFormFile(field: string, filename: string): Writer { + const h = new Headers(); + h.set( + "Content-Disposition", + `form-data; name="${field}"; filename="${filename}"` + ); + h.set("Content-Type", "application/octet-stream"); + return this.createPart(h); + } + + createFormField(field: string): Writer { + const h = new Headers(); + h.set("Content-Disposition", `form-data; name="${field}"`); + h.set("Content-Type", "application/octet-stream"); + return this.createPart(h); + } + + async writeField(field: string, value: string) { + const f = await this.createFormField(field); + await f.write(encoder.encode(value)); + } + + async writeFile(field: string, filename: string, file: Reader) { + const f = await this.createFormFile(field, filename); + await copy(f, file); + } + + private flush(): Promise { + return this.bufWriter.flush(); + } + + /** Close writer. No additional data can be writen to stream */ + async close() { + if (this.isClosed) { + throw new Error("multipart: writer is closed"); + } + if (this.lastPart) { + this.lastPart.close(); + this.lastPart = void 0; + } + await this.writer.write(encoder.encode(`\r\n--${this.boundary}--\r\n`)); + await this.flush(); + this.isClosed = true; + } +} diff --git a/multipart/multipart_test.ts b/multipart/multipart_test.ts new file mode 100644 index 0000000000..3181e45c1b --- /dev/null +++ b/multipart/multipart_test.ts @@ -0,0 +1,208 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +import { assert, test } from "../testing/mod.ts"; +import { + matchAfterPrefix, + MultipartReader, + MultipartWriter, + scanUntilBoundary +} from "./multipart.ts"; +import { Buffer, copy, open, remove } from "deno"; +import * as path from "../fs/path.ts"; +import { FormFile, isFormFile } from "./formfile.ts"; +import { StringWriter } from "../io/writers.ts"; + +const e = new TextEncoder(); +const d = new TextDecoder(); +const boundary = "--abcde"; +const dashBoundary = e.encode("--" + boundary); +const nlDashBoundary = e.encode("\r\n--" + boundary); + +test(function multipartScanUntilBoundary1() { + const data = `--${boundary}`; + const [n, err] = scanUntilBoundary( + e.encode(data), + dashBoundary, + nlDashBoundary, + 0, + "EOF" + ); + assert.equal(n, 0); + assert.equal(err, "EOF"); +}); + +test(function multipartScanUntilBoundary2() { + const data = `foo\r\n--${boundary}`; + const [n, err] = scanUntilBoundary( + e.encode(data), + dashBoundary, + nlDashBoundary, + 0, + "EOF" + ); + assert.equal(n, 3); + assert.equal(err, "EOF"); +}); + +test(function multipartScanUntilBoundary4() { + const data = `foo\r\n--`; + const [n, err] = scanUntilBoundary( + e.encode(data), + dashBoundary, + nlDashBoundary, + 0, + null + ); + assert.equal(n, 3); + assert.equal(err, null); +}); + +test(function multipartScanUntilBoundary3() { + const data = `foobar`; + const [n, err] = scanUntilBoundary( + e.encode(data), + dashBoundary, + nlDashBoundary, + 0, + null + ); + assert.equal(n, data.length); + assert.equal(err, null); +}); + +test(function multipartMatchAfterPrefix1() { + const data = `${boundary}\r`; + const v = matchAfterPrefix(e.encode(data), e.encode(boundary), null); + assert.equal(v, 1); +}); + +test(function multipartMatchAfterPrefix2() { + const data = `${boundary}hoge`; + const v = matchAfterPrefix(e.encode(data), e.encode(boundary), null); + assert.equal(v, -1); +}); + +test(function multipartMatchAfterPrefix3() { + const data = `${boundary}`; + const v = matchAfterPrefix(e.encode(data), e.encode(boundary), null); + assert.equal(v, 0); +}); + +test(async function multipartMultipartWriter() { + const buf = new Buffer(); + const mw = new MultipartWriter(buf); + await mw.writeField("foo", "foo"); + await mw.writeField("bar", "bar"); + const f = await open(path.resolve("./multipart/fixtures/sample.txt"), "r"); + await mw.writeFile("file", "sample.txt", f); + await mw.close(); +}); + +test(function multipartMultipartWriter2() { + const w = new StringWriter(); + assert.throws( + () => new MultipartWriter(w, ""), + Error, + "invalid boundary length" + ); + assert.throws( + () => + new MultipartWriter( + w, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ), + Error, + "invalid boundary length" + ); + assert.throws( + () => new MultipartWriter(w, "aaa aaa"), + Error, + "invalid boundary character" + ); + assert.throws( + () => new MultipartWriter(w, "boundary¥¥"), + Error, + "invalid boundary character" + ); +}); + +test(async function multipartMultipartWriter3() { + const w = new StringWriter(); + const mw = new MultipartWriter(w); + await mw.writeField("foo", "foo"); + await mw.close(); + await assert.throwsAsync( + async () => { + await mw.close(); + }, + Error, + "closed" + ); + await assert.throwsAsync( + async () => { + await mw.writeFile("bar", "file", null); + }, + Error, + "closed" + ); + await assert.throwsAsync( + async () => { + await mw.writeField("bar", "bar"); + }, + Error, + "closed" + ); + assert.throws( + () => { + mw.createFormField("bar"); + }, + Error, + "closed" + ); + assert.throws( + () => { + mw.createFormFile("bar", "file"); + }, + Error, + "closed" + ); +}); + +test(async function multipartMultipartReader() { + // FIXME: path resolution + const o = await open(path.resolve("./multipart/fixtures/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + const form = await mr.readForm(10 << 20); + assert.equal(form["foo"], "foo"); + assert.equal(form["bar"], "bar"); + const file = form["file"] as FormFile; + assert.equal(isFormFile(file), true); + assert.assert(file.content !== void 0); +}); + +test(async function multipartMultipartReader2() { + const o = await open(path.resolve("./multipart/fixtures/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + const form = await mr.readForm(20); // + try { + assert.equal(form["foo"], "foo"); + assert.equal(form["bar"], "bar"); + const file = form["file"] as FormFile; + assert.equal(file.type, "application/octet-stream"); + const f = await open(file.tempfile); + const w = new StringWriter(); + await copy(w, f); + const json = JSON.parse(w.toString()); + assert.equal(json["compilerOptions"]["target"], "es2018"); + f.close(); + } finally { + const file = form["file"] as FormFile; + await remove(file.tempfile); + } +}); diff --git a/strings/strings.ts b/strings/strings.ts new file mode 100644 index 0000000000..266c611656 --- /dev/null +++ b/strings/strings.ts @@ -0,0 +1,15 @@ +/** A default TextEncoder instance */ +export const encoder = new TextEncoder(); + +/** Shorthand for new TextEncoder().encode() */ +export function encode(input?: string): Uint8Array { + return encoder.encode(input); +} + +/** A default TextDecoder instance */ +export const decoder = new TextDecoder(); + +/** Shorthand for new TextDecoder().decode() */ +export function decode(input?: Uint8Array): string { + return decoder.decode(input); +} diff --git a/test.ts b/test.ts index a0f5d67e90..92d916e410 100755 --- a/test.ts +++ b/test.ts @@ -4,12 +4,19 @@ import "colors/test.ts"; import "datetime/test.ts"; import "examples/test.ts"; import "flags/test.ts"; +import "io/bufio_test.ts"; +import "io/ioutil_test.ts"; +import "io/util_test.ts"; +import "io/writers_test.ts"; +import "io/readers_test.ts"; import "fs/path/test.ts"; import "io/test.ts"; import "http/server_test.ts"; import "http/file_server_test.ts"; import "log/test.ts"; import "media_types/test.ts"; +import "multipart/formfile_test.ts"; +import "multipart/multipart_test.ts"; import "prettier/main_test.ts"; import "testing/test.ts"; import "textproto/test.ts";