From 6e0c9a0c32ebdbe66301a11f106ba848ee96b8bd Mon Sep 17 00:00:00 2001 From: Yusuke Sakurai Date: Sun, 12 Apr 2020 14:24:58 +0900 Subject: [PATCH] refactor(std/multipart): make readForm() return value more type safe (#4710) --- std/mime/multipart.ts | 66 ++++++++++++++++--- std/mime/multipart_test.ts | 129 +++++++++++++++++++++++++------------ 2 files changed, 145 insertions(+), 50 deletions(-) diff --git a/std/mime/multipart.ts b/std/mime/multipart.ts index 032d4e29de..c46948226b 100644 --- a/std/mime/multipart.ts +++ b/std/mime/multipart.ts @@ -251,6 +251,17 @@ function skipLWSPChar(u: Uint8Array): Uint8Array { return ret.slice(0, j); } +export interface MultipartFormData { + file(key: string): FormFile | undefined; + value(key: string): string | undefined; + entries(): IterableIterator<[string, string | FormFile | undefined]>; + [Symbol.iterator](): IterableIterator< + [string, string | FormFile | undefined] + >; + /** Remove all tempfiles */ + removeAll(): Promise; +} + /** Reader for parsing multipart/form-data */ export class MultipartReader { readonly newLine = encoder.encode("\r\n"); @@ -268,11 +279,11 @@ export class MultipartReader { * overflowed file data will be written to temporal files. * String field values are never written to files. * null value means parsing or writing to file was failed in some reason. + * @param maxMemory maximum memory size to store file in memory. bytes. @default 1048576 (1MB) * */ - async readForm( - maxMemory: number - ): Promise<{ [key: string]: null | string | FormFile }> { - const result = Object.create(null); + async readForm(maxMemory = 10 << 20): Promise { + const fileMap = new Map(); + const valueMap = new Map(); let maxValueBytes = maxMemory + (10 << 20); const buf = new Buffer(new Uint8Array(maxValueBytes)); for (;;) { @@ -292,11 +303,11 @@ export class MultipartReader { throw new RangeError("message too large"); } const value = buf.toString(); - result[p.formName] = value; + valueMap.set(p.formName, value); continue; } // file - let formFile: FormFile | null = null; + let formFile: FormFile | undefined; const n = await copy(buf, p); const contentType = p.headers.get("content-type"); assert(contentType != null, "content-type must be set"); @@ -333,9 +344,11 @@ export class MultipartReader { maxMemory -= n; maxValueBytes -= n; } - result[p.formName] = formFile; + if (formFile) { + fileMap.set(p.formName, formFile); + } } - return result; + return multipatFormData(fileMap, valueMap); } private currentPart: PartReader | undefined; @@ -399,6 +412,43 @@ export class MultipartReader { } } +function multipatFormData( + fileMap: Map, + valueMap: Map +): MultipartFormData { + function file(key: string): FormFile | undefined { + return fileMap.get(key); + } + function value(key: string): string | undefined { + return valueMap.get(key); + } + function* entries(): IterableIterator< + [string, string | FormFile | undefined] + > { + yield* fileMap; + yield* valueMap; + } + async function removeAll(): Promise { + const promises: Array> = []; + for (const val of fileMap.values()) { + if (!val.tempfile) continue; + promises.push(Deno.remove(val.tempfile)); + } + await Promise.all(promises); + } + return { + file, + value, + entries, + removeAll, + [Symbol.iterator](): IterableIterator< + [string, string | FormFile | undefined] + > { + return entries(); + }, + }; +} + class PartWriter implements Writer { closed = false; private readonly partHeader: string; diff --git a/std/mime/multipart_test.ts b/std/mime/multipart_test.ts index 9b96a2f4cf..8b721a4414 100644 --- a/std/mime/multipart_test.ts +++ b/std/mime/multipart_test.ts @@ -1,16 +1,14 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -const { Buffer, copy, open, remove } = Deno; +const { Buffer, copy, open, test } = Deno; import { assert, assertEquals, assertThrows, assertThrowsAsync, } from "../testing/asserts.ts"; -const { test } = Deno; import * as path from "../path/mod.ts"; import { - FormFile, MultipartReader, MultipartWriter, isFormFile, @@ -173,46 +171,93 @@ test(async function multipartMultipartWriter3(): Promise { ); }); -test(async function multipartMultipartReader(): Promise { - // FIXME: path resolution - const o = await open(path.resolve("./mime/testdata/sample.txt")); - const mr = new MultipartReader( - o, - "--------------------------434049563556637648550474" - ); - const form = await mr.readForm(10 << 20); - assertEquals(form["foo"], "foo"); - assertEquals(form["bar"], "bar"); - const file = form["file"] as FormFile; - assertEquals(isFormFile(file), true); - assert(file.content !== void 0); - o.close(); +test({ + name: "[mime/multipart] readForm() basic", + async fn() { + const o = await open(path.resolve("./mime/testdata/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + const form = await mr.readForm(); + assertEquals(form.value("foo"), "foo"); + assertEquals(form.value("bar"), "bar"); + const file = form.file("file"); + assert(isFormFile(file)); + assert(file.content !== void 0); + o.close(); + }, }); -test(async function multipartMultipartReader2(): Promise { - const o = await open(path.resolve("./mime/testdata/sample.txt")); - const mr = new MultipartReader( - o, - "--------------------------434049563556637648550474" - ); - const form = await mr.readForm(20); // - try { - assertEquals(form["foo"], "foo"); - assertEquals(form["bar"], "bar"); - const file = form["file"] as FormFile; - assertEquals(file.type, "application/octet-stream"); - assert(file.tempfile != null); - const f = await open(file.tempfile); - const w = new StringWriter(); - await copy(w, f); - const json = JSON.parse(w.toString()); - assertEquals(json["compilerOptions"]["target"], "es2018"); - f.close(); - } finally { - const file = form["file"] as FormFile; - if (file.tempfile) { - await remove(file.tempfile); +test({ + name: "[mime/multipart] readForm() should store big file in temp file", + async fn() { + const o = await open(path.resolve("./mime/testdata/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + // use low-memory to write "file" into temp file. + const form = await mr.readForm(20); + try { + assertEquals(form.value("foo"), "foo"); + assertEquals(form.value("bar"), "bar"); + const file = form.file("file"); + assert(file != null); + assertEquals(file.type, "application/octet-stream"); + assert(file.tempfile != null); + const f = await open(file.tempfile); + const w = new StringWriter(); + await copy(w, f); + const json = JSON.parse(w.toString()); + assertEquals(json["compilerOptions"]["target"], "es2018"); + f.close(); + } finally { + await form.removeAll(); + o.close(); } - o.close(); - } + }, +}); + +test({ + name: "[mime/multipart] removeAll() should remove all tempfiles", + async fn() { + const o = await open(path.resolve("./mime/testdata/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + const form = await mr.readForm(20); + const file = form.file("file"); + assert(file != null); + const { tempfile, content } = file; + assert(tempfile != null); + assert(content == null); + const stat = await Deno.stat(tempfile); + assertEquals(stat.size, file.size); + await form.removeAll(); + await assertThrowsAsync(async () => { + await Deno.stat(tempfile); + }, Deno.errors.NotFound); + o.close(); + }, +}); + +test({ + name: "[mime/multipart] entries()", + async fn() { + const o = await open(path.resolve("./mime/testdata/sample.txt")); + const mr = new MultipartReader( + o, + "--------------------------434049563556637648550474" + ); + const form = await mr.readForm(); + const map = new Map(form.entries()); + assertEquals(map.get("foo"), "foo"); + assertEquals(map.get("bar"), "bar"); + const file = map.get("file"); + assert(isFormFile(file)); + assertEquals(file.filename, "tsconfig.json"); + o.close(); + }, });