From 39dba12a061e464fb06bc6a763c84b36b5d1a915 Mon Sep 17 00:00:00 2001 From: Marcos Casagrande Date: Fri, 10 Jul 2020 17:49:35 +0200 Subject: [PATCH] fix(cli/buffer): allow Buffer to store MAX_SIZE bytes (#6570) --- cli/js/buffer.ts | 46 ++++++++++----- cli/tests/unit/buffer_test.ts | 102 ++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 14 deletions(-) diff --git a/cli/js/buffer.ts b/cli/js/buffer.ts index 5c304c127e..3308a9a0c1 100644 --- a/cli/js/buffer.ts +++ b/cli/js/buffer.ts @@ -11,7 +11,7 @@ import { assert } from "./util.ts"; // buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond // what is required to hold the contents of r, readFrom() will not grow the // underlying buffer. -const MIN_READ = 512; +const MIN_READ = 32 * 1024; const MAX_SIZE = 2 ** 32 - 2; // `off` is the offset into `dst` where it will at which to begin writing values @@ -133,17 +133,17 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync { // we instead let capacity get twice as large so we // don't spend all our time copying. copyBytes(this.#buf.subarray(this.#off), this.#buf); - } else if (c > MAX_SIZE - c - n) { + } else if (c + n > MAX_SIZE) { throw new Error("The buffer cannot be grown beyond the maximum size."); } else { // Not enough space anywhere, we need to allocate. - const buf = new Uint8Array(2 * c + n); + const buf = new Uint8Array(Math.min(2 * c + n, MAX_SIZE)); copyBytes(this.#buf.subarray(this.#off), buf); this.#buf = buf; } // Restore this.#off and len(this.#buf). this.#off = 0; - this.#reslice(m + n); + this.#reslice(Math.min(m + n, MAX_SIZE)); return m; }; @@ -157,30 +157,48 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync { async readFrom(r: Reader): Promise { let n = 0; + const tmp = new Uint8Array(MIN_READ); while (true) { - const i = this.#grow(MIN_READ); - this.#reslice(i); - const fub = new Uint8Array(this.#buf.buffer, i); - const nread = await r.read(fub); + const shouldGrow = this.capacity - this.length < MIN_READ; + // read into tmp buffer if there's not enough room + // otherwise read directly into the internal buffer + const buf = shouldGrow + ? tmp + : new Uint8Array(this.#buf.buffer, this.length); + + const nread = await r.read(buf); if (nread === null) { return n; } - this.#reslice(i + nread); + + // write will grow if needed + if (shouldGrow) this.writeSync(buf.subarray(0, nread)); + else this.#reslice(this.length + nread); + n += nread; } } readFromSync(r: ReaderSync): number { let n = 0; + const tmp = new Uint8Array(MIN_READ); while (true) { - const i = this.#grow(MIN_READ); - this.#reslice(i); - const fub = new Uint8Array(this.#buf.buffer, i); - const nread = r.readSync(fub); + const shouldGrow = this.capacity - this.length < MIN_READ; + // read into tmp buffer if there's not enough room + // otherwise read directly into the internal buffer + const buf = shouldGrow + ? tmp + : new Uint8Array(this.#buf.buffer, this.length); + + const nread = r.readSync(buf); if (nread === null) { return n; } - this.#reslice(i + nread); + + // write will grow if needed + if (shouldGrow) this.writeSync(buf.subarray(0, nread)); + else this.#reslice(this.length + nread); + n += nread; } } diff --git a/cli/tests/unit/buffer_test.ts b/cli/tests/unit/buffer_test.ts index dc80c9c6d3..440dd5495d 100644 --- a/cli/tests/unit/buffer_test.ts +++ b/cli/tests/unit/buffer_test.ts @@ -11,11 +11,19 @@ import { unitTest, } from "./test_util.ts"; +const MAX_SIZE = 2 ** 32 - 2; // N controls how many iterations of certain checks are performed. const N = 100; let testBytes: Uint8Array | null; let testString: string | null; +let ignoreMaxSizeTests = false; +try { + new ArrayBuffer(MAX_SIZE); +} catch (e) { + ignoreMaxSizeTests = true; +} + function init(): void { if (testBytes == null) { testBytes = new Uint8Array(N); @@ -167,6 +175,100 @@ unitTest(async function bufferTooLargeByteWrites(): Promise { ); }); +unitTest( + { ignore: ignoreMaxSizeTests }, + function bufferGrowWriteMaxBuffer(): void { + const bufSize = 16 * 1024; + const capacities = [MAX_SIZE, MAX_SIZE - 1]; + for (const capacity of capacities) { + let written = 0; + const buf = new Deno.Buffer(); + const writes = Math.floor(capacity / bufSize); + for (let i = 0; i < writes; i++) + written += buf.writeSync(repeat("x", bufSize)); + + if (written < capacity) { + written += buf.writeSync(repeat("x", capacity - written)); + } + + assertEquals(written, capacity); + } + } +); + +unitTest( + { ignore: ignoreMaxSizeTests }, + async function bufferGrowReadCloseMaxBufferPlus1(): Promise { + const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1)); + const buf = new Deno.Buffer(); + + await assertThrowsAsync( + async () => { + await buf.readFrom(reader); + }, + Error, + "grown beyond the maximum size" + ); + } +); + +unitTest( + { ignore: ignoreMaxSizeTests }, + function bufferGrowReadSyncCloseMaxBufferPlus1(): void { + const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1)); + const buf = new Deno.Buffer(); + + assertThrows( + () => { + buf.readFromSync(reader); + }, + Error, + "grown beyond the maximum size" + ); + } +); + +unitTest( + { ignore: ignoreMaxSizeTests }, + function bufferGrowReadSyncCloseToMaxBuffer(): void { + const capacities = [MAX_SIZE, MAX_SIZE - 1]; + for (const capacity of capacities) { + const reader = new Deno.Buffer(new ArrayBuffer(capacity)); + const buf = new Deno.Buffer(); + buf.readFromSync(reader); + + assertEquals(buf.length, capacity); + } + } +); + +unitTest( + { ignore: ignoreMaxSizeTests }, + async function bufferGrowReadCloseToMaxBuffer(): Promise { + const capacities = [MAX_SIZE, MAX_SIZE - 1]; + for (const capacity of capacities) { + const reader = new Deno.Buffer(new ArrayBuffer(capacity)); + const buf = new Deno.Buffer(); + await buf.readFrom(reader); + assertEquals(buf.length, capacity); + } + } +); + +unitTest( + { ignore: ignoreMaxSizeTests }, + async function bufferReadCloseToMaxBufferWithInitialGrow(): Promise { + const capacities = [MAX_SIZE, MAX_SIZE - 1, MAX_SIZE - 512]; + for (const capacity of capacities) { + const reader = new Deno.Buffer(new ArrayBuffer(capacity)); + const buf = new Deno.Buffer(); + buf.grow(MAX_SIZE); + await buf.readFrom(reader); + assertEquals(buf.length, capacity); + } + } +); + unitTest(async function bufferLargeByteReads(): Promise { init(); assert(testBytes);