mirror of
https://github.com/denoland/deno.git
synced 2024-12-22 15:24:46 -05:00
fix(cli/buffer): allow Buffer to store MAX_SIZE bytes (#6570)
This commit is contained in:
parent
1bcc35b84a
commit
39dba12a06
2 changed files with 134 additions and 14 deletions
|
@ -11,7 +11,7 @@ import { assert } from "./util.ts";
|
|||
// buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond
|
||||
// what is required to hold the contents of r, readFrom() will not grow the
|
||||
// underlying buffer.
|
||||
const MIN_READ = 512;
|
||||
const MIN_READ = 32 * 1024;
|
||||
const MAX_SIZE = 2 ** 32 - 2;
|
||||
|
||||
// `off` is the offset into `dst` where it will at which to begin writing values
|
||||
|
@ -133,17 +133,17 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync {
|
|||
// we instead let capacity get twice as large so we
|
||||
// don't spend all our time copying.
|
||||
copyBytes(this.#buf.subarray(this.#off), this.#buf);
|
||||
} else if (c > MAX_SIZE - c - n) {
|
||||
} else if (c + n > MAX_SIZE) {
|
||||
throw new Error("The buffer cannot be grown beyond the maximum size.");
|
||||
} else {
|
||||
// Not enough space anywhere, we need to allocate.
|
||||
const buf = new Uint8Array(2 * c + n);
|
||||
const buf = new Uint8Array(Math.min(2 * c + n, MAX_SIZE));
|
||||
copyBytes(this.#buf.subarray(this.#off), buf);
|
||||
this.#buf = buf;
|
||||
}
|
||||
// Restore this.#off and len(this.#buf).
|
||||
this.#off = 0;
|
||||
this.#reslice(m + n);
|
||||
this.#reslice(Math.min(m + n, MAX_SIZE));
|
||||
return m;
|
||||
};
|
||||
|
||||
|
@ -157,30 +157,48 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync {
|
|||
|
||||
async readFrom(r: Reader): Promise<number> {
|
||||
let n = 0;
|
||||
const tmp = new Uint8Array(MIN_READ);
|
||||
while (true) {
|
||||
const i = this.#grow(MIN_READ);
|
||||
this.#reslice(i);
|
||||
const fub = new Uint8Array(this.#buf.buffer, i);
|
||||
const nread = await r.read(fub);
|
||||
const shouldGrow = this.capacity - this.length < MIN_READ;
|
||||
// read into tmp buffer if there's not enough room
|
||||
// otherwise read directly into the internal buffer
|
||||
const buf = shouldGrow
|
||||
? tmp
|
||||
: new Uint8Array(this.#buf.buffer, this.length);
|
||||
|
||||
const nread = await r.read(buf);
|
||||
if (nread === null) {
|
||||
return n;
|
||||
}
|
||||
this.#reslice(i + nread);
|
||||
|
||||
// write will grow if needed
|
||||
if (shouldGrow) this.writeSync(buf.subarray(0, nread));
|
||||
else this.#reslice(this.length + nread);
|
||||
|
||||
n += nread;
|
||||
}
|
||||
}
|
||||
|
||||
readFromSync(r: ReaderSync): number {
|
||||
let n = 0;
|
||||
const tmp = new Uint8Array(MIN_READ);
|
||||
while (true) {
|
||||
const i = this.#grow(MIN_READ);
|
||||
this.#reslice(i);
|
||||
const fub = new Uint8Array(this.#buf.buffer, i);
|
||||
const nread = r.readSync(fub);
|
||||
const shouldGrow = this.capacity - this.length < MIN_READ;
|
||||
// read into tmp buffer if there's not enough room
|
||||
// otherwise read directly into the internal buffer
|
||||
const buf = shouldGrow
|
||||
? tmp
|
||||
: new Uint8Array(this.#buf.buffer, this.length);
|
||||
|
||||
const nread = r.readSync(buf);
|
||||
if (nread === null) {
|
||||
return n;
|
||||
}
|
||||
this.#reslice(i + nread);
|
||||
|
||||
// write will grow if needed
|
||||
if (shouldGrow) this.writeSync(buf.subarray(0, nread));
|
||||
else this.#reslice(this.length + nread);
|
||||
|
||||
n += nread;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,11 +11,19 @@ import {
|
|||
unitTest,
|
||||
} from "./test_util.ts";
|
||||
|
||||
const MAX_SIZE = 2 ** 32 - 2;
|
||||
// N controls how many iterations of certain checks are performed.
|
||||
const N = 100;
|
||||
let testBytes: Uint8Array | null;
|
||||
let testString: string | null;
|
||||
|
||||
let ignoreMaxSizeTests = false;
|
||||
try {
|
||||
new ArrayBuffer(MAX_SIZE);
|
||||
} catch (e) {
|
||||
ignoreMaxSizeTests = true;
|
||||
}
|
||||
|
||||
function init(): void {
|
||||
if (testBytes == null) {
|
||||
testBytes = new Uint8Array(N);
|
||||
|
@ -167,6 +175,100 @@ unitTest(async function bufferTooLargeByteWrites(): Promise<void> {
|
|||
);
|
||||
});
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
function bufferGrowWriteMaxBuffer(): void {
|
||||
const bufSize = 16 * 1024;
|
||||
const capacities = [MAX_SIZE, MAX_SIZE - 1];
|
||||
for (const capacity of capacities) {
|
||||
let written = 0;
|
||||
const buf = new Deno.Buffer();
|
||||
const writes = Math.floor(capacity / bufSize);
|
||||
for (let i = 0; i < writes; i++)
|
||||
written += buf.writeSync(repeat("x", bufSize));
|
||||
|
||||
if (written < capacity) {
|
||||
written += buf.writeSync(repeat("x", capacity - written));
|
||||
}
|
||||
|
||||
assertEquals(written, capacity);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
async function bufferGrowReadCloseMaxBufferPlus1(): Promise<void> {
|
||||
const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1));
|
||||
const buf = new Deno.Buffer();
|
||||
|
||||
await assertThrowsAsync(
|
||||
async () => {
|
||||
await buf.readFrom(reader);
|
||||
},
|
||||
Error,
|
||||
"grown beyond the maximum size"
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
function bufferGrowReadSyncCloseMaxBufferPlus1(): void {
|
||||
const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1));
|
||||
const buf = new Deno.Buffer();
|
||||
|
||||
assertThrows(
|
||||
() => {
|
||||
buf.readFromSync(reader);
|
||||
},
|
||||
Error,
|
||||
"grown beyond the maximum size"
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
function bufferGrowReadSyncCloseToMaxBuffer(): void {
|
||||
const capacities = [MAX_SIZE, MAX_SIZE - 1];
|
||||
for (const capacity of capacities) {
|
||||
const reader = new Deno.Buffer(new ArrayBuffer(capacity));
|
||||
const buf = new Deno.Buffer();
|
||||
buf.readFromSync(reader);
|
||||
|
||||
assertEquals(buf.length, capacity);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
async function bufferGrowReadCloseToMaxBuffer(): Promise<void> {
|
||||
const capacities = [MAX_SIZE, MAX_SIZE - 1];
|
||||
for (const capacity of capacities) {
|
||||
const reader = new Deno.Buffer(new ArrayBuffer(capacity));
|
||||
const buf = new Deno.Buffer();
|
||||
await buf.readFrom(reader);
|
||||
assertEquals(buf.length, capacity);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(
|
||||
{ ignore: ignoreMaxSizeTests },
|
||||
async function bufferReadCloseToMaxBufferWithInitialGrow(): Promise<void> {
|
||||
const capacities = [MAX_SIZE, MAX_SIZE - 1, MAX_SIZE - 512];
|
||||
for (const capacity of capacities) {
|
||||
const reader = new Deno.Buffer(new ArrayBuffer(capacity));
|
||||
const buf = new Deno.Buffer();
|
||||
buf.grow(MAX_SIZE);
|
||||
await buf.readFrom(reader);
|
||||
assertEquals(buf.length, capacity);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
unitTest(async function bufferLargeByteReads(): Promise<void> {
|
||||
init();
|
||||
assert(testBytes);
|
||||
|
|
Loading…
Reference in a new issue