0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-10-29 08:58:01 -04:00

Add deno.Buffer (#1121)

Do not confuse this with Node's Buffer. This is a direct port of Go's
bytes.Buffer - it allows buffering of Reader and Writer objects.
This commit is contained in:
Ryan Dahl 2018-11-04 15:36:46 -08:00 committed by GitHub
parent 4e07783663
commit bd88e56cbc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 418 additions and 0 deletions

View file

@ -77,6 +77,7 @@ main_extern = [
ts_sources = [
"js/assets.ts",
"js/blob.ts",
"js/buffer.ts",
"js/chmod.ts",
"js/compiler.ts",
"js/console.ts",

230
js/buffer.ts Normal file
View file

@ -0,0 +1,230 @@
// This code has been ported almost directly from Go's src/bytes/buffer.go
// Copyright 2009 The Go Authors. All rights reserved. BSD license.
// https://github.com/golang/go/blob/master/LICENSE
//import * as io from "./io";
import { Reader, Writer, ReadResult } from "./io";
import { assert } from "./util";
import { TextDecoder } from "./text_encoding";
// MIN_READ is the minimum ArrayBuffer size passed to a read call by
// buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond
// what is required to hold the contents of r, readFrom() will not grow the
// underlying buffer.
const MIN_READ = 512;
const MAX_SIZE = 2 ** 32 - 2;
// `off` is the offset into `dst` where it will at which to begin writing values
// from `src`.
// Returns the number of bytes copied.
function copyBytes(dst: Uint8Array, src: Uint8Array, off = 0): number {
const r = dst.byteLength - off;
if (src.byteLength > r) {
src = src.subarray(0, r);
}
dst.set(src, off);
return src.byteLength;
}
/** A Buffer is a variable-sized buffer of bytes with read() and write()
* methods. Based on https://golang.org/pkg/bytes/#Buffer
*/
export class Buffer implements Reader, Writer {
private buf: Uint8Array; // contents are the bytes buf[off : len(buf)]
private off = 0; // read at buf[off], write at buf[buf.byteLength]
constructor(ab?: ArrayBuffer) {
if (ab == null) {
this.buf = new Uint8Array(0);
} else {
this.buf = new Uint8Array(ab);
}
}
/** bytes() returns a slice holding the unread portion of the buffer.
* The slice is valid for use only until the next buffer modification (that
* is, only until the next call to a method like read(), write(), reset(), or
* truncate()). The slice aliases the buffer content at least until the next
* buffer modification, so immediate changes to the slice will affect the
* result of future reads.
*/
bytes(): Uint8Array {
return this.buf.subarray(this.off);
}
/** toString() returns the contents of the unread portion of the buffer
* as a string. Warning - if multibyte characters are present when data is
* flowing through the buffer, this method may result in incorrect strings
* due to a character being split.
*/
toString(): string {
const decoder = new TextDecoder();
return decoder.decode(this.buf.subarray(this.off));
}
/** empty() returns whether the unread portion of the buffer is empty. */
empty() {
return this.buf.byteLength <= this.off;
}
/** length is a getter that returns the number of bytes of the unread
* portion of the buffer
*/
get length() {
return this.buf.byteLength - this.off;
}
/** Returns the capacity of the buffer's underlying byte slice, that is,
* the total space allocated for the buffer's data.
*/
get capacity(): number {
return this.buf.buffer.byteLength;
}
/** truncate() discards all but the first n unread bytes from the buffer but
* continues to use the same allocated storage. It throws if n is negative or
* greater than the length of the buffer.
*/
truncate(n: number): void {
if (n === 0) {
this.reset();
return;
}
if (n < 0 || n > this.length) {
throw Error("bytes.Buffer: truncation out of range");
}
this._reslice(this.off + n);
}
/** reset() resets the buffer to be empty, but it retains the underlying
* storage for use by future writes. reset() is the same as truncate(0)
*/
reset(): void {
this._reslice(0);
this.off = 0;
}
/** _tryGrowByReslice() is a version of grow for the fast-case
* where the internal buffer only needs to be resliced. It returns the index
* where bytes should be written and whether it succeeded.
* It returns -1 if a reslice was not needed.
*/
private _tryGrowByReslice(n: number): number {
const l = this.buf.byteLength;
if (n <= this.capacity - l) {
this._reslice(l + n);
return l;
}
return -1;
}
private _reslice(len: number): void {
assert(len <= this.buf.buffer.byteLength);
this.buf = new Uint8Array(this.buf.buffer, 0, len);
}
/** read() reads the next len(p) bytes from the buffer or until the buffer
* is drained. The return value n is the number of bytes read. If the
* buffer has no data to return, eof in the response will be true.
*/
async read(p: ArrayBufferView): Promise<ReadResult> {
if (!(p instanceof Uint8Array)) {
throw Error("Only Uint8Array supported");
}
if (this.empty()) {
// Buffer is empty, reset to recover space.
this.reset();
if (p.byteLength === 0) {
// TODO This edge case should be tested by porting TestReadEmptyAtEOF
// from the Go tests.
return { nread: 0, eof: false };
}
return { nread: 0, eof: true };
}
const nread = copyBytes(p, this.buf.subarray(this.off));
this.off += nread;
return { nread, eof: false };
}
async write(p: ArrayBufferView): Promise<number> {
const m = this._grow(p.byteLength);
if (!(p instanceof Uint8Array)) {
throw Error("Only Uint8Array supported");
}
return copyBytes(this.buf, p, m);
}
/** _grow() grows the buffer to guarantee space for n more bytes.
* It returns the index where bytes should be written.
* If the buffer can't grow it will throw with ErrTooLarge.
*/
private _grow(n: number): number {
const m = this.length;
// If buffer is empty, reset to recover space.
if (m === 0 && this.off !== 0) {
this.reset();
}
// Fast: Try to grow by means of a reslice.
const i = this._tryGrowByReslice(n);
if (i >= 0) {
return i;
}
const c = this.capacity;
if (n <= Math.floor(c / 2) - m) {
// We can slide things down instead of allocating a new
// ArrayBuffer. We only need m+n <= c to slide, but
// we instead let capacity get twice as large so we
// don't spend all our time copying.
copyBytes(this.buf, this.buf.subarray(this.off));
} else if (c > MAX_SIZE - c - n) {
throw Error("ErrTooLarge"); // TODO DenoError(TooLarge)
} else {
// Not enough space anywhere, we need to allocate.
const buf = new Uint8Array(2 * c + n);
copyBytes(buf, this.buf.subarray(this.off));
this.buf = buf;
}
// Restore this.off and len(this.buf).
this.off = 0;
this._reslice(m + n);
return m;
}
/** grow() grows the buffer's capacity, if necessary, to guarantee space for
* another n bytes. After grow(n), at least n bytes can be written to the
* buffer without another allocation. If n is negative, grow() will panic. If
* the buffer can't grow it will throw ErrTooLarge.
* Based on https://golang.org/pkg/bytes/#Buffer.Grow
*/
grow(n: number): void {
if (n < 0) {
throw Error("Buffer.grow: negative count");
}
const m = this._grow(n);
this._reslice(m);
}
/** readFrom() reads data from r until EOF and appends it to the buffer,
* growing the buffer as needed. It returns the number of bytes read. If the
* buffer becomes too large, readFrom will panic with ErrTooLarge.
* Based on https://golang.org/pkg/bytes/#Buffer.ReadFrom
*/
async readFrom(r: Reader): Promise<number> {
let n = 0;
while (true) {
try {
const i = this._grow(MIN_READ);
this._reslice(i);
const fub = new Uint8Array(this.buf.buffer, i);
const { nread, eof } = await r.read(fub);
this._reslice(i + nread);
n += nread;
if (eof) {
return n;
}
} catch (e) {
return n;
}
}
}
}

185
js/buffer_test.ts Normal file
View file

@ -0,0 +1,185 @@
// This code has been ported almost directly from Go's src/bytes/buffer_test.go
// Copyright 2009 The Go Authors. All rights reserved. BSD license.
// https://github.com/golang/go/blob/master/LICENSE
import { test, assert, assertEqual } from "./test_util.ts";
import { Buffer } from "deno";
// N controls how many iterations of certain checks are performed.
const N = 100;
let testBytes: Uint8Array | null;
let testString: string | null;
function init() {
if (testBytes == null) {
testBytes = new Uint8Array(N);
for (let i = 0; i < N; i++) {
testBytes[i] = "a".charCodeAt(0) + (i % 26);
}
const decoder = new TextDecoder();
testString = decoder.decode(testBytes);
}
}
function check(buf: Buffer, s: string) {
const bytes = buf.bytes();
assertEqual(buf.length, bytes.byteLength);
const decoder = new TextDecoder();
const bytesStr = decoder.decode(bytes);
assertEqual(bytesStr, s);
assertEqual(buf.length, buf.toString().length);
assertEqual(buf.length, s.length);
}
// Fill buf through n writes of byte slice fub.
// The initial contents of buf corresponds to the string s;
// the result is the final contents of buf returned as a string.
async function fillBytes(
buf: Buffer,
s: string,
n: number,
fub: Uint8Array
): Promise<string> {
check(buf, s);
for (; n > 0; n--) {
let m = await buf.write(fub);
assertEqual(m, fub.byteLength);
const decoder = new TextDecoder();
s += decoder.decode(fub);
check(buf, s);
}
return s;
}
// Empty buf through repeated reads into fub.
// The initial contents of buf corresponds to the string s.
async function empty(buf: Buffer, s: string, fub: Uint8Array): Promise<void> {
check(buf, s);
while (true) {
const r = await buf.read(fub);
if (r.nread == 0) {
break;
}
s = s.slice(r.nread);
check(buf, s);
}
check(buf, "");
}
test(function bufferNewBuffer() {
init();
const buf = new Buffer(testBytes.buffer as ArrayBuffer);
check(buf, testString);
});
test(async function bufferBasicOperations() {
init();
let buf = new Buffer();
for (let i = 0; i < 5; i++) {
check(buf, "");
buf.reset();
check(buf, "");
buf.truncate(0);
check(buf, "");
let n = await buf.write(testBytes.subarray(0, 1));
assertEqual(n, 1);
check(buf, "a");
n = await buf.write(testBytes.subarray(1, 2));
assertEqual(n, 1);
check(buf, "ab");
n = await buf.write(testBytes.subarray(2, 26));
assertEqual(n, 24);
check(buf, testString.slice(0, 26));
buf.truncate(26);
check(buf, testString.slice(0, 26));
buf.truncate(20);
check(buf, testString.slice(0, 20));
await empty(buf, testString.slice(0, 20), new Uint8Array(5));
await empty(buf, "", new Uint8Array(100));
// TODO buf.writeByte()
// TODO buf.readByte()
}
});
test(async function bufferLargeByteWrites() {
init();
const buf = new Buffer();
const limit = 9;
for (let i = 3; i < limit; i += 3) {
const s = await fillBytes(buf, "", 5, testBytes);
await empty(buf, s, new Uint8Array(Math.floor(testString.length / i)));
}
check(buf, "");
});
test(async function bufferLargeByteReads() {
init();
const buf = new Buffer();
for (let i = 3; i < 30; i += 3) {
const n = Math.floor(testBytes.byteLength / i);
const s = await fillBytes(buf, "", 5, testBytes.subarray(0, n));
await empty(buf, s, new Uint8Array(testString.length));
}
check(buf, "");
});
test(function bufferCapWithPreallocatedSlice() {
const buf = new Buffer(new ArrayBuffer(10));
assertEqual(buf.capacity, 10);
});
test(async function bufferReadFrom() {
init();
const buf = new Buffer();
for (let i = 3; i < 30; i += 3) {
const s = await fillBytes(
buf,
"",
5,
testBytes.subarray(0, Math.floor(testBytes.byteLength / i))
);
const b = new Buffer();
await b.readFrom(buf);
const fub = new Uint8Array(testString.length);
await empty(b, s, fub);
}
});
function repeat(c: string, bytes: number): Uint8Array {
assertEqual(c.length, 1);
const ui8 = new Uint8Array(bytes);
ui8.fill(c.charCodeAt(0));
return ui8;
}
test(async function bufferTestGrow() {
const tmp = new Uint8Array(72);
for (let startLen of [0, 100, 1000, 10000, 100000]) {
const xBytes = repeat("x", startLen);
for (let growLen of [0, 100, 1000, 10000, 100000]) {
const buf = new Buffer(xBytes.buffer as ArrayBuffer);
// If we read, this affects buf.off, which is good to test.
const { nread, eof } = await buf.read(tmp);
buf.grow(growLen);
const yBytes = repeat("y", growLen);
await buf.write(yBytes);
// Check that buffer has correct data.
assertEqual(
buf.bytes().subarray(0, startLen - nread),
xBytes.subarray(nread)
);
assertEqual(
buf.bytes().subarray(startLen - nread, startLen - nread + growLen),
yBytes
);
}
}
});

View file

@ -20,6 +20,7 @@ export {
ReadWriteCloser,
ReadWriteSeeker
} from "./io";
export { Buffer } from "./buffer";
export { mkdirSync, mkdir } from "./mkdir";
export { makeTempDirSync, makeTempDir } from "./make_temp_dir";
export { chmodSync, chmod } from "./chmod";

View file

@ -4,6 +4,7 @@
// But it can also be run manually: ./out/debug/deno js/unit_tests.ts
import "../website/app_test.js";
import "./blob_test.ts";
import "./buffer_test.ts";
import "./chmod_test.ts";
import "./compiler_test.ts";
import "./console_test.ts";