From 280856f8d81c4b2e53fa8022aafae5a7c008747f Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 7 Nov 2018 14:17:36 -0500 Subject: [PATCH] First pass at bufio.read tests. Original: https://github.com/denoland/deno_std/commit/1eb57aa3948caf88e9064defc15e076b8a46fbd2 --- buffer.ts | 14 +-------- buffer_test.ts | 10 +++++-- bufio.ts | 72 +++++++++++++++++++++++++++++++++++++++++---- bufio_test.ts | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ file_server.ts | 4 +-- http.ts | 5 ++-- util.ts | 13 +++++++- 7 files changed, 170 insertions(+), 28 deletions(-) diff --git a/buffer.ts b/buffer.ts index 1c4d68e98d..a795bc8423 100644 --- a/buffer.ts +++ b/buffer.ts @@ -4,7 +4,7 @@ //import * as io from "./io"; import { Reader, Writer, ReadResult } from "deno"; -import { assert } from "./util.ts"; +import { assert, copyBytes } from "./util.ts"; // MIN_READ is the minimum ArrayBuffer size passed to a read call by // buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond @@ -13,18 +13,6 @@ import { assert } from "./util.ts"; const MIN_READ = 512; const MAX_SIZE = 2 ** 32 - 2; -// `off` is the offset into `dst` where it will at which to begin writing values -// from `src`. -// Returns the number of bytes copied. -function copyBytes(dst: Uint8Array, src: Uint8Array, off = 0): number { - const r = dst.byteLength - off; - if (src.byteLength > r) { - src = src.subarray(0, r); - } - dst.set(src, off); - return src.byteLength; -} - /** A Buffer is a variable-sized buffer of bytes with read() and write() * methods. Based on https://golang.org/pkg/bytes/#Buffer */ diff --git a/buffer_test.ts b/buffer_test.ts index c614b2e03e..8365b29bd4 100644 --- a/buffer_test.ts +++ b/buffer_test.ts @@ -1,8 +1,12 @@ // This code has been ported almost directly from Go's src/bytes/buffer_test.go // Copyright 2009 The Go Authors. All rights reserved. BSD license. // https://github.com/golang/go/blob/master/LICENSE -import { test, assert, assertEqual } from "./test_util.ts"; -import { Buffer } from "deno"; +import { + test, + assert, + assertEqual +} from "http://deno.land/x/testing/testing.ts"; +import { Buffer } from "./buffer.ts"; // N controls how many iterations of certain checks are performed. const N = 100; @@ -13,7 +17,7 @@ function init() { if (testBytes == null) { testBytes = new Uint8Array(N); for (let i = 0; i < N; i++) { - testBytes[i] = "a".charCodeAt(0) + (i % 26); + testBytes[i] = "a".charCodeAt(0) + i % 26; } const decoder = new TextDecoder(); testString = decoder.decode(testBytes); diff --git a/bufio.ts b/bufio.ts index dc1a2095d9..805a8136ed 100644 --- a/bufio.ts +++ b/bufio.ts @@ -1,9 +1,22 @@ +// Ported to Deno from: +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + import * as deno from "deno"; +import { assert, copyBytes } from "./util.ts"; const DEFAULT_BUF_SIZE = 4096; const MIN_BUF_SIZE = 16; const MAX_CONSECUTIVE_EMPTY_READS = 100; +export class ErrNegativeRead extends Error { + constructor() { + super("bufio: reader returned negative count from Read"); + this.name = "ErrNegativeRead"; + } +} + export class Reader implements deno.Reader { private buf: Uint8Array; private rd: deno.Reader; // Reader provided by caller. @@ -16,7 +29,7 @@ export class Reader implements deno.Reader { if (size < MIN_BUF_SIZE) { size = MIN_BUF_SIZE; } - this._reset(new Uint8Array(size), rd) + this._reset(new Uint8Array(size), rd); } /** Returns the size of the underlying buffer in bytes. */ @@ -42,7 +55,7 @@ export class Reader implements deno.Reader { for (let i = MAX_CONSECUTIVE_EMPTY_READS; i > 0; i--) { const { nread, eof } = await this.rd.read(this.buf.subarray(this.w)); if (nread < 0) { - throw Error("negative read"); + throw new ErrNegativeRead(); } this.w += nread; if (eof) { @@ -69,9 +82,58 @@ export class Reader implements deno.Reader { this.lastCharSize = -1; } + /** reads data into p. + * It returns the number of bytes read into p. + * The bytes are taken from at most one Read on the underlying Reader, + * hence n may be less than len(p). + * At EOF, the count will be zero and err will be io.EOF. + * To read exactly len(p) bytes, use io.ReadFull(b, p). + */ async read(p: ArrayBufferView): Promise { - throw Error("not implemented"); - return { nread: 0, eof: false }; + let rr: deno.ReadResult = { nread: p.byteLength, eof: false }; + if (rr.nread === 0) { + return rr; + } + + if (this.r === this.w) { + /* + if (this.err != null) { + throw this.readErr(); + } + */ + if (p.byteLength >= this.buf.byteLength) { + // Large read, empty buffer. + // Read directly into p to avoid copy. + rr = await this.rd.read(p); + if (rr.nread < 0) { + throw new ErrNegativeRead(); + } + if (rr.nread > 0) { + this.lastByte = p[rr.nread - 1]; + // this.lastRuneSize = -1; + } + return rr; + } + // One read. + // Do not use this.fill, which will loop. + this.r = 0; + this.w = 0; + rr = await this.rd.read(this.buf); + if (rr.nread < 0) { + throw new ErrNegativeRead(); + } + if (rr.nread === 0) { + return rr; + } + this.w += rr.nread; + } + + // copy as much as we can + rr.nread = copyBytes(p as Uint8Array, this.buf.subarray(this.r, this.w), 0); + this.r += rr.nread; + this.lastByte = this.buf[this.r - 1]; + // this.lastRuneSize = -1; + return rr; } /** Returns the next byte [0, 255] or -1 if EOF. */ @@ -88,5 +150,3 @@ export class Reader implements deno.Reader { return c; } } - - diff --git a/bufio_test.ts b/bufio_test.ts index a66052ba16..1600852afd 100644 --- a/bufio_test.ts +++ b/bufio_test.ts @@ -1,3 +1,8 @@ +// Ported to Deno from: +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + import * as deno from "deno"; import { test, assertEqual } from "http://deno.land/x/testing/testing.ts"; import * as bufio from "./bufio.ts"; @@ -30,3 +35,78 @@ test(async function bufioReaderSimple() { const s = await readBytes(b); assertEqual(s, data); }); + +type ReadMaker = { name: string; fn: (r: deno.Reader) => deno.Reader }; + +const readMakers: ReadMaker[] = [ + { name: "full", fn: r => r } + /* + { name: "byte", fn(r) => new iotest.OneByteReader(r) }, + { name: "half", fn(r) => new iotest.HalfReader(r) }, + { name: "data+err", r => new iotest.DataErrReader(r) }, + { name: "timeout", r => new iotest.TimeoutReader(r) }, + */ +]; + +// Call read to accumulate the text of a file +async function reads(buf: bufio.Reader, m: number): Promise { + const b = new Uint8Array(1000); + let nb = 0; + while (true) { + const { nread, eof } = await buf.read(b.subarray(nb, nb + m)); + nb += nread; + if (eof) { + break; + } + } + const decoder = new TextDecoder(); + return decoder.decode(b.subarray(0, nb)); +} + +type BufReader = { name: string; fn: (r: bufio.Reader) => Promise }; + +const bufreaders: BufReader[] = [ + { name: "1", fn: (b: bufio.Reader) => reads(b, 1) } +]; + +const MIN_READ_BUFFER_SIZE = 16; +const bufsizes: number[] = [ + 0, + MIN_READ_BUFFER_SIZE, + 23, + 32, + 46, + 64, + 93, + 128, + 1024, + 4096 +]; + +test(async function bufioReader() { + const texts = new Array(31); + let str = ""; + let all = ""; + for (let i = 0; i < texts.length - 1; i++) { + texts[i] = str + "\n"; + all += texts[i]; + str += String.fromCharCode(i % 26 + 97); + } + texts[texts.length - 1] = all; + + for (let text of texts) { + for (let readmaker of readMakers) { + for (let bufreader of bufreaders) { + for (let bufsize of bufsizes) { + const read = readmaker.fn(stringsReader(text)); + const buf = new bufio.Reader(read, bufsize); + const s = await bufreader.fn(buf); + const debugStr = + `reader=${readmaker.name} ` + + `fn=${bufreader.name} bufsize=${bufsize} want=${text} got=${s}`; + assertEqual(s, text, debugStr); + } + } + } + } +}); diff --git a/file_server.ts b/file_server.ts index 9d3d5366ea..d5deccf3c4 100644 --- a/file_server.ts +++ b/file_server.ts @@ -4,12 +4,12 @@ import { open, cwd } from "deno"; const addr = "0.0.0.0:4500"; const d = cwd(); -listenAndServe(addr, async (req) => { +listenAndServe(addr, async req => { const filename = d + "/" + req.url; let res; try { res = { status: 200, body: open(filename) }; - } catch(e) { + } catch (e) { res = { status: 500, body: "bad" }; } req.respond(res); diff --git a/http.ts b/http.ts index c8b7d8d904..4c209153a9 100644 --- a/http.ts +++ b/http.ts @@ -55,12 +55,12 @@ function readRequest(b: bufio.Reader): ServerRequest { // First line: GET /index.html HTTP/1.0 const s = await tp.readLine(); - const [ method, url, proto ] = parseRequestLine(s); + const [method, url, proto] = parseRequestLine(s); console.log("readRequest", method, url); } // Returns [method, url, proto] -function parseRequestLine(line: string): [ string, string, string ] { +function parseRequestLine(line: string): [string, string, string] { return line.split(" ", 3); } @@ -69,4 +69,3 @@ export function listen(addr: string): Server { const s = new Server(listener); return s; } - diff --git a/util.ts b/util.ts index decf4d043f..84ef8b5a8b 100644 --- a/util.ts +++ b/util.ts @@ -1,6 +1,17 @@ - export function assert(cond: boolean, msg = "assert") { if (!cond) { throw Error(msg); } } + +// `off` is the offset into `dst` where it will at which to begin writing values +// from `src`. +// Returns the number of bytes copied. +export function copyBytes(dst: Uint8Array, src: Uint8Array, off = 0): number { + const r = dst.byteLength - off; + if (src.byteLength > r) { + src = src.subarray(0, r); + } + dst.set(src, off); + return src.byteLength; +}