diff --git a/Makefile b/Makefile index b94f157cc7..4b84880939 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ test: deno test.ts -.PHONY: test +fmt: + prettier *.ts --write + +.PHONY: test fmt diff --git a/bufio.ts b/bufio.ts index 06f2eb4320..e571937344 100644 --- a/bufio.ts +++ b/bufio.ts @@ -4,17 +4,18 @@ // license that can be found in the LICENSE file. import { Reader, ReadResult } from "deno"; -import { assert, copyBytes } from "./util.ts"; +import { assert, charCode, copyBytes } from "./util.ts"; const DEFAULT_BUF_SIZE = 4096; const MIN_BUF_SIZE = 16; const MAX_CONSECUTIVE_EMPTY_READS = 100; +const CR = charCode("\r"); +const LF = charCode("\n"); -export class ErrNegativeRead extends Error { - constructor() { - super("bufio: reader returned negative count from Read"); - this.name = "ErrNegativeRead"; - } +export enum BufState { + Ok, + EOF, + BufferFull } /** BufReader implements buffering for a Reader object. */ @@ -51,7 +52,7 @@ export class BufReader implements Reader { // Reads a new chunk into the buffer. // Returns true if EOF, false on successful read. - private async _fill(): Promise { + private async _fill(): Promise { // Slide existing data to beginning. if (this.r > 0) { this.buf.copyWithin(0, this.r, this.w); @@ -70,17 +71,15 @@ export class BufReader implements Reader { rr = await this.rd.read(this.buf.subarray(this.w)); } catch (e) { this.err = e; - return false; - } - if (rr.nread < 0) { - throw new ErrNegativeRead(); + return BufState.Ok; } + assert(rr.nread >= 0, "negative read"); this.w += rr.nread; if (rr.eof) { - return true; + return BufState.EOF; } if (rr.nread > 0) { - return false; + return BufState.Ok; } } throw Error("No Progress"); @@ -124,9 +123,7 @@ export class BufReader implements Reader { // Large read, empty buffer. // Read directly into p to avoid copy. rr = await this.rd.read(p); - if (rr.nread < 0) { - throw new ErrNegativeRead(); - } + assert(rr.nread >= 0, "negative read"); if (rr.nread > 0) { this.lastByte = p[rr.nread - 1]; // this.lastRuneSize = -1; @@ -140,10 +137,12 @@ export class BufReader implements Reader { // Do not use this.fill, which will loop. this.r = 0; this.w = 0; - rr = await this.rd.read(this.buf); - if (rr.nread < 0) { - throw new ErrNegativeRead(); + try { + rr = await this.rd.read(this.buf); + } catch (e) { + this.err = e; } + assert(rr.nread >= 0, "negative read"); if (rr.nread === 0) { if (this.err) { throw this._readErr(); @@ -189,4 +188,115 @@ export class BufReader implements Reader { async readString(delim: string): Promise { throw new Error("Not implemented"); } + + /** readLine() is a low-level line-reading primitive. Most callers should use + * readBytes('\n') or readString('\n') instead or use a Scanner. + * + * readLine tries to return a single line, not including the end-of-line bytes. + * If the line was too long for the buffer then isPrefix is set and the + * beginning of the line is returned. The rest of the line will be returned + * from future calls. isPrefix will be false when returning the last fragment + * of the line. The returned buffer is only valid until the next call to + * ReadLine. ReadLine either returns a non-nil line or it returns an error, + * never both. + * + * The text returned from ReadLine does not include the line end ("\r\n" or "\n"). + * No indication or error is given if the input ends without a final line end. + * Calling UnreadByte after ReadLine will always unread the last byte read + * (possibly a character belonging to the line end) even if that byte is not + * part of the line returned by ReadLine. + */ + async readLine(): Promise<{ + line?: Uint8Array; + isPrefix: boolean; + state: BufState; + }> { + let line: Uint8Array; + let state: BufState; + try { + [line, state] = await this.readSlice(LF); + } catch (err) { + this.err = err; + } + + if (state === BufState.BufferFull) { + // Handle the case where "\r\n" straddles the buffer. + if (line.byteLength > 0 && line[line.byteLength - 1] === CR) { + // Put the '\r' back on buf and drop it from line. + // Let the next call to ReadLine check for "\r\n". + assert(this.r > 0, "bufio: tried to rewind past start of buffer"); + this.r--; + line = line.subarray(0, line.byteLength - 1); + } + return { line, isPrefix: true, state }; + } + + if (line.byteLength === 0) { + return { line, isPrefix: false, state }; + } + + if (line[line.byteLength - 1] == LF) { + let drop = 1; + if (line.byteLength > 1 && line[line.byteLength - 2] === CR) { + drop = 2; + } + line = line.subarray(0, line.byteLength - drop); + } + return { line, isPrefix: false, state }; + } + + /** readSlice() reads until the first occurrence of delim in the input, + * returning a slice pointing at the bytes in the buffer. The bytes stop + * being valid at the next read. If readSlice() encounters an error before + * finding a delimiter, it returns all the data in the buffer and the error + * itself (often io.EOF). readSlice() fails with error ErrBufferFull if the + * buffer fills without a delim. Because the data returned from readSlice() + * will be overwritten by the next I/O operation, most clients should use + * readBytes() or readString() instead. readSlice() returns err != nil if and + * only if line does not end in delim. + */ + async readSlice(delim: number): Promise<[Uint8Array, BufState]> { + let s = 0; // search start index + let line: Uint8Array; + let state = BufState.Ok; + while (true) { + // Search buffer. + let i = this.buf.subarray(this.r + s, this.w).indexOf(delim); + if (i >= 0) { + i += s; + line = this.buf.subarray(this.r, this.r + i + 1); + this.r += i + 1; + break; + } + + // Pending error? + if (this.err) { + line = this.buf.subarray(this.r, this.w); + this.r = this.w; + throw this._readErr(); + break; + } + + // Buffer full? + if (this.buffered() >= this.buf.byteLength) { + this.r = this.w; + line = this.buf; + state = BufState.BufferFull; + break; + } + + s = this.w - this.r; // do not rescan area we scanned before + + await this._fill(); // buffer is not full + } + + // Handle last byte, if any. + let i = line.byteLength - 1; + if (i >= 0) { + this.lastByte = line[i]; + // this.lastRuneSize = -1 + } + + return [line, state]; + } } diff --git a/bufio_test.ts b/bufio_test.ts index 85be74fdc4..9a3361d6f2 100644 --- a/bufio_test.ts +++ b/bufio_test.ts @@ -5,9 +5,10 @@ import * as deno from "deno"; import { test, assertEqual } from "http://deno.land/x/testing/testing.ts"; -import { BufReader } from "./bufio.ts"; +import { BufReader, BufState } from "./bufio.ts"; import { Buffer } from "./buffer.ts"; import * as iotest from "./iotest.ts"; +import { charCode } from "./util.ts"; async function readBytes(buf: BufReader): Promise { const b = new Uint8Array(1000); @@ -42,7 +43,7 @@ type ReadMaker = { name: string; fn: (r: deno.Reader) => deno.Reader }; const readMakers: ReadMaker[] = [ { name: "full", fn: r => r }, { name: "byte", fn: r => new iotest.OneByteReader(r) }, - { name: "half", fn: r => new iotest.HalfReader(r) }, + { name: "half", fn: r => new iotest.HalfReader(r) } // TODO { name: "data+err", r => new iotest.DataErrReader(r) }, // { name: "timeout", fn: r => new iotest.TimeoutReader(r) }, ]; @@ -50,7 +51,7 @@ const readMakers: ReadMaker[] = [ function readLines(b: BufReader): string { let s = ""; while (true) { - let s1 = b.readString('\n'); + let s1 = b.readString("\n"); if (s1 == null) { break; // EOF } @@ -83,7 +84,7 @@ const bufreaders: NamedBufReader[] = [ { name: "4", fn: (b: BufReader) => reads(b, 4) }, { name: "5", fn: (b: BufReader) => reads(b, 5) }, { name: "7", fn: (b: BufReader) => reads(b, 7) }, - { name: "bytes", fn: readBytes }, + { name: "bytes", fn: readBytes } // { name: "lines", fn: readLines }, ]; @@ -128,3 +129,20 @@ test(async function bufioBufReader() { } } }); + +test(async function bufioBufferFull() { + const longString = + "And now, hello, world! It is the time for all good men to come to the aid of their party"; + const buf = new BufReader(stringsReader(longString), MIN_READ_BUFFER_SIZE); + let [line, state] = await buf.readSlice(charCode("!")); + + const decoder = new TextDecoder(); + let actual = decoder.decode(line); + assertEqual(state, BufState.BufferFull); + assertEqual(actual, "And now, hello, "); + + [line, state] = await buf.readSlice(charCode("!")); + actual = decoder.decode(line); + assertEqual(actual, "world!"); + assertEqual(state, BufState.Ok); +}); diff --git a/util.ts b/util.ts index 84ef8b5a8b..d05bea8c04 100644 --- a/util.ts +++ b/util.ts @@ -15,3 +15,7 @@ export function copyBytes(dst: Uint8Array, src: Uint8Array, off = 0): number { dst.set(src, off); return src.byteLength; } + +export function charCode(s: string): number { + return s.charCodeAt(0); +}