clean up textproto code in std (#4458)

- moved and renamed append() into bytes from ws and textproto
- renamed textproto/readder_tests.ts -> textproto/test.ts

std/bytes/mod.ts

@@ -7,8 +7,8 @@ export function findIndex(a: Uint8Array, pat: Uint8Array): number {
   for (let i = 0; i < a.length; i++) {
     if (a[i] !== s) continue;
     const pin = i;
-    let matched = 1,
+    let matched = 1;
-      j = i;
+    let j = i;
     while (matched < pat.length) {
       j++;
       if (a[j] !== pat[j - pin]) {
@@ -29,8 +29,8 @@ export function findLastIndex(a: Uint8Array, pat: Uint8Array): number {
   for (let i = a.length - 1; i >= 0; i--) {
     if (a[i] !== e) continue;
     const pin = i;
-    let matched = 1,
+    let matched = 1;
-      j = i;
+    let j = i;
     while (matched < pat.length) {
       j--;
       if (a[j] !== pat[pat.length - 1 - (pin - j)]) {
@@ -94,3 +94,11 @@ export function repeat(b: Uint8Array, count: number): Uint8Array {
 
   return nb;
 }
+
+/** Concatenate two binary arrays and return new one  */
+export function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
+  const output = new Uint8Array(a.length + b.length);
+  output.set(a, 0);
+  output.set(b, a.length);
+  return output;
+}

std/bytes/test.ts

@@ -1,9 +1,17 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { findIndex, findLastIndex, equal, hasPrefix, repeat } from "./mod.ts";
+import {
-import { assertEquals, assertThrows } from "../testing/asserts.ts";
+  findIndex,
+  findLastIndex,
+  equal,
+  hasPrefix,
+  repeat,
+  concat
+} from "./mod.ts";
+import { assertEquals, assertThrows, assert } from "../testing/asserts.ts";
+import { encode, decode } from "../strings/mod.ts";
 
-Deno.test(function bytesfindIndex1(): void {
+Deno.test("[bytes] findIndex1", () => {
   const i = findIndex(
     new Uint8Array([1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 3]),
     new Uint8Array([0, 1, 2])
@@ -11,12 +19,12 @@ Deno.test(function bytesfindIndex1(): void {
   assertEquals(i, 2);
 });
 
-Deno.test(function bytesfindIndex2(): void {
+Deno.test("[bytes] findIndex2", () => {
   const i = findIndex(new Uint8Array([0, 0, 1]), new Uint8Array([0, 1]));
   assertEquals(i, 1);
 });
 
-Deno.test(function bytesfindLastIndex1(): void {
+Deno.test("[bytes] findLastIndex1", () => {
   const i = findLastIndex(
     new Uint8Array([0, 1, 2, 0, 1, 2, 0, 1, 3]),
     new Uint8Array([0, 1, 2])
@@ -24,22 +32,22 @@ Deno.test(function bytesfindLastIndex1(): void {
   assertEquals(i, 3);
 });
 
-Deno.test(function bytesfindLastIndex2(): void {
+Deno.test("[bytes] findLastIndex2", () => {
   const i = findLastIndex(new Uint8Array([0, 1, 1]), new Uint8Array([0, 1]));
   assertEquals(i, 0);
 });
 
-Deno.test(function bytesBytesequal(): void {
+Deno.test("[bytes] equal", () => {
   const v = equal(new Uint8Array([0, 1, 2, 3]), new Uint8Array([0, 1, 2, 3]));
   assertEquals(v, true);
 });
 
-Deno.test(function byteshasPrefix(): void {
+Deno.test("[bytes] hasPrefix", () => {
   const v = hasPrefix(new Uint8Array([0, 1, 2]), new Uint8Array([0, 1]));
   assertEquals(v, true);
 });
 
-Deno.test(function bytesrepeat(): void {
+Deno.test("[bytes] repeat", () => {
   // input / output / count / error message
   const repeatTestCase = [
     ["", "", 0],
@@ -71,3 +79,21 @@ Deno.test(function bytesrepeat(): void {
     }
   }
 });
+
+Deno.test("[bytes] concat", () => {
+  const u1 = encode("Hello ");
+  const u2 = encode("World");
+  const joined = concat(u1, u2);
+  assertEquals(decode(joined), "Hello World");
+  assert(u1 !== joined);
+  assert(u2 !== joined);
+});
+
+Deno.test("[bytes] concat empty arrays", () => {
+  const u1 = new Uint8Array();
+  const u2 = new Uint8Array();
+  const joined = concat(u1, u2);
+  assertEquals(joined.byteLength, 0);
+  assert(u1 !== joined);
+  assert(u2 !== joined);
+});

std/textproto/mod.ts

@@ -5,24 +5,14 @@
 
 import { BufReader } from "../io/bufio.ts";
 import { charCode } from "../io/util.ts";
+import { concat } from "../bytes/mod.ts";
+import { decode } from "../strings/mod.ts";
 
-const asciiDecoder = new TextDecoder();
 function str(buf: Uint8Array | null | undefined): string {
   if (buf == null) {
     return "";
   } else {
-    return asciiDecoder.decode(buf);
+    return decode(buf);
-  }
-}
-
-export function append(a: Uint8Array, b: Uint8Array): Uint8Array {
-  if (a == null) {
-    return b;
-  } else {
-    const output = new Uint8Array(a.length + b.length);
-    output.set(a, 0);
-    output.set(b, a.length);
-    return output;
   }
 }
 
@@ -146,9 +136,7 @@ export class TextProtoReader {
         }
         return l;
       }
-
+      line = line ? concat(line, l) : l;
-      // @ts-ignore
-      line = append(line, l);
       if (!more) {
         break;
       }

std/textproto/reader_test.ts

@@ -1,180 +0,0 @@
-// Based on https://github.com/golang/go/blob/master/src/net/textproto/reader_test.go
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-import { BufReader } from "../io/bufio.ts";
-import { TextProtoReader } from "./mod.ts";
-import { stringsReader } from "../io/util.ts";
-import {
-  assert,
-  assertEquals,
-  assertThrows,
-  assertNotEOF
-} from "../testing/asserts.ts";
-const { test } = Deno;
-
-function reader(s: string): TextProtoReader {
-  return new TextProtoReader(new BufReader(stringsReader(s)));
-}
-
-test({
-  ignore: true,
-  name: "[textproto] Reader : DotBytes",
-  fn(): Promise<void> {
-    const _input =
-      "dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n";
-    return Promise.resolve();
-  }
-});
-
-test(async function textprotoReadEmpty(): Promise<void> {
-  const r = reader("");
-  const m = await r.readMIMEHeader();
-  assertEquals(m, Deno.EOF);
-});
-
-test(async function textprotoReader(): Promise<void> {
-  const r = reader("line1\nline2\n");
-  let s = await r.readLine();
-  assertEquals(s, "line1");
-
-  s = await r.readLine();
-  assertEquals(s, "line2");
-
-  s = await r.readLine();
-  assert(s === Deno.EOF);
-});
-
-test({
-  name: "[textproto] Reader : MIME Header",
-  async fn(): Promise<void> {
-    const input =
-      "my-key: Value 1  \r\nLong-key: Even Longer Value\r\nmy-Key: " +
-      "Value 2\r\n\n";
-    const r = reader(input);
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("My-Key"), "Value 1, Value 2");
-    assertEquals(m.get("Long-key"), "Even Longer Value");
-  }
-});
-
-test({
-  name: "[textproto] Reader : MIME Header Single",
-  async fn(): Promise<void> {
-    const input = "Foo: bar\n\n";
-    const r = reader(input);
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("Foo"), "bar");
-  }
-});
-
-test({
-  name: "[textproto] Reader : MIME Header No Key",
-  async fn(): Promise<void> {
-    const input = ": bar\ntest-1: 1\n\n";
-    const r = reader(input);
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("Test-1"), "1");
-  }
-});
-
-test({
-  name: "[textproto] Reader : Large MIME Header",
-  async fn(): Promise<void> {
-    const data: string[] = [];
-    // Go test is 16*1024. But seems it can't handle more
-    for (let i = 0; i < 1024; i++) {
-      data.push("x");
-    }
-    const sdata = data.join("");
-    const r = reader(`Cookie: ${sdata}\r\n\r\n`);
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("Cookie"), sdata);
-  }
-});
-
-// Test that we read slightly-bogus MIME headers seen in the wild,
-// with spaces before colons, and spaces in keys.
-test({
-  name: "[textproto] Reader : MIME Header Non compliant",
-  async fn(): Promise<void> {
-    const input =
-      "Foo: bar\r\n" +
-      "Content-Language: en\r\n" +
-      "SID : 0\r\n" +
-      "Audio Mode : None\r\n" +
-      "Privilege : 127\r\n\r\n";
-    const r = reader(input);
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("Foo"), "bar");
-    assertEquals(m.get("Content-Language"), "en");
-    assertEquals(m.get("SID"), "0");
-    assertEquals(m.get("Privilege"), "127");
-    // Not a legal http header
-    assertThrows((): void => {
-      assertEquals(m.get("Audio Mode"), "None");
-    });
-  }
-});
-
-test({
-  name: "[textproto] Reader : MIME Header Malformed",
-  async fn(): Promise<void> {
-    const input = [
-      "No colon first line\r\nFoo: foo\r\n\r\n",
-      " No colon first line with leading space\r\nFoo: foo\r\n\r\n",
-      "\tNo colon first line with leading tab\r\nFoo: foo\r\n\r\n",
-      " First: line with leading space\r\nFoo: foo\r\n\r\n",
-      "\tFirst: line with leading tab\r\nFoo: foo\r\n\r\n",
-      "Foo: foo\r\nNo colon second line\r\n\r\n"
-    ];
-    const r = reader(input.join(""));
-
-    let err;
-    try {
-      await r.readMIMEHeader();
-    } catch (e) {
-      err = e;
-    }
-    assert(err instanceof Deno.errors.InvalidData);
-  }
-});
-
-test({
-  name: "[textproto] Reader : MIME Header Trim Continued",
-  async fn(): Promise<void> {
-    const input =
-      "" + // for code formatting purpose.
-      "a:\n" +
-      " 0 \r\n" +
-      "b:1 \t\r\n" +
-      "c: 2\r\n" +
-      " 3\t\n" +
-      "  \t 4  \r\n\n";
-    const r = reader(input);
-    let err;
-    try {
-      await r.readMIMEHeader();
-    } catch (e) {
-      err = e;
-    }
-    assert(err instanceof Deno.errors.InvalidData);
-  }
-});
-
-test({
-  name: "[textproto] #409 issue : multipart form boundary",
-  async fn(): Promise<void> {
-    const input = [
-      "Accept: */*\r\n",
-      'Content-Disposition: form-data; name="test"\r\n',
-      " \r\n",
-      "------WebKitFormBoundaryimeZ2Le9LjohiUiG--\r\n\n"
-    ];
-    const r = reader(input.join(""));
-    const m = assertNotEOF(await r.readMIMEHeader());
-    assertEquals(m.get("Accept"), "*/*");
-    assertEquals(m.get("Content-Disposition"), 'form-data; name="test"');
-  }
-});

std/textproto/test.ts

@@ -1,17 +1,180 @@
-// Based on https://github.com/golang/go/blob/891682/src/net/textproto/
+// Based on https://github.com/golang/go/blob/master/src/net/textproto/reader_test.go
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-import { append } from "./mod.ts";
+import { BufReader } from "../io/bufio.ts";
-import { assertEquals } from "../testing/asserts.ts";
+import { TextProtoReader } from "./mod.ts";
+import { stringsReader } from "../io/util.ts";
+import {
+  assert,
+  assertEquals,
+  assertThrows,
+  assertNotEOF
+} from "../testing/asserts.ts";
 const { test } = Deno;
 
-test(function textprotoAppend(): void {
+function reader(s: string): TextProtoReader {
-  const enc = new TextEncoder();
+  return new TextProtoReader(new BufReader(stringsReader(s)));
-  const dec = new TextDecoder();
+}
-  const u1 = enc.encode("Hello ");
+
-  const u2 = enc.encode("World");
+test({
-  const joined = append(u1, u2);
+  ignore: true,
-  assertEquals(dec.decode(joined), "Hello World");
+  name: "[textproto] Reader : DotBytes",
+  fn(): Promise<void> {
+    const _input =
+      "dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n";
+    return Promise.resolve();
+  }
+});
+
+test("[textproto] ReadEmpty", async () => {
+  const r = reader("");
+  const m = await r.readMIMEHeader();
+  assertEquals(m, Deno.EOF);
+});
+
+test("[textproto] Reader", async () => {
+  const r = reader("line1\nline2\n");
+  let s = await r.readLine();
+  assertEquals(s, "line1");
+
+  s = await r.readLine();
+  assertEquals(s, "line2");
+
+  s = await r.readLine();
+  assert(s === Deno.EOF);
+});
+
+test({
+  name: "[textproto] Reader : MIME Header",
+  async fn(): Promise<void> {
+    const input =
+      "my-key: Value 1  \r\nLong-key: Even Longer Value\r\nmy-Key: " +
+      "Value 2\r\n\n";
+    const r = reader(input);
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("My-Key"), "Value 1, Value 2");
+    assertEquals(m.get("Long-key"), "Even Longer Value");
+  }
+});
+
+test({
+  name: "[textproto] Reader : MIME Header Single",
+  async fn(): Promise<void> {
+    const input = "Foo: bar\n\n";
+    const r = reader(input);
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("Foo"), "bar");
+  }
+});
+
+test({
+  name: "[textproto] Reader : MIME Header No Key",
+  async fn(): Promise<void> {
+    const input = ": bar\ntest-1: 1\n\n";
+    const r = reader(input);
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("Test-1"), "1");
+  }
+});
+
+test({
+  name: "[textproto] Reader : Large MIME Header",
+  async fn(): Promise<void> {
+    const data: string[] = [];
+    // Go test is 16*1024. But seems it can't handle more
+    for (let i = 0; i < 1024; i++) {
+      data.push("x");
+    }
+    const sdata = data.join("");
+    const r = reader(`Cookie: ${sdata}\r\n\r\n`);
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("Cookie"), sdata);
+  }
+});
+
+// Test that we read slightly-bogus MIME headers seen in the wild,
+// with spaces before colons, and spaces in keys.
+test({
+  name: "[textproto] Reader : MIME Header Non compliant",
+  async fn(): Promise<void> {
+    const input =
+      "Foo: bar\r\n" +
+      "Content-Language: en\r\n" +
+      "SID : 0\r\n" +
+      "Audio Mode : None\r\n" +
+      "Privilege : 127\r\n\r\n";
+    const r = reader(input);
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("Foo"), "bar");
+    assertEquals(m.get("Content-Language"), "en");
+    assertEquals(m.get("SID"), "0");
+    assertEquals(m.get("Privilege"), "127");
+    // Not a legal http header
+    assertThrows((): void => {
+      assertEquals(m.get("Audio Mode"), "None");
+    });
+  }
+});
+
+test({
+  name: "[textproto] Reader : MIME Header Malformed",
+  async fn(): Promise<void> {
+    const input = [
+      "No colon first line\r\nFoo: foo\r\n\r\n",
+      " No colon first line with leading space\r\nFoo: foo\r\n\r\n",
+      "\tNo colon first line with leading tab\r\nFoo: foo\r\n\r\n",
+      " First: line with leading space\r\nFoo: foo\r\n\r\n",
+      "\tFirst: line with leading tab\r\nFoo: foo\r\n\r\n",
+      "Foo: foo\r\nNo colon second line\r\n\r\n"
+    ];
+    const r = reader(input.join(""));
+
+    let err;
+    try {
+      await r.readMIMEHeader();
+    } catch (e) {
+      err = e;
+    }
+    assert(err instanceof Deno.errors.InvalidData);
+  }
+});
+
+test({
+  name: "[textproto] Reader : MIME Header Trim Continued",
+  async fn(): Promise<void> {
+    const input =
+      "" + // for code formatting purpose.
+      "a:\n" +
+      " 0 \r\n" +
+      "b:1 \t\r\n" +
+      "c: 2\r\n" +
+      " 3\t\n" +
+      "  \t 4  \r\n\n";
+    const r = reader(input);
+    let err;
+    try {
+      await r.readMIMEHeader();
+    } catch (e) {
+      err = e;
+    }
+    assert(err instanceof Deno.errors.InvalidData);
+  }
+});
+
+test({
+  name: "[textproto] #409 issue : multipart form boundary",
+  async fn(): Promise<void> {
+    const input = [
+      "Accept: */*\r\n",
+      'Content-Disposition: form-data; name="test"\r\n',
+      " \r\n",
+      "------WebKitFormBoundaryimeZ2Le9LjohiUiG--\r\n\n"
+    ];
+    const r = reader(input.join(""));
+    const m = assertNotEOF(await r.readMIMEHeader());
+    assertEquals(m.get("Accept"), "*/*");
+    assertEquals(m.get("Content-Disposition"), 'form-data; name="test"');
+  }
 });

std/ws/mod.ts

@@ -9,6 +9,7 @@ import { writeResponse } from "../http/io.ts";
 import { TextProtoReader } from "../textproto/mod.ts";
 import { Deferred, deferred } from "../util/async.ts";
 import { assertNotEOF } from "../testing/asserts.ts";
+import { concat } from "../bytes/mod.ts";
 import Conn = Deno.Conn;
 import Writer = Deno.Writer;
 
@@ -57,20 +58,6 @@ export function isWebSocketPongEvent(
 
 export type WebSocketMessage = string | Uint8Array;
 
-// TODO move this to common/util module
-export function append(a: Uint8Array, b: Uint8Array): Uint8Array {
-  if (a == null || !a.length) {
-    return b;
-  }
-  if (b == null || !b.length) {
-    return a;
-  }
-  const output = new Uint8Array(a.length + b.length);
-  output.set(a, 0);
-  output.set(b, a.length);
-  return output;
-}
-
 export interface WebSocketFrame {
   isLastFrame: boolean;
   opcode: OpCode;
@@ -148,10 +135,10 @@ export async function writeFrame(
     ]);
   }
   if (frame.mask) {
-    header = append(header, frame.mask);
+    header = concat(header, frame.mask);
   }
   unmask(frame.payload, frame.mask);
-  header = append(header, frame.payload);
+  header = concat(header, frame.payload);
   const w = BufWriter.create(writer);
   await w.write(header);
   await w.flush();