foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-11 16:42:21 -05:00
Code Issues Packages 1 Wiki Activity

port ops to JSON: compiler, errors, fetch, files (#2804)

Browse source
This commit is contained in:
Bartek Iwańczuk 2019-08-24 15:02:42 +02:00 committed by Ryan Dahl
parent 5b2baa5c99
commit 79f82cf10e
20 changed files with 320 additions and 637 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cli/main.rs
View file

@ -32,7 +32,6 @@ mod http_body;
mod http_util;
mod import_map;
pub mod msg;
pub mod msg_util;
pub mod ops;
pub mod permissions;
mod progress;

75
cli/msg.fbs
View file

@ -1,23 +1,14 @@
union Any {
Accept,
ApplySourceMap,
Cache,
Chdir,
Chmod,
Chown,
Close,
CopyFile,
CreateWorker,
CreateWorkerRes,
Cwd,
CwdRes,
Dial,
Fetch,
FetchSourceFile,
FetchSourceFileRes,
FetchRes,
FormatError,
FormatErrorRes,
GetRandomValues,
GlobalTimer,
GlobalTimerRes,
@ -38,8 +29,6 @@ union Any {
NewConn,
Now,
NowRes,
Open,
OpenRes,
PermissionRevoke,
Permissions,
PermissionsRes,
@ -215,33 +204,6 @@ table WorkerPostMessage {
// data passed thru the zero-copy data parameter.
}
table FetchSourceFile {
specifier: string;
referrer: string;
}
table FetchSourceFileRes {
// If it's a non-http module, moduleName and filename will be the same.
// For http modules, module_name is its resolved http URL, and filename
// is the location of the locally downloaded source code.
module_name: string;
filename: string;
media_type: MediaType;
data: [ubyte];
}
table ApplySourceMap {
filename: string;
line: int;
column: int;
}
table Cache {
extension: string;
module_id: string;
contents: string;
}
table Chdir {
directory: string;
}
@ -274,29 +236,6 @@ table PermissionsRes {
hrtime: bool;
}
// Note this represents The WHOLE header of an http message, not just the key
// value pairs. That means it includes method and url for Requests and status
// for responses. This is why it is singular "Header" instead of "Headers".
table HttpHeader {
is_request: bool;
// Request only:
method: string;
url: string;
// Response only:
status: uint16;
// Both:
fields: [KeyValue];
}
table Fetch {
header: HttpHeader;
}
table FetchRes {
header: HttpHeader;
body_rid: uint32;
}
table MakeTempDir {
dir: string;
prefix: string;
@ -416,16 +355,6 @@ table Truncate {
len: uint;
}
table Open {
filename: string;
perm: uint;
mode: string;
}
table OpenRes {
rid: uint32;
}
table Read {
rid: uint32;
// (ptr, len) is passed as second parameter to Deno.core.send().
@ -444,10 +373,6 @@ table WriteRes {
nbyte: uint;
}
table Close {
rid: uint32;
}
table Kill {
pid: int32;
signo: int32;

124
cli/msg_util.rs
View file

@ -1,124 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// Helpers for serialization.
use crate::msg;
use deno::ErrBox;
use flatbuffers;
use http::header::HeaderName;
use http::uri::Uri;
use http::Method;
use hyper::header::HeaderMap;
use hyper::header::HeaderValue;
use hyper::Body;
use hyper::Request;
use hyper::Response;
use std::str::FromStr;
type Headers = HeaderMap<HeaderValue>;
pub fn serialize_key_value<'bldr>(
builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
key: &str,
value: &str,
) -> flatbuffers::WIPOffset<msg::KeyValue<'bldr>> {
let key = builder.create_string(&key);
let value = builder.create_string(&value);
msg::KeyValue::create(
builder,
&msg::KeyValueArgs {
key: Some(key),
value: Some(value),
},
)
}
pub fn serialize_request_header<'bldr>(
builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
r: &Request<Body>,
) -> flatbuffers::WIPOffset<msg::HttpHeader<'bldr>> {
let method = builder.create_string(r.method().as_str());
let url = builder.create_string(r.uri().to_string().as_ref());
let mut fields = Vec::new();
for (key, val) in r.headers().iter() {
let kv = serialize_key_value(builder, key.as_ref(), val.to_str().unwrap());
fields.push(kv);
}
let fields = builder.create_vector(fields.as_ref());
msg::HttpHeader::create(
builder,
&msg::HttpHeaderArgs {
is_request: true,
method: Some(method),
url: Some(url),
fields: Some(fields),
..Default::default()
},
)
}
pub fn serialize_fields<'bldr>(
builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
headers: &Headers,
) -> flatbuffers::WIPOffset<
flatbuffers::Vector<
'bldr,
flatbuffers::ForwardsUOffset<msg::KeyValue<'bldr>>,
>,
> {
let mut fields = Vec::new();
for (key, val) in headers.iter() {
let kv = serialize_key_value(builder, key.as_ref(), val.to_str().unwrap());
fields.push(kv);
}
builder.create_vector(fields.as_ref())
}
// Not to be confused with serialize_response which has nothing to do with HTTP.
pub fn serialize_http_response<'bldr>(
builder: &mut flatbuffers::FlatBufferBuilder<'bldr>,
r: &Response<Body>,
) -> flatbuffers::WIPOffset<msg::HttpHeader<'bldr>> {
let status = r.status().as_u16();
let fields = serialize_fields(builder, r.headers());
msg::HttpHeader::create(
builder,
&msg::HttpHeaderArgs {
is_request: false,
status,
fields: Some(fields),
..Default::default()
},
)
}
pub fn deserialize_request(
header_msg: msg::HttpHeader<'_>,
body: Body,
) -> Result<Request<Body>, ErrBox> {
let mut r = Request::new(body);
assert!(header_msg.is_request());
let u = header_msg.url().unwrap();
let u = Uri::from_str(u).map_err(ErrBox::from)?;
*r.uri_mut() = u;
if let Some(method) = header_msg.method() {
let method = Method::from_str(method).unwrap();
*r.method_mut() = method;
}
if let Some(fields) = header_msg.fields() {
let headers = r.headers_mut();
for i in 0..fields.len() {
let kv = fields.get(i);
let key = kv.key().unwrap();
let name = HeaderName::from_bytes(key.as_bytes()).unwrap();
let value = kv.value().unwrap();
let v = HeaderValue::from_str(value).unwrap();
headers.insert(name, v);
}
}
Ok(r)
}

90
cli/ops/compiler.rs
View file

@ -1,86 +1,68 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use super::dispatch_flatbuffers::serialize_response;
use super::utils::*;
use crate::deno_error;
use crate::msg;
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::state::ThreadSafeState;
use crate::tokio_util;
use deno::*;
use flatbuffers::FlatBufferBuilder;
use futures::Future;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct CacheArgs {
module_id: String,
contents: String,
extension: String,
}
pub fn op_cache(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
assert!(data.is_none());
let inner = base.inner_as_cache().unwrap();
let extension = inner.extension().unwrap();
// TODO: rename to something with 'url'
let module_id = inner.module_id().unwrap();
let contents = inner.contents().unwrap();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: CacheArgs = serde_json::from_value(args)?;
let module_specifier = ModuleSpecifier::resolve_url(module_id)
let module_specifier = ModuleSpecifier::resolve_url(&args.module_id)
.expect("Should be valid module specifier");
state.ts_compiler.cache_compiler_output(
&module_specifier,
extension,
contents,
&args.extension,
&args.contents,
)?;
ok_buf(empty_buf())
Ok(JsonOp::Sync(json!({})))
}
#[derive(Deserialize)]
struct FetchSourceFileArgs {
specifier: String,
referrer: String,
}
pub fn op_fetch_source_file(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
if !base.sync() {
return Err(deno_error::no_async_support());
}
assert!(data.is_none());
let inner = base.inner_as_fetch_source_file().unwrap();
let cmd_id = base.cmd_id();
let specifier = inner.specifier().unwrap();
let referrer = inner.referrer().unwrap();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: FetchSourceFileArgs = serde_json::from_value(args)?;
// TODO(ry) Maybe a security hole. Only the compiler worker should have access
// to this. Need a test to demonstrate the hole.
let is_dyn_import = false;
let resolved_specifier =
state.resolve(specifier, referrer, false, is_dyn_import)?;
state.resolve(&args.specifier, &args.referrer, false, is_dyn_import)?;
let fut = state
.file_fetcher
.fetch_source_file_async(&resolved_specifier)
.and_then(move |out| {
let builder = &mut FlatBufferBuilder::new();
let data_off = builder.create_vector(out.source_code.as_slice());
let msg_args = msg::FetchSourceFileResArgs {
module_name: Some(builder.create_string(&out.url.to_string())),
filename: Some(builder.create_string(&out.filename.to_str().unwrap())),
media_type: out.media_type,
data: Some(data_off),
};
let inner = msg::FetchSourceFileRes::create(builder, &msg_args);
Ok(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(inner.as_union_value()),
inner_type: msg::Any::FetchSourceFileRes,
..Default::default()
},
))
});
.fetch_source_file_async(&resolved_specifier);
// WARNING: Here we use tokio_util::block_on() which starts a new Tokio
// runtime for executing the future. This is so we don't inadvernently run
// out of threads in the main runtime.
let result_buf = tokio_util::block_on(fut)?;
Ok(Op::Sync(result_buf))
let out = tokio_util::block_on(fut)?;
Ok(JsonOp::Sync(json!({
"moduleName": out.url.to_string(),
"filename": out.filename.to_str().unwrap(),
"mediaType": out.media_type as i32,
"sourceCode": String::from_utf8(out.source_code).unwrap(),
})))
}

13
cli/ops/dispatch_flatbuffers.rs
View file

@ -6,10 +6,7 @@ use deno::*;
use flatbuffers::FlatBufferBuilder;
use hyper::rt::Future;
use super::compiler::{op_cache, op_fetch_source_file};
use super::errors::{op_apply_source_map, op_format_error};
use super::fetch::op_fetch;
use super::files::{op_close, op_open, op_read, op_seek, op_write};
use super::files::{op_read, op_write};
use super::fs::{
op_chdir, op_chmod, op_chown, op_copy_file, op_cwd, op_link,
op_make_temp_dir, op_mkdir, op_read_dir, op_read_link, op_remove, op_rename,
@ -142,19 +139,13 @@ pub fn serialize_response(
pub fn op_selector_std(inner_type: msg::Any) -> Option<CliDispatchFn> {
match inner_type {
msg::Any::Accept => Some(op_accept),
msg::Any::ApplySourceMap => Some(op_apply_source_map),
msg::Any::Cache => Some(op_cache),
msg::Any::Chdir => Some(op_chdir),
msg::Any::Chmod => Some(op_chmod),
msg::Any::Chown => Some(op_chown),
msg::Any::Close => Some(op_close),
msg::Any::CopyFile => Some(op_copy_file),
msg::Any::CreateWorker => Some(op_create_worker),
msg::Any::Cwd => Some(op_cwd),
msg::Any::Dial => Some(op_dial),
msg::Any::Fetch => Some(op_fetch),
msg::Any::FetchSourceFile => Some(op_fetch_source_file),
msg::Any::FormatError => Some(op_format_error),
msg::Any::GetRandomValues => Some(op_get_random_values),
msg::Any::GlobalTimer => Some(op_global_timer),
msg::Any::GlobalTimerStop => Some(op_global_timer_stop),
@ -168,7 +159,6 @@ pub fn op_selector_std(inner_type: msg::Any) -> Option<CliDispatchFn> {
msg::Any::Metrics => Some(op_metrics),
msg::Any::Mkdir => Some(op_mkdir),
msg::Any::Now => Some(op_now),
msg::Any::Open => Some(op_open),
msg::Any::PermissionRevoke => Some(op_revoke_permission),
msg::Any::Permissions => Some(op_permissions),
msg::Any::Read => Some(op_read),
@ -181,7 +171,6 @@ pub fn op_selector_std(inner_type: msg::Any) -> Option<CliDispatchFn> {
msg::Any::Resources => Some(op_resources),
msg::Any::Run => Some(op_run),
msg::Any::RunStatus => Some(op_run_status),
msg::Any::Seek => Some(op_seek),
msg::Any::Shutdown => Some(op_shutdown),
msg::Any::Stat => Some(op_stat),
msg::Any::Symlink => Some(op_symlink),

96
cli/ops/errors.rs
View file

@ -1,88 +1,56 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use super::dispatch_flatbuffers::serialize_response;
use super::utils::*;
use crate::deno_error;
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::fmt_errors::JSError;
use crate::msg;
use crate::source_maps::get_orig_position;
use crate::source_maps::CachedMaps;
use crate::state::ThreadSafeState;
use deno::*;
use flatbuffers::FlatBufferBuilder;
use std::collections::HashMap;
#[derive(Deserialize)]
struct FormatErrorArgs {
error: String,
}
pub fn op_format_error(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
assert!(data.is_none());
let inner = base.inner_as_format_error().unwrap();
let json_str = inner.error().unwrap();
let error = JSError::from_json(json_str, &state.ts_compiler);
let error_string = error.to_string();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: FormatErrorArgs = serde_json::from_value(args)?;
let error = JSError::from_json(&args.error, &state.ts_compiler);
let mut builder = FlatBufferBuilder::new();
let new_error = builder.create_string(&error_string);
Ok(JsonOp::Sync(json!({
"error": error.to_string(),
})))
}
let inner = msg::FormatErrorRes::create(
&mut builder,
&msg::FormatErrorResArgs {
error: Some(new_error),
},
);
let response_buf = serialize_response(
base.cmd_id(),
&mut builder,
msg::BaseArgs {
inner_type: msg::Any::FormatErrorRes,
inner: Some(inner.as_union_value()),
..Default::default()
},
);
ok_buf(response_buf)
#[derive(Deserialize)]
struct ApplySourceMap {
filename: String,
line: i32,
column: i32,
}
pub fn op_apply_source_map(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
if !base.sync() {
return Err(deno_error::no_async_support());
}
assert!(data.is_none());
let inner = base.inner_as_apply_source_map().unwrap();
let cmd_id = base.cmd_id();
let filename = inner.filename().unwrap();
let line = inner.line();
let column = inner.column();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: ApplySourceMap = serde_json::from_value(args)?;
let mut mappings_map: CachedMaps = HashMap::new();
let (orig_filename, orig_line, orig_column) = get_orig_position(
filename.to_owned(),
line.into(),
column.into(),
args.filename,
args.line.into(),
args.column.into(),
&mut mappings_map,
&state.ts_compiler,
);
let builder = &mut FlatBufferBuilder::new();
let msg_args = msg::ApplySourceMapArgs {
filename: Some(builder.create_string(&orig_filename)),
line: orig_line as i32,
column: orig_column as i32,
};
let res_inner = msg::ApplySourceMap::create(builder, &msg_args);
ok_buf(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(res_inner.as_union_value()),
inner_type: msg::Any::ApplySourceMap,
..Default::default()
},
))
Ok(JsonOp::Sync(json!({
"filename": orig_filename.to_string(),
"line": orig_line as u32,
"column": orig_column as u32,
})))
}

87
cli/ops/fetch.rs
View file

@ -1,38 +1,57 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use super::dispatch_flatbuffers::serialize_response;
use super::utils::CliOpResult;
use super::dispatch_json::{Deserialize, JsonOp, Value};
use crate::http_util;
use crate::msg;
use crate::msg_util;
use crate::resources;
use crate::state::ThreadSafeState;
use deno::*;
use flatbuffers::FlatBufferBuilder;
use http::header::HeaderName;
use http::uri::Uri;
use http::Method;
use hyper;
use hyper::header::HeaderValue;
use hyper::rt::Future;
use hyper::Request;
use std;
use std::convert::From;
use std::str::FromStr;
#[derive(Deserialize)]
struct FetchArgs {
method: Option<String>,
url: String,
headers: Vec<(String, String)>,
}
pub fn op_fetch(
state: &ThreadSafeState,
base: &msg::Base<'_>,
args: Value,
data: Option<PinnedBuf>,
) -> CliOpResult {
let inner = base.inner_as_fetch().unwrap();
let cmd_id = base.cmd_id();
let header = inner.header().unwrap();
assert!(header.is_request());
let url = header.url().unwrap();
) -> Result<JsonOp, ErrBox> {
let args: FetchArgs = serde_json::from_value(args)?;
let url = args.url;
let body = match data {
None => hyper::Body::empty(),
Some(buf) => hyper::Body::from(Vec::from(&*buf)),
};
let req = msg_util::deserialize_request(header, body)?;
let mut req = Request::new(body);
let uri = Uri::from_str(&url).map_err(ErrBox::from)?;
*req.uri_mut() = uri;
let url_ = url::Url::parse(url).map_err(ErrBox::from)?;
if let Some(method) = args.method {
let method = Method::from_str(&method).unwrap();
*req.method_mut() = method;
}
let headers = req.headers_mut();
for header_pair in args.headers {
let name = HeaderName::from_bytes(header_pair.0.as_bytes()).unwrap();
let v = HeaderValue::from_str(&header_pair.1).unwrap();
headers.insert(name, v);
}
let url_ = url::Url::parse(&url).map_err(ErrBox::from)?;
state.check_net_url(&url_)?;
let client = http_util::get_client();
@ -42,32 +61,22 @@ pub fn op_fetch(
.request(req)
.map_err(ErrBox::from)
.and_then(move |res| {
let builder = &mut FlatBufferBuilder::new();
let header_off = msg_util::serialize_http_response(builder, &res);
let status = res.status().as_u16();
let mut res_headers = Vec::new();
for (key, val) in res.headers().iter() {
res_headers.push((key.to_string(), val.to_str().unwrap().to_owned()));
}
let body = res.into_body();
let body_resource = resources::add_hyper_body(body);
let inner = msg::FetchRes::create(
builder,
&msg::FetchResArgs {
header: Some(header_off),
body_rid: body_resource.rid,
},
);
Ok(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(inner.as_union_value()),
inner_type: msg::Any::FetchRes,
..Default::default()
},
))
let json_res = json!({
"bodyRid": body_resource.rid,
"status": status,
"headers": res_headers
});
if base.sync() {
let result_buf = future.wait()?;
Ok(Op::Sync(result_buf))
} else {
Ok(Op::Async(Box::new(future)))
}
futures::future::ok(json_res)
});
Ok(JsonOp::Async(Box::new(future)))
}

98
cli/ops/files.rs
View file

@ -1,5 +1,6 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use super::dispatch_flatbuffers::serialize_response;
use super::dispatch_json::{Deserialize, JsonOp, Value};
use super::utils::*;
use crate::deno_error;
use crate::fs as deno_fs;
@ -14,17 +15,22 @@ use std;
use std::convert::From;
use tokio;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct OpenArgs {
promise_id: Option<u64>,
filename: String,
mode: String,
}
pub fn op_open(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
assert!(data.is_none());
let cmd_id = base.cmd_id();
let inner = base.inner_as_open().unwrap();
let (filename, filename_) =
deno_fs::resolve_from_cwd(inner.filename().unwrap())?;
let mode = inner.mode().unwrap();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: OpenArgs = serde_json::from_value(args)?;
let (filename, filename_) = deno_fs::resolve_from_cwd(&args.filename)?;
let mode = args.mode.as_ref();
let mut open_options = tokio::fs::OpenOptions::new();
@ -75,44 +81,39 @@ pub fn op_open(
}
}
let is_sync = args.promise_id.is_none();
let op = open_options.open(filename).map_err(ErrBox::from).and_then(
move |fs_file| {
let resource = resources::add_fs_file(fs_file);
let builder = &mut FlatBufferBuilder::new();
let inner =
msg::OpenRes::create(builder, &msg::OpenResArgs { rid: resource.rid });
Ok(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(inner.as_union_value()),
inner_type: msg::Any::OpenRes,
..Default::default()
},
))
futures::future::ok(json!(resource.rid))
},
);
if base.sync() {
if is_sync {
let buf = op.wait()?;
Ok(Op::Sync(buf))
Ok(JsonOp::Sync(buf))
} else {
Ok(Op::Async(Box::new(op)))
Ok(JsonOp::Async(Box::new(op)))
}
}
#[derive(Deserialize)]
struct CloseArgs {
rid: i32,
}
pub fn op_close(
_state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
assert!(data.is_none());
let inner = base.inner_as_close().unwrap();
let rid = inner.rid();
match resources::lookup(rid) {
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: CloseArgs = serde_json::from_value(args)?;
match resources::lookup(args.rid as u32) {
None => Err(deno_error::bad_resource()),
Some(resource) => {
resource.close();
ok_buf(empty_buf())
Ok(JsonOp::Sync(json!({})))
}
}
}
@ -202,27 +203,32 @@ pub fn op_write(
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct SeekArgs {
promise_id: Option<u64>,
rid: i32,
offset: i32,
whence: i32,
}
pub fn op_seek(
_state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> CliOpResult {
assert!(data.is_none());
let inner = base.inner_as_seek().unwrap();
let rid = inner.rid();
let offset = inner.offset();
let whence = inner.whence();
args: Value,
_zero_copy: Option<PinnedBuf>,
) -> Result<JsonOp, ErrBox> {
let args: SeekArgs = serde_json::from_value(args)?;
match resources::lookup(rid) {
match resources::lookup(args.rid as u32) {
None => Err(deno_error::bad_resource()),
Some(resource) => {
let op = resources::seek(resource, offset, whence)
.and_then(move |_| Ok(empty_buf()));
if base.sync() {
let op = resources::seek(resource, args.offset, args.whence as u32)
.and_then(move |_| futures::future::ok(json!({})));
if args.promise_id.is_none() {
let buf = op.wait()?;
Ok(Op::Sync(buf))
Ok(JsonOp::Sync(buf))
} else {
Ok(Op::Async(Box::new(op)))
Ok(JsonOp::Async(Box::new(op)))
}
}
}

41
cli/ops/mod.rs
View file

@ -37,6 +37,14 @@ pub const OP_UTIME: OpId = 7;
pub const OP_SET_ENV: OpId = 8;
pub const OP_HOME_DIR: OpId = 9;
pub const OP_START: OpId = 10;
pub const OP_APPLY_SOURCE_MAP: OpId = 11;
pub const OP_FORMAT_ERROR: OpId = 12;
pub const OP_CACHE: OpId = 13;
pub const OP_FETCH_SOURCE_FILE: OpId = 14;
pub const OP_OPEN: OpId = 15;
pub const OP_CLOSE: OpId = 16;
pub const OP_SEEK: OpId = 17;
pub const OP_FETCH: OpId = 18;
pub fn dispatch(
state: &ThreadSafeState,
@ -74,6 +82,39 @@ pub fn dispatch(
OP_START => {
dispatch_json::dispatch(os::op_start, state, control, zero_copy)
}
OP_APPLY_SOURCE_MAP => dispatch_json::dispatch(
errors::op_apply_source_map,
state,
control,
zero_copy,
),
OP_FORMAT_ERROR => dispatch_json::dispatch(
errors::op_format_error,
state,
control,
zero_copy,
),
OP_CACHE => {
dispatch_json::dispatch(compiler::op_cache, state, control, zero_copy)
}
OP_FETCH_SOURCE_FILE => dispatch_json::dispatch(
compiler::op_fetch_source_file,
state,
control,
zero_copy,
),
OP_OPEN => {
dispatch_json::dispatch(files::op_open, state, control, zero_copy)
}
OP_CLOSE => {
dispatch_json::dispatch(files::op_close, state, control, zero_copy)
}
OP_SEEK => {
dispatch_json::dispatch(files::op_seek, state, control, zero_copy)
}
OP_FETCH => {
dispatch_json::dispatch(fetch::op_fetch, state, control, zero_copy)
}
OP_FLATBUFFER => dispatch_flatbuffers::dispatch(state, control, zero_copy),
_ => panic!("bad op_id"),
};

56
js/compiler.ts
View file

@ -7,9 +7,11 @@ import { Console } from "./console";
import { core } from "./core";
import { Diagnostic, fromTypeScriptDiagnostic } from "./diagnostics";
import { cwd } from "./dir";
import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
import * as dispatch from "./dispatch";
import { sendSync } from "./dispatch_json";
import { msg } from "./dispatch_flatbuffers";
import * as os from "./os";
import { TextDecoder, TextEncoder } from "./text_encoding";
import { TextEncoder } from "./text_encoding";
import { getMappedModuleName, parseTypeDirectives } from "./type_directives";
import { assert, notImplemented } from "./util";
import * as util from "./util";
@ -121,35 +123,15 @@ interface EmitResult {
/** Ops to Rust to resolve and fetch a modules meta data. */
function fetchSourceFile(specifier: string, referrer: string): SourceFile {
util.log("fetchSourceFile", { specifier, referrer });
// Send FetchSourceFile message
const builder = flatbuffers.createBuilder();
const specifier_ = builder.createString(specifier);
const referrer_ = builder.createString(referrer);
const inner = msg.FetchSourceFile.createFetchSourceFile(
builder,
specifier_,
referrer_
);
const baseRes = sendSync(builder, msg.Any.FetchSourceFile, inner);
assert(baseRes != null);
assert(
msg.Any.FetchSourceFileRes === baseRes!.innerType(),
`base.innerType() unexpectedly is ${baseRes!.innerType()}`
);
const fetchSourceFileRes = new msg.FetchSourceFileRes();
assert(baseRes!.inner(fetchSourceFileRes) != null);
const dataArray = fetchSourceFileRes.dataArray();
const decoder = new TextDecoder();
const sourceCode = dataArray ? decoder.decode(dataArray) : undefined;
// flatbuffers returns `null` for an empty value, this does not fit well with
// idiomatic TypeScript under strict null checks, so converting to `undefined`
util.log("compiler.fetchSourceFile", { specifier, referrer });
const res = sendSync(dispatch.OP_FETCH_SOURCE_FILE, {
specifier,
referrer
});
return {
moduleName: fetchSourceFileRes.moduleName() || undefined,
filename: fetchSourceFileRes.filename() || undefined,
mediaType: fetchSourceFileRes.mediaType(),
sourceCode,
typeDirectives: parseTypeDirectives(sourceCode)
...res,
typeDirectives: parseTypeDirectives(res.sourceCode)
};
}
@ -171,19 +153,7 @@ function humanFileSize(bytes: number): string {
/** Ops to rest for caching source map and compiled js */
function cache(extension: string, moduleId: string, contents: string): void {
util.log("cache", extension, moduleId);
const builder = flatbuffers.createBuilder();
const extension_ = builder.createString(extension);
const moduleId_ = builder.createString(moduleId);
const contents_ = builder.createString(contents);
const inner = msg.Cache.createCache(
builder,
extension_,
moduleId_,
contents_
);
const baseRes = sendSync(builder, msg.Any.Cache, inner);
assert(baseRes == null);
sendSync(dispatch.OP_CACHE, { extension, moduleId, contents });
}
const encoder = new TextEncoder();

17
js/dispatch.ts
View file

@ -15,6 +15,14 @@ export const OP_UTIME = 7;
export const OP_SET_ENV = 8;
export const OP_HOME_DIR = 9;
export const OP_START = 10;
export const OP_APPLY_SOURCE_MAP = 11;
export const OP_FORMAT_ERROR = 12;
export const OP_CACHE = 13;
export const OP_FETCH_SOURCE_FILE = 14;
export const OP_OPEN = 15;
export const OP_CLOSE = 16;
export const OP_SEEK = 17;
export const OP_FETCH = 18;
export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void {
switch (opId) {
@ -25,10 +33,17 @@ export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void {
case OP_READ:
minimal.asyncMsgFromRust(opId, ui8);
break;
case OP_EXIT:
case OP_IS_TTY:
case OP_ENV:
case OP_EXEC_PATH:
case OP_UTIME:
case OP_OPEN:
case OP_SEEK:
case OP_FETCH:
json.asyncMsgFromRust(opId, ui8);
break;
default:
throw Error("bad opId");
throw Error("bad async opId");
}
}

51
js/error_stack.ts
View file

@ -1,8 +1,8 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
// Some of the code here is adapted directly from V8 and licensed under a BSD
// style license available here: https://github.com/v8/v8/blob/24886f2d1c565287d33d71e4109a53bf0b54b75c/LICENSE.v8
import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
import * as dispatch from "./dispatch";
import { sendSync } from "./dispatch_json";
import { assert } from "./util";
export interface Location {
@ -17,40 +17,6 @@ export interface Location {
column: number;
}
function req(
filename: string,
line: number,
column: number
): [flatbuffers.Builder, msg.Any.ApplySourceMap, flatbuffers.Offset] {
const builder = flatbuffers.createBuilder();
const filename_ = builder.createString(filename);
const inner = msg.ApplySourceMap.createApplySourceMap(
builder,
filename_,
// On this side, line/column are 1 based, but in the source maps, they are
// 0 based, so we have to convert back and forth
line - 1,
column - 1
);
return [builder, msg.Any.ApplySourceMap, inner];
}
function res(baseRes: msg.Base | null): Location {
assert(baseRes != null);
assert(baseRes!.innerType() === msg.Any.ApplySourceMap);
const res = new msg.ApplySourceMap();
assert(baseRes!.inner(res) != null);
const filename = res.filename()!;
assert(filename != null);
return {
filename,
// On this side, line/column are 1 based, but in the source maps, they are
// 0 based, so we have to convert back and forth
line: res.line() + 1,
column: res.column() + 1
};
}
/** Given a current location in a module, lookup the source location and
* return it.
*
@ -75,7 +41,18 @@ function res(baseRes: msg.Base | null): Location {
*/
export function applySourceMap(location: Location): Location {
const { filename, line, column } = location;
return res(sendSync(...req(filename, line, column)));
// On this side, line/column are 1 based, but in the source maps, they are
// 0 based, so we have to convert back and forth
const res = sendSync(dispatch.OP_APPLY_SOURCE_MAP, {
filename,
line: line - 1,
column: column - 1
});
return {
filename: res.filename,
line: res.line + 1,
column: res.column + 1
};
}
/** Mutate the call site so that it returns the location, instead of its

94
js/fetch.ts
View file

@ -1,6 +1,5 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { assert, createResolvable, notImplemented, isTypedArray } from "./util";
import { sendAsync, msg, flatbuffers } from "./dispatch_flatbuffers";
import * as domTypes from "./dom_types";
import { TextDecoder, TextEncoder } from "./text_encoding";
import { DenoBlob, bytesSymbol as blobBytesSymbol } from "./blob";
@ -10,6 +9,8 @@ import { read, close } from "./files";
import { Buffer } from "./buffer";
import { FormData } from "./form_data";
import { URLSearchParams } from "./url_search_params";
import * as dispatch from "./dispatch";
import { sendAsync } from "./dispatch_json";
function getHeaderValueParams(value: string): Map<string, string> {
const params = new Map();
@ -320,67 +321,35 @@ export class Response implements domTypes.Response {
}
}
function msgHttpRequest(
builder: flatbuffers.Builder,
url: string,
method: null | string,
headers: null | domTypes.Headers
): flatbuffers.Offset {
const methodOffset = !method ? 0 : builder.createString(method);
let fieldsOffset: flatbuffers.Offset = 0;
const urlOffset = builder.createString(url);
if (headers) {
const kvOffsets: flatbuffers.Offset[] = [];
for (const [key, val] of headers.entries()) {
const keyOffset = builder.createString(key);
const valOffset = builder.createString(val);
kvOffsets.push(
msg.KeyValue.createKeyValue(builder, keyOffset, valOffset)
);
}
fieldsOffset = msg.HttpHeader.createFieldsVector(builder, kvOffsets);
} else {
}
return msg.HttpHeader.createHttpHeader(
builder,
true,
methodOffset,
urlOffset,
0,
fieldsOffset
);
interface FetchResponse {
bodyRid: number;
status: number;
headers: Array<[string, string]>;
}
function deserializeHeaderFields(m: msg.HttpHeader): Array<[string, string]> {
const out: Array<[string, string]> = [];
for (let i = 0; i < m.fieldsLength(); i++) {
const item = m.fields(i)!;
out.push([item.key()!, item.value()!]);
}
return out;
}
async function getFetchRes(
async function sendFetchReq(
url: string,
method: string | null,
headers: domTypes.Headers | null,
body: ArrayBufferView | undefined
): Promise<msg.FetchRes> {
// Send Fetch message
const builder = flatbuffers.createBuilder();
const headerOff = msgHttpRequest(builder, url, method, headers);
const resBase = await sendAsync(
builder,
msg.Any.Fetch,
msg.Fetch.createFetch(builder, headerOff),
body
);
): Promise<FetchResponse> {
let headerArray: Array<[string, string]> = [];
if (headers) {
headerArray = Array.from(headers.entries());
}
// Decode FetchRes
assert(msg.Any.FetchRes === resBase.innerType());
const inner = new msg.FetchRes();
assert(resBase.inner(inner) != null);
return inner;
let zeroCopy = undefined;
if (body) {
zeroCopy = new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
}
const args = {
method,
url,
headers: headerArray
};
return (await sendAsync(dispatch.OP_FETCH, args, zeroCopy)) as FetchResponse;
}
/** Fetch a resource from the network. */
@ -448,20 +417,13 @@ export async function fetch(
}
while (remRedirectCount) {
const inner = await getFetchRes(url, method, headers, body);
const header = inner.header()!;
const bodyRid = inner.bodyRid();
assert(!header.isRequest());
const status = header.status();
const headersList = deserializeHeaderFields(header);
const fetchResponse = await sendFetchReq(url, method, headers, body);
const response = new Response(
url,
status,
headersList,
bodyRid,
fetchResponse.status,
fetchResponse.headers,
fetchResponse.bodyRid,
redirected
);
if ([301, 302, 303, 307, 308].includes(response.status)) {

52
js/files.ts
View file

@ -12,37 +12,22 @@ import {
} from "./io";
import { sendAsyncMinimal } from "./dispatch_minimal";
import { assert } from "./util";
import { sendAsync, sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
import * as dispatch from "./dispatch";
import {
sendSync as sendSyncJson,
sendAsync as sendAsyncJson
} from "./dispatch_json";
import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
import { OP_READ, OP_WRITE } from "./dispatch";
function reqOpen(
filename: string,
mode: OpenMode
): [flatbuffers.Builder, msg.Any, flatbuffers.Offset] {
const builder = flatbuffers.createBuilder();
const filename_ = builder.createString(filename);
const mode_ = builder.createString(mode);
const inner = msg.Open.createOpen(builder, filename_, 0, mode_);
return [builder, msg.Any.Open, inner];
}
function resOpen(baseRes: null | msg.Base): File {
assert(baseRes != null);
assert(msg.Any.OpenRes === baseRes!.innerType());
const res = new msg.OpenRes();
assert(baseRes!.inner(res) != null);
const rid = res.rid();
// eslint-disable-next-line @typescript-eslint/no-use-before-define
return new File(rid);
}
/** Open a file and return an instance of the `File` object
* synchronously.
*
* const file = Deno.openSync("/foo/bar.txt");
*/
export function openSync(filename: string, mode: OpenMode = "r"): File {
return resOpen(sendSync(...reqOpen(filename, mode)));
const rid = sendSyncJson(dispatch.OP_OPEN, { filename, mode });
return new File(rid);
}
/** Open a file and return an instance of the `File` object.
@ -55,7 +40,8 @@ export async function open(
filename: string,
mode: OpenMode = "r"
): Promise<File> {
return resOpen(await sendAsync(...reqOpen(filename, mode)));
const rid = await sendAsyncJson(dispatch.OP_OPEN, { filename, mode });
return new File(rid);
}
function reqRead(
@ -165,23 +151,13 @@ export async function write(rid: number, p: Uint8Array): Promise<number> {
}
}
function reqSeek(
rid: number,
offset: number,
whence: SeekMode
): [flatbuffers.Builder, msg.Any, flatbuffers.Offset] {
const builder = flatbuffers.createBuilder();
const inner = msg.Seek.createSeek(builder, rid, offset, whence);
return [builder, msg.Any.Seek, inner];
}
/** Seek a file ID synchronously to the given offset under mode given by `whence`.
*
* const file = Deno.openSync("/foo/bar.txt");
* Deno.seekSync(file.rid, 0, 0);
*/
export function seekSync(rid: number, offset: number, whence: SeekMode): void {
sendSync(...reqSeek(rid, offset, whence));
sendSyncJson(dispatch.OP_SEEK, { rid, offset, whence });
}
/** Seek a file ID to the given offset under mode given by `whence`.
@ -196,14 +172,12 @@ export async function seek(
offset: number,
whence: SeekMode
): Promise<void> {
await sendAsync(...reqSeek(rid, offset, whence));
await sendAsyncJson(dispatch.OP_SEEK, { rid, offset, whence });
}
/** Close the file ID. */
export function close(rid: number): void {
const builder = flatbuffers.createBuilder();
const inner = msg.Close.createClose(builder, rid);
sendSync(builder, msg.Any.Close, inner);
sendSyncJson(dispatch.OP_CLOSE, { rid });
}
/** The Deno abstraction for reading and writing files. */

18
js/format_error.ts
View file

@ -1,17 +1,9 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
import { assert } from "./util";
import * as dispatch from "./dispatch";
import { sendSync } from "./dispatch_json";
// TODO(bartlomieju): move to `repl.ts`?
export function formatError(errString: string): string {
const builder = flatbuffers.createBuilder();
const errString_ = builder.createString(errString);
const offset = msg.FormatError.createFormatError(builder, errString_);
const baseRes = sendSync(builder, msg.Any.FormatError, offset);
assert(baseRes != null);
assert(msg.Any.FormatErrorRes === baseRes!.innerType());
const formatErrorResMsg = new msg.FormatErrorRes();
assert(baseRes!.inner(formatErrorResMsg) != null);
const formattedError = formatErrorResMsg.error();
assert(formatError != null);
return formattedError!;
const res = sendSync(dispatch.OP_FORMAT_ERROR, { error: errString });
return res.error;
}

8
tests/error_004_missing_module.ts.out
View file

@ -1,12 +1,12 @@
[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts"
[WILDCARD] js/dispatch_flatbuffers.ts:[WILDCARD]
[WILDCARD] js/dispatch_json.ts:[WILDCARD]
at DenoError (js/errors.ts:[WILDCARD])
at maybeError (js/dispatch_flatbuffers.ts:[WILDCARD])
at maybeThrowError (js/dispatch_flatbuffers.ts:[WILDCARD])
at sendSync (js/dispatch_flatbuffers.ts:[WILDCARD])
at toDenoError (js/dispatch_json.ts:[WILDCARD])
at sendSync$1 (js/dispatch_json.ts:[WILDCARD])
at fetchSourceFile (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNames (js/compiler.ts:[WILDCARD])
at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])

8
tests/error_005_missing_dynamic_import.ts.out
View file

@ -1,11 +1,11 @@
[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/bad-module.ts"
[WILDCARD] js/dispatch_flatbuffers.ts:[WILDCARD]
[WILDCARD] js/dispatch_json.ts:[WILDCARD]
at DenoError (js/errors.ts:[WILDCARD])
at maybeError (js/dispatch_flatbuffers.ts:[WILDCARD])
at maybeThrowError (js/dispatch_flatbuffers.ts:[WILDCARD])
at sendSync (js/dispatch_flatbuffers.ts:[WILDCARD])
at toDenoError (js/dispatch_json.ts:[WILDCARD])
at sendSync$1 (js/dispatch_json.ts:[WILDCARD])
at fetchSourceFile (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])

8
tests/error_006_import_ext_failure.ts.out
View file

@ -1,11 +1,11 @@
[WILDCARD]error: Uncaught NotFound: Cannot resolve module "[WILDCARD]/non-existent"
[WILDCARD] js/dispatch_flatbuffers.ts:[WILDCARD]
[WILDCARD] js/dispatch_json.ts:[WILDCARD]
at DenoError (js/errors.ts:[WILDCARD])
at maybeError (js/dispatch_flatbuffers.ts:[WILDCARD])
at maybeThrowError (js/dispatch_flatbuffers.ts:[WILDCARD])
at sendSync (js/dispatch_flatbuffers.ts:[WILDCARD])
at toDenoError (js/dispatch_json.ts:[WILDCARD])
at sendSync$1 (js/dispatch_json.ts:[WILDCARD])
at fetchSourceFile (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])

11
tests/error_011_bad_module_specifier.ts.out
View file

@ -1,12 +1,11 @@
[WILDCARD]error: Uncaught ImportPrefixMissing: relative import path "bad-module.ts" not prefixed with / or ./ or ../
[WILDCARD] js/dispatch_flatbuffers.ts:[WILDCARD]
[WILDCARD] js/dispatch_json.ts:[WILDCARD]
at DenoError (js/errors.ts:[WILDCARD])
at maybeError (js/dispatch_flatbuffers.ts:[WILDCARD])
at maybeThrowError (js/dispatch_flatbuffers.ts:[WILDCARD])
at sendSync (js/dispatch_flatbuffers.ts:[WILDCARD])
at toDenoError (js/dispatch_json.ts:[WILDCARD])
at sendSync$1 (js/dispatch_json.ts:[WILDCARD])
at fetchSourceFile (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNames (js/compiler.ts:[WILDCARD])
at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD])
at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])

9
tests/error_012_bad_dynamic_import_specifier.ts.out
View file

@ -1,12 +1,11 @@
[WILDCARD]error: Uncaught ImportPrefixMissing: relative import path "bad-module.ts" not prefixed with / or ./ or ../
[WILDCARD] js/dispatch_flatbuffers.ts:[WILDCARD]
[WILDCARD] js/dispatch_json.ts:[WILDCARD]
at DenoError (js/errors.ts:[WILDCARD])
at maybeError (js/dispatch_flatbuffers.ts:[WILDCARD])
at maybeThrowError (js/dispatch_flatbuffers.ts:[WILDCARD])
at sendSync (js/dispatch_flatbuffers.ts:[WILDCARD])
at toDenoError (js/dispatch_json.ts:[WILDCARD])
at sendSync$1 (js/dispatch_json.ts:[WILDCARD])
at fetchSourceFile (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNames (js/compiler.ts:[WILDCARD])
at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])
at processImportedModules ([WILDCARD]typescript.js:[WILDCARD])