foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-28 16:20:57 -05:00
Code Issues Packages 1 Wiki Activity

perf: disable WAL for transpiled source cache (#18084)

Browse source 
Disable Write-Ahead Log for the cached module source database.

This brings SQLite connection cost on startup from 2.5% to 1.6%.
This commit is contained in:
Bartek Iwańczuk 2023-03-22 01:01:15 +01:00 committed by GitHub
parent 7d9653d51f
commit 619806d7a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 95 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
cli/cache/check.rs vendored
View file

@ -7,7 +7,7 @@ use deno_core::error::AnyError;
use deno_runtime::deno_webstorage::rusqlite::params; use deno_runtime::deno_webstorage::rusqlite::params;
use deno_runtime::deno_webstorage::rusqlite::Connection; use deno_runtime::deno_webstorage::rusqlite::Connection;
use super::common::run_sqlite_pragma; use super::common::INITIAL_PRAGMAS;
/// The cache used to tell whether type checking should occur again. /// The cache used to tell whether type checking should occur again.
/// ///
@ -60,8 +60,7 @@ impl TypeCheckCache {
conn: Connection, conn: Connection,
cli_version: String, cli_version: String,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?; initialize(&conn, cli_version)?;
create_tables(&conn, cli_version)?;
Ok(Self(Some(conn))) Ok(Self(Some(conn)))
} }
@ -158,31 +157,24 @@ impl TypeCheckCache {
} }
} }
fn create_tables( fn initialize(conn: &Connection, cli_version: String) -> Result<(), AnyError> {
conn: &Connection, // INT doesn't store up to u64, so use TEXT for check_hash
cli_version: String, let query = format!(
) -> Result<(), AnyError> { "{INITIAL_PRAGMAS}
// INT doesn't store up to u64, so use TEXT CREATE TABLE IF NOT EXISTS checkcache (
conn.execute( check_hash TEXT PRIMARY KEY
"CREATE TABLE IF NOT EXISTS checkcache ( );
check_hash TEXT PRIMARY KEY CREATE TABLE IF NOT EXISTS tsbuildinfo (
)", specifier TEXT PRIMARY KEY,
[], text TEXT NOT NULL
)?; );
conn.execute( CREATE TABLE IF NOT EXISTS info (
"CREATE TABLE IF NOT EXISTS tsbuildinfo ( key TEXT PRIMARY KEY,
specifier TEXT PRIMARY KEY, value TEXT NOT NULL
text TEXT NOT NULL );
)", ",
[], );
)?; conn.execute_batch(&query)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
// delete the cache when the CLI version changes // delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn let data_cli_version: Option<String> = conn

30
cli/cache/common.rs vendored
View file

@ -2,9 +2,6 @@
use std::hash::Hasher; use std::hash::Hasher;
use deno_core::error::AnyError;
use deno_runtime::deno_webstorage::rusqlite::Connection;
/// A very fast insecure hasher that uses the xxHash algorithm. /// A very fast insecure hasher that uses the xxHash algorithm.
#[derive(Default)] #[derive(Default)]
pub struct FastInsecureHasher(twox_hash::XxHash64); pub struct FastInsecureHasher(twox_hash::XxHash64);
@ -47,19 +44,14 @@ impl FastInsecureHasher {
} }
} }
/// Runs the common sqlite pragma. /// Disable write-ahead-logging and tweak some other stuff.
pub fn run_sqlite_pragma(conn: &Connection) -> Result<(), AnyError> { /// We want to favor startup time over cache performance and
// Enable write-ahead-logging and tweak some other stuff /// creating a WAL is expensive on startup.
let initial_pragmas = " pub static INITIAL_PRAGMAS: &str = "
-- enable write-ahead-logging mode PRAGMA journal_mode=OFF;
PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
PRAGMA synchronous=NORMAL; PRAGMA temp_store=memory;
PRAGMA temp_store=memory; PRAGMA page_size=4096;
PRAGMA page_size=4096; PRAGMA mmap_size=6000000;
PRAGMA mmap_size=6000000; PRAGMA optimize;
PRAGMA optimize; ";
";
conn.execute_batch(initial_pragmas)?;
Ok(())
}

40
cli/cache/incremental.rs vendored
View file

@ -12,8 +12,8 @@ use deno_runtime::deno_webstorage::rusqlite::Connection;
use serde::Serialize; use serde::Serialize;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use super::common::run_sqlite_pragma;
use super::common::FastInsecureHasher; use super::common::FastInsecureHasher;
use super::common::INITIAL_PRAGMAS;
/// Cache used to skip formatting/linting a file again when we /// Cache used to skip formatting/linting a file again when we
/// know it is already formatted or has no lint diagnostics. /// know it is already formatted or has no lint diagnostics.
@ -174,8 +174,7 @@ impl SqlIncrementalCache {
state_hash: u64, state_hash: u64,
cli_version: String, cli_version: String,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?; initialize(&conn, cli_version)?;
create_tables(&conn, cli_version)?;
Ok(Self { conn, state_hash }) Ok(Self { conn, state_hash })
} }
@ -238,26 +237,21 @@ impl SqlIncrementalCache {
} }
} }
fn create_tables( fn initialize(conn: &Connection, cli_version: String) -> Result<(), AnyError> {
conn: &Connection, // INT doesn't store up to u64, so use TEXT for source_hash
cli_version: String, let query = format!(
) -> Result<(), AnyError> { "{INITIAL_PRAGMAS}
// INT doesn't store up to u64, so use TEXT CREATE TABLE IF NOT EXISTS incrementalcache (
conn.execute( file_path TEXT PRIMARY KEY,
"CREATE TABLE IF NOT EXISTS incrementalcache ( state_hash TEXT NOT NULL,
file_path TEXT PRIMARY KEY, source_hash TEXT NOT NULL
state_hash TEXT NOT NULL, );
source_hash TEXT NOT NULL CREATE TABLE IF NOT EXISTS info (
)", key TEXT PRIMARY KEY,
[], value TEXT NOT NULL
)?; );"
conn.execute( );
"CREATE TABLE IF NOT EXISTS info ( conn.execute_batch(&query)?;
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
// delete the cache when the CLI version changes // delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn let data_cli_version: Option<String> = conn

64
cli/cache/node.rs vendored
View file

@ -13,7 +13,7 @@ use serde::Serialize;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use super::common::run_sqlite_pragma; use super::common::INITIAL_PRAGMAS;
use super::FastInsecureHasher; use super::FastInsecureHasher;
// todo(dsherret): use deno_ast::CjsAnalysisData directly when upgrading deno_ast // todo(dsherret): use deno_ast::CjsAnalysisData directly when upgrading deno_ast
@ -155,8 +155,7 @@ impl NodeAnalysisCacheInner {
conn: Connection, conn: Connection,
version: String, version: String,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?; initialize(&conn, &version)?;
create_tables(&conn, &version)?;
Ok(Self { conn }) Ok(Self { conn })
} }
@ -260,41 +259,32 @@ impl NodeAnalysisCacheInner {
} }
} }
fn create_tables(conn: &Connection, cli_version: &str) -> Result<(), AnyError> { fn initialize(conn: &Connection, cli_version: &str) -> Result<(), AnyError> {
// INT doesn't store up to u64, so use TEXT for source_hash // INT doesn't store up to u64, so use TEXT for source_hash
conn.execute( let query = format!(
"CREATE TABLE IF NOT EXISTS cjsanalysiscache ( "{INITIAL_PRAGMAS}
specifier TEXT PRIMARY KEY, CREATE TABLE IF NOT EXISTS cjsanalysiscache (
source_hash TEXT NOT NULL, specifier TEXT PRIMARY KEY,
data TEXT NOT NULL source_hash TEXT NOT NULL,
)", data TEXT NOT NULL
[], );
)?; CREATE UNIQUE INDEX IF NOT EXISTS cjsanalysiscacheidx
conn.execute( ON cjsanalysiscache(specifier);
"CREATE UNIQUE INDEX IF NOT EXISTS cjsanalysiscacheidx CREATE TABLE IF NOT EXISTS esmglobalscache (
ON cjsanalysiscache(specifier)", specifier TEXT PRIMARY KEY,
[], source_hash TEXT NOT NULL,
)?; data TEXT NOT NULL
conn.execute( );
"CREATE TABLE IF NOT EXISTS esmglobalscache ( CREATE UNIQUE INDEX IF NOT EXISTS esmglobalscacheidx
specifier TEXT PRIMARY KEY, ON esmglobalscache(specifier);
source_hash TEXT NOT NULL, CREATE TABLE IF NOT EXISTS info (
data TEXT NOT NULL key TEXT PRIMARY KEY,
)", value TEXT NOT NULL
[], );
)?; "
conn.execute( );
"CREATE UNIQUE INDEX IF NOT EXISTS esmglobalscacheidx
ON esmglobalscache(specifier)", conn.execute_batch(&query)?;
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
// delete the cache when the CLI version changes // delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn let data_cli_version: Option<String> = conn

44
cli/cache/parsed_source.rs vendored
View file

@ -19,7 +19,7 @@ use deno_graph::ParsedSourceStore;
use deno_runtime::deno_webstorage::rusqlite::params; use deno_runtime::deno_webstorage::rusqlite::params;
use deno_runtime::deno_webstorage::rusqlite::Connection; use deno_runtime::deno_webstorage::rusqlite::Connection;
use super::common::run_sqlite_pragma; use super::common::INITIAL_PRAGMAS;
use super::FastInsecureHasher; use super::FastInsecureHasher;
#[derive(Clone, Default)] #[derive(Clone, Default)]
@ -162,8 +162,7 @@ impl ParsedSourceCacheModuleAnalyzer {
cli_version: String, cli_version: String,
sources: ParsedSourceCacheSources, sources: ParsedSourceCacheSources,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?; initialize(&conn, cli_version)?;
create_tables(&conn, cli_version)?;
Ok(Self { conn, sources }) Ok(Self { conn, sources })
} }
@ -288,27 +287,24 @@ impl deno_graph::ModuleAnalyzer for ParsedSourceCacheModuleAnalyzer {
} }
} }
fn create_tables( fn initialize(conn: &Connection, cli_version: String) -> Result<(), AnyError> {
conn: &Connection, let query = format!(
cli_version: String, "{INITIAL_PRAGMAS}
) -> Result<(), AnyError> { -- INT doesn't store up to u64, so use TEXT for source_hash
// INT doesn't store up to u64, so use TEXT for source_hash CREATE TABLE IF NOT EXISTS moduleinfocache (
conn.execute( specifier TEXT PRIMARY KEY,
"CREATE TABLE IF NOT EXISTS moduleinfocache ( media_type TEXT NOT NULL,
specifier TEXT PRIMARY KEY, source_hash TEXT NOT NULL,
media_type TEXT NOT NULL, module_info TEXT NOT NULL
source_hash TEXT NOT NULL, );
module_info TEXT NOT NULL CREATE TABLE IF NOT EXISTS info (
)", key TEXT PRIMARY KEY,
[], value TEXT NOT NULL
)?; );
conn.execute( "
"CREATE TABLE IF NOT EXISTS info ( );
key TEXT PRIMARY KEY,
value TEXT NOT NULL conn.execute_batch(&query)?;
)",
[],
)?;
// delete the cache when the CLI version changes // delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn let data_cli_version: Option<String> = conn