Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(compile): code cache #26528

Merged
merged 25 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
7236756
perf(compile): use less memory
dsherret Oct 22, 2024
1928daf
working now with typescript
dsherret Oct 23, 2024
32913e7
Merge branch 'main' into perf_deno_compile_less_memory
dsherret Oct 23, 2024
788f4ab
Tell v8 that something is a string like before.
dsherret Oct 23, 2024
742ae3f
fix byonm issue
dsherret Oct 23, 2024
1ad5f62
Merge branch 'main' into perf_deno_compile_less_memory
dsherret Oct 23, 2024
fd13339
maybe fix ci
dsherret Oct 24, 2024
1e0dabd
do not store data urls in the binary
dsherret Oct 24, 2024
d7cd10b
switch to le because this is not network
dsherret Oct 24, 2024
c156944
review
dsherret Oct 24, 2024
938c3e0
perf(compile): code cache for initial load
dsherret Oct 24, 2024
0a7c050
use distinct strategies for compile
dsherret Oct 24, 2024
0957e09
use distinct strategies for compile
dsherret Oct 24, 2024
bad329a
tests
dsherret Oct 24, 2024
e9e4ad2
support --no-code-cache
dsherret Oct 24, 2024
8a910fd
lint
dsherret Oct 24, 2024
bc08455
Merge branch 'main' into perf_compile_code_cache
dsherret Oct 24, 2024
815be3a
remove unused use
dsherret Oct 24, 2024
ec0d0c7
fix test
dsherret Oct 24, 2024
1140105
Do not subtract with overflow when deserializing.
dsherret Oct 24, 2024
5b13157
maybe fix test failing because they had the same binary name
dsherret Oct 24, 2024
d360dda
Merge branch 'main' into perf_compile_code_cache
dsherret Nov 18, 2024
488f6dc
update after merge
dsherret Nov 18, 2024
0e38fab
lint
dsherret Nov 18, 2024
b1a784b
Merge branch 'main' into perf_compile_code_cache
dsherret Nov 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cli/args/flags.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1939,6 +1939,7 @@ On the first invocation with deno will download the proper binary and cache it i
])
.help_heading(COMPILE_HEADING),
)
.arg(no_code_cache_arg())
.arg(
Arg::new("no-terminal")
.long("no-terminal")
Expand Down Expand Up @@ -4431,6 +4432,8 @@ fn compile_parse(
};
ext_arg_parse(flags, matches);

flags.code_cache_enabled = !matches.get_flag("no-code-cache");

flags.subcommand = DenoSubcommand::Compile(CompileFlags {
source_file,
output,
Expand Down Expand Up @@ -10040,6 +10043,7 @@ mod tests {
include: vec![]
}),
type_check_mode: TypeCheckMode::Local,
code_cache_enabled: true,
..Flags::default()
}
);
Expand All @@ -10048,7 +10052,7 @@ mod tests {
#[test]
fn compile_with_flags() {
#[rustfmt::skip]
let r = flags_from_vec(svec!["deno", "compile", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--unsafely-ignore-certificate-errors", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--allow-read", "--allow-net", "--v8-flags=--help", "--seed", "1", "--no-terminal", "--icon", "favicon.ico", "--output", "colors", "--env=.example.env", "https://examples.deno.land/color-logging.ts", "foo", "bar", "-p", "8080"]);
let r = flags_from_vec(svec!["deno", "compile", "--import-map", "import_map.json", "--no-code-cache", "--no-remote", "--config", "tsconfig.json", "--no-check", "--unsafely-ignore-certificate-errors", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--allow-read", "--allow-net", "--v8-flags=--help", "--seed", "1", "--no-terminal", "--icon", "favicon.ico", "--output", "colors", "--env=.example.env", "https://examples.deno.land/color-logging.ts", "foo", "bar", "-p", "8080"]);
assert_eq!(
r.unwrap(),
Flags {
Expand All @@ -10064,6 +10068,7 @@ mod tests {
}),
import_map_path: Some("import_map.json".to_string()),
no_remote: true,
code_cache_enabled: false,
config_flag: ConfigFlag::Path("tsconfig.json".to_owned()),
type_check_mode: TypeCheckMode::None,
reload: true,
Expand Down
10 changes: 10 additions & 0 deletions cli/cache/code_cache.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.

use std::sync::Arc;

use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_runtime::code_cache;
use deno_runtime::deno_webstorage::rusqlite::params;

use crate::worker::CliCodeCache;

use super::cache_db::CacheDB;
use super::cache_db::CacheDBConfiguration;
use super::cache_db::CacheDBHash;
Expand Down Expand Up @@ -82,6 +86,12 @@ impl CodeCache {
}
}

impl CliCodeCache for CodeCache {
fn as_code_cache(self: Arc<Self>) -> Arc<dyn code_cache::CodeCache> {
self
}
}

impl code_cache::CodeCache for CodeCache {
fn get_sync(
&self,
Expand Down
14 changes: 14 additions & 0 deletions cli/standalone/binary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ use crate::args::NpmInstallDepsProvider;
use crate::args::PermissionFlags;
use crate::args::UnstableConfig;
use crate::cache::DenoDir;
use crate::cache::FastInsecureHasher;
use crate::emit::Emitter;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClientProvider;
Expand Down Expand Up @@ -174,6 +175,7 @@ pub struct SerializedWorkspaceResolver {
pub struct Metadata {
pub argv: Vec<String>,
pub seed: Option<u64>,
pub code_cache_key: Option<u64>,
pub permissions: PermissionFlags,
pub location: Option<Url>,
pub v8_flags: Vec<String>,
Expand Down Expand Up @@ -604,10 +606,21 @@ impl<'a> DenoCompileBinaryWriter<'a> {
VfsBuilder::new(root_path.clone())?
};
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
let mut code_cache_key_hasher = if cli_options.code_cache_enabled() {
Some(FastInsecureHasher::new_deno_versioned())
} else {
None
};
for module in graph.modules() {
if module.specifier().scheme() == "data" {
continue; // don't store data urls as an entry as they're in the code
}
if let Some(hasher) = &mut code_cache_key_hasher {
if let Some(source) = module.source() {
hasher.write(module.specifier().as_str().as_bytes());
hasher.write(source.as_bytes());
}
}
let (maybe_source, media_type) = match module {
deno_graph::Module::Js(m) => {
let source = if m.media_type.is_emittable() {
Expand Down Expand Up @@ -675,6 +688,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
let metadata = Metadata {
argv: compile_flags.args.clone(),
seed: cli_options.seed(),
code_cache_key: code_cache_key_hasher.map(|h| h.finish()),
location: cli_options.location_flag().clone(),
permissions: cli_options.permission_flags().clone(),
v8_flags: cli_options.v8_flags().clone(),
Expand Down
Loading