diff --git a/src/main.rs b/src/main.rs index 9d00bfe0..5e973734 100644 --- a/src/main.rs +++ b/src/main.rs @@ -121,38 +121,40 @@ fn build_project(build_command: BuildCommand) { if metadata.is_dir() { compile_workspace(&mut compiler, filepath, None); - } else { - if filepath.extension().unwrap_or_default() == "h" { - let source_files = compiler.source_files; - - let content = std::fs::read_to_string(filepath) - .map_err(|err| { - eprintln!("{}", err); - exit(1); - }) - .unwrap(); - - let header_key = source_files.add(filepath.into(), content); - - let header_contents = source_files - .get(header_key) - .content() - .chars() - .into_text_stream(header_key) - .into_text(); - - let preprocessed = exit_unless( - c::preprocessor::preprocess(header_contents, &diagnostics), - &source_files, - ); - - println!("{preprocessed:?}"); - return; - } + return; + } - let project_folder = filepath.parent().unwrap(); - compile_single_file_only(&mut compiler, project_folder, filepath); + // Experimental header parsing + if filepath.extension().unwrap_or_default() == "h" { + let source_files = compiler.source_files; + + let content = std::fs::read_to_string(filepath) + .map_err(|err| { + eprintln!("{}", err); + exit(1); + }) + .unwrap(); + + let header_key = source_files.add(filepath.into(), content); + + let header_contents = source_files + .get(header_key) + .content() + .chars() + .into_text_stream(header_key) + .into_text(); + + let preprocessed = exit_unless( + c::preprocessor::preprocess(header_contents, &diagnostics), + &source_files, + ); + + println!("{preprocessed:?}"); + return; } + + let project_folder = filepath.parent().unwrap(); + compile_single_file_only(&mut compiler, project_folder, filepath); } fn exit_unless(result: Result, source_files: &SourceFiles) -> T { diff --git a/src/show.rs b/src/show.rs index c579003c..78dc06fb 100644 --- a/src/show.rs +++ b/src/show.rs @@ -1,19 +1,15 @@ -use crate::source_files::{Source, SourceFiles}; +use crate::source_files::SourceFiles; pub trait Show { fn show(&self, w: &mut dyn std::fmt::Write, source_files: &SourceFiles) -> std::fmt::Result; + + fn eprintln(self: &Self, source_files: &SourceFiles) { + let mut message = String::new(); + self.show(&mut message, source_files).unwrap(); + eprintln!("{}", message); + } } pub fn into_show(show: T) -> Box { Box::new(show) } - -pub fn error_println(message: &str, source: Source, source_files: &SourceFiles) { - eprintln!( - "{}:{}:{}: error: {}", - source_files.get(source.key).filename(), - source.location.line, - source.location.column, - message, - ) -} diff --git a/src/workspace/compile/module/mod.rs b/src/workspace/compile/module/mod.rs index 5d9d752f..95de3da3 100644 --- a/src/workspace/compile/module/mod.rs +++ b/src/workspace/compile/module/mod.rs @@ -1,6 +1,7 @@ use crate::{ ast::Settings, compiler::Compiler, + data_units::ByteUnits, diagnostics::ErrorDiagnostic, inflow::{Inflow, IntoInflow}, lexer::Lexer, @@ -15,19 +16,18 @@ use crate::{ }; use std::path::Path; +pub struct CompiledModule<'a, I: Inflow + 'a> { + pub total_file_size: ByteUnits, + pub remaining_input: Input<'a, I>, + pub settings: Settings, + pub source_file: SourceFileKey, +} + pub fn compile_module_file<'a>( compiler: &Compiler<'a>, _fs: &Fs, path: &Path, -) -> Result< - ( - usize, - Input<'a, impl Inflow + 'a>, - Settings, - SourceFileKey, - ), - Box, -> { +) -> Result + 'a>, Box> { let content = std::fs::read_to_string(path) .map_err(ErrorDiagnostic::plain) .map_err(into_show)?; @@ -64,5 +64,10 @@ pub fn compile_module_file<'a>( ))); }; - Ok((content.len(), input, settings, key)) + Ok(CompiledModule { + total_file_size: ByteUnits::of(content.len().try_into().unwrap()), + remaining_input: input, + settings, + source_file: key, + }) } diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index 3d023795..edcbdab3 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -19,6 +19,7 @@ use crate::{ translate_expr, }, compiler::Compiler, + data_units::ByteUnits, diagnostics::{ErrorDiagnostic, WarningDiagnostic}, exit_unless, inflow::{Inflow, IntoInflow}, @@ -35,7 +36,7 @@ use crate::{ token::Token, }; use append_only_vec::AppendOnlyVec; -use compile::module::compile_module_file; +use compile::module::{compile_module_file, CompiledModule}; use derive_more::IsVariant; use explore::{explore, ExploreResult}; use fs::{Fs, FsNodeId}; @@ -100,18 +101,195 @@ fn explore_constrained( (explore(&fs, project_folder), None) } +struct CompilationStats { + files_processed: AtomicU64, + bytes_processed: AtomicU64, + num_files_failed: AtomicU64, + num_module_files_failed: AtomicU64, +} + +impl CompilationStats { + pub fn new() -> Self { + Self { + files_processed: AtomicU64::new(0), + bytes_processed: AtomicU64::new(0), + num_files_failed: AtomicU64::new(0), + num_module_files_failed: AtomicU64::new(0), + } + } + + pub fn failed_files_estimate(&self) -> u64 { + self.num_files_failed.load(Ordering::Relaxed) + } + + pub fn failed_modules_estimate(&self) -> u64 { + self.num_module_files_failed.load(Ordering::Relaxed) + } + + pub fn bytes_processed_estimate(&self) -> u64 { + self.bytes_processed.load(Ordering::Relaxed) + } + + pub fn files_processed_estimate(&self) -> u64 { + self.files_processed.load(Ordering::Relaxed) + } + + pub fn fail_file(&self) { + self.num_files_failed.fetch_add(1, Ordering::Relaxed); + } + + pub fn fail_module_file(&self) { + self.num_module_files_failed.fetch_add(1, Ordering::Relaxed); + } + + pub fn process_file(&self) { + self.files_processed.fetch_add(1, Ordering::Relaxed); + } + + pub fn process_bytes(&self, count: ByteUnits) { + self.bytes_processed + .fetch_add(count.bytes(), Ordering::Relaxed); + } +} + +struct WorkspaceQueue<'a, I: Inflow> { + code_files: Mutex>>, + module_files: Mutex>, + ast_files: AppendOnlyVec<(FsNodeId, AstFile)>, + module_folders: AppendOnlyVec<(FsNodeId, Settings)>, +} + +impl<'a, I: Inflow> WorkspaceQueue<'a, I> { + pub fn new(normal_files: Vec, module_files: Vec) -> Self { + Self { + code_files: Mutex::new(normal_files.into_iter().map(CodeFile::Normal).collect_vec()), + module_files: Mutex::new(module_files), + ast_files: AppendOnlyVec::new(), + module_folders: AppendOnlyVec::new(), + } + } + + pub fn push_module_folder(&self, folder_fs_node_id: FsNodeId, settings: Settings) { + self.module_folders.push((folder_fs_node_id, settings)); + } + + pub fn push_code_file(&self, code_file: CodeFile<'a, I>) { + self.code_files.lock().unwrap().push(code_file); + } + + pub fn push_code_files(&self, code_files: impl Iterator>) { + self.code_files.lock().unwrap().extend(code_files); + } + + pub fn push_module_files(&self, module_files: impl Iterator) { + self.module_files.lock().unwrap().extend(module_files); + } + + pub fn for_module_files(&self, f: impl Fn(ModuleFile)) { + loop { + // CAREFUL: Lock doesn't immediately drop unless we do it this way (while loop is not equivalent) + let Some(module_file) = self.module_files.lock().unwrap().pop() else { + break; + }; + f(module_file); + } + } + + pub fn for_code_files(&self, f: impl Fn(CodeFile<'a, I>)) { + loop { + // CAREFUL: Lock doesn't immediately drop unless we do it this way (while loop is not equivalent) + let Some(code_file) = self.code_files.lock().unwrap().pop() else { + break; + }; + f(code_file); + } + } +} + +fn queue_dependencies>( + compiler: &Compiler, + fs: &Fs, + mut settings: Settings, + source_file: SourceFileKey, + stats: &CompilationStats, + queue: &WorkspaceQueue, +) -> Settings { + for folder in settings.namespace_to_dependency.values().flatten() { + let infrastructure = compiler + .options + .infrastructure + .as_ref() + .expect("must have infrastructure specified in order to import") + .absolutize() + .expect("failed to get absolute path for compiler infrastructure"); + + let absolute_folder = infrastructure.join("import").join(&**folder); + let already_discovered = fs.find(&absolute_folder).is_some(); + + if !already_discovered { + let Some(ExploreResult { + module_files: new_module_files, + normal_files: new_normal_files, + }) = explore(&fs, &absolute_folder) + else { + ErrorDiagnostic::new( + format!("Dependency '{}' could not be found", &**folder), + Source::new(source_file, Location::new(0, 1)), + ) + .eprintln(compiler.source_files); + stats.fail_module_file(); + return settings; + }; + + queue.push_module_files(new_module_files.into_iter()); + queue.push_code_files(new_normal_files.into_iter().map(CodeFile::Normal)); + } + + let module_fs_node_id = fs.find(&absolute_folder).expect("module loaded"); + settings + .dependency_to_module + .insert(folder.to_string(), module_fs_node_id); + } + + settings +} + +fn process_module_file<'a, 'b: 'a, I: Inflow>( + compiler: &Compiler, + fs: &Fs, + module_file: ModuleFile, + compiled_module: CompiledModule<'a, I>, + stats: &CompilationStats, + queue: &WorkspaceQueue<'a, I>, +) { + let folder_fs_node_id = fs + .get(module_file.fs_node_id) + .parent + .expect("module file has parent"); + + let CompiledModule { + settings, + source_file, + total_file_size, + remaining_input, + } = compiled_module; + + let settings = queue_dependencies(compiler, fs, settings, source_file, stats, queue); + + queue.push_module_folder(folder_fs_node_id, settings); + queue.push_code_file(CodeFile::Module(module_file, remaining_input)); + + stats.process_file(); + stats.process_bytes(total_file_size); +} + pub fn compile_workspace( compiler: &mut Compiler, project_folder: &Path, single_file: Option, ) { - let compiler = compiler; - let start_time = Instant::now(); - let files_processed = AtomicU64::new(0); - let bytes_processed = AtomicU64::new(0); - let num_files_failed = AtomicU64::new(0); - let num_module_files_failed = AtomicU64::new(0); + let stats = CompilationStats::new(); let fs = Fs::new(); let (exploration, guaranteed_entry) = explore_constrained(&fs, project_folder, single_file); @@ -128,112 +306,34 @@ pub fn compile_workspace( exit(1); }; - let num_threads = (normal_files.len() + module_files.len()).min(NUM_THREADS); - let all_modules_done = Barrier::new(num_threads); - let code_files = Mutex::new(normal_files.into_iter().map(CodeFile::Normal).collect_vec()); - let module_files = Mutex::new(module_files); - let ast_files = AppendOnlyVec::new(); - let module_folders = AppendOnlyVec::new(); + let thread_count = (normal_files.len() + module_files.len()).min(NUM_THREADS); + let all_modules_done = Barrier::new(thread_count); + let queue = WorkspaceQueue::new(normal_files, module_files); std::thread::scope(|scope| { - for _ in 0..num_threads { + for _ in 0..thread_count { scope.spawn(|| { // ===== Process module files ===== - loop { - // CAREFUL: Lock doesn't immediately drop unless we do it this way (while loop is not equivalent) - let Some(module_file) = module_files.lock().unwrap().pop() else { - break; - }; - - let (did_bytes, rest_input, mut settings, source_file_key) = + queue.for_module_files(|module_file| { + let compiled_module = match compile_module_file(compiler, &fs, &module_file.path) { Ok(values) => values, Err(err) => { - let mut message = String::new(); - err.show(&mut message, compiler.source_files) - .expect("failed to print error"); - eprintln!("{}", message); - - // SAFETY: This is okay, since we never need to read this until the - // threads have been synchronized - num_module_files_failed.fetch_add(1, Ordering::Relaxed); - continue; + err.eprintln(compiler.source_files); + stats.fail_module_file(); + return; } }; - for folder in settings.namespace_to_dependency.values().flatten() { - let infrastructure = compiler - .options - .infrastructure - .as_ref() - .expect("must have infrastructure specified in order to import") - .absolutize() - .expect("failed to get absolute path for compiler infrastructure"); - - let absolute_folder = infrastructure.join("import").join(&**folder); - - let already_discovered = fs.find(&absolute_folder).is_some(); - - if !already_discovered { - let Some(ExploreResult { - module_files: new_module_files, - normal_files: new_normal_files, - }) = explore(&fs, &absolute_folder) - else { - // TODO: CLEANUP: Clean up this error message code - - let top_source = Source::new(source_file_key, Location::new(1, 1)); - let err = ErrorDiagnostic::new( - format!("Dependency '{}' could not be found", &**folder), - top_source, - ); - - let mut message = String::new(); - err.show(&mut message, compiler.source_files) - .expect("failed to print error"); - eprintln!("{}", message); - - // SAFETY: This is okay, since we never need to read this until the - // threads have been synchronized - num_module_files_failed.fetch_add(1, Ordering::Relaxed); - continue; - }; - - module_files - .lock() - .unwrap() - .extend(new_module_files.into_iter()); - - code_files - .lock() - .unwrap() - .extend(new_normal_files.into_iter().map(CodeFile::Normal)); - } - - let module_fs_node_id = fs.find(&absolute_folder).expect("module loaded"); - - settings - .dependency_to_module - .insert(folder.to_string(), module_fs_node_id); - } - - let folder_fs_node_id = fs - .get(module_file.fs_node_id) - .parent - .expect("module file as parent"); - - module_folders.push((folder_fs_node_id, settings)); - - code_files - .lock() - .unwrap() - .push(CodeFile::Module(module_file, rest_input)); - - // SAFETY: These are okay, since we never need to read them until the - // threads have been synchronized - files_processed.fetch_add(1, Ordering::Relaxed); - bytes_processed.fetch_add(did_bytes.try_into().unwrap(), Ordering::Relaxed); - } + process_module_file( + compiler, + &fs, + module_file, + compiled_module, + &stats, + &queue, + ); + }); // NOTE: This synchronizes the threads, and marks the end of module-related modifications/processing. // `num_module_files_failed` can now be consistently read from... @@ -242,37 +342,23 @@ pub fn compile_workspace( // ==== Don't continue if module files had errors ===== // SAFETY: This is okay, as all the modifications happened before we synchronized // the modifying threads. - if num_module_files_failed.load(Ordering::Relaxed) != 0 { + if stats.failed_modules_estimate() != 0 { return; } // ===== Process normal files ===== - loop { - // CAREFUL: Lock doesn't immediately drop unless we do it this way (while loop is not equivalent) - let Some(code_file) = code_files.lock().unwrap().pop() else { - break; - }; - - let did_bytes = match compile_code_file(compiler, code_file, &ast_files) { - Ok(did_bytes) => did_bytes, + queue.for_code_files(|code_file| { + match compile_code_file(compiler, code_file, &queue.ast_files) { + Ok(did_bytes) => { + stats.process_file(); + stats.process_bytes(did_bytes); + } Err(err) => { - let mut message = String::new(); - err.show(&mut message, compiler.source_files) - .expect("failed to print error"); - eprintln!("{}", message); - - // SAFETY: This is okay, as we don't need to read the value - // until all of the modifying threads are joined. - num_files_failed.fetch_add(1, Ordering::Relaxed); - continue; + err.eprintln(compiler.source_files); + stats.fail_file(); } }; - - // SAFETY: These are okay, as we don't need to read these values - // until all of the modifying threads are joined. - files_processed.fetch_add(1, Ordering::Relaxed); - bytes_processed.fetch_add(did_bytes.try_into().unwrap(), Ordering::Relaxed); - } + }); }); } }); @@ -280,7 +366,7 @@ pub fn compile_workspace( let in_how_many_seconds = start_time.elapsed().as_millis() as f64 / 1000.0; // SAFETY: This is okay since all modifying threads were joined (and thereby synchronized) - let num_module_files_failed = num_module_files_failed.load(Ordering::Relaxed); + let num_module_files_failed = stats.failed_modules_estimate(); if num_module_files_failed != 0 { eprintln!( "error: {num_module_files_failed} module file(s) were determined to have errors in {in_how_many_seconds:.2} seconds", @@ -290,7 +376,7 @@ pub fn compile_workspace( } // SAFETY: This is okay since all modifying threads were joined (and thereby synchronized) - let num_files_failed = num_files_failed.load(Ordering::Relaxed); + let num_files_failed = stats.failed_files_estimate(); if num_files_failed != 0 { eprintln!( "error: {num_files_failed} file(s) were determined to have errors in {in_how_many_seconds:.2} seconds", @@ -304,13 +390,12 @@ pub fn compile_workspace( exit(1); }; - let module_folders = HashMap::::from_iter(module_folders.into_iter()); - let mut files = IndexMap::from_iter(ast_files.into_iter()); + let module_folders = HashMap::::from_iter(queue.module_folders.into_iter()); + let mut files = IndexMap::from_iter(queue.ast_files.into_iter()); if compiler.options.interpret { if let Some(guaranteed_entry) = guaranteed_entry { - let ast_file = files.get_mut(&guaranteed_entry).unwrap(); - setup_build_system_interpreter_symbols(ast_file); + setup_build_system_interpreter_symbols(files.get_mut(&guaranteed_entry).unwrap()); } else { eprintln!( "error: experimental manual interpreter does not properly handle multiple files yet" @@ -385,12 +470,10 @@ pub fn compile_workspace( // SAFETY: This is okay, as we synchronized by joining let bytes_processed = - humansize::make_format(humansize::DECIMAL)(bytes_processed.load(Ordering::Relaxed)); + humansize::make_format(humansize::DECIMAL)(stats.bytes_processed_estimate()); // SAFETY: This is okay, as we synchronized by joining - let files_processed = files_processed - .load(Ordering::Relaxed) - .separate_with_commas(); + let files_processed = stats.files_processed_estimate().separate_with_commas(); println!( "Compiled {} from {} files in {:.2} seconds", @@ -404,7 +487,7 @@ fn compile_code_file<'a, I: Inflow>( compiler: &Compiler, code_file: CodeFile<'a, I>, out_ast_files: &AppendOnlyVec<(FsNodeId, AstFile)>, -) -> Result> { +) -> Result> { match code_file { CodeFile::Normal(normal_file) => compile_normal_file(compiler, &normal_file, out_ast_files), CodeFile::Module(module_file, rest) => { @@ -417,17 +500,17 @@ fn compile_rest_module_file<'a, I: Inflow>( module_file: &ModuleFile, rest_input: Input<'a, I>, out_ast_files: &AppendOnlyVec<(FsNodeId, AstFile)>, -) -> Result> { +) -> Result> { let mut parser = Parser::new(rest_input); out_ast_files.push((module_file.fs_node_id, parser.parse().map_err(into_show)?)); - Ok(0) // No new bytes processed + Ok(ByteUnits::ZERO) // No new bytes processed } fn compile_normal_file( compiler: &Compiler, normal_file: &NormalFile, out_ast_files: &AppendOnlyVec<(FsNodeId, AstFile)>, -) -> Result> { +) -> Result> { let path = &normal_file.path; let content = std::fs::read_to_string(path) @@ -459,7 +542,7 @@ fn compile_normal_file( } } - Ok(content.len()) + Ok(ByteUnits::of(content.len().try_into().unwrap())) } fn header(