diff --git a/apps/framework-cli/src/cli/local_webserver.rs b/apps/framework-cli/src/cli/local_webserver.rs index cf28804d8..fb1d7bf35 100644 --- a/apps/framework-cli/src/cli/local_webserver.rs +++ b/apps/framework-cli/src/cli/local_webserver.rs @@ -15,7 +15,7 @@ use crate::framework::data_model::config::EndpointIngestionFormat; use crate::infrastructure::stream::redpanda; use crate::infrastructure::stream::redpanda::ConfiguredProducer; -use crate::framework::typescript::ts_node::CliMessage; +use crate::framework::typescript::bin::CliMessage; use crate::project::Project; use bytes::Buf; use http_body_util::BodyExt; diff --git a/apps/framework-cli/src/cli/routines.rs b/apps/framework-cli/src/cli/routines.rs index 4ca095561..85d3a725c 100644 --- a/apps/framework-cli/src/cli/routines.rs +++ b/apps/framework-cli/src/cli/routines.rs @@ -369,8 +369,10 @@ pub async fn start_development_mode( let topics = fetch_topics(&project.redpanda_config).await?; - let mut function_process_registry = - FunctionProcessRegistry::new(project.redpanda_config.clone()); + let mut function_process_registry = FunctionProcessRegistry::new( + project.redpanda_config.clone(), + project.project_location.clone(), + ); // Once the below function is optimized to act on events, this // will need to get refactored out. @@ -385,6 +387,7 @@ pub async fn start_development_mode( let mut blocks_process_registry = AggregationProcessRegistry::new( project.language, project.blocks_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), false, ); @@ -393,6 +396,7 @@ pub async fn start_development_mode( let mut aggregations_process_registry = AggregationProcessRegistry::new( project.language, project.aggregations_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), true, ); @@ -402,6 +406,7 @@ pub async fn start_development_mode( project.language, project.clickhouse_config.clone(), project.consumption_dir(), + project.project_location.clone(), ); process_consumption_changes( &project, @@ -544,8 +549,10 @@ pub async fn start_production_mode( .start_all(&framework_object_versions, &version_syncs, metrics.clone()) .await; - let mut function_process_registry = - FunctionProcessRegistry::new(project.redpanda_config.clone()); + let mut function_process_registry = FunctionProcessRegistry::new( + project.redpanda_config.clone(), + project.project_location.clone(), + ); // Once the below function is optimized to act on events, this // will need to get refactored out. process_streaming_func_changes( @@ -558,6 +565,7 @@ pub async fn start_production_mode( let mut blocks_process_registry = AggregationProcessRegistry::new( project.language, project.blocks_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), false, ); @@ -565,6 +573,7 @@ pub async fn start_production_mode( let mut aggregations_process_registry = AggregationProcessRegistry::new( project.language, project.aggregations_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), true, ); @@ -574,6 +583,7 @@ pub async fn start_production_mode( project.language, project.clickhouse_config.clone(), project.consumption_dir(), + project.project_location.clone(), ); process_consumption_changes( &project, diff --git a/apps/framework-cli/src/framework/core/code_loader.rs b/apps/framework-cli/src/framework/core/code_loader.rs index adb60a82c..52357de51 100644 --- a/apps/framework-cli/src/framework/core/code_loader.rs +++ b/apps/framework-cli/src/framework/core/code_loader.rs @@ -166,6 +166,7 @@ pub async fn get_framework_objects_from_schema_file( let data_models_configs = data_model::config::get( path, + &project.project_location, framework_objects .enums .iter() diff --git a/apps/framework-cli/src/framework/core/primitive_map.rs b/apps/framework-cli/src/framework/core/primitive_map.rs index e061f3dd9..03edfb4c0 100644 --- a/apps/framework-cli/src/framework/core/primitive_map.rs +++ b/apps/framework-cli/src/framework/core/primitive_map.rs @@ -198,6 +198,7 @@ impl PrimitiveMap { let data_models_configs = data_model::config::get( path, + &project.project_location, file_objects .enums .iter() diff --git a/apps/framework-cli/src/framework/data_model/config.rs b/apps/framework-cli/src/framework/data_model/config.rs index a62b8d1cd..8ac7b5baa 100644 --- a/apps/framework-cli/src/framework/data_model/config.rs +++ b/apps/framework-cli/src/framework/data_model/config.rs @@ -113,10 +113,11 @@ async fn parse_python_model_file( pub async fn get( path: &Path, + project_path: &Path, enums: HashSet<&str>, ) -> Result, ModelConfigurationError> { if path.extension() == Some(OsStr::new("ts")) { - let config = get_data_model_configs(path, enums).await?; + let config = get_data_model_configs(path, project_path, enums).await?; info!("Data Model configuration for {:?}: {:?}", path, config); Ok(config) } else if path.extension() == Some(OsStr::new("py")) diff --git a/apps/framework-cli/src/framework/typescript.rs b/apps/framework-cli/src/framework/typescript.rs index 2a5316696..cfeae6557 100644 --- a/apps/framework-cli/src/framework/typescript.rs +++ b/apps/framework-cli/src/framework/typescript.rs @@ -1,8 +1,8 @@ pub mod aggregation; +pub mod bin; pub mod consumption; pub mod export_collectors; pub mod generator; pub mod parser; pub mod streaming; pub mod templates; -pub mod ts_node; diff --git a/apps/framework-cli/src/framework/typescript/aggregation.rs b/apps/framework-cli/src/framework/typescript/aggregation.rs index 50f02d509..035eaa37d 100644 --- a/apps/framework-cli/src/framework/typescript/aggregation.rs +++ b/apps/framework-cli/src/framework/typescript/aggregation.rs @@ -6,10 +6,10 @@ use tokio::process::Child; use crate::framework::aggregations::model::AggregationError; use crate::infrastructure::olap::clickhouse::config::ClickHouseConfig; -use super::ts_node; +use super::bin; -const AGGREGATION_RUNNER_WRAPPER: &str = include_str!("ts_scripts/aggregation.ts"); -const BLOCKS_RUNNER_WRAPPER: &str = include_str!("ts_scripts/blocks.ts"); +const BLOCKS_RUNNER_BIN: &str = "blocks"; +const AGGREGATIONS_RUNNER_BIN: &str = "aggregations"; // TODO: Abstract away ClickhouseConfig to support other databases // TODO: Bubble up compilation errors to the user @@ -17,6 +17,7 @@ pub fn run( clickhouse_config: ClickHouseConfig, aggregations_path: &Path, is_blocks: bool, + project_path: &Path, ) -> Result { let host_port = clickhouse_config.host_port.to_string(); let use_ssl = clickhouse_config.use_ssl.to_string(); @@ -31,9 +32,9 @@ pub fn run( ]; let mut aggregation_process = if is_blocks { - ts_node::run(BLOCKS_RUNNER_WRAPPER, &args)? + bin::run(BLOCKS_RUNNER_BIN, project_path, &args)? } else { - ts_node::run(AGGREGATION_RUNNER_WRAPPER, &args)? + bin::run(AGGREGATIONS_RUNNER_BIN, project_path, &args)? }; let stdout = aggregation_process diff --git a/apps/framework-cli/src/framework/typescript/bin.rs b/apps/framework-cli/src/framework/typescript/bin.rs new file mode 100644 index 000000000..5cfda6eb7 --- /dev/null +++ b/apps/framework-cli/src/framework/typescript/bin.rs @@ -0,0 +1,35 @@ +use crate::{cli::display::MessageType, utilities::constants::TSCONFIG_JSON}; +use serde::Deserialize; +use std::{path::Path, process::Stdio}; + +use tokio::process::{Child, Command}; + +#[derive(Deserialize)] +pub struct CliMessage { + pub message_type: MessageType, + pub action: String, + pub message: String, +} + +const RUNNER_COMMAND: &str = "moose-runner"; + +pub fn run( + binary_command: &str, + project_path: &Path, + args: &[&str], +) -> Result { + let mut command = Command::new(RUNNER_COMMAND); + + command.arg(binary_command); + + command.env("TS_NODE_PROJECT", project_path.join(TSCONFIG_JSON)); + + for arg in args { + command.arg(arg); + } + + command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() +} diff --git a/apps/framework-cli/src/framework/typescript/consumption.rs b/apps/framework-cli/src/framework/typescript/consumption.rs index 6b43ac814..95de98b8c 100644 --- a/apps/framework-cli/src/framework/typescript/consumption.rs +++ b/apps/framework-cli/src/framework/typescript/consumption.rs @@ -6,15 +6,16 @@ use tokio::process::Child; use crate::infrastructure::olap::clickhouse::config::ClickHouseConfig; use crate::infrastructure::processes::consumption_registry::ConsumptionError; -use super::ts_node; +use super::bin; -const CONSUMPTION_RUNNER_WRAPPER: &str = include_str!("ts_scripts/consumption-api.ts"); +const CONSUMPTION_RUNNER_BIN: &str = "consumption-apis"; // TODO: Abstract away ClickhouseConfig to support other databases // TODO: Bubble up compilation errors to the user pub fn run( clickhouse_config: ClickHouseConfig, consumption_path: &Path, + project_path: &Path, ) -> Result { let host_port = clickhouse_config.host_port.to_string(); let use_ssl = clickhouse_config.use_ssl.to_string(); @@ -28,7 +29,7 @@ pub fn run( &use_ssl, ]; - let mut consumption_process = ts_node::run(CONSUMPTION_RUNNER_WRAPPER, &args)?; + let mut consumption_process = bin::run(CONSUMPTION_RUNNER_BIN, project_path, &args)?; let stdout = consumption_process .stdout diff --git a/apps/framework-cli/src/framework/typescript/export_collectors.rs b/apps/framework-cli/src/framework/typescript/export_collectors.rs index fa6106968..0bb8f6af4 100644 --- a/apps/framework-cli/src/framework/typescript/export_collectors.rs +++ b/apps/framework-cli/src/framework/typescript/export_collectors.rs @@ -4,10 +4,10 @@ use std::path::Path; use serde_json::Value; use tokio::io::AsyncReadExt; -use super::ts_node::run; +use super::bin; use crate::framework::data_model::config::{ConfigIdentifier, DataModelConfig}; -const MODULE_EXPORT_SERIALIZER: &str = include_str!("ts_scripts/moduleExportSerializer.ts"); +const EXPORT_SERIALIZER_BIN: &str = "export-serializer"; #[derive(Debug, thiserror::Error)] #[error("Failed to run code")] @@ -21,13 +21,13 @@ pub enum ExportCollectorError { }, } -async fn collect_exports(file: &Path) -> Result { +async fn collect_exports(file: &Path, project_path: &Path) -> Result { let file_path_str = file.to_str().ok_or(ExportCollectorError::Other { message: "Did not get a proper file path to load exports from".to_string(), })?; let args = vec![file_path_str]; - let process = run(MODULE_EXPORT_SERIALIZER, &args)?; + let process = bin::run(EXPORT_SERIALIZER_BIN, project_path, &args)?; let mut stdout = process .stdout @@ -57,9 +57,10 @@ async fn collect_exports(file: &Path) -> Result { pub async fn get_data_model_configs( file: &Path, + project_path: &Path, enums: HashSet<&str>, ) -> Result, ExportCollectorError> { - let exports = collect_exports(file).await?; + let exports = collect_exports(file, project_path).await?; match exports { Value::Object(map) => { diff --git a/apps/framework-cli/src/framework/typescript/parser.rs b/apps/framework-cli/src/framework/typescript/parser.rs index 86af4275e..be4e37f64 100644 --- a/apps/framework-cli/src/framework/typescript/parser.rs +++ b/apps/framework-cli/src/framework/typescript/parser.rs @@ -37,13 +37,15 @@ pub fn extract_data_model_from_file( let internal = project.internal_dir().unwrap(); let output_dir = internal.join("serialized_datamodels"); + log::info!("Extracting data model from file: {:?}", path); + fs::write( internal.join(TSCONFIG_JSON), json!({ "compilerOptions":{ "outDir": "dist", // relative path, so .moose/dist "plugins": [{ - "transform": "../node_modules/@514labs/moose-lib/dist/toDataModels.js" + "transform": "../node_modules/@514labs/moose-lib/dist/dataModels/toDataModels.js" }], "strict":true }, @@ -60,15 +62,25 @@ pub fn extract_data_model_from_file( } }, )?; - let ts_return_code = Command::new("npx") - .arg("tspc") + + let ts_return_code = Command::new("moose-tspc") .arg("--project") .arg(format!(".moose/{}", TSCONFIG_JSON)) .env("NPM_CONFIG_UPDATE_NOTIFIER", "false") .current_dir(&project.project_location) - .spawn()? + .spawn() + .map_err(|err| { + log::error!("Error while starting moose-tspc: {}", err); + TypescriptParsingError::TypescriptCompilerError(Some(err)) + })? .wait() - .map_err(|err| TypescriptParsingError::TypescriptCompilerError(Some(err)))?; + .map_err(|err| { + log::error!("Error while running moose-tspc: {}", err); + TypescriptParsingError::TypescriptCompilerError(Some(err)) + })?; + + log::info!("Typescript compiler return code: {:?}", ts_return_code); + if !ts_return_code.success() { return Err(TypescriptParsingError::TypescriptCompilerError(None)); } @@ -87,6 +99,7 @@ pub fn extract_data_model_from_file( let mut output_json = serde_json::from_slice::(&output) .map_err(|_| TypescriptParsingError::TypescriptCompilerError(None))?; + if let Some(error_type) = output_json.get("error_type") { if let Some(error_type) = error_type.as_str() { if error_type == "unknown_type" { @@ -169,18 +182,18 @@ mod tests { .wait() .unwrap(); - Command::new("rm") - .arg("-rf") - .arg("./tests/test_project/node_modules/@514labs/moose-lib/dist/") + Command::new("npm") + .arg("link") + .current_dir("../../packages/ts-moose-lib") .spawn() .unwrap() .wait() .unwrap(); - Command::new("cp") - .arg("-r") - .arg("../../packages/ts-moose-lib/dist/") - .arg("./tests/test_project/node_modules/@514labs/moose-lib/dist/") + Command::new("npm") + .arg("link") + .arg("@514labs/moose-lib") + .current_dir("./tests/test_project") .spawn() .unwrap() .wait() diff --git a/apps/framework-cli/src/framework/typescript/streaming.rs b/apps/framework-cli/src/framework/typescript/streaming.rs index ad58f9de3..420f23757 100644 --- a/apps/framework-cli/src/framework/typescript/streaming.rs +++ b/apps/framework-cli/src/framework/typescript/streaming.rs @@ -5,9 +5,9 @@ use tokio::process::Child; use crate::infrastructure::stream::redpanda::RedpandaConfig; -use super::ts_node; +use super::bin; -const FUNCTION_RUNNER_WRAPPER: &str = include_str!("ts_scripts/streaming-function.ts"); +const FUNCTION_RUNNER_BIN: &str = "streaming-functions"; // TODO: we currently refer redpanda configuration here. If we want to be able to // abstract this to other type of streaming engine, we will need to be able to abstract this away. @@ -17,6 +17,7 @@ pub fn run( target_topic: &str, target_topic_config: &str, streaming_function_file: &Path, + project_path: &Path, // TODO Remove the anyhow type here ) -> Result { let mut args = vec![ @@ -48,7 +49,7 @@ pub fn run( args.push(redpanda_config.security_protocol.as_ref().unwrap()); } - let mut streaming_function_process = ts_node::run(FUNCTION_RUNNER_WRAPPER, &args)?; + let mut streaming_function_process = bin::run(FUNCTION_RUNNER_BIN, project_path, &args)?; let stdout = streaming_function_process .stdout diff --git a/apps/framework-cli/src/framework/typescript/ts_node.rs b/apps/framework-cli/src/framework/typescript/ts_node.rs deleted file mode 100644 index 80248fc8f..000000000 --- a/apps/framework-cli/src/framework/typescript/ts_node.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::cli::display::MessageType; -use serde::Deserialize; -use std::process::Stdio; - -use tokio::process::{Child, Command}; - -#[derive(Deserialize)] -pub struct CliMessage { - pub message_type: MessageType, - pub action: String, - pub message: String, -} - -pub fn run(script: &str, args: &[&str]) -> Result { - let mut command = Command::new("npx"); - - command - .env("NPM_CONFIG_UPDATE_NOTIFIER", "false") - .arg("--yes") - .arg("ts-node") - .arg("-e") - .arg(script) - .arg("--"); - - for arg in args { - command.arg(arg); - } - - command - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() -} diff --git a/apps/framework-cli/src/framework/typescript/ts_scripts/moduleExportSerializer.ts b/apps/framework-cli/src/framework/typescript/ts_scripts/moduleExportSerializer.ts deleted file mode 100644 index f289ddf1b..000000000 --- a/apps/framework-cli/src/framework/typescript/ts_scripts/moduleExportSerializer.ts +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env ts-node - -import process from "process"; - -const target_model = process.argv[1]; - -async function read_exports() { - const exports_list = await import(target_model); - console.log(JSON.stringify(exports_list)); -} - -read_exports(); diff --git a/apps/framework-cli/src/framework/typescript/tsconfig.json b/apps/framework-cli/src/framework/typescript/tsconfig.json deleted file mode 100644 index 0f7c40e1a..000000000 --- a/apps/framework-cli/src/framework/typescript/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "outDir": "dist", - "plugins": [ - { - "transform": "../node_modules/@514labs/moose-lib/dist/toDataModels.js" - } - ], - "strict": true - }, - "include": ["app/datamodels/**/*", ""] -} diff --git a/apps/framework-cli/src/infrastructure/processes/aggregations_registry.rs b/apps/framework-cli/src/infrastructure/processes/aggregations_registry.rs index 08379ce05..9d316fc51 100644 --- a/apps/framework-cli/src/infrastructure/processes/aggregations_registry.rs +++ b/apps/framework-cli/src/infrastructure/processes/aggregations_registry.rs @@ -16,6 +16,7 @@ pub struct AggregationProcessRegistry { registry: HashMap, language: SupportedLanguages, dir: PathBuf, + project_path: PathBuf, clickhouse_config: ClickHouseConfig, is_aggregation: bool, } @@ -24,6 +25,7 @@ impl AggregationProcessRegistry { pub fn new( language: SupportedLanguages, dir: PathBuf, + project_path: PathBuf, clickhouse_config: ClickHouseConfig, is_aggregation: bool, ) -> Self { @@ -31,6 +33,7 @@ impl AggregationProcessRegistry { registry: HashMap::new(), language, dir, + project_path, clickhouse_config, is_aggregation, } @@ -45,6 +48,7 @@ impl AggregationProcessRegistry { self.clickhouse_config.clone(), &self.dir, !self.is_aggregation, + &self.project_path, )?, SupportedLanguages::Python => python::aggregation::run( self.clickhouse_config.clone(), diff --git a/apps/framework-cli/src/infrastructure/processes/consumption_registry.rs b/apps/framework-cli/src/infrastructure/processes/consumption_registry.rs index fe83ae89b..a54f590f9 100644 --- a/apps/framework-cli/src/infrastructure/processes/consumption_registry.rs +++ b/apps/framework-cli/src/infrastructure/processes/consumption_registry.rs @@ -24,6 +24,7 @@ pub struct ConsumptionProcessRegistry { clickhouse_config: ClickHouseConfig, dir: PathBuf, language: SupportedLanguages, + project_path: PathBuf, } impl ConsumptionProcessRegistry { @@ -31,12 +32,14 @@ impl ConsumptionProcessRegistry { language: SupportedLanguages, clickhouse_config: ClickHouseConfig, dir: PathBuf, + project_path: PathBuf, ) -> Self { Self { api_process: Option::None, language, dir, clickhouse_config, + project_path, } } @@ -47,9 +50,11 @@ impl ConsumptionProcessRegistry { SupportedLanguages::Python => { python::consumption::run(self.clickhouse_config.clone(), &self.dir) } - SupportedLanguages::Typescript => { - typescript::consumption::run(self.clickhouse_config.clone(), &self.dir) - } + SupportedLanguages::Typescript => typescript::consumption::run( + self.clickhouse_config.clone(), + &self.dir, + &self.project_path, + ), }?; self.api_process = Some(child); diff --git a/apps/framework-cli/src/infrastructure/processes/functions_registry.rs b/apps/framework-cli/src/infrastructure/processes/functions_registry.rs index 1704a684c..37206bd2b 100644 --- a/apps/framework-cli/src/infrastructure/processes/functions_registry.rs +++ b/apps/framework-cli/src/infrastructure/processes/functions_registry.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use log::info; use tokio::process::Child; @@ -29,13 +29,15 @@ pub enum FunctionRegistryError { pub struct FunctionProcessRegistry { registry: HashMap, kafka_config: RedpandaConfig, + project_path: PathBuf, } impl FunctionProcessRegistry { - pub fn new(kafka_config: RedpandaConfig) -> Self { + pub fn new(kafka_config: RedpandaConfig, project_path: PathBuf) -> Self { Self { registry: HashMap::new(), kafka_config, + project_path, } } @@ -58,6 +60,7 @@ impl FunctionProcessRegistry { &function_process.target_topic, &function_process.target_topic_config_json(), &function_process.executable, + &self.project_path, )?) } else { Err(FunctionRegistryError::UnsupportedFunctionLanguage { diff --git a/apps/framework-cli/src/infrastructure/processes/process_registry.rs b/apps/framework-cli/src/infrastructure/processes/process_registry.rs index bd42d5a4c..a19a38392 100644 --- a/apps/framework-cli/src/infrastructure/processes/process_registry.rs +++ b/apps/framework-cli/src/infrastructure/processes/process_registry.rs @@ -13,16 +13,21 @@ pub struct ProcessRegistries { impl ProcessRegistries { pub fn new(project: &Project) -> Self { - let functions = FunctionProcessRegistry::new(project.redpanda_config.clone()); + let functions = FunctionProcessRegistry::new( + project.redpanda_config.clone(), + project.project_location.clone(), + ); let aggregations = AggregationProcessRegistry::new( project.language, project.aggregations_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), true, ); let blocks = AggregationProcessRegistry::new( project.language, project.blocks_dir(), + project.project_location.clone(), project.clickhouse_config.clone(), false, ); @@ -30,6 +35,7 @@ impl ProcessRegistries { project.language, project.clickhouse_config.clone(), project.consumption_dir(), + project.project_location.clone(), ); Self { diff --git a/apps/framework-cli/src/project/typescript_project.rs b/apps/framework-cli/src/project/typescript_project.rs index cceacc648..32814b9b3 100644 --- a/apps/framework-cli/src/project/typescript_project.rs +++ b/apps/framework-cli/src/project/typescript_project.rs @@ -35,17 +35,10 @@ impl Default for TypescriptProject { ("moose".to_string(), "moose-cli".to_string()), ("build".to_string(), "moose-cli build --docker".to_string()), ]), - dependencies: HashMap::from([ - ("@514labs/moose-lib".to_string(), "latest".to_string()), - ("@clickhouse/client-web".to_string(), "1.1.0".to_string()), - ("fastq".to_string(), "1.17.1".to_string()), - ("kafkajs".to_string(), "2.2.4".to_string()), - ]), + dependencies: HashMap::from([("@514labs/moose-lib".to_string(), "latest".to_string())]), dev_dependencies: HashMap::from([ ("@514labs/moose-cli".to_string(), "latest".to_string()), - ("typescript".to_string(), "~5.4.0".to_string()), ("@types/node".to_string(), "^20.12.12".to_string()), - ("ts-patch".to_string(), "~3.2.0".to_string()), ]), } } diff --git a/packages/ts-moose-lib/package.json b/packages/ts-moose-lib/package.json index 82d6b9ff7..f18e202d7 100644 --- a/packages/ts-moose-lib/package.json +++ b/packages/ts-moose-lib/package.json @@ -8,7 +8,8 @@ "dist" ], "bin": { - "ts-node": "node_modules/ts-node/dist/bin.js" + "moose-tspc": "./dist/moose-tspc.js", + "moose-runner": "./dist/moose-runner.js" }, "scripts": { "typecheck": "tsc --noEmit", @@ -20,7 +21,9 @@ "@clickhouse/client-web": "1.5.0", "fastq": "1.17.1", "kafkajs": "2.2.4", - "ts-patch": "~3.2.0" + "ts-patch": "~3.2.0", + "ts-node": "10.9.2", + "typescript": "~5.4.0" }, "devDependencies": { "@repo/ts-config": "workspace:*", @@ -28,7 +31,6 @@ "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", "eslint": "^8.46.0", - "tsup": "^8.0.2", - "typescript": "~5.4.0" + "tsup": "^8.0.2" } } diff --git a/apps/framework-cli/src/framework/typescript/ts_scripts/aggregation.ts b/packages/ts-moose-lib/src/blocks/aggregation.ts old mode 100644 new mode 100755 similarity index 93% rename from apps/framework-cli/src/framework/typescript/ts_scripts/aggregation.ts rename to packages/ts-moose-lib/src/blocks/aggregation.ts index daf08a961..7be1fa8bb --- a/apps/framework-cli/src/framework/typescript/ts_scripts/aggregation.ts +++ b/packages/ts-moose-lib/src/blocks/aggregation.ts @@ -1,12 +1,9 @@ import process from "node:process"; import { ClickHouseClient } from "@clickhouse/client-web"; import fastq, { queueAsPromised } from "fastq"; -import { - getFileName, - walkDir, - getClickhouseClient, - cliLog, -} from "@514labs/moose-lib"; +import { cliLog, getClickhouseClient, getFileName, walkDir } from "../commons"; + +// DEPRECATED we need to remove this file when all the aggregations are converted to use the new blocks interface MvQuery { select: string; @@ -29,6 +26,8 @@ class DependencyError extends Error { const AGGREGATIONS_DIR_PATH = process.argv[1]; const [ + , + , , , CLICKHOUSE_DB, @@ -99,7 +98,7 @@ const asyncWorker = async (task: MvQueueTask) => { await createAggregation(task.chClient, task.path); }; -const main = async () => { +export const runAggregations = async () => { const chClient = getClickhouseClient(clickhouseConfig); console.log(`Connected`); @@ -130,5 +129,3 @@ const main = async () => { await new Promise((resolve) => setTimeout(resolve, 1000)); } }; - -main(); diff --git a/packages/ts-moose-lib/src/blocks.ts b/packages/ts-moose-lib/src/blocks/helpers.ts similarity index 100% rename from packages/ts-moose-lib/src/blocks.ts rename to packages/ts-moose-lib/src/blocks/helpers.ts diff --git a/apps/framework-cli/src/framework/typescript/ts_scripts/blocks.ts b/packages/ts-moose-lib/src/blocks/runner.ts old mode 100644 new mode 100755 similarity index 92% rename from apps/framework-cli/src/framework/typescript/ts_scripts/blocks.ts rename to packages/ts-moose-lib/src/blocks/runner.ts index d344f9ac6..ea103ea6d --- a/apps/framework-cli/src/framework/typescript/ts_scripts/blocks.ts +++ b/packages/ts-moose-lib/src/blocks/runner.ts @@ -1,7 +1,8 @@ import process from "node:process"; import { ClickHouseClient } from "@clickhouse/client-web"; import fastq, { queueAsPromised } from "fastq"; -import { getClickhouseClient, walkDir, Blocks } from "@514labs/moose-lib"; +import { getClickhouseClient, walkDir } from "../commons"; +import { Blocks } from "./helpers"; interface BlocksQueueTask { chClient: ClickHouseClient; @@ -17,6 +18,8 @@ class DependencyError extends Error { } const [ + , + , , BLOCKS_DIR_PATH, CLICKHOUSE_DB, @@ -66,7 +69,7 @@ const asyncWorker = async (task: BlocksQueueTask) => { await createBlocks(task.chClient, task.blocks); }; -const main = async () => { +export const runBlocks = async () => { const chClient = getClickhouseClient(clickhouseConfig); console.log(`Connected`); @@ -87,7 +90,7 @@ const main = async () => { for (const path of blocksFiles) { console.log(`Adding to queue: ${path}`); - const blocks = (await import(path)).default as Blocks; + const blocks = require(path).default as Blocks; queue.push({ chClient, @@ -100,5 +103,3 @@ const main = async () => { await new Promise((resolve) => setTimeout(resolve, 1000)); } }; - -main(); diff --git a/packages/ts-moose-lib/src/commons.ts b/packages/ts-moose-lib/src/commons.ts index e08bfecd1..efbf74952 100644 --- a/packages/ts-moose-lib/src/commons.ts +++ b/packages/ts-moose-lib/src/commons.ts @@ -66,6 +66,7 @@ type CliLogData = { action: string; message: string; }; + export const cliLog: (log: CliLogData) => void = (log) => { const req = http.request({ port: 5000, @@ -76,3 +77,14 @@ export const cliLog: (log: CliLogData) => void = (log) => { req.write(JSON.stringify({ message_type: "Info", ...log })); req.end(); }; + +/** + * Method to change .ts, .cts, and .mts to .js, .cjs, and .mjs + * This is needed because 'import' does not support .ts, .cts, and .mts + */ +export function mapTstoJs(filePath: string): string { + return filePath + .replace(/\.ts$/, ".js") + .replace(/\.cts$/, ".cjs") + .replace(/\.mts$/, ".mjs"); +} diff --git a/packages/ts-moose-lib/src/consumption-helpers.ts b/packages/ts-moose-lib/src/consumption-apis/helpers.ts similarity index 100% rename from packages/ts-moose-lib/src/consumption-helpers.ts rename to packages/ts-moose-lib/src/consumption-apis/helpers.ts diff --git a/apps/framework-cli/src/framework/typescript/ts_scripts/consumption-api.ts b/packages/ts-moose-lib/src/consumption-apis/runner.ts old mode 100644 new mode 100755 similarity index 91% rename from apps/framework-cli/src/framework/typescript/ts_scripts/consumption-api.ts rename to packages/ts-moose-lib/src/consumption-apis/runner.ts index 619537f5e..f6dd0a9bb --- a/apps/framework-cli/src/framework/typescript/ts_scripts/consumption-api.ts +++ b/packages/ts-moose-lib/src/consumption-apis/runner.ts @@ -1,15 +1,16 @@ import http from "http"; import process from "node:process"; -import { getClickhouseClient, MooseClient, sql } from "@514labs/moose-lib"; +import { getClickhouseClient } from "../commons"; +import { MooseClient, sql } from "./helpers"; export const antiCachePath = (path: string) => `${path}?num=${Math.random().toString()}&time=${Date.now()}`; -const CONSUMPTION_DIR_PATH = process.argv[1]; - const [ , , + , + CONSUMPTION_DIR_PATH, CLICKHOUSE_DB, CLICKHOUSE_HOST, CLICKHOUSE_PORT, @@ -55,7 +56,7 @@ const apiHandler = async ( {}, ); - const userFuncModule = await import(pathName); + const userFuncModule = require(pathName); const result = await userFuncModule.default(paramsObject, { client: new MooseClient(getClickhouseClient(clickhouseConfig)), @@ -84,7 +85,7 @@ const apiHandler = async ( } }; -const startApiService = async () => { +export const runConsumptionApis = async () => { console.log("Starting API service"); const server = http.createServer(apiHandler); @@ -100,5 +101,3 @@ const startApiService = async () => { console.log("Server running on port 4001"); }); }; - -startApiService(); diff --git a/packages/ts-moose-lib/src/dataModelTypes.ts b/packages/ts-moose-lib/src/dataModels/dataModelTypes.ts similarity index 100% rename from packages/ts-moose-lib/src/dataModelTypes.ts rename to packages/ts-moose-lib/src/dataModels/dataModelTypes.ts diff --git a/packages/ts-moose-lib/src/enumConvert.ts b/packages/ts-moose-lib/src/dataModels/enumConvert.ts similarity index 100% rename from packages/ts-moose-lib/src/enumConvert.ts rename to packages/ts-moose-lib/src/dataModels/enumConvert.ts diff --git a/packages/ts-moose-lib/src/toDataModels.ts b/packages/ts-moose-lib/src/dataModels/toDataModels.ts similarity index 100% rename from packages/ts-moose-lib/src/toDataModels.ts rename to packages/ts-moose-lib/src/dataModels/toDataModels.ts diff --git a/packages/ts-moose-lib/src/typeConvert.ts b/packages/ts-moose-lib/src/dataModels/typeConvert.ts similarity index 100% rename from packages/ts-moose-lib/src/typeConvert.ts rename to packages/ts-moose-lib/src/dataModels/typeConvert.ts diff --git a/packages/ts-moose-lib/src/index.ts b/packages/ts-moose-lib/src/index.ts index 1d74c50fb..c445e621b 100644 --- a/packages/ts-moose-lib/src/index.ts +++ b/packages/ts-moose-lib/src/index.ts @@ -1,4 +1,4 @@ -import { MooseClient, sql } from "./consumption-helpers"; +import { MooseClient, sql } from "./consumption-apis/helpers"; export interface Aggregation { select: string; orderBy: string; @@ -28,6 +28,6 @@ export type DataModelConfig = Partial<{ }; }>; -export * from "./blocks"; +export * from "./blocks/helpers"; export * from "./commons"; -export * from "./consumption-helpers"; +export * from "./consumption-apis/helpers"; diff --git a/packages/ts-moose-lib/src/moduleExportSerializer.ts b/packages/ts-moose-lib/src/moduleExportSerializer.ts new file mode 100755 index 000000000..7d695fe32 --- /dev/null +++ b/packages/ts-moose-lib/src/moduleExportSerializer.ts @@ -0,0 +1,8 @@ +import process from "process"; + +const target_model = process.argv[3]; + +export async function runExportSerializer() { + const exports_list = require(target_model); + console.log(JSON.stringify(exports_list)); +} diff --git a/packages/ts-moose-lib/src/moose-runner.ts b/packages/ts-moose-lib/src/moose-runner.ts new file mode 100644 index 000000000..a6dab97ca --- /dev/null +++ b/packages/ts-moose-lib/src/moose-runner.ts @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +// This file is use to run the proper runners for moose based on the +// the arguments passed to the file. +// It regiters ts-node to be able to interpret user code. + +import { register } from "ts-node"; +register({ + esm: true, + experimentalTsImportSpecifiers: true, +}); + +import { runBlocks } from "./blocks/runner"; +import { runAggregations } from "./blocks/aggregation"; +import { runConsumptionApis } from "./consumption-apis/runner"; +import { runStreamingFunctions } from "./streaming-functions/runner"; +import { runExportSerializer } from "./moduleExportSerializer"; + +// We register ts-node to be able to interpret TS user code. + +switch (process.argv[2]) { + case "export-serializer": + runExportSerializer(); + break; + case "blocks": + runBlocks(); + break; + case "aggregations": + runAggregations(); + break; + case "consumption-apis": + runConsumptionApis(); + break; + case "streaming-functions": + runStreamingFunctions(); + break; + default: + console.error("Invalid argument"); + process.exit(1); +} diff --git a/packages/ts-moose-lib/src/moose-tspc.ts b/packages/ts-moose-lib/src/moose-tspc.ts new file mode 100644 index 000000000..073816a7d --- /dev/null +++ b/packages/ts-moose-lib/src/moose-tspc.ts @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import { getLiveModule } from "ts-patch"; +import { runInThisContext } from "vm"; +import path from "path"; + +const { js, tsModule } = getLiveModule("tsc.js"); + +const script = runInThisContext(` + (function (exports, require, module, __filename, __dirname) { + ${js} + }); + `); + +script.call( + exports, + exports, + require, + module, + tsModule.modulePath, + path.dirname(tsModule.modulePath), +); diff --git a/apps/framework-cli/src/framework/typescript/ts_scripts/streaming-function.ts b/packages/ts-moose-lib/src/streaming-functions/runner.ts old mode 100644 new mode 100755 similarity index 51% rename from apps/framework-cli/src/framework/typescript/ts_scripts/streaming-function.ts rename to packages/ts-moose-lib/src/streaming-functions/runner.ts index c3669728e..d6787be16 --- a/apps/framework-cli/src/framework/typescript/ts_scripts/streaming-function.ts +++ b/packages/ts-moose-lib/src/streaming-functions/runner.ts @@ -1,18 +1,8 @@ import { Consumer, Kafka, KafkaMessage, Producer, SASLOptions } from "kafkajs"; import { Buffer } from "node:buffer"; import process from "node:process"; -import { cliLog } from "@514labs/moose-lib"; import http from "http"; - -const SOURCE_TOPIC = process.argv[1]; -const TARGET_TOPIC = process.argv[2]; -const TARGET_TOPIC_CONFIG = process.argv[3]; -const FUNCTION_FILE_PATH = process.argv[4]; -const BROKER = process.argv[5]; -const SASL_USERNAME = process.argv[6]; -const SASL_PASSWORD = process.argv[7]; -const SASL_MECHANISM = process.argv[8]; -const SECURITY_PROTOCOL = process.argv[9]; +import { cliLog } from "../commons"; type CliLogData = { count: number; @@ -20,92 +10,84 @@ type CliLogData = { function_name: string; direction: "In" | "Out"; }; -export const metricsLog: (log: CliLogData) => void = (log) => { - const req = http.request({ - port: 5000, - method: "POST", - path: "/metrics-logs", - }); // no callback, fire and forget - - req.write(JSON.stringify({ ...log })); - req.end(); -}; type StreamingFunction = (data: unknown) => unknown | Promise; type SlimKafkaMessage = { value: string }; -const logPrefix = `${SOURCE_TOPIC} -> ${TARGET_TOPIC}`; -const log = (message: string): void => { - console.log(`${logPrefix}: ${message}`); -}; - -const error = (message: string): void => { - console.error(`${logPrefix}: ${message}`); -}; - -const warn = (message: string): void => { - console.warn(`${logPrefix}: ${message}`); -}; - -log("Initializing streaming function..."); -if (SOURCE_TOPIC === undefined) { - error("Missing source topic"); - process.exit(1); +interface StreamingFunctionArgs { + sourceTopic: string; + targetTopic: string; + targetTopicConfig: string; + functionFilePath: string; + broker: string; + saslUsername?: string; + saslPassword?: string; + saslMechanism?: string; + securityProtocol?: string; } -if (TARGET_TOPIC === undefined) { - error("Missing target topic"); - process.exit(1); +interface Logger { + logPrefix: string; + log: (message: string) => void; + error: (message: string) => void; + warn: (message: string) => void; } -if (FUNCTION_FILE_PATH === undefined) { - error("Missing streaming function file path"); - process.exit(1); -} - -if (BROKER === undefined) { - error("Missing argument: BROKER"); - process.exit(1); -} - -// TODO - when having optional arguments, we should use a proper -// command line parser to handle this. As is a user would not be able to -// only provide a SASL_USERNAME and not a SASL_PASSWORD. -if (SASL_USERNAME === undefined) { - warn("No argument: SASL_USERNAME"); -} - -if (SASL_PASSWORD === undefined) { - warn("No argument: SASL_USERNAME"); -} - -if (SASL_MECHANISM === undefined) { - warn("No argument: SASL_MECHANISM"); -} - -if (SECURITY_PROTOCOL === undefined) { - warn("No argument: SECURITY_PROTOCOL"); -} - -log(`Streaming function configuration loaded with file ${FUNCTION_FILE_PATH}`); +const parseArgs = (): StreamingFunctionArgs => { + const SOURCE_TOPIC = process.argv[3]; + const TARGET_TOPIC = process.argv[4]; + const TARGET_TOPIC_CONFIG = process.argv[5]; + const FUNCTION_FILE_PATH = process.argv[6]; + const BROKER = process.argv[7]; + const SASL_USERNAME = process.argv[8]; + const SASL_PASSWORD = process.argv[9]; + const SASL_MECHANISM = process.argv[10]; + const SECURITY_PROTOCOL = process.argv[11]; + + return { + sourceTopic: SOURCE_TOPIC, + targetTopic: TARGET_TOPIC, + targetTopicConfig: TARGET_TOPIC_CONFIG, + functionFilePath: FUNCTION_FILE_PATH, + broker: BROKER, + saslUsername: SASL_USERNAME, + saslPassword: SASL_PASSWORD, + saslMechanism: SASL_MECHANISM, + securityProtocol: SECURITY_PROTOCOL, + }; +}; -const getSaslConfig = (): SASLOptions | undefined => { - const mechanism = SASL_MECHANISM ? SASL_MECHANISM.toLowerCase() : ""; +const buildSaslConfig = ( + logger: Logger, + args: StreamingFunctionArgs, +): SASLOptions | undefined => { + const mechanism = args.saslMechanism ? args.saslMechanism.toLowerCase() : ""; switch (mechanism) { case "plain": case "scram-sha-256": case "scram-sha-512": return { mechanism: mechanism, - username: SASL_USERNAME || "", - password: SASL_PASSWORD || "", + username: args.saslUsername || "", + password: args.saslPassword || "", }; default: - log(`Unsupported SASL mechanism: ${SASL_MECHANISM}`); + logger.warn(`Unsupported SASL mechanism: ${args.saslMechanism}`); return undefined; } }; +export const metricsLog: (log: CliLogData) => void = (log) => { + const req = http.request({ + port: 5000, + method: "POST", + path: "/metrics-logs", + }); // no callback, fire and forget + + req.write(JSON.stringify({ ...log })); + req.end(); +}; + const jsonDateReviver = (key: string, value: unknown): unknown => { const iso8601Format = /^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$/; @@ -117,40 +99,37 @@ const jsonDateReviver = (key: string, value: unknown): unknown => { return value; }; -const kafka = new Kafka({ - clientId: "streaming-function-consumer", - brokers: [BROKER], - ssl: SECURITY_PROTOCOL === "SASL_SSL", - sasl: getSaslConfig(), -}); - -const streamingFuncId = `flow-${SOURCE_TOPIC}-${TARGET_TOPIC}`; -const consumer: Consumer = kafka.consumer({ - groupId: streamingFuncId, -}); -const producer: Producer = kafka.producer({ transactionalId: streamingFuncId }); - -const startProducer = async (): Promise => { +const startProducer = async ( + logger: Logger, + producer: Producer, +): Promise => { await producer.connect(); - log("Producer is running..."); + logger.log("Producer is running..."); }; -const stopProducer = async (): Promise => { +const stopProducer = async ( + logger: Logger, + producer: Producer, +): Promise => { await producer.disconnect(); - log("Producer is shutting down..."); + logger.log("Producer is shutting down..."); }; -const stopConsumer = async (): Promise => { +const stopConsumer = async ( + logger: Logger, + consumer: Consumer, +): Promise => { await consumer.disconnect(); - log("Consumer is shutting down..."); + logger.log("Consumer is shutting down..."); }; const handleMessage = async ( + logger: Logger, streamingFunction: StreamingFunction, message: KafkaMessage, ): Promise => { if (message.value === undefined || message.value === null) { - log(`Received message with no value, skipping...`); + logger.log(`Received message with no value, skipping...`); return null; } @@ -168,9 +147,9 @@ const handleMessage = async ( } } catch (e) { // TODO: Track failure rate - error(`Failed to transform data`); + logger.error(`Failed to transform data`); if (e instanceof Error) { - error(e.message); + logger.error(e.message); } } @@ -178,7 +157,9 @@ const handleMessage = async ( }; const sendMessages = async ( - targetTopic: string, + logger: Logger, + args: StreamingFunctionArgs, + producer: Producer, messages: SlimKafkaMessage[], maxMessageSize: number, ): Promise => { @@ -191,8 +172,10 @@ const sendMessages = async ( if (chunkSize + messageSize > maxMessageSize) { // Send the current chunk before adding the new message - await producer.send({ topic: targetTopic, messages: chunks }); - log(`Sent ${chunks.length} transformed data to ${targetTopic}`); + await producer.send({ topic: args.targetTopic, messages: chunks }); + logger.log( + `Sent ${chunks.length} transformed data to ${args.targetTopic}`, + ); // Start a new chunk chunks = [message]; @@ -210,62 +193,65 @@ const sendMessages = async ( // Send the last chunk if (chunks.length > 0) { - await producer.send({ topic: targetTopic, messages: chunks }); - log(`Sent final ${chunks.length} transformed data to ${targetTopic}`); + await producer.send({ topic: args.targetTopic, messages: chunks }); + logger.log( + `Sent final ${chunks.length} transformed data to ${args.targetTopic}`, + ); } } catch (e) { - error(`Failed to send transformed data`); + logger.error(`Failed to send transformed data`); if (e instanceof Error) { - error(e.message); + logger.error(e.message); } } }; -setTimeout(() => sendMessageMetricsOut(), 1000); - let count_in = 0; let count_out = 0; let bytes = 0; -const sendMessageMetricsIn = () => { +const sendMessageMetricsIn = (logger: Logger) => { metricsLog({ count: count_in, - function_name: logPrefix, + function_name: logger.logPrefix, bytes: bytes, direction: "In", }); count_in = 0; bytes = 0; - setTimeout(() => sendMessageMetricsIn(), 1000); + setTimeout(() => sendMessageMetricsIn(logger), 1000); }; -const sendMessageMetricsOut = () => { +const sendMessageMetricsOut = (logger: Logger) => { metricsLog({ count: count_out, // We actually only read bytes from the in direction bytes: bytes, - function_name: logPrefix, + function_name: logger.logPrefix, direction: "Out", }); count_out = 0; - setTimeout(() => sendMessageMetricsOut(), 1000); + setTimeout(() => sendMessageMetricsOut(logger), 1000); }; const startConsumer = async ( - sourceTopic: string, - targetTopic: string, + logger: Logger, + args: StreamingFunctionArgs, + consumer: Consumer, + producer: Producer, + streamingFuncId: string, maxMessageSize: number, ): Promise => { await consumer.connect(); - log( - `Starting consumer group '${streamingFuncId}' with source topic: ${sourceTopic} and target topic: ${targetTopic}`, + logger.log( + `Starting consumer group '${streamingFuncId}' with source topic: ${args.sourceTopic} and target topic: ${args.targetTopic}`, ); let streamingFunctionImport; try { - streamingFunctionImport = await import( - FUNCTION_FILE_PATH.substring(0, FUNCTION_FILE_PATH.length - 3) + streamingFunctionImport = require( + args.functionFilePath.substring(0, args.functionFilePath.length - 3), ); } catch (e) { cliLog({ action: "Code failed to load", message: `${e}` }); @@ -274,7 +260,7 @@ const startConsumer = async ( const streamingFunction: StreamingFunction = streamingFunctionImport.default; await consumer.subscribe({ - topics: [sourceTopic], + topics: [args.sourceTopic], // to read records sent before subscriber is created fromBeginning: true, }); @@ -285,13 +271,13 @@ const startConsumer = async ( cliLog({ action: "Received", - message: `${logPrefix} ${batch.messages.length} message(s)`, + message: `${logger.logPrefix} ${batch.messages.length} message(s)`, }); - log(`Received ${batch.messages.length} message(s)`); + logger.log(`Received ${batch.messages.length} message(s)`); const messages = ( await Promise.all( batch.messages.map((message) => - handleMessage(streamingFunction, message), + handleMessage(logger, streamingFunction, message), ), ) ).flat(); @@ -300,7 +286,9 @@ const startConsumer = async ( if (filteredMessages.length > 0) { await sendMessages( - targetTopic, + logger, + args, + producer, filteredMessages as SlimKafkaMessage[], maxMessageSize, ); @@ -308,11 +296,9 @@ const startConsumer = async ( }, }); - log("Consumer is running..."); + logger.log("Consumer is running..."); }; -setTimeout(() => sendMessageMetricsIn(), 1000); - /** * message.max.bytes is a broker setting that applies to all topics. * max.message.bytes is a per-topic setting. @@ -332,41 +318,80 @@ const getMaxMessageSize = (config: Record): number => { return Math.min(maxMessageBytes, messageMaxBytes); }; -const startStreaming = async ( - sourceTopic: string, - targetTopic: string, - targetTopicConfigJson: string, -): Promise => { +const buildLogger = (args: StreamingFunctionArgs): Logger => { + const logPrefix = `${args.sourceTopic} -> ${args.targetTopic}`; + const logger: Logger = { + logPrefix: logPrefix, + log: (message: string): void => { + console.log(`${logPrefix}: ${message}`); + }, + error: (message: string): void => { + console.error(`${logPrefix}: ${message}`); + }, + warn: (message: string): void => { + console.warn(`${logPrefix}: ${message}`); + }, + }; + return logger; +}; + +export const runStreamingFunctions = async (): Promise => { + const args = parseArgs(); + const logger = buildLogger(args); + + setTimeout(() => sendMessageMetricsIn(logger), 1000); + setTimeout(() => sendMessageMetricsOut(logger), 1000); + + const kafka = new Kafka({ + clientId: "streaming-function-consumer", + brokers: [args.broker], + ssl: args.securityProtocol === "SASL_SSL", + sasl: buildSaslConfig(logger, args), + }); + + const streamingFuncId = `flow-${args.sourceTopic}-${args.targetTopic}`; + const consumer: Consumer = kafka.consumer({ + groupId: streamingFuncId, + }); + const producer: Producer = kafka.producer({ + transactionalId: streamingFuncId, + }); + + process.on("SIGTERM", async () => { + logger.log("Received SIGTERM, shutting down..."); + await stopConsumer(logger, consumer); + await stopProducer(logger, producer); + process.exit(0); + }); + try { - await startProducer(); + await startProducer(logger, producer); try { - const targetTopicConfig = JSON.parse(targetTopicConfigJson) as Record< + const targetTopicConfig = JSON.parse(args.targetTopicConfig) as Record< string, unknown >; const maxMessageSize = getMaxMessageSize(targetTopicConfig); - await startConsumer(sourceTopic, targetTopic, maxMessageSize); + await startConsumer( + logger, + args, + consumer, + producer, + streamingFuncId, + maxMessageSize, + ); } catch (e) { - error("Failed to start kafka consumer: "); + logger.error("Failed to start kafka consumer: "); if (e instanceof Error) { - error(e.message); + logger.error(e.message); } } } catch (e) { - error("Failed to start kafka producer: "); + logger.error("Failed to start kafka producer: "); if (e instanceof Error) { - error(e.message); + logger.error(e.message); } } }; - -process.on("SIGTERM", async () => { - log("Received SIGTERM, shutting down..."); - await stopConsumer(); - await stopProducer(); - process.exit(0); -}); - -startStreaming(SOURCE_TOPIC, TARGET_TOPIC, TARGET_TOPIC_CONFIG); diff --git a/packages/ts-moose-lib/tsup.config.ts b/packages/ts-moose-lib/tsup.config.ts index 9b83f166e..48dd45763 100644 --- a/packages/ts-moose-lib/tsup.config.ts +++ b/packages/ts-moose-lib/tsup.config.ts @@ -1,7 +1,12 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/toDataModels.ts"], + entry: [ + "src/index.ts", + "src/dataModels/toDataModels.ts", + "src/moose-tspc.ts", + "src/moose-runner.ts", + ], format: ["cjs", "esm"], // Build for commonJS and ESmodules dts: true, // Generate declaration file (.d.ts) splitting: false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92371323c..4cd06020f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,7 +99,7 @@ importers: version: 2.3.0 tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.3) + version: 1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5))) devDependencies: "@headlessui/react": specifier: ^1.7.18 @@ -136,7 +136,7 @@ importers: version: 8.4.38 tailwindcss: specifier: ^3.3.0 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)) typescript: specifier: ^5 version: 5.4.5 @@ -231,7 +231,7 @@ importers: version: 10.4.19(postcss@8.4.38) jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@18.19.33) + version: 29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) jest-environment-jsdom: specifier: ^29.7.0 version: 29.7.0 @@ -243,7 +243,7 @@ importers: version: link:../../packages/tailwind-config tailwindcss: specifier: ^3.4.0 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) typescript: specifier: ^4.9.5 version: 4.9.5 @@ -346,7 +346,7 @@ importers: version: 2.3.0 tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.3) + version: 1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5))) devDependencies: "@headlessui/react": specifier: ^1.7.18 @@ -383,7 +383,7 @@ importers: version: 1.12.0 tailwindcss: specifier: ^3.3.0 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)) typescript: specifier: ^5 version: 5.4.5 @@ -455,7 +455,7 @@ importers: version: 2.4.0 tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.3) + version: 1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5))) devDependencies: "@types/d3-array": specifier: ^3.2.1 @@ -480,7 +480,7 @@ importers: version: 8.4.38 tailwindcss: specifier: ^3.4.1 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)) typescript: specifier: ^5 version: 5.4.5 @@ -580,7 +580,7 @@ importers: version: 2.3.0 tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.3) + version: 1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5))) zod: specifier: ^3.22.4 version: 3.23.8 @@ -614,7 +614,7 @@ importers: version: 8.4.38 tailwindcss: specifier: ^3.3.0 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)) typescript: specifier: ^5 version: 5.4.5 @@ -629,7 +629,7 @@ importers: version: 2.3.0 tailwindcss: specifier: ^3.4.1 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)) devDependencies: "@sentry/nextjs": specifier: ^7.106.0 @@ -723,10 +723,10 @@ importers: version: 2.3.0 tailwindcss: specifier: ^3.4.1 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)) tailwindcss-animate: specifier: ^1.0.7 - version: 1.0.7(tailwindcss@3.4.3) + version: 1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5))) devDependencies: "@headlessui/react": specifier: ^1.7.18 @@ -809,7 +809,7 @@ importers: devDependencies: tailwindcss: specifier: ^3.2.4 - version: 3.4.3 + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.13)(typescript@5.4.5)) packages/ts-config: {} @@ -824,9 +824,15 @@ importers: kafkajs: specifier: 2.2.4 version: 2.2.4 + ts-node: + specifier: 10.9.2 + version: 10.9.2(@types/node@20.12.13)(typescript@5.4.5) ts-patch: specifier: ~3.2.0 version: 3.2.0 + typescript: + specifier: ~5.4.0 + version: 5.4.5 devDependencies: "@repo/ts-config": specifier: workspace:* @@ -845,10 +851,7 @@ importers: version: 8.57.0 tsup: specifier: ^8.0.2 - version: 8.0.2(postcss@8.4.38)(typescript@5.4.5) - typescript: - specifier: ~5.4.0 - version: 5.4.5 + version: 8.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.13)(typescript@5.4.5))(typescript@5.4.5) packages: "@514labs/moose-cli-darwin-arm64@0.3.205": @@ -1585,6 +1588,13 @@ packages: } engines: { node: ">=16" } + "@cspotcode/source-map-support@0.8.1": + resolution: + { + integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==, + } + engines: { node: ">=12" } + "@esbuild/aix-ppc64@0.19.12": resolution: { @@ -2185,6 +2195,12 @@ packages: integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==, } + "@jridgewell/trace-mapping@0.3.9": + resolution: + { + integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==, + } + "@mdx-js/loader@3.0.1": resolution: { @@ -4357,6 +4373,30 @@ packages: } engines: { node: ">= 10" } + "@tsconfig/node10@1.0.11": + resolution: + { + integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==, + } + + "@tsconfig/node12@1.0.11": + resolution: + { + integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==, + } + + "@tsconfig/node14@1.0.3": + resolution: + { + integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==, + } + + "@tsconfig/node16@1.0.4": + resolution: + { + integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==, + } + "@types/acorn@4.0.6": resolution: { @@ -5330,6 +5370,12 @@ packages: integrity: sha512-Wk7TEzl1KqvTGs/uyhmHO/3XLd3t1UeU4IstvPXVzGPM522cTjqjNZ99esCkcL52sjqjo8e8CTBcWhkxvGzoAw==, } + arg@4.1.3: + resolution: + { + integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==, + } + arg@5.0.2: resolution: { @@ -6041,6 +6087,12 @@ packages: engines: { node: ^14.15.0 || ^16.10.0 || >=18.0.0 } hasBin: true + create-require@1.1.1: + resolution: + { + integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==, + } + cross-spawn@5.1.0: resolution: { @@ -6554,6 +6606,13 @@ packages: } engines: { node: ^14.15.0 || ^16.10.0 || >=18.0.0 } + diff@4.0.2: + resolution: + { + integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==, + } + engines: { node: ">=0.3.1" } + diff@5.2.0: resolution: { @@ -9107,6 +9166,12 @@ packages: } engines: { node: ">=10" } + make-error@1.3.6: + resolution: + { + integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==, + } + makeerror@1.0.12: resolution: { @@ -11794,6 +11859,23 @@ packages: integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==, } + ts-node@10.9.2: + resolution: + { + integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==, + } + hasBin: true + peerDependencies: + "@swc/core": ">=1.2.50" + "@swc/wasm": ">=1.2.50" + "@types/node": "*" + typescript: ">=2.7" + peerDependenciesMeta: + "@swc/core": + optional: true + "@swc/wasm": + optional: true + ts-patch@3.2.0: resolution: { @@ -12234,6 +12316,12 @@ packages: engines: { node: ">=8" } hasBin: true + v8-compile-cache-lib@3.0.1: + resolution: + { + integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==, + } + v8-compile-cache@2.4.0: resolution: { @@ -12591,6 +12679,13 @@ packages: } engines: { node: ">=12" } + yn@3.1.1: + resolution: + { + integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==, + } + engines: { node: ">=6" } + yocto-queue@0.1.0: resolution: { @@ -12723,10 +12818,10 @@ snapshots: "@aws-crypto/sha1-browser": 3.0.0 "@aws-crypto/sha256-browser": 3.0.0 "@aws-crypto/sha256-js": 3.0.0 - "@aws-sdk/client-sso-oidc": 3.574.0(@aws-sdk/client-sts@3.574.0) - "@aws-sdk/client-sts": 3.574.0 + "@aws-sdk/client-sso-oidc": 3.574.0 + "@aws-sdk/client-sts": 3.574.0(@aws-sdk/client-sso-oidc@3.574.0) "@aws-sdk/core": 3.572.0 - "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0) "@aws-sdk/middleware-bucket-endpoint": 3.568.0 "@aws-sdk/middleware-expect-continue": 3.572.0 "@aws-sdk/middleware-flexible-checksums": 3.572.0 @@ -12781,13 +12876,13 @@ snapshots: transitivePeerDependencies: - aws-crt - "@aws-sdk/client-sso-oidc@3.574.0(@aws-sdk/client-sts@3.574.0)": + "@aws-sdk/client-sso-oidc@3.574.0": dependencies: "@aws-crypto/sha256-browser": 3.0.0 "@aws-crypto/sha256-js": 3.0.0 - "@aws-sdk/client-sts": 3.574.0 + "@aws-sdk/client-sts": 3.574.0(@aws-sdk/client-sso-oidc@3.574.0) "@aws-sdk/core": 3.572.0 - "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0) "@aws-sdk/middleware-host-header": 3.567.0 "@aws-sdk/middleware-logger": 3.568.0 "@aws-sdk/middleware-recursion-detection": 3.567.0 @@ -12824,7 +12919,6 @@ snapshots: "@smithy/util-utf8": 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - "@aws-sdk/client-sts" - aws-crt "@aws-sdk/client-sso@3.572.0": @@ -12870,13 +12964,13 @@ snapshots: transitivePeerDependencies: - aws-crt - "@aws-sdk/client-sts@3.574.0": + "@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)": dependencies: "@aws-crypto/sha256-browser": 3.0.0 "@aws-crypto/sha256-js": 3.0.0 - "@aws-sdk/client-sso-oidc": 3.574.0(@aws-sdk/client-sts@3.574.0) + "@aws-sdk/client-sso-oidc": 3.574.0 "@aws-sdk/core": 3.572.0 - "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-node": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0) "@aws-sdk/middleware-host-header": 3.567.0 "@aws-sdk/middleware-logger": 3.568.0 "@aws-sdk/middleware-recursion-detection": 3.567.0 @@ -12913,6 +13007,7 @@ snapshots: "@smithy/util-utf8": 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - "@aws-sdk/client-sso-oidc" - aws-crt "@aws-sdk/core@3.572.0": @@ -12944,13 +13039,13 @@ snapshots: "@smithy/util-stream": 2.2.0 tslib: 2.6.2 - "@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0))": + "@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0)": dependencies: - "@aws-sdk/client-sts": 3.574.0 + "@aws-sdk/client-sts": 3.574.0(@aws-sdk/client-sso-oidc@3.574.0) "@aws-sdk/credential-provider-env": 3.568.0 "@aws-sdk/credential-provider-process": 3.572.0 "@aws-sdk/credential-provider-sso": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0) - "@aws-sdk/credential-provider-web-identity": 3.568.0(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-web-identity": 3.568.0(@aws-sdk/client-sts@3.574.0) "@aws-sdk/types": 3.567.0 "@smithy/credential-provider-imds": 2.3.0 "@smithy/property-provider": 2.2.0 @@ -12961,14 +13056,14 @@ snapshots: - "@aws-sdk/client-sso-oidc" - aws-crt - "@aws-sdk/credential-provider-node@3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0))": + "@aws-sdk/credential-provider-node@3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0)": dependencies: "@aws-sdk/credential-provider-env": 3.568.0 "@aws-sdk/credential-provider-http": 3.568.0 - "@aws-sdk/credential-provider-ini": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-ini": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0)(@aws-sdk/client-sts@3.574.0) "@aws-sdk/credential-provider-process": 3.572.0 "@aws-sdk/credential-provider-sso": 3.572.0(@aws-sdk/client-sso-oidc@3.574.0) - "@aws-sdk/credential-provider-web-identity": 3.568.0(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0)) + "@aws-sdk/credential-provider-web-identity": 3.568.0(@aws-sdk/client-sts@3.574.0) "@aws-sdk/types": 3.567.0 "@smithy/credential-provider-imds": 2.3.0 "@smithy/property-provider": 2.2.0 @@ -13001,9 +13096,9 @@ snapshots: - "@aws-sdk/client-sso-oidc" - aws-crt - "@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.574.0(@aws-sdk/client-sso-oidc@3.574.0))": + "@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.574.0)": dependencies: - "@aws-sdk/client-sts": 3.574.0 + "@aws-sdk/client-sts": 3.574.0(@aws-sdk/client-sso-oidc@3.574.0) "@aws-sdk/types": 3.567.0 "@smithy/property-provider": 2.2.0 "@smithy/types": 2.12.0 @@ -13119,7 +13214,7 @@ snapshots: "@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.574.0)": dependencies: - "@aws-sdk/client-sso-oidc": 3.574.0(@aws-sdk/client-sts@3.574.0) + "@aws-sdk/client-sso-oidc": 3.574.0 "@aws-sdk/types": 3.567.0 "@smithy/property-provider": 2.2.0 "@smithy/shared-ini-file-loader": 2.4.0 @@ -13397,6 +13492,10 @@ snapshots: dependencies: "@clickhouse/client-common": 0.3.1 + "@cspotcode/source-map-support@0.8.1": + dependencies: + "@jridgewell/trace-mapping": 0.3.9 + "@esbuild/aix-ppc64@0.19.12": optional: true @@ -13685,7 +13784,7 @@ snapshots: jest-util: 29.7.0 slash: 3.0.0 - "@jest/core@29.7.0": + "@jest/core@29.7.0(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5))": dependencies: "@jest/console": 29.7.0 "@jest/reporters": 29.7.0 @@ -13699,7 +13798,7 @@ snapshots: exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@20.14.5) + jest-config: 29.7.0(@types/node@20.14.5)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -13860,6 +13959,11 @@ snapshots: "@jridgewell/resolve-uri": 3.1.2 "@jridgewell/sourcemap-codec": 1.4.15 + "@jridgewell/trace-mapping@0.3.9": + dependencies: + "@jridgewell/resolve-uri": 3.1.2 + "@jridgewell/sourcemap-codec": 1.4.15 + "@mdx-js/loader@3.0.1(webpack@5.93.0)": dependencies: "@mdx-js/mdx": 3.0.1 @@ -15542,6 +15646,14 @@ snapshots: "@tootallnate/once@2.0.0": {} + "@tsconfig/node10@1.0.11": {} + + "@tsconfig/node12@1.0.11": {} + + "@tsconfig/node14@1.0.3": {} + + "@tsconfig/node16@1.0.4": {} + "@types/acorn@4.0.6": dependencies: "@types/estree": 1.0.5 @@ -16290,6 +16402,8 @@ snapshots: arg@1.0.0: {} + arg@4.1.3: {} + arg@5.0.2: {} argparse@1.0.10: @@ -16706,13 +16820,13 @@ snapshots: dependencies: layout-base: 1.0.2 - create-jest@29.7.0(@types/node@18.19.33): + create-jest@29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): dependencies: "@jest/types": 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@18.19.33) + jest-config: 29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -16721,6 +16835,8 @@ snapshots: - supports-color - ts-node + create-require@1.1.1: {} + cross-spawn@5.1.0: dependencies: lru-cache: 4.1.5 @@ -17035,6 +17151,8 @@ snapshots: diff-sequences@29.6.3: {} + diff@4.0.2: {} + diff@5.2.0: {} dir-glob@3.0.1: @@ -17279,7 +17397,7 @@ snapshots: eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) eslint-plugin-react: 7.34.1(eslint@8.57.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) @@ -17297,7 +17415,7 @@ snapshots: eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) eslint-plugin-react: 7.34.1(eslint@8.57.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) @@ -17315,7 +17433,7 @@ snapshots: eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) eslint-plugin-react: 7.34.1(eslint@8.57.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) @@ -17365,7 +17483,7 @@ snapshots: enhanced-resolve: 5.16.1 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.5 is-core-module: 2.13.1 @@ -17425,7 +17543,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -18559,16 +18677,16 @@ snapshots: - babel-plugin-macros - supports-color - jest-cli@29.7.0(@types/node@18.19.33): + jest-cli@29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): dependencies: - "@jest/core": 29.7.0 + "@jest/core": 29.7.0(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) "@jest/test-result": 29.7.0 "@jest/types": 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@18.19.33) + create-jest: 29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) exit: 0.1.2 import-local: 3.1.0 - jest-config: 29.7.0(@types/node@18.19.33) + jest-config: 29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -18578,7 +18696,7 @@ snapshots: - supports-color - ts-node - jest-config@29.7.0(@types/node@18.19.33): + jest-config@29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): dependencies: "@babel/core": 7.24.5 "@jest/test-sequencer": 29.7.0 @@ -18604,11 +18722,12 @@ snapshots: strip-json-comments: 3.1.1 optionalDependencies: "@types/node": 18.19.33 + ts-node: 10.9.2(@types/node@18.19.33)(typescript@4.9.5) transitivePeerDependencies: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@20.14.5): + jest-config@29.7.0(@types/node@20.14.5)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): dependencies: "@babel/core": 7.24.5 "@jest/test-sequencer": 29.7.0 @@ -18634,6 +18753,7 @@ snapshots: strip-json-comments: 3.1.1 optionalDependencies: "@types/node": 20.14.5 + ts-node: 10.9.2(@types/node@18.19.33)(typescript@4.9.5) transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -18874,12 +18994,12 @@ snapshots: merge-stream: 2.0.0 supports-color: 8.1.1 - jest@29.7.0(@types/node@18.19.33): + jest@29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): dependencies: - "@jest/core": 29.7.0 + "@jest/core": 29.7.0(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) "@jest/types": 29.6.3 import-local: 3.1.0 - jest-cli: 29.7.0(@types/node@18.19.33) + jest-cli: 29.7.0(@types/node@18.19.33)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) transitivePeerDependencies: - "@types/node" - babel-plugin-macros @@ -19129,6 +19249,8 @@ snapshots: dependencies: semver: 7.6.2 + make-error@1.3.6: {} + makeerror@1.0.12: dependencies: tmpl: 1.0.5 @@ -20362,12 +20484,45 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.4.38 - postcss-load-config@4.0.2(postcss@8.4.38): + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): + dependencies: + lilconfig: 3.1.1 + yaml: 2.4.2 + optionalDependencies: + postcss: 8.4.38 + ts-node: 10.9.2(@types/node@18.19.33)(typescript@4.9.5) + + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)): + dependencies: + lilconfig: 3.1.1 + yaml: 2.4.2 + optionalDependencies: + postcss: 8.4.38 + ts-node: 10.9.2(@types/node@20.12.12)(typescript@5.4.5) + + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.13)(typescript@5.4.5)): + dependencies: + lilconfig: 3.1.1 + yaml: 2.4.2 + optionalDependencies: + postcss: 8.4.38 + ts-node: 10.9.2(@types/node@20.12.13)(typescript@5.4.5) + + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.13)(typescript@5.4.5)): dependencies: lilconfig: 3.1.1 yaml: 2.4.2 optionalDependencies: postcss: 8.4.38 + ts-node: 10.9.2(@types/node@20.14.13)(typescript@5.4.5) + + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)): + dependencies: + lilconfig: 3.1.1 + yaml: 2.4.2 + optionalDependencies: + postcss: 8.4.38 + ts-node: 10.9.2(@types/node@20.14.5)(typescript@5.4.5) postcss-nested@6.0.1(postcss@8.4.38): dependencies: @@ -21101,11 +21256,96 @@ snapshots: tailwind-merge@2.4.0: {} - tailwindcss-animate@1.0.7(tailwindcss@3.4.3): + tailwindcss-animate@1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5))): dependencies: - tailwindcss: 3.4.3 + tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)) - tailwindcss@3.4.3: + tailwindcss-animate@1.0.7(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5))): + dependencies: + tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)) + + tailwindcss@3.4.3(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)): + dependencies: + "@alloc/quick-lru": 5.2.0 + arg: 5.0.2 + chokidar: 3.6.0 + didyoumean: 1.2.2 + dlv: 1.1.3 + fast-glob: 3.3.2 + glob-parent: 6.0.2 + is-glob: 4.0.3 + jiti: 1.21.0 + lilconfig: 2.1.0 + micromatch: 4.0.5 + normalize-path: 3.0.0 + object-hash: 3.0.0 + picocolors: 1.0.0 + postcss: 8.4.38 + postcss-import: 15.1.0(postcss@8.4.38) + postcss-js: 4.0.1(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5)) + postcss-nested: 6.0.1(postcss@8.4.38) + postcss-selector-parser: 6.0.16 + resolve: 1.22.8 + sucrase: 3.35.0 + transitivePeerDependencies: + - ts-node + + tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)): + dependencies: + "@alloc/quick-lru": 5.2.0 + arg: 5.0.2 + chokidar: 3.6.0 + didyoumean: 1.2.2 + dlv: 1.1.3 + fast-glob: 3.3.2 + glob-parent: 6.0.2 + is-glob: 4.0.3 + jiti: 1.21.0 + lilconfig: 2.1.0 + micromatch: 4.0.5 + normalize-path: 3.0.0 + object-hash: 3.0.0 + picocolors: 1.0.0 + postcss: 8.4.38 + postcss-import: 15.1.0(postcss@8.4.38) + postcss-js: 4.0.1(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5)) + postcss-nested: 6.0.1(postcss@8.4.38) + postcss-selector-parser: 6.0.16 + resolve: 1.22.8 + sucrase: 3.35.0 + transitivePeerDependencies: + - ts-node + + tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.13)(typescript@5.4.5)): + dependencies: + "@alloc/quick-lru": 5.2.0 + arg: 5.0.2 + chokidar: 3.6.0 + didyoumean: 1.2.2 + dlv: 1.1.3 + fast-glob: 3.3.2 + glob-parent: 6.0.2 + is-glob: 4.0.3 + jiti: 1.21.0 + lilconfig: 2.1.0 + micromatch: 4.0.5 + normalize-path: 3.0.0 + object-hash: 3.0.0 + picocolors: 1.0.0 + postcss: 8.4.38 + postcss-import: 15.1.0(postcss@8.4.38) + postcss-js: 4.0.1(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.13)(typescript@5.4.5)) + postcss-nested: 6.0.1(postcss@8.4.38) + postcss-selector-parser: 6.0.16 + resolve: 1.22.8 + sucrase: 3.35.0 + transitivePeerDependencies: + - ts-node + + tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)): dependencies: "@alloc/quick-lru": 5.2.0 arg: 5.0.2 @@ -21124,7 +21364,7 @@ snapshots: postcss: 8.4.38 postcss-import: 15.1.0(postcss@8.4.38) postcss-js: 4.0.1(postcss@8.4.38) - postcss-load-config: 4.0.2(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5)) postcss-nested: 6.0.1(postcss@8.4.38) postcss-selector-parser: 6.0.16 resolve: 1.22.8 @@ -21228,6 +21468,100 @@ snapshots: ts-interface-checker@0.1.13: {} + ts-node@10.9.2(@types/node@18.19.33)(typescript@4.9.5): + dependencies: + "@cspotcode/source-map-support": 0.8.1 + "@tsconfig/node10": 1.0.11 + "@tsconfig/node12": 1.0.11 + "@tsconfig/node14": 1.0.3 + "@tsconfig/node16": 1.0.4 + "@types/node": 18.19.33 + acorn: 8.12.1 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 4.9.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true + + ts-node@10.9.2(@types/node@20.12.12)(typescript@5.4.5): + dependencies: + "@cspotcode/source-map-support": 0.8.1 + "@tsconfig/node10": 1.0.11 + "@tsconfig/node12": 1.0.11 + "@tsconfig/node14": 1.0.3 + "@tsconfig/node16": 1.0.4 + "@types/node": 20.12.12 + acorn: 8.12.1 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.4.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true + + ts-node@10.9.2(@types/node@20.12.13)(typescript@5.4.5): + dependencies: + "@cspotcode/source-map-support": 0.8.1 + "@tsconfig/node10": 1.0.11 + "@tsconfig/node12": 1.0.11 + "@tsconfig/node14": 1.0.3 + "@tsconfig/node16": 1.0.4 + "@types/node": 20.12.13 + acorn: 8.12.1 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.4.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + + ts-node@10.9.2(@types/node@20.14.13)(typescript@5.4.5): + dependencies: + "@cspotcode/source-map-support": 0.8.1 + "@tsconfig/node10": 1.0.11 + "@tsconfig/node12": 1.0.11 + "@tsconfig/node14": 1.0.3 + "@tsconfig/node16": 1.0.4 + "@types/node": 20.14.13 + acorn: 8.12.1 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.4.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true + + ts-node@10.9.2(@types/node@20.14.5)(typescript@5.4.5): + dependencies: + "@cspotcode/source-map-support": 0.8.1 + "@tsconfig/node10": 1.0.11 + "@tsconfig/node12": 1.0.11 + "@tsconfig/node14": 1.0.3 + "@tsconfig/node16": 1.0.4 + "@types/node": 20.14.5 + acorn: 8.12.1 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.4.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true + ts-patch@3.2.0: dependencies: chalk: 4.1.2 @@ -21248,7 +21582,7 @@ snapshots: tslib@2.6.2: {} - tsup@8.0.2(postcss@8.4.38)(typescript@5.4.5): + tsup@8.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.13)(typescript@5.4.5))(typescript@5.4.5): dependencies: bundle-require: 4.1.0(esbuild@0.19.12) cac: 6.7.14 @@ -21258,7 +21592,7 @@ snapshots: execa: 5.1.1 globby: 11.1.0 joycon: 3.1.1 - postcss-load-config: 4.0.2(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.13)(typescript@5.4.5)) resolve-from: 5.0.0 rollup: 4.18.0 source-map: 0.8.0-beta.0 @@ -21532,6 +21866,8 @@ snapshots: kleur: 4.1.5 sade: 1.8.1 + v8-compile-cache-lib@3.0.1: {} + v8-compile-cache@2.4.0: {} v8-to-istanbul@9.2.0: @@ -21786,6 +22122,8 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yn@3.1.1: {} + yocto-queue@0.1.0: {} zod@3.23.8: {}