From 5a081f8957029ea7f00109b7d2eab7ae41b4b8a0 Mon Sep 17 00:00:00 2001 From: William Henderson Date: Sun, 10 Oct 2021 12:44:21 +0100 Subject: [PATCH 1/4] Added `include` directive to configuration --- humphrey-server/src/config/config.rs | 10 +-- humphrey-server/src/config/error.rs | 15 ++-- humphrey-server/src/config/traceback.rs | 4 +- humphrey-server/src/config/tree.rs | 107 +++++++++++++++++++----- 4 files changed, 101 insertions(+), 35 deletions(-) diff --git a/humphrey-server/src/config/config.rs b/humphrey-server/src/config/config.rs index 9218ba6..5b9e237 100644 --- a/humphrey-server/src/config/config.rs +++ b/humphrey-server/src/config/config.rs @@ -112,8 +112,8 @@ pub enum BlacklistMode { impl Config { /// Attempts to load the configuration. pub fn load() -> Result { - if let Ok(config_string) = load_config_file() { - let tree = parse_conf(&config_string).map_err(|e| e.to_string())?; + if let Ok((filename, config_string)) = load_config_file() { + let tree = parse_conf(&config_string, &filename).map_err(|e| e.to_string())?; let config = Self::from_tree(tree)?; Ok(config) @@ -280,17 +280,17 @@ impl Config { } /// Loads the configuration file. -fn load_config_file() -> Result { +fn load_config_file() -> Result<(String, String), ()> { let path = args().nth(1).unwrap_or_else(|| "humphrey.conf".into()); - if let Ok(mut file) = File::open(path) { + if let Ok(mut file) = File::open(&path) { // The file can be opened let mut string = String::new(); if file.read_to_string(&mut string).is_ok() { // The file can be read - Ok(string) + Ok((path, string)) } else { Err(()) } diff --git a/humphrey-server/src/config/error.rs b/humphrey-server/src/config/error.rs index 313fb43..ca59e49 100644 --- a/humphrey-server/src/config/error.rs +++ b/humphrey-server/src/config/error.rs @@ -5,13 +5,18 @@ use std::fmt::Display; #[derive(Debug, PartialEq, Eq)] pub struct ConfigError { message: &'static str, - line: i64, + file: String, + line: u64, } impl ConfigError { /// Creates a new configuration error object. - pub fn new(message: &'static str, line: i64) -> Self { - Self { message, line } + pub fn new(message: &'static str, file: &str, line: u64) -> Self { + Self { + message, + file: file.to_string(), + line, + } } } @@ -19,8 +24,8 @@ impl Display for ConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "Configuration error on line {}: {}", - self.line, self.message + "Configuration error at {} line {}: {}", + self.file, self.line, self.message ) } } diff --git a/humphrey-server/src/config/traceback.rs b/humphrey-server/src/config/traceback.rs index 07deca5..be71f0f 100644 --- a/humphrey-server/src/config/traceback.rs +++ b/humphrey-server/src/config/traceback.rs @@ -5,7 +5,7 @@ where T: Iterator, { inner_iterator: T, - current_line: i64, + current_line: u64, } impl TracebackIterator @@ -13,7 +13,7 @@ where T: Iterator, { /// Gets the current index of the iterator. - pub fn current_line(&self) -> i64 { + pub fn current_line(&self) -> u64 { self.current_line } } diff --git a/humphrey-server/src/config/tree.rs b/humphrey-server/src/config/tree.rs index 5292032..7b8c2b3 100644 --- a/humphrey-server/src/config/tree.rs +++ b/humphrey-server/src/config/tree.rs @@ -3,6 +3,8 @@ use crate::config::traceback::TracebackIterator; use humphrey::krauss::wildcard_match; use std::collections::HashMap; +use std::fs::File; +use std::io::Read; use std::str::Lines; /// Represents a node in the configuration syntax tree. @@ -91,7 +93,7 @@ impl ConfigNode { } /// Parses an entire configuration string. -pub fn parse_conf(conf: &str) -> Result { +pub fn parse_conf(conf: &str, filename: &str) -> Result { let mut lines = TracebackIterator::from(conf.lines()); // Attemps to find the start of the configuration @@ -100,24 +102,28 @@ pub fn parse_conf(conf: &str) -> Result { if let Some(line) = lines.next() { line_content = clean_up(line); } else { - return Err(ConfigError::new("Could not find `server` section", 0)); + return Err(ConfigError::new( + "Could not find `server` section", + filename, + 0, + )); } } // Parses the main section - parse_section("server", &mut lines) + parse_section("server", &mut lines, filename) } /// Recursively parses a section of the configuration. fn parse_section( name: &str, lines: &mut TracebackIterator, + filename: &str, ) -> Result { - let mut section_open: bool = true; let mut values: Vec = Vec::new(); - // While this section has not been closed - while section_open { + // While this section has not ended + loop { // Attempt to read a line if let Some(line) = lines.next() { @@ -130,41 +136,59 @@ fn parse_section( if section_name.starts_with("route ") && section_name != "route {" { // If the section is a route section, parse it as such let route_name = section_name.splitn(2, ' ').last().unwrap().trim(); - let section = parse_section(route_name, lines)?; + let section = parse_section(route_name, lines, filename)?; if let ConfigNode::Section(route_name, inner_values) = section { values.push(ConfigNode::Route(route_name, inner_values)); } } else { // If the section is just a regular section, parse it in the normal way - values.push(parse_section(section_name, lines)?); + values.push(parse_section(section_name, lines, filename)?); } } else if line == "}" { // If the line indicates the end of this section, return the parsed section - section_open = false; + break; } else if !line.is_empty() { // If the line is not empty, attempt to parse the value let parts: Vec<&str> = line.splitn(2, ' ').collect(); - quiet_assert(parts.len() == 2, "Syntax error", lines)?; + quiet_assert(parts.len() == 2, "Syntax error", filename, lines)?; let key = parts[0].trim(); let value = parts[1].trim(); - if wildcard_match("\"*\"", value) { - values.push(ConfigNode::String( - key.into(), - value[1..value.len() - 1].into(), - )) - } else if value.parse::().is_ok() { - values.push(ConfigNode::Number(key.into(), value.into())) - } else if value.parse::().is_ok() { - values.push(ConfigNode::Boolean(key.into(), value.into())) - } else if let Ok(size) = parse_size(value) { - values.push(ConfigNode::Number(key.into(), size.to_string())) + // If this is just a regular value + if key != "include" { + if wildcard_match("\"*\"", value) { + values.push(ConfigNode::String( + key.into(), + value[1..value.len() - 1].into(), + )) + } else if value.parse::().is_ok() { + values.push(ConfigNode::Number(key.into(), value.into())) + } else if value.parse::().is_ok() { + values.push(ConfigNode::Boolean(key.into(), value.into())) + } else if let Ok(size) = parse_size(value) { + values.push(ConfigNode::Number(key.into(), size.to_string())) + } else { + return Err(ConfigError::new( + "Could not parse value", + filename, + lines.current_line(), + )); + } + } else if wildcard_match("\"*\"", value) { + let include_result = + include(&value[1..value.len() - 1], filename, lines.current_line()); + if let Ok(included_nodes) = include_result { + values.extend(included_nodes); + } else { + return Err(include_result.unwrap_err()); + } } else { return Err(ConfigError::new( - "Could not parse value", + "Invalid include value, it takes a file path in quotation marks as its value", + filename, lines.current_line(), )); } @@ -174,6 +198,7 @@ fn parse_section( return Err(ConfigError::new( "Unexpected end of file, expected `}`", + filename, lines.current_line(), )); } @@ -182,6 +207,41 @@ fn parse_section( Ok(ConfigNode::Section(name.into(), values)) } +/// Attempts to include the configuration file at the specified path into the tree, +/// returning a `Vec` of `ConfigNode`s. If unsuccessful, returns a descriptive error. +fn include(path: &str, containing_file: &str, line: u64) -> Result, ConfigError> { + if let Ok(mut file) = File::open(path) { + let mut buf = String::new(); + if file.read_to_string(&mut buf).is_ok() { + buf.push_str("\n}"); + + let mut iter = TracebackIterator::from(buf.lines()); + let parsed_node = parse_section("temp_included_section", &mut iter, path)?; + + match parsed_node { + ConfigNode::Section(_, children) => Ok(children), + _ => Err(ConfigError::new( + "Internal parser error", + containing_file, + line, + )), + } + } else { + Err(ConfigError::new( + "Could not read included file", + containing_file, + line, + )) + } + } else { + Err(ConfigError::new( + "Could not open included file", + containing_file, + line, + )) + } +} + /// Cleans up a line by removing comments and trailing whitespace. fn clean_up(line: &str) -> &str { line.splitn(2, '#').next().unwrap().trim() @@ -217,6 +277,7 @@ fn parse_size(size: &str) -> Result { fn quiet_assert( condition: bool, message: &'static str, + filename: &str, iter: &mut TracebackIterator, ) -> Result<(), ConfigError> where @@ -224,6 +285,6 @@ where { match condition { true => Ok(()), - false => Err(ConfigError::new(message, iter.current_line())), + false => Err(ConfigError::new(message, filename, iter.current_line())), } } From e7faf3806094642c49750727d369bd771e4d9d72 Mon Sep 17 00:00:00 2001 From: William Henderson Date: Sun, 10 Oct 2021 12:44:44 +0100 Subject: [PATCH 2/4] Added testcases for `include` directive --- humphrey-server/src/tests/config.rs | 2 +- humphrey-server/src/tests/include.rs | 92 +++++++++++++++++++ humphrey-server/src/tests/invalid_config.rs | 27 ++++-- humphrey-server/src/tests/mod.rs | 1 + .../src/tests/testcases/include_route.conf | 9 ++ .../src/tests/testcases/nested_include_1.conf | 5 + .../src/tests/testcases/nested_include_2.conf | 4 + .../tests/testcases/nested_include_root.conf | 9 ++ .../src/tests/testcases/routes.conf | 5 + humphrey-server/src/tests/tree.rs | 8 +- 10 files changed, 151 insertions(+), 11 deletions(-) create mode 100644 humphrey-server/src/tests/include.rs create mode 100644 humphrey-server/src/tests/testcases/include_route.conf create mode 100644 humphrey-server/src/tests/testcases/nested_include_1.conf create mode 100644 humphrey-server/src/tests/testcases/nested_include_2.conf create mode 100644 humphrey-server/src/tests/testcases/nested_include_root.conf create mode 100644 humphrey-server/src/tests/testcases/routes.conf diff --git a/humphrey-server/src/tests/config.rs b/humphrey-server/src/tests/config.rs index b963c98..e1d883d 100644 --- a/humphrey-server/src/tests/config.rs +++ b/humphrey-server/src/tests/config.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; #[test] fn test_parse_config() { - let tree = parse_conf(CONF).unwrap(); + let tree = parse_conf(CONF, "valid.conf").unwrap(); let conf = Config::from_tree(tree).unwrap(); #[cfg(feature = "plugins")] diff --git a/humphrey-server/src/tests/include.rs b/humphrey-server/src/tests/include.rs new file mode 100644 index 0000000..d3b7e40 --- /dev/null +++ b/humphrey-server/src/tests/include.rs @@ -0,0 +1,92 @@ +use humphrey_server::config::tree::parse_conf; +use humphrey_server::config::{ + BlacklistConfig, BlacklistMode, CacheConfig, Config, LoadBalancerMode, LoggingConfig, + RouteConfig, +}; +use humphrey_server::logger::LogLevel; +use humphrey_server::proxy::{EqMutex, LoadBalancer}; +use humphrey_server::rand::Lcg; + +use std::env::set_current_dir; +use std::path::Path; + +#[test] +fn include_route() { + // Set current directory to testcases directory so the parser can find the included file + let testcases_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/tests/testcases"); + set_current_dir(testcases_path).unwrap(); + + let string = include_str!("./testcases/include_route.conf"); + let config = Config::from_tree(parse_conf(string, "include_route.conf").unwrap()); + + let expected_conf = Ok(Config { + address: "0.0.0.0".into(), + port: 80, + threads: 32, + websocket_proxy: None, + routes: vec![RouteConfig::Serve { + matches: "/*".into(), + directory: "/var/www".into(), + }], + #[cfg(feature = "plugins")] + plugins: Vec::new(), + logging: LoggingConfig { + level: LogLevel::Warn, + console: true, + file: None, + }, + cache: CacheConfig { + size_limit: 0, + time_limit: 0, + }, + blacklist: BlacklistConfig { + list: Vec::new(), + mode: BlacklistMode::Block, + }, + }); + + assert_eq!(config, expected_conf); +} + +#[test] +fn nested_include() { + // Set current directory to testcases directory so the parser can find the included files + let testcases_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/tests/testcases"); + set_current_dir(testcases_path).unwrap(); + + let string = include_str!("./testcases/nested_include_root.conf"); + let config = Config::from_tree(parse_conf(string, "nested_include_root.conf").unwrap()); + + let expected_conf = Ok(Config { + address: "0.0.0.0".into(), + port: 80, + threads: 32, + websocket_proxy: None, + routes: vec![RouteConfig::Proxy { + matches: "/test".into(), + load_balancer: EqMutex::new(LoadBalancer { + targets: vec!["127.0.0.1".into()], + mode: LoadBalancerMode::Random, + index: 0, + lcg: Lcg::new(), + }), + }], + #[cfg(feature = "plugins")] + plugins: Vec::new(), + logging: LoggingConfig { + level: LogLevel::Warn, + console: true, + file: None, + }, + cache: CacheConfig { + size_limit: 0, + time_limit: 0, + }, + blacklist: BlacklistConfig { + list: Vec::new(), + mode: BlacklistMode::Block, + }, + }); + + assert_eq!(config, expected_conf); +} diff --git a/humphrey-server/src/tests/invalid_config.rs b/humphrey-server/src/tests/invalid_config.rs index 7b1a87f..9c5aa67 100644 --- a/humphrey-server/src/tests/invalid_config.rs +++ b/humphrey-server/src/tests/invalid_config.rs @@ -4,29 +4,44 @@ use humphrey_server::config::tree::parse_conf; #[test] fn value_error() { let string = include_str!("./testcases/value_error.conf"); - let config = parse_conf(string); + let config = parse_conf(string, "value_error.conf"); - assert_eq!(config, Err(ConfigError::new("Could not parse value", 34))); + assert_eq!( + config, + Err(ConfigError::new( + "Could not parse value", + "value_error.conf", + 34 + )) + ); } #[test] fn eof_error() { let string = include_str!("./testcases/eof_error.conf"); - let config = parse_conf(string); + let config = parse_conf(string, "eof_error.conf"); assert_eq!( config, - Err(ConfigError::new("Unexpected end of file, expected `}`", 44)) + Err(ConfigError::new( + "Unexpected end of file, expected `}`", + "eof_error.conf", + 44 + )) ); } #[test] fn empty_file_error() { let string = include_str!("./testcases/empty_file_error.conf"); - let config = parse_conf(string); + let config = parse_conf(string, "empty_file_error.conf"); assert_eq!( config, - Err(ConfigError::new("Could not find `server` section", 0)) + Err(ConfigError::new( + "Could not find `server` section", + "empty_file_error.conf", + 0 + )) ); } diff --git a/humphrey-server/src/tests/mod.rs b/humphrey-server/src/tests/mod.rs index b272d17..dc80410 100644 --- a/humphrey-server/src/tests/mod.rs +++ b/humphrey-server/src/tests/mod.rs @@ -1,3 +1,4 @@ pub mod config; +pub mod include; pub mod invalid_config; pub mod tree; diff --git a/humphrey-server/src/tests/testcases/include_route.conf b/humphrey-server/src/tests/testcases/include_route.conf new file mode 100644 index 0000000..e52ccd0 --- /dev/null +++ b/humphrey-server/src/tests/testcases/include_route.conf @@ -0,0 +1,9 @@ +# Configuration test case with the "include" directive. + +server { + address "0.0.0.0" + port 80 + threads 32 + + include "routes.conf" +} \ No newline at end of file diff --git a/humphrey-server/src/tests/testcases/nested_include_1.conf b/humphrey-server/src/tests/testcases/nested_include_1.conf new file mode 100644 index 0000000..a41739b --- /dev/null +++ b/humphrey-server/src/tests/testcases/nested_include_1.conf @@ -0,0 +1,5 @@ +# Nested level 1 for the nested include test. + +route /test { + include "nested_include_2.conf" +} \ No newline at end of file diff --git a/humphrey-server/src/tests/testcases/nested_include_2.conf b/humphrey-server/src/tests/testcases/nested_include_2.conf new file mode 100644 index 0000000..84eb5d8 --- /dev/null +++ b/humphrey-server/src/tests/testcases/nested_include_2.conf @@ -0,0 +1,4 @@ +# Nested level 2 for the nested include test. + +proxy "127.0.0.1" +load_balancer_mode "random" \ No newline at end of file diff --git a/humphrey-server/src/tests/testcases/nested_include_root.conf b/humphrey-server/src/tests/testcases/nested_include_root.conf new file mode 100644 index 0000000..69e7aad --- /dev/null +++ b/humphrey-server/src/tests/testcases/nested_include_root.conf @@ -0,0 +1,9 @@ +# Configuration test case with the "include" directive used inside an included file. + +server { + address "0.0.0.0" + port 80 + threads 32 + + include "nested_include_1.conf" +} \ No newline at end of file diff --git a/humphrey-server/src/tests/testcases/routes.conf b/humphrey-server/src/tests/testcases/routes.conf new file mode 100644 index 0000000..315630f --- /dev/null +++ b/humphrey-server/src/tests/testcases/routes.conf @@ -0,0 +1,5 @@ +# Section included in "include.conf" + +route /* { + directory "/var/www" +} \ No newline at end of file diff --git a/humphrey-server/src/tests/tree.rs b/humphrey-server/src/tests/tree.rs index d0e7122..b1adf17 100644 --- a/humphrey-server/src/tests/tree.rs +++ b/humphrey-server/src/tests/tree.rs @@ -43,7 +43,7 @@ fn test_build_tree() { ]) ]); - let parsed_conf = parse_conf(CONF).unwrap(); + let parsed_conf = parse_conf(CONF, "valid.conf").unwrap(); assert_eq!(parsed_conf, expected_parsed_conf); } @@ -51,7 +51,7 @@ fn test_build_tree() { #[test] #[rustfmt::skip] fn test_flatten_config() { - let parsed_conf = parse_conf(CONF).unwrap(); + let parsed_conf = parse_conf(CONF, "valid.conf").unwrap(); let mut expected_hashmap: HashMap = HashMap::new(); expected_hashmap.insert("server.address".into(), ConfigNode::String("address".into(), "0.0.0.0".into())); @@ -74,7 +74,7 @@ fn test_flatten_config() { #[test] #[rustfmt::skip] fn test_get_routes() { - let parsed_conf = parse_conf(CONF).unwrap(); + let parsed_conf = parse_conf(CONF, "valid.conf").unwrap(); let mut expected_map: Vec<(String, HashMap)> = Vec::new(); @@ -95,7 +95,7 @@ fn test_get_routes() { #[test] #[rustfmt::skip] fn test_get_plugins() { - let parsed_conf = parse_conf(CONF).unwrap(); + let parsed_conf = parse_conf(CONF, "valid.conf").unwrap(); let mut expected_map: Vec<(String, HashMap)> = Vec::new(); From ef3af23e5b2561fa7627c1c3e65afcbcc94720ae Mon Sep 17 00:00:00 2001 From: William Henderson Date: Sun, 10 Oct 2021 12:45:00 +0100 Subject: [PATCH 3/4] Updated README example to use `include` directive --- humphrey-server/README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/humphrey-server/README.md b/humphrey-server/README.md index 819b8d3..b34e1f0 100644 --- a/humphrey-server/README.md +++ b/humphrey-server/README.md @@ -12,7 +12,7 @@ Humphrey is a very fast, robust and flexible HTTP/1.1 web server, with support f To install the binary, run `cargo install humphrey_server` and it will be automatically downloaded, compiled and added to your path as `humphrey`. Alternatively, you can find precompiled binaries from the [latest GitHub release](https://github.com/w-henderson/Humphrey/releases). ## Configuration -The Humphrey executable is run with a maximum of one argument, which specifies the path to the configuration file (defaulting to `humphrey.conf` in the current directory). The configuration file is where all configuration for Humphrey and any plugins is stored. The syntax is similar to Nginx, with comments starting with a `#`. Below is an example of a configuration file with every supported field specified. Unless specified otherwise, all fields are optional. +The Humphrey executable is run with a maximum of one argument, which specifies the path to the configuration file (defaulting to `humphrey.conf` in the current directory). The configuration file is where all configuration for Humphrey and any plugins is stored. The syntax is similar to Nginx, with comments starting with a `#`. Other configuration files can be included with the `include` directive, like in Nginx. Below is an example of a configuration file with every supported field specified. Unless specified otherwise, all fields are optional. ```conf server { @@ -21,6 +21,10 @@ server { threads 32 # Number of threads to use for the server websocket "localhost:1234" # Where to proxy WebSocket connections to + plugins { + include "php.conf" # Include PHP configuration (see below) + } + blacklist { file "conf/blacklist.txt" # Text file containing blacklisted addresses, one per line mode "block" # Method of enforcing the blacklist, "block" or "forbidden" (which returns 403 Forbidden) @@ -51,15 +55,13 @@ server { ## Using with PHP To use Humphrey with PHP, compile the [PHP plugin in the plugins folder](https://github.com/w-henderson/Humphrey/tree/master/plugins/php) and add the path to the output file to your plugin configuration (also available precompiled in the GitHub releases). You'll need Humphrey installed with plugins enabled (using `cargo install humphrey_server --features plugins`) and you'll also need PHP-CGI or PHP-FPM. Start the PHP server first, and specify its address in the Humphrey configuration file as specified below. Ensure your PHP configuration allows for multithreading if you set more than one thread in the configuration. Finally, you can start Humphrey in the normal way and it will work with PHP. -Add the following in the server section of your configuration file: +In the `php.conf` file which is included in the main configuration through the `include` directive: ```conf -plugins { - php { - library "plugins/php/target/release/php.dll" # Path to compiled library, `.dll` on Windows and `.so` on Linux - address "127.0.0.1" # Address to connect to the PHP-CGI interpreter - port 9000 # Port of the interpreter - threads 8 # Number of threads to connect to the interpreter with - } +php { + library "plugins/php/target/release/php.dll" # Path to compiled library, `.dll` on Windows and `.so` on Linux + address "127.0.0.1" # Address to connect to the PHP-CGI interpreter + port 9000 # Port of the interpreter + threads 8 # Number of threads to connect to the interpreter with } ``` From cc02374930a001ec24e5389a469a9035a5e321b7 Mon Sep 17 00:00:00 2001 From: William Henderson Date: Sun, 10 Oct 2021 12:46:07 +0100 Subject: [PATCH 4/4] Incremented server version number --- humphrey-server/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/humphrey-server/Cargo.toml b/humphrey-server/Cargo.toml index 93e33bc..22cc341 100644 --- a/humphrey-server/Cargo.toml +++ b/humphrey-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "humphrey_server" -version = "0.2.1" +version = "0.2.2" edition = "2018" license = "MIT" homepage = "https://github.com/w-henderson/Humphrey"