Enable windows and macos builds in CI

Makefile

@@ -1,7 +1,7 @@
 # Use directory-local cargo root to install version-specific executables into.
 export CARGO_HOME = $(shell pwd)/.cargo
 
-# the series of builds, tests and checks that runs for pull requests. requires docker.
+# the series of builds, tests and checks that runs for pull requests.
 .PHONY: ci
 ci: check-licenses build integ
 
@@ -28,9 +28,18 @@ build:
 	cargo build --locked -p tuftool
 	cargo test --locked
 
-# checks tough tests with and without the http feature. http testing requires docker.
+
+# installs noxious-server
+# We currently build from a forked version, until such a point that the following are resolved:
+# https://github.com/oguzbilgener/noxious/issues/13
+# https://github.com/oguzbilgener/noxious/pull/14
+.PHONY: noxious
+noxious:
+	cargo install --locked --git https://github.com/cbgbt/noxious.git --tag v1.0.5
+
+# checks tough tests with and without the http feature. http testing requires noxious-server.
 .PHONY: integ
-integ:
+integ: noxious
 	set +e
-	cd tough && cargo test --features '' --locked
-	cd tough && cargo test --all-features --locked
+	cargo test --manifest-path tough/Cargo.toml --features '' --locked
+	cargo test --manifest-path tough/Cargo.toml --all-features --locked

README.md

@@ -5,14 +5,7 @@
 **tuftool** is a Rust command-line utility for generating and signing TUF repositories.
 
 ## Integration Testing
-Integration tests require `docker`.
-
-### Windows❗ Warnings
-- Tests can break on Windows if Git's `autocrlf` feature changes line endings.
-  This is due to the fact that some tests require files to have a *precise* byte size and hash signature.
-  *We have mitigated this with a `.gitattributes` file in the test data directory*.
-
-- Cygwin **must** be installed at `C:\cygwin64\` and have the `make` package installed for integration tests to work properly. 
+Integration tests require, `noxious`, which is installed when running `make integ`.
 
 ## Documentation
 See [tough - Rust](https://docs.rs/tough/) for the latest `tough` library documentation.

deny.toml

@@ -73,6 +73,8 @@ skip = [
     { name = "fastrand", version = "=1.9" },
     # several dependencies are using an old version of bitflags
     { name = "bitflags", version = "=1.3" },
+    # noxious, used for testing, is using an old version of tokio-util
+    { name = "tokio-util", version = "=0.6.10" },
 ]
 
 skip-tree = [

integ/failure-server/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "failure-server"
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+axum = "0.6"
+anyhow = "1.0"
+noxious-client = "1.0"
+rand = "0.8"
+serde_json = "1.0"
+tempfile = "3.8"
+tokio = "1.0"
+tower = { version = "0.4", features = ["util"] }
+tower-fault = "0.0.5"
+tower-http = { version = "0.4", features = ["fs"] }

integ/failure-server/src/lib.rs

@@ -0,0 +1,80 @@
+//! This module sets up 2 HTTP servers.
+//!   * ToxicStaticHttpServer: serves TUF repo files on port 10101, with occasional random 503s.
+//!   * ToxicTcpProxy: proxies to the TUF repo on port 10102, with occasional toxic behavior.
+use anyhow::Result;
+use noxious_client::{StreamDirection, Toxic, ToxicKind};
+use std::path::Path;
+use std::thread::sleep;
+use std::time::Duration;
+use toxic::{ToxicStaticHttpServer, ToxicTcpProxy};
+
+mod toxic;
+
+const STATIC_HTTP_SERVER_LISTEN: &str = "127.0.0.1:10101";
+const TCP_PROXY_LISTEN: &str = "127.0.0.1:10102";
+const TCP_PROXY_CONFIG_API_LISTEN: &str = "127.0.0.1:8472";
+
+pub struct IntegServers {
+    toxic_tcp_proxy: ToxicTcpProxy,
+    toxic_static_http_server: ToxicStaticHttpServer,
+}
+
+impl IntegServers {
+    pub fn new<P: AsRef<Path>>(tuf_reference_repo: P) -> Result<Self> {
+        let tuf_reference_repo = tuf_reference_repo.as_ref().to_owned();
+
+        let toxic_tcp_proxy = ToxicTcpProxy::new(
+            "toxictuf".to_string(),
+            TCP_PROXY_LISTEN,
+            STATIC_HTTP_SERVER_LISTEN,
+            TCP_PROXY_CONFIG_API_LISTEN,
+        )?
+        .with_toxic(Toxic {
+            name: "slowclose".to_string(),
+            kind: ToxicKind::SlowClose { delay: 500 },
+            toxicity: 0.75,
+            direction: StreamDirection::Downstream,
+        })
+        .with_toxic(Toxic {
+            name: "timeout".to_string(),
+            kind: ToxicKind::Timeout { timeout: 100 },
+            toxicity: 0.5,
+            direction: StreamDirection::Downstream,
+        });
+
+        let toxic_static_http_server =
+            ToxicStaticHttpServer::new(STATIC_HTTP_SERVER_LISTEN, tuf_reference_repo)?;
+
+        Ok(Self {
+            toxic_tcp_proxy,
+            toxic_static_http_server,
+        })
+    }
+
+    pub async fn run(&mut self) -> Result<()> {
+        // Make sure we're starting from scratch
+        self.teardown()?;
+
+        self.toxic_static_http_server.start()?;
+        self.toxic_tcp_proxy.start().await?;
+        sleep(Duration::from_secs(1)); // give the servers a chance to start
+
+        println!("**********************************************************************");
+        println!("the toxic tuf repo is available at {TCP_PROXY_LISTEN}");
+
+        Ok(())
+    }
+
+    pub fn teardown(&mut self) -> Result<()> {
+        self.toxic_tcp_proxy.stop()?;
+        self.toxic_static_http_server.stop()?;
+
+        Ok(())
+    }
+}
+
+impl Drop for IntegServers {
+    fn drop(&mut self) {
+        self.teardown().ok();
+    }
+}

integ/failure-server/src/toxic/http_server.rs

@@ -0,0 +1,96 @@
+//! A simple filesystem HTTP server that introduces chaos at the HTTP layer.
+//!
+//! Chaos includes:
+//! * Occasional additional request latency
+//! * Occasional 503 responses
+use super::ToSocketAddrsExt;
+use anyhow::{Context, Result};
+use axum::{
+    http::{Request, StatusCode},
+    middleware::{self, Next},
+    response::Response,
+    Router,
+};
+use std::fmt::Debug;
+use std::net::{SocketAddr, ToSocketAddrs};
+use std::path::{Path, PathBuf};
+use tower_fault::latency::LatencyLayer;
+use tower_http::services::ServeDir;
+
+const ERR_503_PROBABILITY: f64 = 0.5;
+const LATENCY_PROBABILITY: f64 = 0.1;
+
+/// An HTTP server which serves static files from a directory.
+///
+/// The server implementation is "toxic" in that it introduces artificial faults at the HTTP layer.
+#[derive(Debug)]
+pub(crate) struct ToxicStaticHttpServer {
+    /// The proxy's listen address. Written to `ProxyConfig`.
+    listen: SocketAddr,
+
+    /// The path to serve static content from.
+    serve_dir: PathBuf,
+
+    /// Running server, if any
+    running_server: Option<tokio::task::JoinHandle<Result<()>>>,
+}
+
+impl ToxicStaticHttpServer {
+    pub(crate) fn new<T, P>(listen: T, serve_dir: P) -> Result<Self>
+    where
+        T: ToSocketAddrs + Debug,
+        P: AsRef<Path>,
+    {
+        let listen = listen.parse_only_one_address()?;
+        let serve_dir = serve_dir.as_ref().to_owned();
+        let running_server = None;
+
+        Ok(Self {
+            listen,
+            serve_dir,
+            running_server,
+        })
+    }
+
+    /// Starts the HTTP server.
+    pub(crate) fn start(&mut self) -> Result<()> {
+        // Stop any existing server
+        self.stop().ok();
+
+        // Chance to inject 50 to 200 milliseconds of latency
+        let latency_layer = LatencyLayer::new(LATENCY_PROBABILITY, 50..200);
+        // Chance to return an HTTP 503 error
+        let error_layer = middleware::from_fn(maybe_return_error);
+
+        let app = Router::new()
+            .nest_service("/", ServeDir::new(&self.serve_dir))
+            .layer(error_layer)
+            .layer(latency_layer);
+        let server = axum::Server::bind(&self.listen).serve(app.into_make_service());
+
+        self.running_server = Some(tokio::spawn(async {
+            server.await.context("Failed to run ToxicStaticHttpServer")
+        }));
+
+        Ok(())
+    }
+
+    /// Attempts to kill the running server, if there is one.
+    ///
+    /// Succeeds if the server is killed successfully or if it isn't/was never running.
+    pub(crate) fn stop(&mut self) -> Result<()> {
+        if let Some(server) = self.running_server.take() {
+            server.abort();
+        }
+        Ok(())
+    }
+}
+
+/// Middleware for chaotically returning a 503 error.
+async fn maybe_return_error<B>(req: Request<B>, next: Next<B>) -> Result<Response, StatusCode> {
+    if rand::random::<f64>() < ERR_503_PROBABILITY {
+        Err(StatusCode::SERVICE_UNAVAILABLE)
+    } else {
+        Ok(next.run(req).await)
+    }
+}

integ/failure-server/src/toxic/mod.rs

@@ -0,0 +1,35 @@
+use anyhow::{Context, Result};
+use std::fmt::Debug;
+use std::net::{SocketAddr, ToSocketAddrs};
+
+pub(crate) use http_server::ToxicStaticHttpServer;
+pub(crate) use tcp_proxy::ToxicTcpProxy;
+
+mod http_server;
+mod tcp_proxy;
+
+/// Attempts to read exactly one `SocketAddr` from a `ToSocketAddrs`.
+///
+/// Returns an error if more than one SocketAddr is present.
+trait ToSocketAddrsExt {
+    fn parse_only_one_address(self) -> Result<SocketAddr>;
+}
+
+impl<T: ToSocketAddrs + Debug> ToSocketAddrsExt for T {
+    fn parse_only_one_address(self) -> Result<SocketAddr> {
+        let mut addresses = self
+            .to_socket_addrs()
+            .context(format!("Failed to parse {self:?} as socket address"))?;
+
+        let address = addresses
+            .next()
+            .context(format!("Did not parse any addresses from {self:?}"))?;
+
+        anyhow::ensure!(
+            addresses.next().is_none(),
+            format!("Listen address ({:?}) must parse to one address.", address)
+        );
+
+        Ok(address)
+    }
+}