Merge branch 'master' into feat/accept-range-selectors

lycheeverse · Aug 22, 2023 · 4bdf33d · 4bdf33d
2 parents f8aa68b + 007650f
commit 4bdf33d
Show file tree

Hide file tree

Showing 17 changed files with 215 additions and 120 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,13 +1,21 @@
 version: 2
 updates:
-- package-ecosystem: cargo
-  directory: "/"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Group dependency updates into a single pull request.
+    groups:
+      dependencies:
+        patterns:
+          - "*"
 
-- package-ecosystem: github-actions
-  directory: /
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Group dependency updates into a single pull request.
+    groups:
+      dependencies:
+        patterns:
+          - "*"
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/builder/Cargo.toml b/examples/builder/Cargo.toml
@@ -12,7 +12,7 @@ lychee-lib = { path = "../../lychee-lib", version = "0.13.0", default-features =
 tokio = { version = "1.32.0", features = ["full"] }
 regex = "1.9.3"
 http = "0.2.9"
-reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] }
+reqwest = { version = "0.11.19", default-features = false, features = ["gzip"] }
 
 [features]
 email-check = ["lychee-lib/email-check"]

diff --git a/examples/collect_links/Cargo.toml b/examples/collect_links/Cargo.toml
@@ -13,7 +13,7 @@ tokio = { version = "1.32.0", features = ["full"] }
 regex = "1.9.3"
 http = "0.2.9"
 tokio-stream = "0.1.14"
-reqwest = { version = "0.11.18", default-features = false, features = ["gzip"] }
+reqwest = { version = "0.11.19", default-features = false, features = ["gzip"] }
 
 [features]
 email-check = ["lychee-lib/email-check"]

diff --git a/fixtures/fragments/file.html b/fixtures/fragments/file.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>For Testing Fragments</title>
+  </head>
+  <body>
+    <section id="in-the-beginning">
+      <p>
+        To start
+        <a href="file1.md#fragment-1">
+          let's run away.
+        </a>
+      </p>
+    </section>
+    <section>
+      <p id="a-word">Word</p>
+      <a href="#in-the-beginning">back we go</a>
+      <a href="#in-the-end">doesn't exist</a>
+    </section>
+  </body>
+</html>
diff --git a/fixtures/fragments/file1.md b/fixtures/fragments/file1.md
@@ -21,11 +21,11 @@ This is a test file for the fragment loader.
 Explicit fragment links are currently not supported.
 Therefore we put the test into a code block for now to prevent false positives.
 
-```
-<a name="explicit-fragment"></a>
+<a id="explicit-fragment"></a>
 
 [Link to explicit fragment](#explicit-fragment)
-```
+
+[To the html doc](file.html#a-word)
 
 ## Custom Fragments
 

diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml
@@ -21,11 +21,11 @@ lychee-lib = { path = "../lychee-lib", version = "0.13.0", default-features = fa
 
 anyhow = "1.0.75"
 assert-json-diff = "2.0.2"
-clap = { version = "4.3.21", features = ["env", "derive"] }
+clap = { version = "4.3.23", features = ["env", "derive"] }
 console = "0.15.7"
 const_format = "0.2.31"
 csv = "1.2.2"
-dashmap = { version = "5.5.0", features = ["serde"] }
+dashmap = { version = "5.5.1", features = ["serde"] }
 env_logger = "0.10.0"
 futures = "0.3.28"
 headers = "0.3.8"
@@ -38,15 +38,15 @@ once_cell = "1.18.0"
 openssl-sys = { version = "0.9.91", optional = true }
 pad = "0.1.6"
 regex = "1.9.3"
-reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "json"] }
+reqwest = { version = "0.11.19", default-features = false, features = ["gzip", "json"] }
 reqwest_cookie_store = "0.6.0"
 # Make build work on Apple Silicon.
 # See https://github.com/briansmith/ring/issues/1163
 # This is necessary for the homebrew build
 # https://github.com/Homebrew/homebrew-core/pull/70216
 ring = "0.16.20"
 secrecy = { version = "0.8.0", features = ["serde"] }
-serde = { version = "1.0.183", features = ["derive"] }
+serde = { version = "1.0.185", features = ["derive"] }
 serde_json = "1.0.105"
 strum = { version = "0.25.0", features = ["derive"] }
 supports-color = "2.0.0"
@@ -59,7 +59,7 @@ toml = "0.7.6"
 assert_cmd = "2.0.12"
 predicates = "3.0.3"
 pretty_assertions = "1.4.0"
-tempfile = "3.7.1"
+tempfile = "3.8.0"
 tracing-subscriber = { version = "0.3.17", default-features = false, features = ["fmt", "registry", "env-filter"] }
 uuid = { version = "1.4.1", features = ["v4"] }
 wiremock = "0.5.19"

diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs
@@ -14,6 +14,7 @@ mod cli {
     use lychee_lib::{InputSource, ResponseBody};
     use predicates::str::{contains, is_empty};
     use pretty_assertions::assert_eq;
+    use regex::Regex;
     use serde::Serialize;
     use serde_json::Value;
     use tempfile::NamedTempFile;
@@ -1229,18 +1230,33 @@ mod cli {
 
     #[test]
     fn test_suggests_url_alternatives() -> Result<()> {
-        let mut cmd = main_command();
-        let input = fixtures_path().join("INTERNET_ARCHIVE.md");
-
-        cmd.arg("--suggest")
-            .arg(input)
-            .assert()
-            .failure()
-            .code(2)
-            .stdout(contains("Suggestions"))
-            .stdout(contains("http://web.archive.org/web/"));
+        for _ in 0..3 {
+            // This can be flaky. Try up to 3 times
+            let mut cmd = main_command();
+            let input = fixtures_path().join("INTERNET_ARCHIVE.md");
+
+            cmd.arg("--no-progress").arg("--suggest").arg(input);
+
+            // Run he command and check if the output contains the expected
+            // suggestions
+            let assert = cmd.assert();
+            let output = assert.get_output();
+
+            // We're looking for a suggestion that
+            // - starts with http://web.archive.org/web/
+            // - ends with google.com/jobs.html
+            let re = Regex::new(r"http://web\.archive\.org/web/.*google\.com/jobs\.html").unwrap();
+            if re.is_match(&String::from_utf8_lossy(&output.stdout)) {
+                // Test passed
+                return Ok(());
+            } else {
+                // Wait for a second before retrying
+                std::thread::sleep(std::time::Duration::from_secs(1));
+            }
+        }
 
-        Ok(())
+        // If we reached here, it means the test did not pass after multiple attempts
+        Err("Did not get the expected command output after multiple attempts.".into())
     }
 
     #[tokio::test]
@@ -1411,19 +1427,23 @@ mod cli {
             .arg(input)
             .assert()
             .failure()
+            .stderr(contains("fixtures/fragments/file1.md#fragment-1"))
             .stderr(contains("fixtures/fragments/file1.md#fragment-2"))
             .stderr(contains("fixtures/fragments/file2.md#custom-id"))
             .stderr(contains("fixtures/fragments/file1.md#missing-fragment"))
             .stderr(contains("fixtures/fragments/file2.md#fragment-1"))
             .stderr(contains("fixtures/fragments/file1.md#kebab-case-fragment"))
             .stderr(contains("fixtures/fragments/file2.md#missing-fragment"))
             .stderr(contains("fixtures/fragments/empty_file#fragment"))
+            .stderr(contains("fixtures/fragments/file.html#a-word"))
+            .stderr(contains("fixtures/fragments/file.html#in-the-beginning"))
+            .stderr(contains("fixtures/fragments/file.html#in-the-end"))
             .stderr(contains(
                 "fixtures/fragments/file1.md#kebab-case-fragment-1",
             ))
-            .stdout(contains("8 Total"))
-            .stdout(contains("6 OK"))
-            // 2 failures because of missing fragments
-            .stdout(contains("2 Errors"));
+            .stdout(contains("13 Total"))
+            .stdout(contains("10 OK"))
+            // 3 failures because of missing fragments
+            .stdout(contains("3 Errors"));
     }
 }
diff --git a/lychee-lib/Cargo.toml b/lychee-lib/Cargo.toml
@@ -35,16 +35,16 @@ pulldown-cmark = "0.9.3"
 regex = "1.9.3"
 # Use trust-dns to avoid lookup failures on high concurrency
 # https://github.com/seanmonstar/reqwest/issues/296
-reqwest = { version = "0.11.18", default-features = false, features = ["gzip", "trust-dns", "cookies"] }
+reqwest = { version = "0.11.19", default-features = false, features = ["gzip", "trust-dns", "cookies"] }
 reqwest_cookie_store = "0.6.0"
 # Make build work on Apple Silicon.
 # See https://github.com/briansmith/ring/issues/1163
 # This is necessary for the homebrew build
 # https://github.com/Homebrew/homebrew-core/pull/70216
 ring = "0.16.20"
 secrecy = "0.8.0"
-serde = { version = "1.0.183", features = ["derive"] }
-serde_with = "3.2.0"
+serde = { version = "1.0.185", features = ["derive"] }
+serde_with = "3.3.0"
 shellexpand = "3.1.0"
 thiserror = "1.0.47"
 tokio = { version = "1.32.0", features = ["full"] }
@@ -57,7 +57,7 @@ features = ["runtime-tokio"]
 
 [dev-dependencies]
 doc-comment = "0.3.3"
-tempfile = "3.7.1"
+tempfile = "3.8.0"
 wiremock = "0.5.19"
 serde_json = "1.0.105"
 rstest = "0.18.1"