urls_from_html now strips

medialab · Jul 19, 2023 · 2e56ad2 · 2e56ad2
1 parent b3a6c6d
commit 2e56ad2
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/test/urls_from_html_test.py b/test/urls_from_html_test.py
@@ -67,6 +67,11 @@ def test_edge_cases(self):
                 '<a href="" target="_blank">charte</a><a href=\'\' target="_blank">charte</a><a href= target="_blank">charte</a>'
             )
         ) == ["", "", ""]
+        assert list(
+            urls_from_html(
+                "<a \nhref='http://lemonde.fr  '></a><a \nhref='    http://lemonde.fr  '></a><a \nhref='    http://lemonde.fr'></a>"
+            )
+        ) == ["http://lemonde.fr", "http://lemonde.fr", "http://lemonde.fr"]
         assert list(urls_from_html(HTML_WITH_SCRIPT_TAGS)) == ["http://lemonde.fr"]
         assert list(urls_from_html(HTML_WITH_PREFIXED_HREF)) == ["http://lemonde.fr"]
 

diff --git a/ural/urls_from_html.py b/ural/urls_from_html.py
@@ -58,6 +58,11 @@ def __urls_finditer_binary(string, encoding="utf-8", errors="strict"):
 
 def urls_from_html(string, encoding="utf-8", errors="strict"):
     if isinstance(string, bytes):
-        return __urls_finditer_binary(string, encoding=encoding, errors=errors)
+        iterator = __urls_finditer_binary(string, encoding=encoding, errors=errors)
+    else:
+        iterator = __urls_finditer(string)
 
-    return __urls_finditer(string)
+    for url in iterator:
+        url = url.strip()
+
+        yield url