Add rss url checker

test/should_be_rss_url_test.py

@@ -0,0 +1,48 @@
+from ural.should_be_rss_url import should_be_rss_url
+
+TESTS = [
+    (
+        "https://bienpublic.com/politique/2022/12/22/sncf-emmanuel-macron-rattrape-par-le-socialdlvr.it/SfK3Hp",
+        False,
+    ),
+    (
+        "https://lequipe.fr/Cyclisme-sur-route/Article/Dans-la-ferme-d-arnaud-de-lie-la-nouvelle-star-du-cyclisme-belge/1371577",
+        False,
+    ),
+    ("https://wikiart.org/en/edward-hopper/not_detected_235603", False),
+    (
+        "https://play.google.com/store/apps/details?gl=US&hl=fr&id=com.bluetoothscannersc123c4e4e913r1z350999386162e5199.s3.amazonaws.com/web123w9b6n646d6w35099947387d007c9123p470ex620e555baaa745f37ecb6_fr.html",
+        False,
+    ),
+    (
+        "https://clermont.snes.edu/Acces-au-corps-des-agreges-par-liste-d-aptitude.htmlbit.ly/3UhuVXS",
+        False,
+    ),
+    ("https://trib.al/gc5KIo5", False),
+    ("https://rougememoire.com/article/quizsrfc-edition-journee-des-abonnes", False),
+    (
+        "https://changera3.blogspot.com/2022/12/charles-sannat-la-france-en-urgence.html",
+        False,
+    ),
+    (
+        "https://apnews.com/article/entertainment-richmond-6c847bb2e6e52a7048d44d2aacc2ef85?taid=63974840c89e5500011f71e0",
+        False,
+    ),
+    ("https://legifrance.gouv.fr/jorf/texte_jo/JORFTEXT000046652536", False),
+    ("https://www.htmhell.dev/feed.xml", True),
+    ("http://feeds.feedburner.com/2ality?format=xml", True),
+    ("http://webkit.org/blog/feed/", True),
+    ("http://blog.portswigger.net/feeds/posts/default", True),
+    ("https://sindresorhus.com/rss.xml", True),
+    ("https://blog.firsov.net/feeds/posts/default?alt=rss", True),
+    ("http://jakearchibald.com/posts.rss", True),
+    ("https://www.debugbear.com/blog/feed/rss", True),
+    ("https://marcysutton.com/rss.xml", True),
+    ("http://feeds.feedburner.com/kizuruen", True),
+]
+
+
+class TestShouldBeRss(object):
+    def test_basics(self):
+        for url, test in TESTS:
+            assert should_be_rss_url(url) == test

ural/should_be_rss_url.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# =============================================================================
+# Ural Should Be RSS URL
+# =============================================================================
+#
+# Function returning whether the given url should be a rss feed url.
+#
+import re
+
+from ural.utils import urlpathsplit
+from ural.utils import urlsplit
+import ural.get_hostname
+
+PATTERN_QUERY = re.compile(r"(?:format|type)\=(?:xml|feed|atom|rss)", re.I)
+PATTERN_FILE_OBVIOUS = re.compile(
+    r"(?:news|feed|rss|atom|index|blog)s?\.(?:xml|atom|rss|php|json)$", re.I
+)
+PATTERN_FILE = re.compile(r".+\.(?:rss|atom)")
+PATTERN_KEYWORD = re.compile(
+    r"(?:\/|\.|-|_)(?:rss|feed|atom)s?[0-9]*(((?:s|\/|\.|-|_))||^)", re.I
+)
+
+
+def should_be_rss_url(url):
+    query = urlsplit(url).query
+    path = list(reversed(urlpathsplit(url)))
+    path = path[0:3:1]
+    hostname = ural.get_hostname(url)
+
+    if not hostname:
+        return False
+    if query and PATTERN_QUERY.findall(query):
+        return True
+    if path and PATTERN_FILE_OBVIOUS.findall(path[0]):
+        return True
+    if path and PATTERN_FILE.findall(path[0]):
+        return True
+    if PATTERN_KEYWORD.findall(url):
+        return True
+    return False

ural/should_be_rss_url.pyi

@@ -0,0 +1,3 @@
+def should_be_rss_url(
+    url: str,
+) -> bool: ...