first commit

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: ⬆
+  # Python
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: ⬆

.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Github actions init
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Update Pip
+        run: pip install --upgrade pip
+
+      - name: Install Dependencies
+        run: pip install -r ruff mypy
+
+      - name: Install
+        run: pip install -e .
+
+      - name: Lint
+        run: bash scripts/lint.sh

.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - created
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - name: Github actions init
+        uses: actions/checkout@v4
+        with:
+          # To force fetching tags
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build distribution
+        run: python -m build
+
+      - name: Publish
+        uses: pypa/[email protected]

.github/workflows/release.yml

@@ -0,0 +1,45 @@
+name: Release
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Github actions init
+        uses: actions/checkout@v4
+        with:
+          # To force fetching tags
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build
+        run: python -m build
+
+      - name: Read VERSION file
+        id: getversion
+        run: echo "version=$(cat src/reachable/VERSION.md)" >> $GITHUB_OUTPUT
+
+      - name: Changelog
+        run: git log $(git describe --tags --abbrev=0)..HEAD --format="%s %h" > LATEST-CHANGES.md
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            dist/reachable-${{ steps.getversion.outputs.version }}-py3-none-any.whl
+            dist/reachable-${{ steps.getversion.outputs.version }}.tar.gz
+          tag_name: v${{ steps.getversion.outputs.version }}
+          body_path: LATEST-CHANGES.md
+          token: ${{ secrets.PAT_REACHABLE }}

.gitignore

@@ -0,0 +1,7 @@
+.venv
+__pycache__
+dist
+*.egg-info
+.pypirc
+.ruff_cache
+.pytest_cache

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,84 @@
+**Reachable** checks if a URL exists and is reachable.
+
+# Features
+- Use `HEAD`request instead of `GET` to save some bandwidth
+- Follow redirects
+- Handle local redirects (without full URL in `location` header)
+- Record all the URLs of the redirection chain
+- Check if redirected URL match the TLD of source URL
+- Detect Cloudflare protection
+- Avoid basic bot detectors
+    - Use randome Chrome user agent
+    - Wait between consecutive requests to the same host
+    - Include `Host` header
+- Use of HTTP/2
+
+# Installation
+You can install it with pip :
+```bash
+pip install reachable
+```
+Or clone this repository and simply run :
+```bash
+cd reachable/
+pip install -e .
+```
+
+# Usage
+
+## Simple URL
+```python
+result = is_reachable("https://google.com")
+```
+
+The output will look like this:
+```json
+{
+    "url": "https://www.google.com/",
+    "response": null, 
+    "status_code": 200,
+    "success": true,
+    "error_name": null,
+    "cloudflare_protection": false,
+    "redirect": {
+        "chain": ["https://www.google.com/"],
+        "final_url": "https://www.google.com/",
+        "tld_match": true
+    }
+}
+```
+
+## Multiple URLs
+```python
+result = is_reachable(["https://google.com", "http://bing.com"])
+```
+
+The output will look like this:
+```json
+[
+    {
+        "url": "https://www.google.com/",
+        "response": null, 
+        "status_code": 200,
+        "success": true,
+        "error_name": null,
+        "cloudflare_protection": false,
+        "redirect": {
+            "chain": ["https://www.google.com/"],
+            "final_url": "https://www.google.com/",
+            "tld_match": true
+        }
+    },
+    {
+        "url": "https://www.bing.com/?toWww=1&redig=16A78C94",
+        "response": null,
+        "status_code": 200,
+        "success": true,
+        "error_name": null,
+        "cloudflare_protection": false,
+        "redirect": {"chain": ["https://www.bing.com:443/?toWww=1&redig=16A78C94"],
+        "final_url": "https://www.bing.com/?toWww=1&redig=16A78C94",
+        "tld_match": true
+    }
+]
+```

pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "reachable"
+description = "Check if a URL is reachable"
+dynamic = ["version"]
+readme = "README.md"
+authors = [{ name = "Alex Mili" }]
+license = { file = "LICENSE" }
+requires-python = ">=3.8"
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Intended Audience :: Developers",
+]
+keywords = []
+dependencies = ["httpx[http2]", "tldextract", "fake-useragent", "tqdm"]
+
+[project.urls]
+Homepage = "https://github.com/AlexMili/Reachable"
+Issues = "https://github.com/AlexMili/Reachable/issues"
+Repository = "https://github.com/AlexMili/Reachable"
+Documentation = "https://github.com/AlexMili/Reachable"
+
+
+[tool.hatch.build.targets.wheel]
+packages = ["./src/reachable/"]
+
+[tool.hatch.version]
+path = "src/reachable/VERSION.md"
+pattern = "(?P<version>.*)"
+
+[tool.ruff.lint.isort]
+lines-after-imports = 2
+known-first-party = ["reachable"]
+
+[tool.mypy]
+strict = true
+exclude = [".venv", "test", "build", "dist"]
+ignore_missing_imports = true
+show_error_codes = true

requirements-test.txt

@@ -0,0 +1,2 @@
+pytest
+coverage

scripts/lint.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+# mypy src/reachable
+ruff check src/reachable

scripts/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+pytest --cov=reachable --cov-report=xml test/

src/reachable/VERSION.md

@@ -0,0 +1 @@
+0.1.0

src/reachable/client.py

@@ -0,0 +1,82 @@
+import httpx
+import tldextract
+from fake_useragent import UserAgent
+from typing import Union
+
+
+ua = UserAgent(browsers=["chrome"], os="windows", platforms="pc", min_version=120)
+
+
+class Client:
+    def __init__(self, headers: Union[dict, None] = None, include_host: bool = False):
+        transport = httpx.HTTPTransport(retries=3)
+        timeout = httpx.Timeout(10.0, connect=60.0, read=10.0)
+        if headers is None:
+            headers = {
+                "User-Agent": ua.random,
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US;q=0.7,en;q=0.3",
+                "Accept-Encoding": "gzip, deflate, br, zstd",
+                "DNT": "1",
+                "Connection": "keep-alive",
+                "TE": "trailers",
+            }
+
+        self.client = httpx.Client(
+            transport=transport,
+            timeout=timeout,
+            headers=headers,
+            http2=True,
+        )
+        self.include_host = include_host
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        retries: int = 3,
+        headers: Union[dict, None] = None,
+        include_host: bool = False,
+        content=None,
+    ):
+        include_host = include_host | self.include_host
+        data = None
+        tried: int = 0
+
+        if include_host is True and headers is None:
+            headers = {"Host": tldextract.extract(url).fqdn}
+        elif include_host is True and "Host" not in headers:
+            headers["Host"] = tldextract.extract(url).fqdn
+
+        while data is None and tried < retries:
+            try:
+                data = self.client.request(
+                    method, url, headers=headers, content=content
+                )
+            except httpx.ReadTimeout:
+                tried += 1
+
+        return data
+
+    def get(
+        self, url, retries: int = 3, headers: dict = None, include_host: bool = False
+    ):
+        return self.request("get", url, retries, headers, include_host)
+
+    def post(
+        self,
+        url,
+        retries: int = 3,
+        headers: dict = None,
+        content=None,
+        include_host: bool = False,
+    ):
+        return self.request("post", url, retries, headers, include_host, content)
+
+    def head(
+        self, url, retries: int = 3, headers: dict = None, include_host: bool = False
+    ):
+        return self.request("head", url, retries, headers, include_host)
+
+    def close(self):
+        self.client.close()