diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index a39e441..d0b098d 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -6,38 +6,40 @@
version: 2
updates:
- # Maintain dependencies for GitHub Actions
- - package-ecosystem: "github-actions"
- directory: "/"
- schedule:
- interval: "weekly"
- target-branch: "dev"
- commit-message:
- prefix: "chore"
- include: "scope"
- labels:
- - "Type: Maintenance"
-
# Maintain dependencies for go modules
- package-ecosystem: "gomod"
directory: "/"
schedule:
- interval: "daily"
+ interval: "weekly"
target-branch: "dev"
commit-message:
prefix: "chore"
include: "scope"
labels:
- "Type: Maintenance"
+ allow:
+ - dependency-name: "github.com/projectdiscovery/*"
- # Maintain dependencies for docker
- - package-ecosystem: "docker"
- directory: "/"
- schedule:
- interval: "weekly"
- target-branch: "dev"
- commit-message:
- prefix: "chore"
- include: "scope"
- labels:
- - "Type: Maintenance"
\ No newline at end of file
+# # Maintain dependencies for GitHub Actions
+# - package-ecosystem: "github-actions"
+# directory: "/"
+# schedule:
+# interval: "weekly"
+# target-branch: "dev"
+# commit-message:
+# prefix: "chore"
+# include: "scope"
+# labels:
+# - "Type: Maintenance"
+#
+# # Maintain dependencies for docker
+# - package-ecosystem: "docker"
+# directory: "/"
+# schedule:
+# interval: "weekly"
+# target-branch: "dev"
+# commit-message:
+# prefix: "chore"
+# include: "scope"
+# labels:
+# - "Type: Maintenance"
\ No newline at end of file
diff --git a/.github/release.yml b/.github/release.yml
new file mode 100644
index 0000000..1073bdc
--- /dev/null
+++ b/.github/release.yml
@@ -0,0 +1,17 @@
+changelog:
+ exclude:
+ authors:
+ - dependabot
+ categories:
+ - title: 🎉 New Features
+ labels:
+ - "Type: Enhancement"
+ - title: 🐞 Bug Fixes
+ labels:
+ - "Type: Bug"
+ - title: 🔨 Maintenance
+ labels:
+ - "Type: Maintenance"
+ - title: Other Changes
+ labels:
+ - "*"
\ No newline at end of file
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 0000000..381a372
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,26 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 14
+
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 30
+
+# Issues with these labels will never be considered stale
+# exemptLabels:
+# - pinned
+# - security
+
+# Only issues or pull requests with all of these labels are check if stale.
+onlyLabels:
+ - "Status: Abandoned"
+ - "Type: Question"
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+ This issue has been automatically marked as stale because it has not had
+ activity in the past 2 weeks. It will be closed if no further activity occurs. Thank you
+ for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: This issue is being closed after 30 days of inactivity. Thank you.
\ No newline at end of file
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 0dc2651..c4c8bd0 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -13,12 +13,12 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
- os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-latest]
+ os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v4
with:
- go-version: 1.19
+ go-version: 1.21.x
- name: Check out code
uses: actions/checkout@v3
@@ -31,6 +31,10 @@ jobs:
run: go test ./...
working-directory: .
+ - name: Running example
+ run: go run .
+ working-directory: examples/
+
- name: Integration Tests Linux, macOS
if: runner.os == 'Linux' || runner.os == 'macOS'
env:
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 0cff8f6..3080047 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -11,7 +11,7 @@ on:
jobs:
analyze:
name: Analyze
- runs-on: ubuntu-latest-16-cores
+ runs-on: ubuntu-latest
permissions:
actions: read
contents: read
diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml
new file mode 100644
index 0000000..84b26e1
--- /dev/null
+++ b/.github/workflows/dep-auto-merge.yml
@@ -0,0 +1,26 @@
+name: 🤖 dep auto merge
+
+on:
+ pull_request:
+ branches:
+ - dev
+ workflow_dispatch:
+
+permissions:
+ pull-requests: write
+ issues: write
+ repository-projects: write
+
+jobs:
+ automerge:
+ runs-on: ubuntu-latest
+ if: github.actor == 'dependabot[bot]'
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ token: ${{ secrets.DEPENDABOT_PAT }}
+
+ - uses: ahmadnassri/action-dependabot-auto-merge@v2
+ with:
+ github-token: ${{ secrets.DEPENDABOT_PAT }}
+ target: all
\ No newline at end of file
diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml
index 7d2e49b..e6787a3 100644
--- a/.github/workflows/functional-test.yml
+++ b/.github/workflows/functional-test.yml
@@ -13,12 +13,12 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
- os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-latest]
+ os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v4
with:
- go-version: 1.19
+ go-version: 1.21.x
- name: Check out code
uses: actions/checkout@v3
diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
index 0fdf3ea..d8fee0d 100644
--- a/.github/workflows/lint-test.yml
+++ b/.github/workflows/lint-test.yml
@@ -10,16 +10,16 @@ on:
jobs:
lint:
name: Lint Test
- runs-on: ubuntu-latest-16-cores
+ runs-on: ubuntu-latest
steps:
- name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v4
with:
- go-version: 1.19
+ go-version: 1.21.x
- name: Checkout code
uses: actions/checkout@v3
- name: Run golangci-lint
- uses: golangci/golangci-lint-action@v3.4.0
+ uses: golangci/golangci-lint-action@v3.6.0
with:
version: latest
args: --timeout 5m
diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml
index 2dc5a16..884e0f4 100644
--- a/.github/workflows/release-binary.yml
+++ b/.github/workflows/release-binary.yml
@@ -16,14 +16,14 @@ jobs:
fetch-depth: 0
- name: "Set up Go"
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v4
with:
- go-version: 1.19
+ go-version: 1.21.x
- name: "Create release on GitHub"
uses: goreleaser/goreleaser-action@v4
with:
- args: "release --rm-dist"
+ args: "release --clean"
version: latest
workdir: .
env:
diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml
new file mode 100644
index 0000000..c915185
--- /dev/null
+++ b/.github/workflows/release-test.yml
@@ -0,0 +1,29 @@
+name: 🔨 Release Test
+
+on:
+ pull_request:
+ paths:
+ - '**.go'
+ - '**.mod'
+ workflow_dispatch:
+
+jobs:
+ release-test:
+ runs-on: ubuntu-latest-16-cores
+ steps:
+ - name: "Check out code"
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Set up Go
+ uses: actions/setup-go@v4
+ with:
+ go-version: 1.21.x
+
+ - name: release test
+ uses: goreleaser/goreleaser-action@v4
+ with:
+ args: "release --clean --snapshot"
+ version: latest
+ workdir: .
diff --git a/.gitignore b/.gitignore
index 028e9b1..064196c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
.idea/
.vscode/
+dist
cmd/httpx/httpx
integration_tests/httpx
integration_tests/integration-test
@@ -7,3 +8,5 @@ cmd/functional-test/httpx_dev
cmd/functional-test/functional-test
cmd/functional-test/httpx
cmd/functional-test/*.cfg
+
+.devcontainer
\ No newline at end of file
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 78f2618..852f6cb 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -28,8 +28,7 @@ builds:
archives:
- format: zip
- replacements:
- darwin: macOS
+ name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
checksum:
algorithm: sha256
diff --git a/Dockerfile b/Dockerfile
index 101d761..33474c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
# Base
-FROM golang:1.20.2-alpine AS builder
+FROM golang:1.21.4-alpine AS builder
RUN apk add --no-cache git build-base gcc musl-dev
WORKDIR /app
@@ -7,9 +7,9 @@ COPY . /app
RUN go mod download
RUN go build ./cmd/httpx
-FROM alpine:3.17.2
-RUN apk -U upgrade --no-cache \
- && apk add --no-cache bind-tools ca-certificates
+FROM alpine:3.18.2
+RUN apk upgrade --no-cache \
+ && apk add --no-cache bind-tools ca-certificates chromium
COPY --from=builder /app/httpx /usr/local/bin/
-ENTRYPOINT ["httpx"]
\ No newline at end of file
+ENTRYPOINT ["httpx"]
diff --git a/README.md b/README.md
index 0c47187..a462253 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@
Features •
Installation •
Usage •
- Running httpx •
+ Documentation •
Notes •
Join Discord
@@ -62,12 +62,19 @@
# Installation Instructions
-`httpx` requires **go1.19** to install successfully. Run the following command to get the repo:
+`httpx` requires **go1.21** to install successfully. Run the following command to get the repo:
```sh
go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
```
+To learn more about installing httpx, see https://docs.projectdiscovery.io/tools/httpx/install.
+
+| :exclamation: **Disclaimer** |
+|---------------------------------|
+| **This project is in active development**. Expect breaking changes with releases. Review the changelog before updating. |
+| This project was primarily built to be used as a standalone CLI tool. **Running it as a service may pose security risks.** It's recommended to use with caution and additional security measures. |
+
# Usage
```sh
@@ -99,6 +106,7 @@ PROBES:
-lc, -line-count display response body line count
-wc, -word-count display response body word count
-title display page title
+ -bp, -body-preview display first N characters of response body (default 100)
-server, -web-server display server name
-td, -tech-detect display technology in use based on wappalyzer dataset
-method display http request method
@@ -106,18 +114,26 @@ PROBES:
-ip display host ip
-cname display host cname
-asn display host asn information
- -cdn display cdn in use
+ -cdn display cdn/waf in use
-probe display probe status
+HEADLESS:
+ -ss, -screenshot enable saving screenshot of the page using headless browser
+ -system-chrome enable using local installed chrome for screenshot
+ -ho, -headless-options string[] start headless chrome with additional options
+ -esb, -exclude-screenshot-bytes enable excluding screenshot bytes from json output
+ -ehb, -exclude-headless-body enable excluding headless header from json output
+ -st, -screenshot-timeout int set timeout for screenshot in seconds (default 10)
+
MATCHERS:
-mc, -match-code string match response with specified status code (-mc 200,302)
-ml, -match-length string match response with specified content length (-ml 100,102)
-mlc, -match-line-count string match response body with specified line count (-mlc 423,532)
-mwc, -match-word-count string match response body with specified word count (-mwc 43,55)
-mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000)
- -ms, -match-string string match response with specified string (case insensitive) (-ms admin)
+ -ms, -match-string string match response with specified string (-ms admin)
-mr, -match-regex string match response with specified regex (-mr admin)
- -mcdn, -match-cdn string[] match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
+ -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
-mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1')
-mdc, -match-condition string match response with dsl expression condition
@@ -127,15 +143,17 @@ EXTRACTOR:
FILTERS:
-fc, -filter-code string filter response with specified status code (-fc 403,401)
+ -fep, -filter-error-page filter response with ML based error page detection
-fl, -filter-length string filter response with specified content length (-fl 23,33)
-flc, -filter-line-count string filter response body with specified line count (-flc 423,532)
-fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532)
- -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000)
+ -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000)
-fs, -filter-string string filter response with specified string (-fs admin)
-fe, -filter-regex string filter response with specified regex (-fe admin)
- -fcdn, -filter-cdn string[] filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
+ -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
-frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1')
-fdc, -filter-condition string filter response with dsl expression condition
+ -strip strips all tags in response. supported formats: html,xml (default html)
RATE-LIMIT:
-t, -threads int number of threads to use (default 50)
@@ -154,19 +172,27 @@ MISCELLANEOUS:
-vhost probe and display server supporting VHOST
-ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter
+UPDATE:
+ -up, -update update httpx to latest version
+ -duc, -disable-update-check disable automatic httpx update check
+
OUTPUT:
-o, -output string file to write output results
+ -oa, -output-all filename to write output results in all formats
-sr, -store-response store http response to output directory
-srd, -store-response-dir string store http response to custom directory
-csv store output in csv format
-csvo, -csv-output-encoding string define output encoding
- -json store output in JSONL(ines) format
- -irr, -include-response include http request/response in JSON output (-json only)
+ -j, -json store output in JSONL(ines) format
+ -irh, -include-response-header include http response (headers) in JSON output (-json only)
+ -irr, -include-response include http request/response (headers + body) in JSON output (-json only)
-irrb, -include-response-base64 include base64 encoded http request/response in JSON output (-json only)
-include-chain include redirect http chain in JSON output (-json only)
-store-chain include http redirect chain in responses (-sr only)
+ -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only)
CONFIGURATIONS:
+ -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
-r, -resolvers string[] list of custom resolver (file or comma separated)
-allow string[] allowed list of IP/CIDR's to process (file or comma separated)
-deny string[] denied list of IP/CIDR's to process (file or comma separated)
@@ -179,13 +205,17 @@ CONFIGURATIONS:
-fr, -follow-redirects follow http redirects
-maxr, -max-redirects int max number of redirects to follow per host (default 10)
-fhr, -follow-host-redirects follow redirects on the same host
+ -rhsts, -respect-hsts respect HSTS response headers for redirect requests
-vhost-input get a list of vhosts as input
-x string request methods to probe, use 'all' to probe all HTTP methods
-body string post body to include in http request
-s, -stream stream mode - start elaborating input targets without sorting
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
- -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443
+ -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
-ztls use ztls library with autofallback to standard one for tls13
+ -no-decode avoid decoding body
+ -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
+ -no-stdin Disable Stdin processing
DEBUG:
-health-check, -hc run diagnostic check up
@@ -204,322 +234,39 @@ OPTIMIZATIONS:
-nf, -no-fallback display both probed protocol (HTTPS and HTTP)
-nfs, -no-fallback-scheme probe with protocol scheme specified in input
-maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30)
- -ec, -exclude-cdn skip full port scans for CDNs (only checks for 80,443)
+ -e, -exclude string[] exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
-retries int number of retries
- -timeout int timeout in seconds (default 5)
- -delay duration duration between each http request (eg: 200ms, 1s) (default -1ns)
+ -timeout int timeout in seconds (default 10)
+ -delay value duration between each http request (eg: 200ms, 1s) (default -1ns)
-rsts, -response-size-to-save int max response size to save in bytes (default 2147483647)
-rstr, -response-size-to-read int max response size to read in bytes (default 2147483647)
```
-# Running httpX
-
-### URL Probe
-
-This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver.
-
-```console
-cat hosts.txt | httpx
-
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_| v1.1.1
- /_/
-
- projectdiscovery.io
-
-[WRN] Use with caution. You are responsible for your actions
-[WRN] Developers assume no liability and are not responsible for any misuse or damage.
-
-https://mta-sts.managed.hackerone.com
-https://mta-sts.hackerone.com
-https://mta-sts.forwarding.hackerone.com
-https://docs.hackerone.com
-https://www.hackerone.com
-https://resources.hackerone.com
-https://api.hackerone.com
-https://support.hackerone.com
-```
-
-### File Input
-
-This will run the tool with the `-probe` flag against all the hosts in **hosts.txt** and return URLs with probed status.
-
-```console
-httpx -list hosts.txt -silent -probe
-
-http://ns.hackerone.com [FAILED]
-https://docs.hackerone.com [SUCCESS]
-https://mta-sts.hackerone.com [SUCCESS]
-https://mta-sts.managed.hackerone.com [SUCCESS]
-http://email.hackerone.com [FAILED]
-https://mta-sts.forwarding.hackerone.com [SUCCESS]
-http://links.hackerone.com [FAILED]
-https://api.hackerone.com [SUCCESS]
-https://www.hackerone.com [SUCCESS]
-http://events.hackerone.com [FAILED]
-https://support.hackerone.com [SUCCESS]
-https://gslink.hackerone.com [SUCCESS]
-http://o1.email.hackerone.com [FAILED]
-http://info.hackerone.com [FAILED]
-https://resources.hackerone.com [SUCCESS]
-http://o2.email.hackerone.com [FAILED]
-http://o3.email.hackerone.com [FAILED]
-http://go.hackerone.com [FAILED]
-http://a.ns.hackerone.com [FAILED]
-http://b.ns.hackerone.com [FAILED]
-```
-
-### CIDR Input
-
-```console
-echo 173.0.84.0/24 | httpx -silent
-
-https://173.0.84.29
-https://173.0.84.43
-https://173.0.84.31
-https://173.0.84.44
-https://173.0.84.12
-https://173.0.84.4
-https://173.0.84.36
-https://173.0.84.45
-https://173.0.84.14
-https://173.0.84.25
-https://173.0.84.46
-https://173.0.84.24
-https://173.0.84.32
-https://173.0.84.9
-https://173.0.84.13
-https://173.0.84.6
-https://173.0.84.16
-https://173.0.84.34
-```
-### AS Number Input
-```console
-echo AS14421 | httpx -silent
-
-https://216.101.17.248
-https://216.101.17.249
-https://216.101.17.250
-https://216.101.17.251
-https://216.101.17.252
-```
-
-### Tool Chain
-
-
-```console
-subfinder -d hackerone.com -silent| httpx -title -tech-detect -status-code
-
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.1.1
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions
-Developers assume no liability and are not responsible for any misuse or damage.
-https://mta-sts.managed.hackerone.com [404] [Page not found · GitHub Pages] [Varnish,GitHub Pages,Ruby on Rails]
-https://mta-sts.hackerone.com [404] [Page not found · GitHub Pages] [Varnish,GitHub Pages,Ruby on Rails]
-https://mta-sts.forwarding.hackerone.com [404] [Page not found · GitHub Pages] [GitHub Pages,Ruby on Rails,Varnish]
-https://docs.hackerone.com [200] [HackerOne Platform Documentation] [Ruby on Rails,jsDelivr,Gatsby,React,webpack,Varnish,GitHub Pages]
-https://support.hackerone.com [301,302,301,200] [HackerOne] [Cloudflare,Ruby on Rails,Ruby]
-https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.]
-```
-
-### Favicon Hash
-
-
-```console
-subfinder -d hackerone.com -silent | httpx -favicon
-
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.1.5
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions.
-Developers assume no liability and are not responsible for any misuse or damage.
-https://docs.hackerone.com/favicon.ico [595148549]
-https://hackerone.com/favicon.ico [595148549]
-https://mta-sts.managed.hackerone.com/favicon.ico [-1700323260]
-https://mta-sts.forwarding.hackerone.com/favicon.ico [-1700323260]
-https://support.hackerone.com/favicon.ico [-1279294674]
-https://gslink.hackerone.com/favicon.ico [1506877856]
-https://resources.hackerone.com/favicon.ico [-1840324437]
-https://api.hackerone.com/favicon.ico [566218143]
-https://mta-sts.hackerone.com/favicon.ico [-1700323260]
-https://www.hackerone.com/favicon.ico [778073381]
-```
-
-### [JARM Fingerprint](https://github.com/salesforce/jarm)
+# Running httpx
-
-```console
-subfinder -d hackerone.com -silent | httpx -jarm
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.2.1
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions.
-Developers assume no liability and are not responsible for any misuse or damage.
-https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
-https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
-https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
-https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
-https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7]
-https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
-https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
-https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5]
-```
-
-### ASN Fingerprint
-
-
-```console
-subfinder -d hackerone.com -silent | httpx -asn
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.2.1
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions.
-Developers assume no liability and are not responsible for any misuse or damage.
-https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US]
-https://gslink.hackerone.com [AS16509, AMAZON-02, US]
-https://www.hackerone.com [AS13335, CLOUDFLARENET, US]
-https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US]
-https://resources.hackerone.com [AS16509, AMAZON-02, US]
-https://support.hackerone.com [AS13335, CLOUDFLARENET, US]
-https://mta-sts.hackerone.com [AS54113, FASTLY, US]
-https://docs.hackerone.com [AS54113, FASTLY, US]
-https://api.hackerone.com [AS13335, CLOUDFLARENET, US]
-```
-
-
-### File/Path Bruteforce
-
-
-```console
-httpx -l urls.txt -path /v1/api -sc
-
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.1.5
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions.
-Developers assume no liability and are not responsible for any misuse or damage.
-https://mta-sts.managed.hackerone.com/v1/api [404]
-https://mta-sts.hackerone.com/v1/api [404]
-https://mta-sts.forwarding.hackerone.com/v1/api [404]
-https://docs.hackerone.com/v1/api [404]
-https://api.hackerone.com/v1/api [401]
-https://hackerone.com/v1/api [302]
-https://support.hackerone.com/v1/api [404]
-https://resources.hackerone.com/v1/api [301]
-https://gslink.hackerone.com/v1/api [404]
-http://www.hackerone.com/v1/api [301]
-```
-
-### Docker Run
-
-```console
-cat sub_domains.txt | docker run -i projectdiscovery/httpx
-
- __ __ __ _ __
- / /_ / /_/ /_____ | |/ /
- / __ \/ __/ __/ __ \| /
- / / / / /_/ /_/ /_/ / |
-/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.1.2
-
- projectdiscovery.io
-
-Use with caution. You are responsible for your actions
-Developers assume no liability and are not responsible for any misuse or damage.
-https://mta-sts.forwarding.hackerone.com
-https://mta-sts.hackerone.com
-https://mta-sts.managed.hackerone.com
-https://www.hackerone.com
-https://api.hackerone.com
-https://gslink.hackerone.com
-https://resources.hackerone.com
-https://docs.hackerone.com
-https://support.hackerone.com
-```
+For details about running httpx, see https://docs.projectdiscovery.io/tools/httpx/running.
### Using `httpx` as a library
-`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
-
-```go
-package main
-
-import (
- "log"
-
- "github.com/projectdiscovery/goflags"
- "github.com/projectdiscovery/gologger"
- "github.com/projectdiscovery/gologger/levels"
- "github.com/projectdiscovery/httpx/runner"
-)
-
-func main() {
- gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
-
- options := runner.Options{
- Methods: "GET",
- InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
- //InputFile: "./targetDomains.txt", // path to file containing the target domains list
- }
-
- if err := options.ValidateOptions(); err != nil {
- log.Fatal(err)
- }
-
- httpxRunner, err := runner.New(&options)
- if err != nil {
- log.Fatal(err)
- }
- defer httpxRunner.Close()
-
- httpxRunner.RunEnumeration()
-}
-```
-
+`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder
# Notes
-- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable.
-- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results
+- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
+- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
-- The following flags should be used for specific use cases instead of running them as default with other probes:
- * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path`
-- When using the `-json` flag, all the default probe results are included in the JSON output.
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
-- Invalid custom resolvers/files are ignored.
+- The following flags should be used for specific use cases instead of running them as default with other probes:
+ - `-ports`
+ - `-path`
+ - `-vhost`
+ - `-screenshot`
+ - `-csp-probe`
+ - `-tls-probe`
+ - `-favicon`
+ - `-http2`
+ - `-pipeline`
+ - `-tls-impersonate`
+
# Acknowledgement
diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt
index 7a525c9..34a53b4 100644
--- a/cmd/functional-test/testcases.txt
+++ b/cmd/functional-test/testcases.txt
@@ -13,8 +13,11 @@ scanme.sh {{binary}} -silent -tls-grab
scanme.sh {{binary}} -silent -unsafe
scanme.sh {{binary}} -silent -x all
scanme.sh {{binary}} -silent -body 'a=b'
-scanme.sh {{binary}} -silent -exclude-cdn
+scanme.sh {{binary}} -silent -exclude cdn
scanme.sh {{binary}} -silent -ports https:443
scanme.sh {{binary}} -silent -ztls
+scanme.sh {{binary}} -silent -jarm
https://scanme.sh?a=1*1 {{binary}} -silent
-https://scanme.sh:443 {{binary}} -asn
\ No newline at end of file
+https://scanme.sh:443 {{binary}} -asn
+scanme.sh {{binary}} -silent -tls-impersonate
+example.com {{binary}} -silent -bp -strip
\ No newline at end of file
diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
index 654d50c..caa1448 100644
--- a/cmd/integration-test/http.go
+++ b/cmd/integration-test/http.go
@@ -5,10 +5,12 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "os"
"strings"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/httpx/internal/testutils"
+ fileutil "github.com/projectdiscovery/utils/file"
)
var httpTestcases = map[string]testutils.TestCase{
@@ -17,19 +19,20 @@ var httpTestcases = map[string]testutils.TestCase{
"Raw HTTP GET Request": &standardHttpGet{unsafe: true},
"Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"},
"Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"},
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path
- "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag
- "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path
+ "Regression test for unwanted chars": &titleUnwantedChars{}, // new line scanning with title flag, Regression test for: https://github.com/projectdiscovery/httpx/issues/433
+ "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file
"Standard HTTP GET Request with match response time": &standardHttpGet{mrt: true, inputValue: "\"<10s\""},
"Standard HTTP GET Request with filter response time": &standardHttpGet{frt: true, inputValue: "\">3ms\""},
"Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}},
"Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}},
"Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}},
+ "Output All": &outputAll{},
}
type standardHttpGet struct {
@@ -254,14 +257,14 @@ func (h *issue414) Execute() error {
return nil
}
-type issue433 struct{}
+type titleUnwantedChars struct{}
-func (h *issue433) Execute() error {
+func (h *titleUnwantedChars) Execute() error {
var ts *httptest.Server
router := httprouter.New()
uriPath := "/index"
router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
- htmlResponse := "Project\n\r Discovery\n - Httpx>test data"
+ htmlResponse := "\v\fProject\n\r Discovery\n - Httpx\t>test data"
fmt.Fprint(w, htmlResponse)
}))
ts = httptest.NewServer(router)
@@ -377,3 +380,42 @@ func (h *outputFilterCondition) Execute() error {
}
return nil
}
+
+type outputAll struct {
+}
+
+func (h *outputAll) Execute() error {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+ w.Header().Add("Content-Type", "application/json")
+ w.WriteHeader(200)
+ fmt.Fprint(w, `{"status": "ok"}`)
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+
+ fileName := "test_output_all"
+ _, hErr := testutils.RunHttpxAndGetResults(ts.URL, false, []string{"-o", fileName, "-oa"}...)
+ if hErr != nil {
+ return hErr
+ }
+
+ expectedFiles := []string{fileName, fileName + ".json", fileName + ".csv"}
+ var actualFiles []string
+
+ for _, file := range expectedFiles {
+ if fileutil.FileExists(file) {
+ actualFiles = append(actualFiles, file)
+ }
+ }
+ if len(actualFiles) != 3 {
+ return errIncorrectResultsCount(actualFiles)
+ }
+
+ for _, file := range actualFiles {
+ _ = os.Remove(file)
+ }
+
+ return nil
+}
diff --git a/common/errorpageclassifier/clf.gob b/common/errorpageclassifier/clf.gob
new file mode 100644
index 0000000..f697862
Binary files /dev/null and b/common/errorpageclassifier/clf.gob differ
diff --git a/common/errorpageclassifier/dataset.txt b/common/errorpageclassifier/dataset.txt
new file mode 100644
index 0000000..1f9ee4e
--- /dev/null
+++ b/common/errorpageclassifier/dataset.txt
@@ -0,0 +1,201 @@
+The Forum page seems to have a glitch. Our technicians are on it.||error
+There was a problem with the Product Details page. Try reloading.||error
+Error 500: The E-books page is experiencing a problem.||error
+Unfortunately, the Video Tutorials page is down for maintenance.||error
+Our Archive page is currently unavailable. We apologize for the inconvenience.||error
+We're having trouble loading the Membership Details page.||error
+An error occurred while trying to access the Profile Settings page.||error
+Error 404: The Team page could not be found.||error
+Our Project Highlights page seems to be having some technical issues.||error
+We're sorry, but we can't seem to find the Donations page.||error
+You've landed on our Forum page. Engage in interesting discussions.||nonerror
+Welcome to the Product Details page. Learn more about our products here.||nonerror
+You are now on our E-books page. Enjoy a wealth of knowledge.||nonerror
+This is the Video Tutorials page. Learn with our easy-to-follow videos.||nonerror
+Welcome to our Archive. Dive into our rich history.||nonerror
+You're now on the Membership Details page. See the benefits of joining us.||nonerror
+This is your Profile Settings page. Update your personal details as needed.||nonerror
+You're on the Team page. Meet the people behind our organization.||nonerror
+Welcome to our Project Highlights page. See what we've been up to.||nonerror
+You've landed on the Donations page. Every contribution helps us do more.||nonerror
+500 - Server Error This is highly unusual! Our tech team have been notified and are working on it.||error
+Sorry this page is currently under maintenance.||error
+Access Denied - You don't have permission to access this page.||error
+This page seems to be missing 404 Error!||error
+Sorry something went wrong. Please try again later.||error
+We're sorry this page could not be found 404.||error
+The page you requested could not be found on our site.||error
+500 - Internal server error. There is a problem with the resource you are looking for and it cannot be displayed.||error
+Error 401 Unauthorized: Access is denied due to invalid credentials.||error
+Bad request 400. Your browser sent a request that this server could not understand.||error
+This is a 404 error page||error
+Sorry this page does not exist||error
+Error 500: Internal Server Error||error
+Oops! That page can’t be found.Try searching from the field above or go to the home page.||error
+An error has occurred while processing your request. It happens to the best of us! Don't worry! There are no bugs without a fix! Let's try again! What were you looking for? If you are an adventurer search this site! If difficulties persist please contact the website administrator and report the error below. 404 Page not found||error
+Whoops our bad... The page you requested was not found and we have a fine guess why. If you typed the URL directly please make sure the spelling is correct. If you clicked on a link to get here the link is outdated. What can you do? Have no fear help is near! There are many ways you can get back on track with Magento Store. Go back to the previous page. Use the search bar at the top of the page to search for your products. Follow these links to get you back on track! Store Home My Account||error
+404 - Page not found Unfortunately the requested page could not be found.||error
+PAGE NOT FOUND The page you're looking for doesn't seem to exist anymore… Return to the homepage||error
+Who moved my... lemon? Oh no - looks like we can't find the page you are looking for. But you know the saying; when life gives you lemons... okay we can't find a clever way to end that sentence but we do have 2 suggestions to help you find what you were looking for: Go to the front page Or Search for a specific topic If something you need really is missing we would love it if you would let us know ❤️️||error
+404—page not found||error
+Apologies but there's a 503 Service Unavailable error. The server cannot handle the request.||error
+Sorry you don't have access rights to this page. Error 403: Forbidden.||error
+404 - Oops! The page you are looking for has been misplaced.||error
+Sorry the server encountered an unexpected condition that prevented it from fulfilling the request. Error 500: Internal Server Error.||error
+Whoa! The page you're looking for seems to have vanished. Error 404.||error
+Sorry this page has moved or doesn't exist anymore. Error 404.||error
+Sorry but your request timed out. Please try again. Error 504: Gateway Timeout.||error
+We're sorry but an unknown error occurred while processing your request.||error
+Error 502: Bad Gateway. The server encountered a temporary error and could not complete your request.||error
+The requested resource could not be found on this server. Please verify your request and try again. Error 404.||error
+This Help Center page is temporarily unavailable.||error
+Privacy Policy page not found. Please try again later.||error
+There seems to be an error on our Services page. We're working to fix it.||error
+An error occurred while loading the Search Results page.||error
+Category page not found. It might have been removed or relocated.||error
+There was a problem loading the Cart page. Please try again.||error
+Our Terms of Service page is currently down for maintenance.||error
+We're sorry, but the Sitemap is not available at the moment.||error
+We're having trouble loading the Reviews page.||error
+An error occurred while trying to access the Partners page.||error
+Settings page is currently unavailable. We apologize for the inconvenience.||error
+Error 404: Resources page not found.||error
+Our Press Releases page seems to be having some technical issues.||error
+We're sorry, but we can't seem to find the Case Studies page.||error
+There was a problem loading the Community page. Please refresh the page.||error
+Error 503: The Subscriptions page is temporarily unavailable.||error
+There's a problem with our Customer Support page. We're on it.||error
+We're having trouble finding the Notifications page. It may have been moved.||error
+There was a problem with the Feedback page. Try again later.||error
+Our Transactions page is currently experiencing some issues. We appreciate your patience.||error
+Your request has been successfully submitted.||nonerror
+You have successfully logged out.||nonerror
+Congratulations on successfully completing the course!||nonerror
+The payment has been processed successfully.||nonerror
+Thank you for your feedback!||nonerror
+Your download will start shortly.||nonerror
+Profile updated successfully.||nonerror
+Thanks for contacting us! We'll get back to you as soon as possible.||nonerror
+Sign-up successful. Welcome to our community!||nonerror
+Your booking has been confirmed. Check your email for details.||nonerror
+Welcome! Your registration was successful.||nonerror
+Congratulations! You've successfully updated your profile.||nonerror
+Great! Your order was placed successfully. We'll send you an email confirmation soon.||nonerror
+Welcome back! Your login was successful.||nonerror
+Success! You've added the item to your cart.||nonerror
+Your request was sent successfully. We'll get back to you as soon as possible.||nonerror
+Great job! Your settings have been saved.||nonerror
+Your message has been submitted successfully. We appreciate your feedback.||nonerror
+Thank you for subscribing to our newsletter!||nonerror
+Great news! Your transaction was successful.||nonerror
+Welcome to our homepage. Feel free to browse around||nonerror
+Thanks for signing up! You're now a registered user.||nonerror
+Your order has been placed successfully! You'll receive a confirmation email shortly||nonerror
+Congratulations your account has been successfully created||nonerror
+Thank you for your inquiry. We will respond to your message within 24 hours||nonerror
+You've successfully added the item to your cart!||nonerror
+Success! Your password has been updated||nonerror
+Welcome back! You have successfully logged in||nonerror
+Great job! Your profile has been updated||nonerror
+Your message was sent successfully. We'll get back to you shortly||nonerror
+Welcome to our website. Explore and enjoy our services.||nonerror
+Thank you for visiting our About Us page. Learn more about our journey and team.||nonerror
+You are now browsing our Products page. Check out our latest offerings.||nonerror
+This is our Contact Us page. Feel free to reach out with any queries or feedback.||nonerror
+You have reached the end of the page. Scroll up to continue browsing.||nonerror
+Welcome to the News section. Stay updated with our latest announcements.||nonerror
+Now viewing: Image Gallery. Enjoy a visual tour of our activities.||nonerror
+You're on our FAQ page. Get answers to common questions.||nonerror
+Welcome to the Blog section. Engage with our thoughts and insights.||nonerror
+This is the Discussion Forum. Join in, ask questions, or help others.||nonerror
+You're on the Login page. Enter your credentials to access your account.||nonerror
+Welcome to the Sign-Up page. Join our community today.||nonerror
+This is your User Dashboard. Manage your account and settings here.||nonerror
+You've reached the Checkout page. Review your order and proceed to payment.||nonerror
+Welcome to the Download section . Access our digital resources here.||nonerror
+This is the Careers page. Explore job opportunities with us.||nonerror
+You're viewing the Events Calendar. Keep track of upcoming activities.||nonerror
+This is the User Profile page. Update your information as needed.||nonerror
+Welcome to our Testimonials page. Read reviews and stories from our users.||nonerror
+You are now on the Home page. Start exploring from here.||nonerror
+Welcome to home page||nonerror
+You're now on our Help Center page. Find answers to common questions here.||nonerror
+Welcome to our Privacy Policy page. Learn how we protect your personal information.||nonerror
+You've landed on the Services page. Explore what we have to offer.||nonerror
+This is the Search Results page. Did you find what you were looking for?||nonerror
+Now browsing the Category page. View all items in this category.||nonerror
+You're now on the Cart page. Review your selections before proceeding to checkout.||nonerror
+Welcome to our Terms of Service page. Understand our conditions for providing services.||nonerror
+You are currently on our Sitemap. Navigate our website with ease.||nonerror
+You are on the Reviews page. Check out what others have to say about us.||nonerror
+Now viewing the Partners page. Meet the organizations we collaborate with.||nonerror
+You're on the Settings page. Customize your user experience.||nonerror
+This is our Resources page. Access useful documents and guides.||nonerror
+You've landed on the Press Releases page. Stay updated with our latest news.||nonerror
+Welcome to our Case Studies page. Discover our past projects and achievements.||nonerror
+You're now on the Community page. Connect and interact with other members.||nonerror
+You are currently on the Subscriptions page. Manage your preferences here.||nonerror
+Now viewing the Customer Support page. We're here to help.||nonerror
+This is the Notifications page. Keep track of your updates and alerts.||nonerror
+You've landed on the Feedback page. Share your thoughts with us.||nonerror
+Welcome to the Transactions page. Monitor your past and current transactions.||nonerror
+500 - Server Error This is highly unusual! Our tech team have been notified and are working on it.||error
+Sorry this page is currently under maintenance.||error
+Access Denied - You don't have permission to access this page.||error
+This page seems to be missing 404 Error!||error
+Sorry something went wrong. Please try again later.||error
+We're sorry this page could not be found 404.||error
+The page you requested could not be found on our site.||error
+500 - Internal server error. There is a problem with the resource you are looking for and it cannot be displayed.||error
+Error 401 Unauthorized: Access is denied due to invalid credentials.||error
+Bad request 400. Your browser sent a request that this server could not understand.||error
+Your request has been successfully submitted.||nonerror
+You have successfully logged out.||nonerror
+Congratulations on successfully completing the course!||nonerror
+The payment has been processed successfully.||nonerror
+Thank you for your feedback!||nonerror
+Your download will start shortly.||nonerror
+Profile updated successfully.||nonerror
+Thanks for contacting us! We'll get back to you as soon as possible.||nonerror
+Sign-up successful. Welcome to our community!||nonerror
+Your booking has been confirmed. Check your email for details.||nonerror
+This is a 404 error page||error
+Sorry this page does not exist||error
+Error 500: Internal Server Error||error
+Oops! That page can’t be found.Try searching from the field above or go to the home page.||error
+An error has occurred while processing your request. It happens to the best of us! Don't worry! There are no bugs without a fix! Let's try again! What were you looking for? If you are an adventurer search this site! If difficulties persist please contact the website administrator and report the error below. 404 Page not found||error
+Whoops our bad... The page you requested was not found and we have a fine guess why. If you typed the URL directly please make sure the spelling is correct. If you clicked on a link to get here the link is outdated. What can you do? Have no fear help is near! There are many ways you can get back on track with Magento Store. Go back to the previous page. Use the search bar at the top of the page to search for your products. Follow these links to get you back on track! Store Home | My Account||error
+404 - Page not found Unfortunately the requested page could not be found.||error
+PAGE NOT FOUND The page you're looking for doesn't seem to exist anymore… Return to the homepage||error
+Who moved my... lemon? Oh no - looks like we can't find the page you are looking for. But you know the saying; when life gives you lemons... okay we can't find a clever way to end that sentence but we do have 2 suggestions to help you find what you were looking for: Go to the front page Or Search for a specific topic If something you need really is missing we would love it if you would let us know ❤️️||error
+404—page not found||error
+Apologies but there's a 503 Service Unavailable error. The server cannot handle the request.||error
+Sorry you don't have access rights to this page. Error 403: Forbidden.||error
+404 - Oops! The page you are looking for has been misplaced.||error
+Sorry the server encountered an unexpected condition that prevented it from fulfilling the request. Error 500: Internal Server Error.||error
+Whoa! The page you're looking for seems to have vanished. Error 404.||error
+Sorry this page has moved or doesn't exist anymore. Error 404.||error
+Sorry but your request timed out. Please try again. Error 504: Gateway Timeout.||error
+We're sorry but an unknown error occurred while processing your request.||error
+Error 502: Bad Gateway. The server encountered a temporary error and could not complete your request.||error
+The requested resource could not be found on this server. Please verify your request and try again. Error 404.||error
+Welcome! Your registration was successful.||nonerror
+Congratulations! You've successfully updated your profile.||nonerror
+Great! Your order was placed successfully. We'll send you an email confirmation soon.||nonerror
+Welcome back! Your login was successful.||nonerror
+Success! You've added the item to your cart.||nonerror
+Your request was sent successfully. We'll get back to you as soon as possible.||nonerror
+Great job! Your settings have been saved.||nonerror
+Your message has been submitted successfully. We appreciate your feedback.||nonerror
+Thank you for subscribing to our newsletter!||nonerror
+Great news! Your transaction was successful.||nonerror
+Welcome to our homepage. Feel free to browse around||nonerror
+Thanks for signing up! You're now a registered user.||nonerror
+Your order has been placed successfully! You'll receive a confirmation email shortly||nonerror
+Congratulations your account has been successfully created||nonerror
+Thank you for your inquiry. We will respond to your message within 24 hours||nonerror
+You've successfully added the item to your cart!||nonerror
+Success! Your password has been updated||nonerror
+Welcome back! You have successfully logged in||nonerror
+Great job! Your profile has been updated||nonerror
+Your message was sent successfully. We'll get back to you shortly||nonerror
\ No newline at end of file
diff --git a/common/errorpageclassifier/errorpageclassifier.go b/common/errorpageclassifier/errorpageclassifier.go
new file mode 100644
index 0000000..d916d7c
--- /dev/null
+++ b/common/errorpageclassifier/errorpageclassifier.go
@@ -0,0 +1,39 @@
+package errorpageclassifier
+
+import (
+ _ "embed"
+
+ "github.com/jaytaylor/html2text"
+ "github.com/projectdiscovery/utils/ml/naive_bayes"
+)
+
+//go:embed clf.gob
+var classifierData []byte
+
+type ErrorPageClassifier struct {
+ classifier *naive_bayes.NaiveBayesClassifier
+}
+
+func New() *ErrorPageClassifier {
+ classifier, err := naive_bayes.NewClassifierFromFileData(classifierData)
+ if err != nil {
+ panic(err)
+ }
+ return &ErrorPageClassifier{classifier: classifier}
+}
+
+func (n *ErrorPageClassifier) Classify(html string) string {
+ text := htmlToText(html)
+ if text == "" {
+ return "other"
+ }
+ return n.classifier.Classify(text)
+}
+
+func htmlToText(html string) string {
+ text, err := html2text.FromString(html, html2text.Options{TextOnly: true})
+ if err != nil {
+ panic(err)
+ }
+ return text
+}
diff --git a/common/errorpageclassifier/errorpageclassifier_test.go b/common/errorpageclassifier/errorpageclassifier_test.go
new file mode 100644
index 0000000..35923b2
--- /dev/null
+++ b/common/errorpageclassifier/errorpageclassifier_test.go
@@ -0,0 +1,53 @@
+package errorpageclassifier
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestErrorPageClassifier(t *testing.T) {
+ t.Run("test creation of new ErrorPageClassifier", func(t *testing.T) {
+ epc := New()
+ assert.NotNil(t, epc)
+ })
+
+ t.Run("test classification non error page text", func(t *testing.T) {
+ epc := New()
+ assert.Equal(t, "nonerror", epc.Classify(`
+
+
+
+ Terms of Service
+
+
+ Welcome to our Terms of Service page.
+ Understand our conditions for providing services.
+
+
+ `))
+ })
+
+ t.Run("test classification on error page text", func(t *testing.T) {
+ epc := New()
+ assert.Equal(t, "error", epc.Classify(`
+
+
+ Error 403: Forbidden
+
+
+
+
+
Error 403: Forbidden
+
Sorry you don't have access rights to this page.
+
+
+
+ |
+ Response Info
+ |
+
+ Screenshot
+ |
+
+
+
+ {{ $ExtractTitle := .Options.ExtractTitle }}
+ {{ $OutputStatusCode := .Options.StatusCode }}
+ {{ $OutputContentLength := .Options.ContentLength }}
+ {{ $Favicon := .Options.Favicon }}
+ {{ $OutputResponseTime := .Options.OutputResponseTime }}
+ {{ $OutputLinesCount := .Options.OutputLinesCount }}
+ {{ $OutputWordsCount := .Options.OutputWordsCount }}
+ {{ $OutputServerHeader := .Options.OutputServerHeader }}
+ {{ $TechDetect := .Options.TechDetect }}
+ {{range .Output}}
+ {{if ne .ScreenshotPath ""}}
+
+ |
+
+ |
+
+
+ 
+
+ |
+
+ {{end}}
+ {{end}}
+
+
+
+
+
diff --git a/static/static.go b/static/static.go
new file mode 100644
index 0000000..82bf659
--- /dev/null
+++ b/static/static.go
@@ -0,0 +1,8 @@
+package static
+
+import (
+ _ "embed"
+)
+
+//go:embed html-summary.html
+var HtmlTemplate string