Implemented a redirect strategy which respects HSTS headers and modif…

…ies the URL accordingly if necessary

common/httpx/httpx.go

@@ -75,6 +75,27 @@ func New(options *Options) (*HTTPX, error) {
 		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
 			// add custom cookies if necessary
 			httpx.setCustomCookies(redirectedRequest)
+
+			//Add redirect policy which takes HSTS into account.
+			//Since the net/http/client doesn't take it into account
+			//it is possible to modify it here.
+			//If during redirect the scheme switches from HTTPS to HTTP
+			//but the Strict-Transport-Security header is present the request
+			//would go to the specified location. This could mean that it is not
+			//followed the same way as a browser. There exist some cases in the wild.
+			if httpx.Options.RespectHSTS {
+				location := redirectedRequest.Response.Header.Get("Location")
+				hsts := redirectedRequest.Response.Header.Get("Strict-Transport-Security")
+				url, err := redirectedRequest.URL.Parse(location)
+				if err != nil {
+				} else {
+					if url.Scheme == "http" && hsts != "" {
+						url.Scheme = "https"
+					}
+				}
+				redirectedRequest.URL = url
+			}
+
 			if len(previousRequests) >= options.MaxRedirects {
 				// https://github.com/golang/go/issues/10069
 				return http.ErrUseLastResponse

common/httpx/option.go

@@ -24,6 +24,7 @@ type Options struct {
 	VHostSimilarityRatio int
 	FollowRedirects      bool
 	FollowHostRedirects  bool
+	RespectHSTS          bool
 	MaxRedirects         int
 	Unsafe               bool
 	TLSGrab              bool

runner/options.go

@@ -186,6 +186,7 @@ type Options struct {
 	Location                  bool
 	ContentLength             bool
 	FollowRedirects           bool
+	RespectHSTS               bool
 	StoreResponse             bool
 	JSONOutput                bool
 	CSVOutput                 bool
@@ -409,6 +410,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"),
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "max number of redirects to follow per host"),
 		flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
+		flagSet.BoolVarP(&options.RespectHSTS, "respect-hsts", "rhsts", false, "respect HSTS response headers for redirect requests"),
 		flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "get a list of vhosts as input"),
 		flagSet.StringVar(&options.Methods, "x", "", "request methods to probe, use 'all' to probe all HTTP methods"),
 		flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"),

runner/runner.go

@@ -106,6 +106,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.RetryMax = options.Retries
 	httpxOptions.FollowRedirects = options.FollowRedirects
 	httpxOptions.FollowHostRedirects = options.FollowHostRedirects
+	httpxOptions.RespectHSTS = options.RespectHSTS
 	httpxOptions.MaxRedirects = options.MaxRedirects
 	httpxOptions.HTTPProxy = options.HTTPProxy
 	httpxOptions.Unsafe = options.Unsafe