From af9cbec832e907a4065b228bfb1083e3a376d330 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 05:40:59 +0000 Subject: [PATCH 01/17] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.58 to 1.0.59. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.58...v1.0.59) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31992dc6a2..6a2eeb4f20 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.47 - github.com/projectdiscovery/retryabledns v1.0.58 + github.com/projectdiscovery/retryabledns v1.0.59 github.com/projectdiscovery/retryablehttp-go v1.0.58 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 78bd017040..fe64290d48 100644 --- a/go.sum +++ b/go.sum @@ -875,8 +875,8 @@ github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5 github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= -github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= +github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= +github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= From 2800cb6744e09836517f28d54552938d1c4fa993 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 05:41:05 +0000 Subject: [PATCH 02/17] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.120 to 0.0.122. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.120...v0.0.122) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31992dc6a2..17660b4e65 100644 --- a/go.mod +++ b/go.mod @@ -95,7 +95,7 @@ require ( github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.49 github.com/projectdiscovery/utils v0.0.92 - github.com/projectdiscovery/wappalyzergo v0.0.120 + github.com/projectdiscovery/wappalyzergo v0.0.122 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 78bd017040..7f46ae808f 100644 --- a/go.sum +++ b/go.sum @@ -891,8 +891,8 @@ github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOE github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0= github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= -github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= -github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c= +github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= From c2c0f0672f6554418bc5b85e98b78ed367594bad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 06:06:04 +0000 Subject: [PATCH 03/17] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.58 to 1.0.60. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.58...v1.0.60) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6a2eeb4f20..4b9692a0d2 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.69 + github.com/projectdiscovery/fastdialer v0.0.70 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.47 github.com/projectdiscovery/retryabledns v1.0.59 - github.com/projectdiscovery/retryablehttp-go v1.0.58 + github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index fe64290d48..36745c65e6 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= -github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= +github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U= +github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -877,8 +877,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= -github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= +github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= +github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= From 16ce116fed5c3a6b4adc321e2f262276db69e4b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 06:31:25 +0000 Subject: [PATCH 04/17] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.69 to 0.0.71. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.69...v0.0.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9dd94c74c..1002e41efd 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.70 + github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.47 diff --git a/go.sum b/go.sum index 69ad9c9002..dbc2f68aeb 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U= -github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4= +github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= +github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= From 4170e1cbb8653683977c3637b64c5203953bf9e4 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 20 May 2024 00:48:40 +0530 Subject: [PATCH 05/17] more goroutine leak fixes to nuclei (#5188) * more goroutine leak fixes to nuclei * run only dns templates for test * updated httpx to dev * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- go.mod | 28 +++++++-------- go.sum | 55 +++++++++++++++-------------- lib/tests/sdk_test.go | 2 +- pkg/js/libs/postgres/postgres.go | 15 +++++--- pkg/protocols/http/httputils/spm.go | 2 ++ pkg/protocols/http/request.go | 24 +++++++++++-- pkg/utils/http_probe.go | 1 + 7 files changed, 77 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 1002e41efd..36d655cd1b 100644 --- a/go.mod +++ b/go.mod @@ -21,9 +21,9 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/fastdialer v0.0.71 - github.com/projectdiscovery/hmap v0.0.41 - github.com/projectdiscovery/interactsh v1.1.9 - github.com/projectdiscovery/rawhttp v0.1.47 + github.com/projectdiscovery/hmap v0.0.42 + github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 + github.com/projectdiscovery/rawhttp v0.1.49 github.com/projectdiscovery/retryabledns v1.0.59 github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/yamldoc-go v1.0.4 @@ -38,9 +38,9 @@ require ( github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.24.0 + golang.org/x/net v0.25.0 golang.org/x/oauth2 v0.11.0 - golang.org/x/text v0.14.0 + golang.org/x/text v0.15.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -78,30 +78,30 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.52 + github.com/projectdiscovery/dsl v0.0.56 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.50 + github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 - github.com/projectdiscovery/httpx v1.6.0 + github.com/projectdiscovery/httpx v1.6.1 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.39 + github.com/projectdiscovery/ratelimit v0.0.40 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.49 - github.com/projectdiscovery/utils v0.0.92 - github.com/projectdiscovery/wappalyzergo v0.0.122 + github.com/projectdiscovery/utils v0.0.93 + github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 - golang.org/x/term v0.19.0 + golang.org/x/term v0.20.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 ) @@ -303,10 +303,10 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/mod v0.14.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index dbc2f68aeb..2a8f67849f 100644 --- a/go.sum +++ b/go.sum @@ -835,8 +835,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= -github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= +github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= +github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= @@ -845,20 +845,20 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.50 h1:vhpM3mDTnXTktypUuVKWxgRpuBFJE+wzOr56rnOl4Bc= -github.com/projectdiscovery/goflags v0.1.50/go.mod h1:3A8u7Q7e59S7f6QlJmMvKMXD/Tp+bfWre64gUpt6a6Q= +github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= +github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= -github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= -github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ= -github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM= -github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc= -github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I= +github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= +github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= +github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc= +github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss= +github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4= +github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= @@ -869,10 +869,10 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc= -github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0= -github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5gfgc4V9AWU= -github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g= +github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU= +github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE= +github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM= +github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= @@ -889,10 +889,10 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg= github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0= -github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= -github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= -github.com/projectdiscovery/wappalyzergo v0.0.122 h1:xfNJ7VNzU/OGlgYtsyB5ppuOHdfWzU2B8cYATwTz54c= -github.com/projectdiscovery/wappalyzergo v0.0.122/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= +github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= +github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= +github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1191,8 +1191,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1285,8 +1285,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1388,8 +1388,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1401,8 +1401,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1417,8 +1417,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index ee688d797e..b20c163d1a 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -29,7 +29,7 @@ func TestSimpleNuclei(t *testing.T) { goleak.VerifyNone(t, knownLeaks...) }() ne, err := nuclei.NewNucleiEngine( - nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // filter dns templates nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), ) require.Nil(t, err) diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go index 3b52604f39..5aea415010 100644 --- a/pkg/js/libs/postgres/postgres.go +++ b/pkg/js/libs/postgres/postgres.go @@ -104,6 +104,7 @@ func executeQuery(host string, port int, username string, password string, dbNam if err != nil { return nil, err } + defer db.Close() rows, err := db.Query(query) if err != nil { @@ -143,12 +144,16 @@ func connect(host string, port int, username string, password string, dbName str target := net.JoinHostPort(host, fmt.Sprintf("%d", port)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := pg.Connect(&pg.Options{ - Addr: target, - User: username, - Password: password, - Database: dbName, - }) + Addr: target, + User: username, + Password: password, + Database: dbName, + IdleCheckFrequency: -1, + }).WithContext(ctx).WithTimeout(10 * time.Second) defer db.Close() _, err := db.Exec("select 1") diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go index bb0b337c77..22fdd8f93b 100644 --- a/pkg/protocols/http/httputils/spm.go +++ b/pkg/protocols/http/httputils/spm.go @@ -123,9 +123,11 @@ func (h *StopAtFirstMatchHandler[T]) MatchCallback(fn func()) { // run runs the internal handler func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) { defer h.internalWg.Done() + for { select { case <-ctx.Done(): + return case val, ok := <-h.ResultChan: if !ok { return diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index df914a785c..c372e3262b 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -114,7 +114,13 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous } shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch) - spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, maxErrorsWhenParallel, shouldStop) + + childCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + spmHandler := httputils.NewNonBlockingSPMHandler[error](childCtx, maxErrorsWhenParallel, shouldStop) + defer spmHandler.Cancel() + gotMatches := &atomic.Bool{} // wrappedCallback is a callback that wraps the original callback // to implement stop at first match logic @@ -195,7 +201,13 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV // Stop-at-first-match logic while executing requests // parallely using threads shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch) - spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop) + defer spmHandler.Cancel() + // wrappedCallback is a callback that wraps the original callback // to implement stop at first match logic wrappedCallback := func(event *output.InternalWrappedEvent) { @@ -327,7 +339,13 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu // Stop-at-first-match logic while executing requests // parallely using threads shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch) - spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop) + defer spmHandler.Cancel() + // wrappedCallback is a callback that wraps the original callback // to implement stop at first match logic wrappedCallback := func(event *output.InternalWrappedEvent) { diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go index 1ee4657039..9a0877f187 100644 --- a/pkg/utils/http_probe.go +++ b/pkg/utils/http_probe.go @@ -29,6 +29,7 @@ func ProbeURL(input string, httpxclient *httpx.HTTPX) string { if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { continue } + return formedURL } return "" From bfddce4fbb7112f2dce1c1df25fef1d966cd059b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 May 2024 19:19:49 +0000 Subject: [PATCH 06/17] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.49 to 0.0.52. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.49...v0.0.52) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 3 +-- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 36d655cd1b..9516a8bee9 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 - github.com/projectdiscovery/useragent v0.0.49 + github.com/projectdiscovery/useragent v0.0.52 github.com/projectdiscovery/utils v0.0.93 github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/redis/go-redis/v9 v9.1.0 @@ -203,7 +203,6 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.6.1 // indirect github.com/sashabaranov/go-openai v1.15.3 // indirect diff --git a/go.sum b/go.sum index 2a8f67849f..a6542ad9b5 100644 --- a/go.sum +++ b/go.sum @@ -887,8 +887,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= -github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg= -github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0= +github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= +github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= @@ -949,7 +949,6 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= From 9cc335a34cc9e29ffbc0991003bae9b87514709c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 19 May 2024 23:12:21 +0200 Subject: [PATCH 07/17] first interface ipv4 address (#5186) --- pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 67820ec074..ae5ac6cb9d 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -180,6 +180,7 @@ func interfaceAddress(interfaceName string) (net.IP, error) { if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { if ipnet.IP.To4() != nil { address = ipnet.IP + break } } } From 48ed1dde38398e01122dc80db76d80eea3075092 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 05:06:38 +0000 Subject: [PATCH 08/17] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.93 to 0.0.94. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.93...v0.0.94) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9516a8bee9..25e4f38a33 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.93 + github.com/projectdiscovery/utils v0.0.94 github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index a6542ad9b5..151484cfb3 100644 --- a/go.sum +++ b/go.sum @@ -889,8 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= -github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= +github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= +github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From e92746ddcda9458bffbadbca16183f0dec9472f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 05:06:44 +0000 Subject: [PATCH 09/17] chore(deps): bump github.com/projectdiscovery/rawhttp Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.49 to 0.1.50. - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.49...v0.1.50) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9516a8bee9..1c8cb6965b 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/hmap v0.0.42 github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 - github.com/projectdiscovery/rawhttp v0.1.49 + github.com/projectdiscovery/rawhttp v0.1.50 github.com/projectdiscovery/retryabledns v1.0.59 github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/yamldoc-go v1.0.4 diff --git a/go.sum b/go.sum index a6542ad9b5..5042b9b1bc 100644 --- a/go.sum +++ b/go.sum @@ -871,8 +871,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU= github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE= -github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM= -github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94= +github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM= +github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= From 8cffeecd4f23ffdee0e8949ef3b4efe56e7c1c08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 05:33:04 +0000 Subject: [PATCH 10/17] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.40 to 0.0.42. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.40...v0.0.42) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ba2b86d801..7143ae29fe 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( github.com/projectdiscovery/httpx v1.6.1 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.40 + github.com/projectdiscovery/ratelimit v0.0.42 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 diff --git a/go.sum b/go.sum index b2fc867f65..70d8985d60 100644 --- a/go.sum +++ b/go.sum @@ -869,8 +869,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU= -github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE= +github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4= +github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y= github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM= github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= From f266542dd457b168fb03cdebb969b23a75930ffc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 05:33:16 +0000 Subject: [PATCH 11/17] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.71 to 0.0.72. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.71...v0.0.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ba2b86d801..13a195a3ba 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.71 + github.com/projectdiscovery/fastdialer v0.0.72 github.com/projectdiscovery/hmap v0.0.42 github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 github.com/projectdiscovery/rawhttp v0.1.50 diff --git a/go.sum b/go.sum index b2fc867f65..f05fdf4020 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= -github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= -github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= +github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= +github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= From f6332583b7f62c063b3050db18059c786d2e8e0b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 21 May 2024 18:12:01 +0200 Subject: [PATCH 12/17] making ssl errors non fatal (#5203) * making ssl errors non fatal * adding test --- cmd/integration-test/ssl.go | 21 ++++++++++++ .../protocols/ssl/multi-req.yaml | 34 +++++++++++++++++++ pkg/tmplexec/multiproto/multi.go | 14 +++++++- 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 integration_tests/protocols/ssl/multi-req.yaml diff --git a/cmd/integration-test/ssl.go b/cmd/integration-test/ssl.go index 70b8c48445..24f6e9b903 100644 --- a/cmd/integration-test/ssl.go +++ b/cmd/integration-test/ssl.go @@ -13,6 +13,7 @@ var sslTestcases = []TestCaseInfo{ {Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}}, {Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}}, {Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}}, + {Path: "protocols/ssl/multi-req.yaml", TestCase: &sslMultiReq{}}, } type sslBasic struct{} @@ -118,3 +119,23 @@ func (h *sslWithVars) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type sslMultiReq struct{} + +func (h *sslMultiReq) Execute(filePath string) error { + ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) { + defer conn.Close() + data := make([]byte, 4) + if _, err := conn.Read(data); err != nil { + return + } + }) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V") + if err != nil { + return err + } + + return expectResultsCount(results, 2) +} diff --git a/integration_tests/protocols/ssl/multi-req.yaml b/integration_tests/protocols/ssl/multi-req.yaml new file mode 100644 index 0000000000..670ca633d9 --- /dev/null +++ b/integration_tests/protocols/ssl/multi-req.yaml @@ -0,0 +1,34 @@ +id: multi-req + +info: + name: Multi-Request + author: pdteam + severity: info + +ssl: + - address: "{{Host}}:{{Port}}" + min_version: ssl30 + max_version: ssl30 + + extractors: + - type: json + json: + - " .tls_version" + + - address: "{{Host}}:{{Port}}" + min_version: tls10 + max_version: tls10 + + extractors: + - type: json + json: + - " .tls_version" + + - address: "{{Host}}:{{Port}}" + min_version: tls11 + max_version: tls11 + + extractors: + - type: json + json: + - " .tls_version" \ No newline at end of file diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index 58858f971e..7bbc2a1403 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -8,6 +8,8 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/scan" + "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + stringsutil "github.com/projectdiscovery/utils/strings" ) // Mutliprotocol is a template executer engine that executes multiple protocols @@ -110,9 +112,19 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll() err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback) - // if error skip execution of next protocols + // in case of fatal error skip execution of next protocols if err != nil { + // always log errors ctx.LogError(err) + + // for some classes of protocols (i.e ssl) errors like tls handshake are a legitimate behavior so we don't stop execution + // connection failures are already tracked by the internal host error cache + // we use strings comparison as the error is not formalized into instance within the standard library + // within a flow instead we consider ssl errors as fatal, since a specific logic was requested + if req.Type() == types.SSLProtocol && stringsutil.ContainsAnyI(err.Error(), "protocol version not supported", "could not do tls handshake") { + continue + } + return err } } From 46e4810efb29d869cb66bca59c5dcd37e413183a Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Wed, 22 May 2024 14:54:01 +0200 Subject: [PATCH 13/17] Fixing nil ptr on error --- cmd/nuclei/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index a1123b9ffc..e93a4c4441 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -141,7 +141,10 @@ func main() { nucleiRunner.Close() gologger.Info().Msgf("Creating resume file: %s\n", resumeFileName) err := nucleiRunner.SaveResumeConfig(resumeFileName) - return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file") + if err != nil { + return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file") + } + return nil }) } From 23bd0336fb4fa5ee6f522c1643458d2d7b0aa022 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 25 May 2024 00:29:04 +0530 Subject: [PATCH 14/17] multiple bug fixes + performance improvements (#5148) * prototype errkit * complete errkit implementation * add cause to all timeouts * fix request timeout annotation @timeout * increase responseHeaderTimeout to 8 for stability * rawhttp error related improvements * feat: add port status caching * add port status caching to http * migrate to new utils/errkit * remote dialinterface + error cause * debug dir support using .gitignore debug-* * make nuclei easy to debug * debug dir update .gitignore * temp change (to revert) * Revert "temp change (to revert)" This reverts commit d3131f777713b9f80e2275142e80f36340a76d36. * use available context instead of new one * bump fastdialer * fix hosterrorscache + misc improvements * add 'address' field in error log * fix js vague errors + pgwrap driver * fix max host error + misc updates * update tests as per changes * fix request annotation context * remove closed dialer reference * fix sdk panic issue * bump retryablehttp-go,utils,fastdialer --------- Co-authored-by: Sandeep Singh --- .gitignore | 2 +- go.mod | 45 ++++--- go.sum | 112 ++++++++-------- internal/runner/options.go | 8 +- internal/runner/runner.go | 6 +- lib/multi.go | 1 - lib/sdk_private.go | 2 - pkg/catalog/config/constants.go | 12 +- pkg/catalog/config/nucleiconfig.go | 48 ++++++- pkg/core/executors.go | 2 +- pkg/core/workflow_execute.go | 2 +- pkg/js/compiler/compiler.go | 11 +- pkg/js/compiler/init.go | 9 +- pkg/js/libs/postgres/memo.postgres.go | 2 +- pkg/js/libs/postgres/postgres.go | 8 +- pkg/js/utils/pgwrap/pgwrap.go | 53 ++++++++ pkg/output/output.go | 75 +++++++++-- pkg/output/output_test.go | 4 +- pkg/protocols/code/code.go | 6 +- .../common/contextargs/contextargs.go | 24 ++++ pkg/protocols/common/contextargs/metainput.go | 52 ++++++++ .../common/hosterrorscache/hosterrorscache.go | 120 +++++++++++++++--- .../hosterrorscache/hosterrorscache_test.go | 35 +++-- pkg/protocols/common/protocolinit/init.go | 2 +- pkg/protocols/common/protocolstate/state.go | 1 + pkg/protocols/headless/engine/page_actions.go | 5 +- .../http/httpclientpool/clientpool.go | 27 +++- pkg/protocols/http/httpclientpool/options.go | 54 ++++++++ pkg/protocols/http/request.go | 102 +++++++++++---- pkg/protocols/http/request_annotations.go | 19 ++- pkg/protocols/http/request_fuzz.go | 7 +- pkg/protocols/javascript/js.go | 28 ++-- pkg/protocols/network/request.go | 48 ++++++- pkg/protocols/websocket/websocket.go | 3 +- pkg/templates/cluster.go | 4 +- pkg/testutils/testutils.go | 2 + pkg/tmplexec/generic/exec.go | 2 +- pkg/types/nucleierr/kinds.go | 28 ++++ 38 files changed, 767 insertions(+), 204 deletions(-) create mode 100644 pkg/js/utils/pgwrap/pgwrap.go create mode 100644 pkg/protocols/http/httpclientpool/options.go create mode 100644 pkg/types/nucleierr/kinds.go diff --git a/.gitignore b/.gitignore index 9386137b54..b9e561e3a6 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,5 @@ integration_tests/fuzzplayground /nuclei-stats /nuclei-stats-* /scan-charts - +/**/debug-* diff --git a/go.mod b/go.mod index e775ff43d7..537cc9ce65 100644 --- a/go.mod +++ b/go.mod @@ -16,16 +16,16 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.57 + github.com/miekg/dns v1.1.59 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.72 - github.com/projectdiscovery/hmap v0.0.42 - github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 - github.com/projectdiscovery/rawhttp v0.1.50 - github.com/projectdiscovery/retryabledns v1.0.59 - github.com/projectdiscovery/retryablehttp-go v1.0.60 + github.com/projectdiscovery/fastdialer v0.1.0 + github.com/projectdiscovery/hmap v0.0.43 + github.com/projectdiscovery/interactsh v1.1.9 + github.com/projectdiscovery/rawhttp v0.1.49 + github.com/projectdiscovery/retryabledns v1.0.60 + github.com/projectdiscovery/retryablehttp-go v1.0.62 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -35,7 +35,7 @@ require ( github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 github.com/valyala/fasttemplate v1.2.2 - github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 + github.com/weppos/publicsuffix-go v0.30.2 github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.25.0 @@ -73,7 +73,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/labstack/echo/v4 v4.10.2 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa - github.com/lib/pq v1.10.1 + github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.22 github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 @@ -94,12 +94,12 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.94 + github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 - github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc + github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 golang.org/x/term v0.20.0 gopkg.in/yaml.v3 v3.0.1 @@ -134,7 +134,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.3.8 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect @@ -175,7 +175,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kataras/jwt v0.1.10 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/logrusorgru/aurora/v4 v4.0.0 // indirect @@ -203,16 +203,15 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/refraction-networking/utls v1.6.1 // indirect + github.com/refraction-networking/utls v1.6.6 // indirect github.com/sashabaranov/go-openai v1.15.3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/tidwall/btree v1.7.0 // indirect - github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/buntdb v1.3.1 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -233,7 +232,7 @@ require ( github.com/zeebo/blake3 v0.2.3 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect mellium.im/sasl v0.3.1 // indirect ) @@ -298,16 +297,16 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect - go.etcd.io/bbolt v1.3.8 // indirect + github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db // indirect + go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a - golang.org/x/mod v0.14.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.17.0 + golang.org/x/tools v0.21.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 7da521271a..c176b49953 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= @@ -367,8 +367,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= @@ -394,8 +392,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= @@ -630,8 +626,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -662,8 +658,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM= -github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= -github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -714,8 +710,8 @@ github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3r github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= -github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= @@ -773,8 +769,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= @@ -837,8 +831,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= -github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= -github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= +github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= +github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -853,12 +847,12 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= -github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= +github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg= +github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc= github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc= github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss= -github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663 h1:0ih5JJyTht2sv4J4lbfSPaUkKdFDci+Bwxg6vsyd2F4= -github.com/projectdiscovery/interactsh v1.1.10-0.20240519152711-6a0cb98b1663/go.mod h1:gTrQCQfYlhoIM4X11+zwL7uRuraOUCoiE4Z6z1QD86g= +github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc= +github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= @@ -871,14 +865,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4= github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y= -github.com/projectdiscovery/rawhttp v0.1.50 h1:JQw0jBnUEcU2ZoTi8TBQpxHKykxkoBSfh9WETwggZIM= -github.com/projectdiscovery/rawhttp v0.1.50/go.mod h1:h3PiVqB8w7U/yOK+TCAJb1/zu0FHPCFM2ZQ1k2qm9cg= +github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM= +github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= -github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= -github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= +github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ= +github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= +github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY= +github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -889,8 +883,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= -github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= +github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= +github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -927,13 +921,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= -github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA= -github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo= +github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig= +github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1015,17 +1007,17 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg= -github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50= +github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ= +github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= -github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o= +github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -1068,8 +1060,8 @@ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY= -github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg= -github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= +github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU= +github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= @@ -1132,15 +1124,15 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w= -github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s= -github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g= github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE= github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1188,8 +1180,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1202,8 +1194,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1226,8 +1218,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1283,7 +1275,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1310,8 +1303,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1383,9 +1376,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1398,8 +1390,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1415,7 +1407,7 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -1476,8 +1468,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/runner/options.go b/internal/runner/options.go index cad5202f8a..3d36f51123 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" @@ -114,7 +115,12 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) - err := protocolinit.Init(options) + err := protocolstate.Init(options) + if err != nil { + gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err) + } + + err = protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index a0c46f6d03..24010d7537 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -359,6 +359,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, // Close releases all the resources and cleans up func (r *Runner) Close() { + // dump hosterrors cache + if r.hostErrors != nil { + r.hostErrors.Close() + } if r.output != nil { r.output.Close() } @@ -464,7 +468,7 @@ func (r *Runner) RunEnumeration() error { Parser: r.parser, } - if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" { + if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { // Go StdLib style experimental/debug feature switch executorOpts.ExportReqURLPattern = true } diff --git a/lib/multi.go b/lib/multi.go index 5b2c7d6776..7c713e8aa0 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -123,7 +123,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t return err } } - defer tmpEngine.Close() // create ephemeral nuclei objects/instances/types using base nuclei engine unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts) if err != nil { diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 13e8746510..cfe9c88a25 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -25,7 +25,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -119,7 +118,6 @@ func (e *NucleiEngine) init() error { e.parser = templates.NewParser() sharedInit.Do(func() { - _ = protocolstate.Init(e.opts) _ = protocolinit.Init(e.opts) }) diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 736671127d..3918a0124a 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.7` + Version = `v3.2.8-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" @@ -68,3 +68,13 @@ func trimDevIfExists(version string) string { } return version } + +// similar to go pattern of enabling debug related features +// we add custom/extra switches for debugging purposes +const ( + // DebugArgHostErrorStats is used to print host error stats + // when it is closed + DebugArgHostErrorStats = "host-error-stats" + // DebugExportReqURLPattern is used to export request URL pattern + DebugExportURLPattern = "req-url-pattern" +) diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index 619d7fd47a..062393e542 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -46,9 +46,10 @@ type Config struct { LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` // internal / unexported fields - disableUpdates bool `json:"-"` // disable updates both version check and template updates - homeDir string `json:"-"` // User Home Directory - configDir string `json:"-"` // Nuclei Global Config Directory + disableUpdates bool `json:"-"` // disable updates both version check and template updates + homeDir string `json:"-"` // User Home Directory + configDir string `json:"-"` // Nuclei Global Config Directory + debugArgs []string `json:"-"` // debug args } // IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates. @@ -329,6 +330,46 @@ func (c *Config) copyIgnoreFile() { } } +// IsDebugArgEnabled checks if debug arg is enabled +// this could be a feature specific to debugging like PPROF or printing stats +// of max host error etc +func (c *Config) IsDebugArgEnabled(arg string) bool { + for _, v := range c.debugArgs { + if v == arg { + return true + } + } + return false +} + +// parseDebugArgs from string +func (c *Config) parseDebugArgs(data string) { + // use space as seperator instead of commas + tmp := strings.Fields(data) + for _, v := range tmp { + key := v + value := "" + // if it is key value pair then split it + if strings.Contains(v, "=") { + parts := strings.SplitN(v, "=", 2) + if len(parts) != 2 { + continue + } + key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + } + if value == "false" || value == "0" { + // if false or disabled then skip + continue + } + switch key { + case DebugArgHostErrorStats: + c.debugArgs = append(c.debugArgs, DebugArgHostErrorStats) + case DebugExportURLPattern: + c.debugArgs = append(c.debugArgs, DebugExportURLPattern) + } + } +} + func init() { // first attempt to migrate all files from old config directory to new config directory goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition @@ -377,6 +418,7 @@ func init() { // and even if it is changed we don't follow it since it is not expected behavior // If custom templates are in default locations only then they are loaded while running nuclei DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory) + DefaultConfig.parseDebugArgs(env.GetEnvOrDefault("NUCLEI_ARGS", "")) } // Add Default Config adds default when .templates-config.json file is not present diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 86302bb713..0e91c3883b 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -107,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ currentInfo.Unlock() // Skip if the host has had errors - if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) { + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) { return true } diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go index d0d3ede004..028df4d01f 100644 --- a/pkg/core/workflow_execute.go +++ b/pkg/core/workflow_execute.go @@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan } if err != nil { if w.Options.HostErrorsCache != nil { - w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) + w.Options.HostErrorsCache.MarkFailed(ctx.Input, err) } if len(template.Executers) == 1 { mainErr = err diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 05eacda6c6..367bd013b2 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -10,9 +10,15 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" stringsutil "github.com/projectdiscovery/utils/strings" ) +var ( + // ErrJSExecDeadline is the error returned when alloted time for script execution exceeds + ErrJSExecDeadline = errkit.New("js engine execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() +) + // Compiler provides a runtime to execute goja runtime // based javascript scripts efficiently while also // providing them access to custom modules defined in libs/. @@ -107,7 +113,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, } // execute with context and timeout - ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second) + ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline) defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { @@ -119,6 +125,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, return ExecuteProgram(program, args, opts) }) if err != nil { + if val, ok := err.(*goja.Exception); ok { + err = val.Unwrap() + } return nil, err } var res ExecuteResult diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go index 8a6d4a04be..8a171dc3da 100644 --- a/pkg/js/compiler/init.go +++ b/pkg/js/compiler/init.go @@ -1,6 +1,8 @@ package compiler -import "github.com/projectdiscovery/nuclei/v3/pkg/types" +import ( + "github.com/projectdiscovery/nuclei/v3/pkg/types" +) // jsprotocolInit @@ -9,6 +11,7 @@ var ( JsProtocolTimeout = 10 PoolingJsVmConcurrency = 100 NonPoolingVMConcurrency = 20 + JsTimeoutMultiplier = 1.5 ) // Init initializes the javascript protocol @@ -21,7 +24,9 @@ func Init(opts *types.Options) error { // 100 is reasonable default opts.JsConcurrency = 100 } - JsProtocolTimeout = opts.Timeout + // we have dialer timeout set to 10s so js needs to be at least + // 15s to return the actual error if not it will be a dialer timeout + JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier) PoolingJsVmConcurrency = opts.JsConcurrency PoolingJsVmConcurrency -= NonPoolingVMConcurrency return nil diff --git a/pkg/js/libs/postgres/memo.postgres.go b/pkg/js/libs/postgres/memo.postgres.go index 786d87133e..9c61356b07 100755 --- a/pkg/js/libs/postgres/memo.postgres.go +++ b/pkg/js/libs/postgres/memo.postgres.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - _ "github.com/lib/pq" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils" diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go index 5aea415010..f20ea85a73 100644 --- a/pkg/js/libs/postgres/postgres.go +++ b/pkg/js/libs/postgres/postgres.go @@ -9,10 +9,11 @@ import ( "time" "github.com/go-pg/pg" - _ "github.com/lib/pq" "github.com/praetorian-inc/fingerprintx/pkg/plugins" postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql" utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils" + "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" + _ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" ) @@ -100,7 +101,7 @@ func executeQuery(host string, port int, username string, password string, dbNam target := net.JoinHostPort(host, fmt.Sprintf("%d", port)) connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", username, password, target, dbName) - db, err := sql.Open("postgres", connStr) + db, err := sql.Open(pgwrap.PGWrapDriver, connStr) if err != nil { return nil, err } @@ -152,6 +153,9 @@ func connect(host string, port int, username string, password string, dbName str User: username, Password: password, Database: dbName, + Dialer: func(network, addr string) (net.Conn, error) { + return protocolstate.Dialer.Dial(context.Background(), network, addr) + }, IdleCheckFrequency: -1, }).WithContext(ctx).WithTimeout(10 * time.Second) defer db.Close() diff --git a/pkg/js/utils/pgwrap/pgwrap.go b/pkg/js/utils/pgwrap/pgwrap.go new file mode 100644 index 0000000000..d1b82f7ab3 --- /dev/null +++ b/pkg/js/utils/pgwrap/pgwrap.go @@ -0,0 +1,53 @@ +package pgwrap + +import ( + "context" + "database/sql" + "database/sql/driver" + "net" + "time" + + "github.com/lib/pq" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" +) + +const ( + PGWrapDriver = "pgwrap" +) + +type pgDial struct { + fd *fastdialer.Dialer +} + +func (p *pgDial) Dial(network, address string) (net.Conn, error) { + return p.fd.Dial(context.TODO(), network, address) +} + +func (p *pgDial) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + ctx, cancel := context.WithTimeoutCause(context.Background(), timeout, fastdialer.ErrDialTimeout) + defer cancel() + return p.fd.Dial(ctx, network, address) +} + +func (p *pgDial) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return p.fd.Dial(ctx, network, address) +} + +// Unfortunately lib/pq does not provide easy to customize or +// replace dialer so we need to hijack it by wrapping it in our own +// driver and register it as postgres driver + +// PgDriver is the Postgres database driver. +type PgDriver struct{} + +// Open opens a new connection to the database. name is a connection string. +// Most users should only use it through database/sql package from the standard +// library. +func (d PgDriver) Open(name string) (driver.Conn, error) { + return pq.DialOpen(&pgDial{fd: protocolstate.Dialer}, name) +} + +func init() { + sql.Register(PGWrapDriver, &PgDriver{}) +} diff --git a/pkg/output/output.go b/pkg/output/output.go index 044d164f1f..1ee710ff3b 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "io" + "log/slog" "os" "path/filepath" "regexp" @@ -27,9 +28,12 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/nuclei/v3/pkg/utils" + "github.com/projectdiscovery/utils/errkit" fileutil "github.com/projectdiscovery/utils/file" osutils "github.com/projectdiscovery/utils/os" + urlutil "github.com/projectdiscovery/utils/url" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -299,10 +303,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error { // JSONLogRequest is a trace/error log request written to file type JSONLogRequest struct { - Template string `json:"template"` - Input string `json:"input"` - Error string `json:"error"` - Type string `json:"type"` + Template string `json:"template"` + Type string `json:"type"` + Input string `json:"input"` + Address string `json:"address"` + Error string `json:"error"` + Kind string `json:"kind,omitempty"` + Attrs interface{} `json:"attrs,omitempty"` } // Request writes a log the requests trace log @@ -315,12 +322,43 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques Input: input, Type: requestType, } - if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil { - request.Error = unwrappedErr.Error() - } else { + parsed, _ := urlutil.ParseAbsoluteURL(input, false) + if parsed != nil { + request.Address = parsed.Hostname() + port := parsed.Port() + if port == "" { + switch parsed.Scheme { + case urlutil.HTTP: + port = "80" + case urlutil.HTTPS: + port = "443" + } + } + request.Address += ":" + port + } + errX := errkit.FromError(requestErr) + if errX == nil { request.Error = "none" + } else { + request.Kind = errkit.ErrKindUnknown.String() + var cause error + if len(errX.Errors()) > 1 { + cause = errX.Errors()[0] + } + if cause == nil { + cause = errX + } + cause = tryParseCause(cause) + request.Error = cause.Error() + request.Kind = errkit.GetErrorKind(requestErr, nucleierr.ErrTemplateLogic).String() + if len(errX.Attrs()) > 0 { + request.Attrs = slog.GroupValue(errX.Attrs()...) + } + } + // check if address slog attr is avaiable in error if set use it + if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil { + request.Address = val.String() } - data, err := jsoniter.Marshal(request) if err != nil { return @@ -452,3 +490,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string, } } + +// tryParseCause tries to parse the cause of given error +// this is legacy support due to use of errorutil in existing libraries +// but this should not be required once all libraries are updated +func tryParseCause(err error) error { + if err == nil { + return nil + } + msg := err.Error() + if strings.HasPrefix(msg, "ReadStatusLine:") { + // last index is actual error (from rawhttp) + parts := strings.Split(msg, ":") + return errkit.New(strings.TrimSpace(parts[len(parts)-1])) + } + if strings.Contains(msg, "read ") { + // same here + parts := strings.Split(msg, ":") + return errkit.New(strings.TrimSpace(parts[len(parts)-1])) + } + return err +} diff --git a/pkg/output/output_test.go b/pkg/output/output_test.go index 7c96c05297..a21f7e8898 100644 --- a/pkg/output/output_test.go +++ b/pkg/output/output_test.go @@ -30,7 +30,7 @@ func TestStandardWriterRequest(t *testing.T) { require.NoError(t, err) w.Request("path", "input", "http", nil) - require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String()) + require.Equal(t, `{"template":"path","type":"http","input":"input","address":"input:","error":"none"}`, traceWriter.String()) require.Empty(t, errorWriter.String()) }) @@ -47,7 +47,7 @@ func TestStandardWriterRequest(t *testing.T) { fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), ) - require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String()) + require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","type":"http","input":"https://example.com/tcpconfig.html","address":"example.com:443","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","kind":"unknown-error"}`, errorWriter.String()) }) } diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 70dda72548..f103f50d0c 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -32,6 +32,7 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" ) @@ -41,7 +42,10 @@ const ( ) var ( + // pythonEnvRegexCompiled is the compiled regex for python environment variables pythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex) + // ErrCodeExecutionDeadline is the error returned when alloted time for script execution exceeds + ErrCodeExecutionDeadline = errkit.New("code execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) // Request is a request for the SSL protocol @@ -214,7 +218,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) { diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go index 8e8f0361c4..ed004ae370 100644 --- a/pkg/protocols/common/contextargs/contextargs.go +++ b/pkg/protocols/common/contextargs/contextargs.go @@ -37,6 +37,13 @@ func New(ctx context.Context) *Context { return NewWithInput(ctx, "") } +// NewWithMetaInput creates a new contextargs instance with meta input +func NewWithMetaInput(ctx context.Context, input *MetaInput) *Context { + n := New(ctx) + n.MetaInput = input + return n +} + // Create a new contextargs instance with input string func NewWithInput(ctx context.Context, input string) *Context { jar, err := cookiejar.New(nil) @@ -174,3 +181,20 @@ func (ctx *Context) Clone() *Context { } return newCtx } + +// GetCopyIfHostOutdated returns a new contextargs if the host is outdated +func GetCopyIfHostOutdated(ctx *Context, url string) *Context { + if ctx.MetaInput.Input == "" { + newctx := ctx.Clone() + newctx.MetaInput.Input = url + return newctx + } + orig, _ := urlutil.Parse(ctx.MetaInput.Input) + newURL, _ := urlutil.Parse(url) + if orig != nil && newURL != nil && orig.Host != newURL.Host { + newCtx := ctx.Clone() + newCtx.MetaInput.Input = newURL.Host + return newCtx + } + return ctx +} diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go index afda2fda26..72eac6247d 100644 --- a/pkg/protocols/common/contextargs/metainput.go +++ b/pkg/protocols/common/contextargs/metainput.go @@ -4,11 +4,13 @@ import ( "bytes" "crypto/md5" "fmt" + "net" "strings" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" urlutil "github.com/projectdiscovery/utils/url" + "github.com/segmentio/ksuid" ) // MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput) @@ -47,6 +49,56 @@ func (metaInput *MetaInput) URL() (*urlutil.URL, error) { return instance, nil } +// Port returns the port of the target +// if port is not present then empty string is returned +func (metaInput *MetaInput) Port() string { + target, err := urlutil.ParseAbsoluteURL(metaInput.Input, false) + if err != nil { + return "" + } + return target.Port() +} + +// Address return the remote address of target +// Note: it does not resolve the domain to ip +// it is meant to be used by hosterrorsCache if invalid metainput +// is provided then it uses a random ksuid as hostname to avoid skipping valid targets +func (metaInput *MetaInput) Address() string { + var hostname, port string + target, err := urlutil.ParseAbsoluteURL(metaInput.Target(), false) + if err != nil { + if metaInput.CustomIP == "" { + // since this is used in hosterrorscache we add a random id + // which will never be used to avoid skipping valid targets + hostname = fmt.Sprintf("invalid-%s", ksuid.New().String()) + } + } else { + hostname = target.Hostname() + port = target.Port() + if port == "" { + switch target.Scheme { + case urlutil.HTTP: + port = "80" + case urlutil.HTTPS: + port = "443" + default: + port = "80" + } + } + } + if metaInput.CustomIP != "" { + hostname = metaInput.CustomIP + } + if port == "" { + if strings.HasPrefix(hostname, "http://") { + port = "80" + } else if strings.HasPrefix(hostname, "https://") { + port = "443" + } + } + return net.JoinHostPort(hostname, port) +} + // ID returns a unique id/hash for metainput func (metaInput *MetaInput) ID() string { if metaInput.CustomIP != "" { diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 3abd1ec55f..e40f13dde3 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -10,17 +10,25 @@ import ( "github.com/bluele/gcache" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" + "github.com/projectdiscovery/utils/errkit" ) // CacheInterface defines the signature of the hosterrorscache so that // users of Nuclei as embedded lib may implement their own cache type CacheInterface interface { - SetVerbose(verbose bool) // log verbosely - Close() // close the cache - Check(value string) bool // return true if the host should be skipped - MarkFailed(value string, err error) // record a failure (and cause) for the host + SetVerbose(verbose bool) // log verbosely + Close() // close the cache + Check(ctx *contextargs.Context) bool // return true if the host should be skipped + MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host } +var ( + _ CacheInterface = (*Cache)(nil) +) + // Cache is a cache for host based errors. It allows skipping // certain hosts based on an error threshold. // @@ -34,8 +42,10 @@ type Cache struct { } type cacheItem struct { - errors atomic.Int32 sync.Once + errors atomic.Int32 + isPermanentErr bool + cause error // optional cause } const DefaultMaxHostsCount = 10000 @@ -55,6 +65,16 @@ func (c *Cache) SetVerbose(verbose bool) { // Close closes the host errors cache func (c *Cache) Close() { + if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) { + items := c.failedTargets.GetALL(false) + for k, v := range items { + val, ok := v.(*cacheItem) + if !ok { + continue + } + gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load()) + } + } c.failedTargets.Purge() } @@ -88,14 +108,19 @@ func (c *Cache) normalizeCacheValue(value string) string { // - URL: https?:// type // - Host:port type // - host type -func (c *Cache) Check(value string) bool { - finalValue := c.normalizeCacheValue(value) +func (c *Cache) Check(ctx *contextargs.Context) bool { + finalValue := c.GetKeyFromContext(ctx, nil) existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue) if err != nil { return false } existingCacheItemValue := existingCacheItem.(*cacheItem) + if existingCacheItemValue.isPermanentErr { + // skipping permanent errors is expected so verbose instead of info + gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause) + return true + } if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) { existingCacheItemValue.Do(func() { @@ -107,15 +132,22 @@ func (c *Cache) Check(value string) bool { } // MarkFailed marks a host as failed previously -func (c *Cache) MarkFailed(value string, err error) { +func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) { if !c.checkError(err) { return } - finalValue := c.normalizeCacheValue(value) + finalValue := c.GetKeyFromContext(ctx, err) existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue) if err != nil || existingCacheItem == nil { newItem := &cacheItem{errors: atomic.Int32{}} newItem.errors.Store(1) + if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) { + // skip this address altogether + // permanent errors are always permanent hence this is created once + // and never updated so no need to synchronize + newItem.isPermanentErr = true + newItem.cause = err + } _ = c.failedTargets.Set(finalValue, newItem) return } @@ -124,19 +156,77 @@ func (c *Cache) MarkFailed(value string, err error) { _ = c.failedTargets.Set(finalValue, existingCacheItemValue) } -var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`) +// GetKeyFromContext returns the key for the cache from the context +func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string { + // Note: + // ideally any changes made to remote addr in template like {{Hostname}}:81 etc + // should be reflected in contextargs but it is not yet reflected in some cases + // and needs refactor of ScanContext + ContextArgs to achieve that + // i.e why we use real address from error if present + address := ctx.MetaInput.Address() + // get address override from error + if err != nil { + tmp := errkit.GetAttrValue(err, "address") + if tmp.Any() != nil { + address = tmp.String() + } + } + finalValue := c.normalizeCacheValue(address) + return finalValue +} + +var reCheckError = regexp.MustCompile(`(no address found for host|could not resolve host|connection refused|connection reset by peer|could not connect to any address found for host|timeout awaiting response headers)`) // checkError checks if an error represents a type that should be // added to the host skipping table. +// it first parses error and extracts the cause and checks for blacklisted +// or common errors that should be skipped func (c *Cache) checkError(err error) bool { if err == nil { return false } - errString := err.Error() - for _, msg := range c.TrackError { - if strings.Contains(errString, msg) { - return true + kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic) + switch kind { + case nucleierr.ErrTemplateLogic: + // these are errors that are not related to the target + // and are due to template logic + return false + case errkit.ErrKindNetworkTemporary: + // these should not be counted as host errors + return false + case errkit.ErrKindNetworkPermanent: + // these should be counted as host errors + return true + case errkit.ErrKindDeadline: + // these should not be counted as host errors + return false + default: + // parse error for furthur processing + errX := errkit.FromError(err) + tmp := errX.Cause() + cause := tmp.Error() + if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") { + // this is a FP and should not be counted as a host error + // because server closes connection when it reads corrupted bytes which we send via rawhttp + return false + } + if strings.HasPrefix(cause, "ReadStatusLine:") { + // error is present in last part when using rawhttp + // this will be fixed once errkit is used everywhere + lastIndex := strings.LastIndex(cause, ":") + if lastIndex == -1 { + lastIndex = 0 + } + if lastIndex >= len(cause)-1 { + lastIndex = 0 + } + cause = cause[lastIndex+1:] + } + for _, msg := range c.TrackError { + if strings.Contains(cause, msg) { + return true + } } + return reCheckError.MatchString(cause) } - return reCheckError.MatchString(errString) } diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index a4730366e0..6fd04b475b 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -1,11 +1,13 @@ package hosterrorscache import ( + "context" "fmt" "sync" "sync/atomic" "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/stretchr/testify/require" ) @@ -13,8 +15,8 @@ func TestCacheCheck(t *testing.T) { cache := New(3, DefaultMaxHostsCount, nil) for i := 0; i < 100; i++ { - cache.MarkFailed("test", fmt.Errorf("could not resolve host")) - got := cache.Check("test") + cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host")) + got := cache.Check(newCtxArgs("test")) if i < 2 { // till 3 the host is not flagged to skip require.False(t, got) @@ -24,7 +26,7 @@ func TestCacheCheck(t *testing.T) { } } - value := cache.Check("test") + value := cache.Check(newCtxArgs("test")) require.Equal(t, true, value, "could not get checked value") } @@ -32,8 +34,8 @@ func TestTrackErrors(t *testing.T) { cache := New(3, DefaultMaxHostsCount, []string{"custom error"}) for i := 0; i < 100; i++ { - cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error")) - got := cache.Check("custom") + cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error")) + got := cache.Check(newCtxArgs("custom")) if i < 2 { // till 3 the host is not flagged to skip require.False(t, got) @@ -42,7 +44,7 @@ func TestTrackErrors(t *testing.T) { require.True(t, got) } } - value := cache.Check("custom") + value := cache.Check(newCtxArgs("custom")) require.Equal(t, true, value, "could not get checked value") } @@ -73,16 +75,18 @@ func TestCacheMarkFailed(t *testing.T) { tests := []struct { host string - expected int + expected int32 }{ {"http://example.com:80", 1}, {"example.com:80", 2}, - {"example.com", 1}, + // earlier if port is not provided then port was omitted + // but from now it will default to appropriate http scheme based port with 80 as default + {"example.com:443", 1}, } for _, test := range tests { - normalizedCacheValue := cache.normalizeCacheValue(test.host) - cache.MarkFailed(test.host, fmt.Errorf("no address found for host")) + normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil) + cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host")) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) require.Nil(t, err) require.NotNil(t, failedTarget) @@ -102,7 +106,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { }{ {"http://example.com:80", 200}, {"example.com:80", 200}, - {"example.com", 100}, + {"example.com:443", 100}, } // the cache is not atomic during items creation, so we pre-create them with counter to zero @@ -120,14 +124,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - cache.MarkFailed(currentTest.host, fmt.Errorf("could not resolve host")) + cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host")) }() } } wg.Wait() for _, test := range tests { - require.True(t, cache.Check(test.host)) + require.True(t, cache.Check(newCtxArgs(test.host))) normalizedCacheValue := cache.normalizeCacheValue(test.host) failedTarget, err := cache.failedTargets.Get(normalizedCacheValue) @@ -139,3 +143,8 @@ func TestCacheMarkFailedConcurrent(t *testing.T) { require.EqualValues(t, test.expected, value.errors.Load()) } } + +func newCtxArgs(value string) *contextargs.Context { + ctx := contextargs.NewWithInput(context.TODO(), value) + return ctx +} diff --git a/pkg/protocols/common/protocolinit/init.go b/pkg/protocols/common/protocolinit/init.go index 2ab5d7ca9c..59f7dd40b0 100644 --- a/pkg/protocols/common/protocolinit/init.go +++ b/pkg/protocols/common/protocolinit/init.go @@ -13,7 +13,6 @@ import ( // Init initializes the client pools for the protocols func Init(options *types.Options) error { - if err := protocolstate.Init(options); err != nil { return err } @@ -40,4 +39,5 @@ func Init(options *types.Options) error { func Close() { protocolstate.Dialer.Close() + protocolstate.Dialer = nil } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index ae5ac6cb9d..89675e7696 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -210,6 +210,7 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) { func Close() { if Dialer != nil { Dialer.Close() + Dialer = nil } StopActiveMemGuardian() } diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 348cab0aec..57cc371093 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -24,6 +24,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http" contextutil "github.com/projectdiscovery/utils/context" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" @@ -35,6 +36,8 @@ import ( var ( errinvalidArguments = errorutil.New("invalid arguments provided") ErrLFAccessDenied = errorutil.New("Use -allow-local-file-access flag to enable local file access") + // ErrActionExecDealine is the error returned when alloted time for action execution exceeds + ErrActionExecDealine = errkit.New("headless action execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) const ( @@ -619,7 +622,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro // Just wait the event to happen waitFunc := func() (err error) { // execute actual wait event - ctx, cancel := context.WithTimeout(context.Background(), maxDuration) + ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, ErrActionExecDealine) defer cancel() err = contextutil.ExecFunc(ctx, p.page.WaitEvent(waitEvent)) return diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 3e2baf55a5..4200e25b78 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -35,9 +35,10 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - // ResponseHeaderTimeout is the timeout for response headers + // MaxResponseHeaderTimeout is the timeout for response headers // to be read from the server (this prevents infinite hang started by server if any) - ResponseHeaderTimeout = time.Duration(5) * time.Second + // Note: this will be overridden temporarily when using @timeout request annotation + MaxResponseHeaderTimeout = time.Duration(10) * time.Second // HttpTimeoutMultiplier is the multiplier for the http timeout HttpTimeoutMultiplier = 3 ) @@ -53,6 +54,9 @@ func Init(options *types.Options) error { if normalClient != nil { return nil } + if options.Timeout > 10 { + MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second + } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } @@ -111,6 +115,8 @@ type Configuration struct { RedirectFlow RedirectFlow // Connection defines custom connection configuration Connection *ConnectionConfiguration + // ResponseHeaderTimeout is the timeout for response body to be read from the server + ResponseHeaderTimeout time.Duration } // Hash returns the hash of the configuration to allow client pooling @@ -129,13 +135,15 @@ func (c *Configuration) Hash() string { builder.WriteString(strconv.FormatBool(c.DisableCookie)) builder.WriteString("c") builder.WriteString(strconv.FormatBool(c.Connection != nil)) + builder.WriteString("r") + builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10)) hash := builder.String() return hash } // HasStandardOptions checks whether the configuration requires custom settings func (c *Configuration) HasStandardOptions() bool { - return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout + return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0 } // GetRawHTTP returns the rawhttp request client @@ -182,6 +190,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl maxIdleConns := 0 maxConnsPerHost := 0 maxIdleConnsPerHost := -1 + // do not split given timeout into chunks for retry + // because this won't work on slow hosts + retryableHttpOptions.NoAdjustTimeout = true if configuration.Threads > 0 || options.ScanStrategy == scanstrategy.HostSpray.String() { // Single host @@ -235,6 +246,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return nil, errors.Wrap(err, "could not create client certificate") } + // responseHeaderTimeout is max timeout for response headers to be read + responseHeaderTimeout := MaxResponseHeaderTimeout + if configuration.ResponseHeaderTimeout != 0 { + responseHeaderTimeout = configuration.ResponseHeaderTimeout + } + transport := &http.Transport{ ForceAttemptHTTP2: options.ForceAttemptHTTP2, DialContext: Dialer.Dial, @@ -252,7 +269,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl MaxConnsPerHost: maxConnsPerHost, TLSClientConfig: tlsConfig, DisableKeepAlives: disableKeepAlives, - ResponseHeaderTimeout: ResponseHeaderTimeout, + ResponseHeaderTimeout: responseHeaderTimeout, } if types.ProxyURL != "" { @@ -351,7 +368,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed } } -func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error { +func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error { if maxRedirects == 0 { if len(via) > defaultMaxRedirects { return http.ErrUseLastResponse diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go new file mode 100644 index 0000000000..b6e60b6367 --- /dev/null +++ b/pkg/protocols/http/httpclientpool/options.go @@ -0,0 +1,54 @@ +package httpclientpool + +import ( + "io" + "net/http" + "strings" + "time" + + "github.com/projectdiscovery/rawhttp" +) + +// WithCustomTimeout is a configuration for custom timeout +type WithCustomTimeout struct { + Timeout time.Duration +} + +// RawHttpRequestOpts is a configuration for raw http request +type RawHttpRequestOpts struct { + // Method is the http method to use + Method string + // URL is the url to request + URL string + // Path is request path to use + Path string + // Headers is the headers to use + Headers map[string][]string + // Body is the body to use + Body io.Reader + // Options is more client related options + Options *rawhttp.Options +} + +// SendRawRequest sends a raw http request with the provided options and returns http response +func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) { + resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options) + if err != nil { + cause := err.Error() + if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") { + // this error is caused when rawhttp client sends a corrupted or malformed request packet to server + // some servers may attempt gracefully shutdown but most will just abruptly close the connection which results + // in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow + req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body) + req.Header = opts.Headers + resp = &http.Response{ + Request: req, + StatusCode: http.StatusBadRequest, + Status: http.StatusText(http.StatusBadRequest), + Body: io.NopCloser(strings.NewReader("")), + } + return resp, nil + } + } + return resp, err +} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index c372e3262b..7b6c6cee24 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -35,8 +35,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/rawhttp" convUtil "github.com/projectdiscovery/utils/conversion" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" httpUtils "github.com/projectdiscovery/utils/http" "github.com/projectdiscovery/utils/reader" @@ -55,7 +57,9 @@ const ( var ( MaxBodyRead = int64(10 * 1024 * 1024) // 10MB // ErrMissingVars is error occured when variables are missing - ErrMissingVars = errors.New("stop execution due to unresolved variables") + ErrMissingVars = errkit.New("stop execution due to unresolved variables").SetKind(nucleierr.ErrTemplateLogic).Build() + // ErrHttpEngineRequestDeadline is error occured when request deadline set by http request engine is exceeded + ErrHttpEngineRequestDeadline = errkit.New("http request engine deadline exceeded").SetKind(errkit.ErrKindDeadline).Build() ) // Type returns the type of the protocol request @@ -147,15 +151,15 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } }) for i := 0; i < request.RaceNumberRequests; i++ { - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { // stop sending more requests condition is met break } @@ -163,7 +167,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous // execute http request go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { // stop sending more requests condition is met return } @@ -232,8 +236,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } @@ -261,7 +265,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV } // break if stop at first match is found or host is unresponsive - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) { break } @@ -277,17 +281,23 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV if input.MetaInput.Input == "" { input.MetaInput.Input = generatedHttpRequest.URL() } + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) + if request.isUnresponsiveAddress(updatedInput) { + // skip on unresponsive host no need to continue + spmHandler.Cancel() + return nil + } spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() { return } // putting ratelimiter here prevents any unnecessary waiting if any request.options.RateLimitTake() // after ratelimit take, check if we need to stop - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() { return } @@ -370,8 +380,8 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu return } // marks thsi host as unresponsive if applicable - request.markUnresponsiveHost(input, err) - if request.isUnresponsiveHost(input) { + request.markUnresponsiveAddress(input, err) + if request.isUnresponsiveAddress(input) { // stop all inflight requests spmHandler.Cancel() } @@ -389,7 +399,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu default: } - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) || spmHandler.Cancelled() { // skip if first match is found break } @@ -403,11 +413,17 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu if input.MetaInput.Input == "" { input.MetaInput.Input = generatedHttpRequest.URL() } + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) + if request.isUnresponsiveAddress(updatedInput) { + // skip on unresponsive host no need to continue + spmHandler.Cancel() + return nil + } generatedHttpRequest.pipelinedClient = pipeClient spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) { + if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) { // skip if first match is found return } @@ -466,7 +482,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.RateLimitTake() ctx := request.newContext(input) - ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options)) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline) defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) @@ -477,6 +493,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return true, err } + // ideally if http template used a custom port or hostname + // we would want to update it in input but currently templateCtx logic + // is closely tied to contextargs.Context so we are temporarily creating + // a copy and using it to check for host errors etc + // but this should be replaced once templateCtx is refactored properly + updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL()) if generatedHttpRequest.customCancelFunction != nil { defer generatedHttpRequest.customCancelFunction() @@ -487,7 +509,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa input.MetaInput.Input = generatedHttpRequest.URL() } // Check if hosts keep erroring - if request.isUnresponsiveHost(input) { + if request.isUnresponsiveAddress(updatedInput) { return true, nil } var gotMatches bool @@ -527,7 +549,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } if execReqErr != nil { // if applicable mark the host as unresponsive - request.markUnresponsiveHost(input, execReqErr) + request.markUnresponsiveAddress(updatedInput, execReqErr) requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL()) request.options.Progress.IncrementFailedRequestsBy(1) } else { @@ -580,7 +602,7 @@ const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occurred func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) { // Check if hosts keep erroring - if request.isUnresponsiveHost(input) { + if request.isUnresponsiveAddress(input) { return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input) } @@ -713,7 +735,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ inputUrl = url.String() } formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path) - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + + // send rawhttp request and get response + resp, err = httpclientpool.SendRawRequest(generatedRequest.original.rawhttpClient, &httpclientpool.RawHttpRequestOpts{ + Method: generatedRequest.rawRequest.Method, + URL: inputUrl, + Path: generatedRequest.rawRequest.Path, + Headers: generators.ExpandMapValues(generatedRequest.rawRequest.Headers), + Body: io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), + Options: &options, + }) } else { //** For Normal requests **// hostname = generatedRequest.request.URL.Host @@ -731,17 +762,34 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if errSignature := request.handleSignature(generatedRequest); errSignature != nil { return errSignature } - httpclient := request.httpClient + + // this will be assigned/updated if this specific request has a custom configuration + var modifiedConfig *httpclientpool.Configuration + + // check for cookie related configuration if input.CookieJar != nil { connConfiguration := request.connConfiguration connConfiguration.Connection.SetCookieJar(input.CookieJar) - client, err := httpclientpool.Get(request.options.Options, connConfiguration) + modifiedConfig = connConfiguration + } + // check for request updatedTimeout annotation + updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout) + if ok { + if modifiedConfig == nil { + modifiedConfig = request.connConfiguration + } + modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout + } + + if modifiedConfig != nil { + client, err := httpclientpool.Get(request.options.Options, modifiedConfig) if err != nil { return errors.Wrap(err, "could not get http client") } httpclient = client } + resp, err = httpclient.Do(generatedRequest.request) } } @@ -1087,20 +1135,20 @@ func (request *Request) newContext(input *contextargs.Context) context.Context { return input.Context() } -// markUnresponsiveHost checks if the error is a unreponsive host error and marks it -func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) { +// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it +func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) { if err == nil { return } if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + request.options.HostErrorsCache.MarkFailed(input, err) } } -// isUnresponsiveHost checks if the error is a unreponsive based on its execution history -func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool { +// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history +func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool { if request.options.HostErrorsCache != nil { - return request.options.HostErrorsCache.Check(input.MetaInput.ID()) + return request.options.HostErrorsCache.Check(input) } return false } diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go index 67bf7c1671..c0f4b69774 100644 --- a/pkg/protocols/http/request_annotations.go +++ b/pkg/protocols/http/request_annotations.go @@ -10,7 +10,9 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/errkit" iputil "github.com/projectdiscovery/utils/ip" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -27,6 +29,15 @@ var ( reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`) // @once sets the request to be executed only once for a specific URL reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`) + + // ErrTimeoutAnnotationDeadline is the error returned when a specific amount of time was exceeded for a request + // which was alloted using @timeout annotation this usually means that vulnerability was not found + // in rare case it could also happen due to network congestion + // the assigned class is TemplateLogic since this in almost every case means that server is not vulnerable + ErrTimeoutAnnotationDeadline = errkit.New("timeout annotation deadline exceeded").SetKind(nucleierr.ErrTemplateLogic).Build() + // ErrRequestTimeoutDeadline is the error returned when a specific amount of time was exceeded for a request + // this happens when the request execution exceeds alloted time + ErrRequestTimeoutDeadline = errkit.New("request timeout deadline exceeded when notimeout is set").SetKind(errkit.ErrKindDeadline).Build() ) type flowMark int @@ -120,12 +131,16 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req value := strings.TrimSpace(duration[1]) if parsed, err := time.ParseDuration(value); err == nil { //nolint:govet // cancelled automatically by withTimeout - ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed) + // global timeout is overridden by annotation by replacing context + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline) + // add timeout value to context + ctx = context.WithValue(ctx, httpclientpool.WithCustomTimeout{}, httpclientpool.WithCustomTimeout{Timeout: parsed}) request = request.Clone(ctx) } } else { //nolint:govet // cancelled automatically by withTimeout - ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options)) + // global timeout is overridden by annotation by replacing context + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline) request = request.Clone(ctx) } } diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index bc5f9caa9b..8d980b4bf3 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -6,7 +6,6 @@ package http // -> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result] import ( - "context" "fmt" "net/http" "strings" @@ -133,7 +132,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value return request.executeGeneratedFuzzingRequest(gr, input, callback) }, Values: values, - BaseRequest: baseRequest.Clone(context.TODO()), + BaseRequest: baseRequest.Clone(input.Context()), }) if err == nil { applicable = true @@ -158,7 +157,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) hasInteractMarkers := len(gr.InteractURLs) > 0 - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) { return false } request.options.RateLimitTake() @@ -201,7 +200,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, } if requestErr != nil { if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr) + request.options.HostErrorsCache.MarkFailed(input, requestErr) } gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr) } diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 84457b5fca..71f9f53413 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -32,6 +32,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" syncutil "github.com/projectdiscovery/utils/sync" @@ -360,7 +361,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV } if request.generator != nil && request.Threads > 1 { - request.executeRequestParallel(context.Background(), hostPort, hostname, input, payloadValues, callback) + request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback) return nil } @@ -387,11 +388,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV } callback(result) }, requestOptions); err != nil { - _ = err - // Review: should we log error here? - // it is technically not error as it is expected to fail - // gologger.Warning().Msgf("Could not execute request: %s\n", err) - // do not return even if error occured + if errkit.IsNetworkPermanentErr(err) { + // gologger.Verbose().Msgf("Could not execute request: %s\n", err) + return err + } } // If this was a match, and we want to stop at first match, skip all further requests. shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch @@ -408,8 +408,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo if threads == 0 { threads = 1 } - ctx, cancel := context.WithCancel(ctxParent) - defer cancel() + ctx, cancel := context.WithCancelCause(ctxParent) + defer cancel(nil) requestOptions := request.options gotmatches := &atomic.Bool{} @@ -453,16 +453,15 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo } callback(result) }, requestOptions); err != nil { - _ = err - // Review: should we log error here? - // it is technically not error as it is expected to fail - // gologger.Warning().Msgf("Could not execute request: %s\n", err) - // do not return even if error occured + if errkit.IsNetworkPermanentErr(err) { + cancel(err) + return + } } // If this was a match, and we want to stop at first match, skip all further requests. if shouldStopAtFirstMatch && gotmatches.Load() { - cancel() + cancel(nil) return } }() @@ -507,7 +506,6 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte results = compiler.ExecuteResult{"success": false, "error": err.Error()} } request.options.Progress.IncrementRequests() - requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort) diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 1123f40569..62807abf93 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -1,7 +1,6 @@ package network import ( - "context" "encoding/hex" "fmt" "net" @@ -64,7 +63,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err errs = append(errs, err) continue } - conn, err := protocolstate.Dialer.Dial(context.TODO(), "tcp", addr) + conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr) if err != nil { errs = append(errs, err) continue @@ -119,6 +118,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var address string var err error + if request.isUnresponsiveAddress(input) { + // skip on unresponsive address no need to continue + return nil + } if request.SelfContained { address = "" @@ -173,6 +176,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA request.options.Progress.IncrementFailedRequestsBy(1) return err } + updatedTarget := input.Clone() + updatedTarget.MetaInput.Input = actualAddress // if request threads matches global payload concurrency we follow it shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency @@ -206,11 +211,19 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA m.Unlock() } } + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return nil + } value = generators.MergeMaps(value, payloads) swg.Add() go func(vars map[string]interface{}) { defer swg.Done() + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return + } if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil { m.Lock() multiErr = multierr.Append(multiErr, err) @@ -240,13 +253,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac if host, _, err := net.SplitHostPort(actualAddress); err == nil { hostname = host } + updatedTarget := input.Clone() + updatedTarget.MetaInput.Input = actualAddress + + if request.isUnresponsiveAddress(updatedTarget) { + // skip on unresponsive address no need to continue + return nil + } if shouldUseTLS { - conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress) } else { - conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress) } if err != nil { + // adds it to unresponsive address list if applicable + request.markUnresponsiveAddress(updatedTarget, err) request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") @@ -475,3 +497,21 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte } return b[:count], nil } + +// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it +func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) { + if err == nil { + return + } + if request.options.HostErrorsCache != nil { + request.options.HostErrorsCache.MarkFailed(input, err) + } +} + +// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history +func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool { + if request.options.HostErrorsCache != nil { + return request.options.HostErrorsCache.Check(input) + } + return false +} diff --git a/pkg/protocols/websocket/websocket.go b/pkg/protocols/websocket/websocket.go index 0146b57c49..a956de0e3f 100644 --- a/pkg/protocols/websocket/websocket.go +++ b/pkg/protocols/websocket/websocket.go @@ -1,7 +1,6 @@ package websocket import ( - "context" "crypto/tls" "fmt" "io" @@ -228,7 +227,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context, parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) addressToDial = parsedAddress.String() - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) + conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial) if err != nil { requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 97e7b67bd0..603378e986 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -296,7 +296,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input, err) } return results, err } @@ -330,7 +330,7 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.Input, err) + e.options.HostErrorsCache.MarkFailed(ctx.Input, err) } return scanCtx.GenerateResult(), err } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 68410f7f1d..4ce40ace96 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" @@ -27,6 +28,7 @@ import ( // Init initializes the protocols and their configurations func Init(options *types.Options) { + _ = protocolstate.Init(options) _ = protocolinit.Init(options) } diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index 29ace4477d..3b337f6b20 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -85,7 +85,7 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { if err != nil { ctx.LogError(err) if g.options.HostErrorsCache != nil { - g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err) + g.options.HostErrorsCache.MarkFailed(ctx.Input, err) } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err) } diff --git a/pkg/types/nucleierr/kinds.go b/pkg/types/nucleierr/kinds.go new file mode 100644 index 0000000000..9042beaade --- /dev/null +++ b/pkg/types/nucleierr/kinds.go @@ -0,0 +1,28 @@ +package nucleierr + +import ( + "strings" + + "github.com/projectdiscovery/utils/errkit" +) + +var ( + // ErrTemplateLogic are errors that occured due to missing variable or something similar in template logic + // so this is more of a virtual error that is expected due to template logic + ErrTemplateLogic = errkit.NewPrimitiveErrKind("TemplateLogic", "Error expected due to template logic", isTemplateLogicKind) +) + +// isTemplateLogicKind checks if an error is of template logic kind +func isTemplateLogicKind(err *errkit.ErrorX) bool { + if err == nil || err.Cause() == nil { + return false + } + v := err.Cause().Error() + switch { + case strings.Contains(v, "timeout annotation deadline exceeded"): + return true + case strings.Contains(v, "stop execution due to unresolved variables"): + return true + } + return false +} From 8e0bb8fe3407dde569d7b7241dcdec1430a763ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 19:00:12 +0000 Subject: [PATCH 15/17] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.56 to 0.0.57 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.56 to 0.0.57. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.56...v0.0.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 537cc9ce65..8368b1afd3 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.56 + github.com/projectdiscovery/dsl v0.0.57 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.52 diff --git a/go.sum b/go.sum index c176b49953..1c56ada6a6 100644 --- a/go.sum +++ b/go.sum @@ -829,8 +829,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= -github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= +github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= +github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= From f43588dcd287245133b7ed62f5b32bed1b10746f Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Sat, 25 May 2024 02:52:49 +0530 Subject: [PATCH 16/17] version update --- pkg/catalog/config/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 3918a0124a..bfd95f1b88 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.8-dev` + Version = `v3.2.8` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" From 718aca6e0e7e612a263f1a0965a631149914dd99 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Sat, 25 May 2024 04:03:25 +0530 Subject: [PATCH 17/17] go mod update --- go.mod | 12 ++++++------ go.sum | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 8368b1afd3..9006cea29b 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.25.0 - golang.org/x/oauth2 v0.11.0 + golang.org/x/oauth2 v0.18.0 golang.org/x/text v0.15.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -81,7 +81,7 @@ require ( github.com/projectdiscovery/dsl v0.0.57 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.52 + github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 @@ -92,7 +92,7 @@ require ( github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/uncover v1.0.7 + github.com/projectdiscovery/uncover v1.0.8 github.com/projectdiscovery/useragent v0.0.52 github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.0 @@ -260,7 +260,7 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.1 // indirect @@ -289,7 +289,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/trivago/tgo v1.0.7 - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect @@ -307,7 +307,7 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.0 - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1c56ada6a6..a65794fa25 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -839,8 +840,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= -github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= +github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= +github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -879,8 +880,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A= -github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= +github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0= +github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= @@ -1048,8 +1049,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -1288,8 +1289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1498,8 +1499,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=