Feat/save responses to output directory provided in new -srd flag and also print if in json if -json flag enabled fixes #128, #129, #130

cmd/cariddi/main.go

@@ -90,6 +90,12 @@ func main() {
 		StoreResp:     flags.StoreResp,
 	}
 
+	config.OutputDir = output.CariddiOutputFolder
+	if flags.StoredRespDir != "" {
+		config.OutputDir = flags.StoredRespDir
+		config.StoreResp = true
+	}
+
 	// Read the targets from standard input.
 	targets := input.ScanTargets()
 
@@ -118,18 +124,18 @@ func main() {
 	// Create output files if needed (txt / html).
 	config.Txt = ""
 	if flags.TXTout != "" {
-		config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt")
+		config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt", config.OutputDir)
 	}
 
 	var ResultHTML = ""
 	if flags.HTMLout != "" {
-		ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html")
+		ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html", config.OutputDir)
 		output.BannerHTML(ResultHTML)
 		output.HeaderHTML("Results", ResultHTML)
 	}
 
 	if config.StoreResp {
-		fileUtils.CreateIndexOutputFile("index.responses.txt")
+		fileUtils.CreateIndexOutputFile("index.responses.txt", config.OutputDir)
 	}
 
 	// Read headers if needed
@@ -167,13 +173,13 @@ func main() {
 	// IF TXT OUTPUT >
 	if flags.TXTout != "" {
 		output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints,
-			finalExtensions, finalErrors, finalInfos)
+			finalExtensions, finalErrors, finalInfos, config.OutputDir)
 	}
 
 	// IF HTML OUTPUT >
 	if flags.HTMLout != "" {
 		output.HTMLOutput(flags, ResultHTML, finalResults, finalSecret,
-			finalEndpoints, finalExtensions, finalErrors, finalInfos)
+			finalEndpoints, finalExtensions, finalErrors, finalInfos, config.OutputDir)
 	}
 
 	// If needed print secrets.

internal/file/file.go

@@ -44,9 +44,9 @@ const (
 
 // CreateOutputFolder creates the output folder
 // If it fails exits with an error message.
-func CreateOutputFolder() {
+func CreateOutputFolder(outputDir string) {
 	// Create a folder/directory at a full qualified path
-	err := os.Mkdir("output-cariddi", Permission0755)
+	err := os.MkdirAll(outputDir, Permission0755)
 	if err != nil {
 		fmt.Println("Can't create output folder.")
 		os.Exit(1)
@@ -56,9 +56,9 @@ func CreateOutputFolder() {
 // CreateHostOutputFolder creates the host output folder
 // for the HTTP responses.
 // If it fails exits with an error message.
-func CreateHostOutputFolder(host string) {
+func CreateHostOutputFolder(host string, outputDir string) {
 	// Create a folder/directory at a full qualified path
-	err := os.MkdirAll(filepath.Join("output-cariddi", host), Permission0755)
+	err := os.MkdirAll(filepath.Join(outputDir, host), Permission0755)
 	if err != nil {
 		fmt.Println("Can't create host output folder.")
 		os.Exit(1)
@@ -71,21 +71,21 @@ func CreateHostOutputFolder(host string) {
 // already exists, if yes asks the user if cariddi has to overwrite it;
 // if no cariddi creates it.
 // Whenever an instruction fails, it exits with an error message.
-func CreateOutputFile(target string, subcommand string, format string) string {
+func CreateOutputFile(target string, subcommand string, format string, outputDir string) string {
 	target = ReplaceBadCharacterOutput(target)
 
 	var filename string
 	if subcommand != "" {
-		filename = filepath.Join("output-cariddi", target+"."+subcommand+"."+format)
+		filename = filepath.Join(outputDir, target+"."+subcommand+"."+format)
 	} else {
-		filename = filepath.Join("output-cariddi", target+"."+format)
+		filename = filepath.Join(outputDir, target+"."+format)
 	}
 
 	_, err := os.Stat(filename)
 
 	if os.IsNotExist(err) {
-		if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
-			CreateOutputFolder()
+		if _, err := os.Stat(fmt.Sprintf("%s/", outputDir)); os.IsNotExist(err) {
+			CreateOutputFolder(outputDir)
 		}
 		// If the file doesn't exist, create it.
 		f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
@@ -119,15 +119,15 @@ func CreateOutputFile(target string, subcommand string, format string) string {
 // It creates the output folder if needed, then checks if the index output file
 // already exists, if no cariddi creates it.
 // Whenever an instruction fails, it exits with an error message.
-func CreateIndexOutputFile(filename string) {
+func CreateIndexOutputFile(filename string, outputDir string) {
 	_, err := os.Stat(filename)
 
 	if os.IsNotExist(err) {
-		if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
-			CreateOutputFolder()
+		if _, err := os.Stat(fmt.Sprintf("%s/", outputDir)); os.IsNotExist(err) {
+			CreateOutputFolder(outputDir)
 		}
 		// If the file doesn't exist, create it.
-		filename = filepath.Join("output-cariddi", filename)
+		filename = filepath.Join(outputDir, filename)
 
 		f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
 		if err != nil {
@@ -161,6 +161,7 @@ func ReadFile(inputFile string) []string {
 	for scanner.Scan() {
 		text = append(text, scanner.Text())
 	}
+
 	file.Close()
 
 	return text

pkg/crawler/colly.go

@@ -134,8 +134,12 @@ func New(scan *Scan) *Results {
 			fmt.Println(r.Request.URL)
 		}
 
+		var outputPath string
+
 		if scan.StoreResp {
-			err := output.StoreHTTPResponse(r)
+			var err error
+			outputPath, err = output.StoreHTTPResponse(r, scan.OutputDir)
+
 			if err != nil {
 				log.Println(err)
 			}
@@ -193,7 +197,7 @@ func New(scan *Scan) *Results {
 
 		if scan.JSON {
 			jsonOutput, err := output.GetJSONString(
-				r, secrets, parameters, filetype, errors, infos,
+				r, secrets, parameters, filetype, errors, infos, outputPath,
 			)
 
 			if err == nil {

pkg/crawler/options.go

@@ -59,6 +59,7 @@ type Scan struct {
 	FileType      int
 	Headers       map[string]string
 	StoreResp     bool
+	OutputDir     string
 
 	// Settings
 	Concurrency int

pkg/crawler/useragents.go

@@ -35,6 +35,10 @@ import (
 	"time"
 )
 
+const (
+	maxRandomValue = 100
+)
+
 // genOsString generates a random OS string for a User Agent.
 func genOsString() string {
 	source := rand.NewSource(time.Now().UnixNano())
@@ -151,7 +155,7 @@ func GenerateRandomUserAgent() string {
 	source := rand.NewSource(time.Now().UnixNano())
 	rng := rand.New(source)
 
-	decision := rng.Intn(100)
+	decision := rng.Intn(maxRandomValue)
 
 	var ua string
 	if decision%2 == 0 {

pkg/input/check.go

@@ -29,6 +29,7 @@ package input
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	fileUtils "github.com/edoardottt/cariddi/internal/file"
@@ -47,6 +48,33 @@ func CheckOutputFile(input string) bool {
 	return true
 }
 
+// IsValidOutputPath checks if the directory is valid and, if created, cleans it up.
+func CheckOutputPath(outputPath string) bool {
+	// Convert to absolute path if necessary
+	absPath, err := filepath.Abs(outputPath)
+	if err != nil {
+		return false
+	}
+
+	// Check if the directory already exists
+	_, err = os.Stat(absPath)
+	if err == nil {
+		// Directory exists, so it's valid, no need to delete
+		return true
+	}
+
+	// If the directory does not exist, try to create it
+	err = os.MkdirAll(absPath, os.ModePerm)
+	if err != nil {
+		return false
+	}
+
+	// Since we created the directory, clean it up
+	defer os.RemoveAll(absPath)
+
+	return true
+}
+
 // CheckFlags checks the flags taken as input.
 func CheckFlags(flags Input) {
 	if flags.TXTout != "" {
@@ -127,4 +155,11 @@ func CheckFlags(flags Input) {
 		fmt.Println("	- cat urls | cariddi -headersfile headers.txt")
 		os.Exit(1)
 	}
+
+	if flags.StoredRespDir != "" {
+		if !CheckOutputPath(flags.StoredRespDir) {
+			fmt.Println("Validation failed for srd flag; there may be errors creating the output directory.")
+			os.Exit(1)
+		}
+	}
 }

pkg/input/check_test.go

@@ -0,0 +1,55 @@
+package input_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/edoardottt/cariddi/pkg/input"
+)
+
+// TestCheckOutputPath tests the CheckOutputPath function for valid and invalid cases.
+func TestCheckOutputPath(t *testing.T) {
+	// Create a temporary directory for testing
+	tmpDir, err := os.MkdirTemp("", "testdir")
+	defer os.RemoveAll(tmpDir) // Cleanup after test
+
+	if err != nil {
+		t.Fatalf("Failed to create temporary directory: %v", err)
+	}
+
+	// Valid case: Existing directory
+	if !input.CheckOutputPath(tmpDir) {
+		t.Errorf("CheckOutputPath(%s) = false; want true", tmpDir)
+	}
+
+	// Verify that the existing directory is still present after check
+	_, err = os.Stat(tmpDir)
+	if err != nil {
+		t.Errorf("Existing directory %s should still be present after CheckOutputPath, but got error: %v", tmpDir, err)
+	}
+
+	// Cross-platform invalid cases
+	invalidPaths := []string{
+		// Null character is invalid on all platforms
+		filepath.Join(tmpDir, "invalid\000path"),
+	}
+
+	for _, invalidPath := range invalidPaths {
+		if input.CheckOutputPath(invalidPath) {
+			t.Errorf("CheckOutputPath(%s) = true; want false", invalidPath)
+		}
+	}
+
+	// Valid case: New directory creation and cleanup
+	newDir := filepath.Join(tmpDir, "newdir")
+	if !input.CheckOutputPath(newDir) {
+		t.Errorf("CheckOutputPath(%s) = false; want true", newDir)
+	}
+
+	// After check, the directory should be removed
+	_, err = os.Stat(newDir)
+	if err == nil || !os.IsNotExist(err) {
+		t.Errorf("CheckOutputPath should remove the directory, but it still exists: %s", newDir)
+	}
+}

pkg/input/flags.go

@@ -94,6 +94,8 @@ type Input struct {
 	UserAgent string
 	// StoreResp stores HTTP responses.
 	StoreResp bool
+	// StoredRespDir stores HTTP responses to the directory provided.
+	StoredRespDir string
 }
 
 // ScanFlag defines all the options taken
@@ -142,6 +144,8 @@ func ScanFlag() Input {
 
 	storeRespPtr := flag.Bool("sr", false, "Store HTTP responses.")
 
+	storedRespDirPtr := flag.String("srd", "", "Stores HTTP responses to the directory provided.")
+
 	flag.Parse()
 
 	result := Input{
@@ -173,6 +177,7 @@ func ScanFlag() Input {
 		*debugPtr,
 		*userAgentPtr,
 		*storeRespPtr,
+		*storedRespDirPtr,
 	}
 
 	return result

pkg/output/jsonl.go

@@ -44,6 +44,7 @@ type JSONData struct {
 	ContentType   string          `json:"content_type,omitempty"`
 	ContentLength int             `json:"content_length,omitempty"`
 	Matches       *MatcherResults `json:"matches,omitempty"`
+	OutputPath    string          `json:"output_path,omitempty"`
 	// Host          string `json:"host"` # TODO: Available in Colly 2.x
 }
 
@@ -67,6 +68,7 @@ func GetJSONString(
 	filetype *scanner.FileType,
 	errors []scanner.ErrorMatched,
 	infos []scanner.InfoMatched,
+	outputPath string,
 ) ([]byte, error) {
 	// Parse response headers
 	headers := r.Headers
@@ -136,6 +138,7 @@ func GetJSONString(
 		ContentType:   contentType,
 		ContentLength: contentLength,
 		Matches:       matcherResults,
+		OutputPath:    outputPath,
 		// Host:          "", // TODO
 	}