Allow private tlds only

internal/runner/options.go

@@ -9,16 +9,20 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/tldfinder/pkg/agent"
+	"github.com/projectdiscovery/tldfinder/pkg/registry"
 	"github.com/projectdiscovery/tldfinder/pkg/resolve"
 	"github.com/projectdiscovery/tldfinder/pkg/source"
+	"github.com/projectdiscovery/tldfinder/pkg/utils"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
 	logutil "github.com/projectdiscovery/utils/log"
 	updateutils "github.com/projectdiscovery/utils/update"
 	"github.com/weppos/publicsuffix-go/publicsuffix"
+	"golang.org/x/exp/slices"
 )
 
 var (
@@ -180,8 +184,6 @@ func ParseOptions() *Options {
 		os.Exit(0)
 	}
 
-	options.preProcessOptions()
-
 	if !options.Silent {
 		showBanner()
 	}
@@ -202,6 +204,8 @@ func ParseOptions() *Options {
 		os.Exit(0)
 	}
 
+	options.preProcessOptions()
+
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
 	err = options.validateOptions()
@@ -256,19 +260,26 @@ func listSources(options *Options) {
 
 func (options *Options) preProcessOptions() {
 	for i, query := range options.Domain {
-		query, _ = sanitize(query)
+		query, _ = utils.Sanitize(query)
 		options.Domain[i] = options.parseQuery(query)
-
 	}
 }
 
 func (options *Options) parseQuery(query string) string {
 	if options.DiscoveryMode == source.DNSMode {
+		var tld string
 		if parsed, err := publicsuffix.Parse(query); err == nil {
-			query = parsed.TLD
+			tld = parsed.TLD
+		} else {
+			tld = strings.Trim(query, ".")
+		}
+		if slices.Contains(registry.PrivateTLDs, tld) {
+			query = tld
 		} else {
-			query = strings.Trim(query, ".")
+			gologger.Print().Msgf("[%v] Skipping \"%s\": only private TLDs are allowed (e.g., .google)\n", aurora.NewAurora(!options.NoColor).BrightYellow("WRN"), query)
+			query = ""
 		}
+
 	} else if options.DiscoveryMode == source.TLDMode {
 		if parsed, err := publicsuffix.Parse(query); err == nil {
 			query = parsed.SLD

internal/runner/runner.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/projectdiscovery/tldfinder/pkg/agent"
 	"github.com/projectdiscovery/tldfinder/pkg/resolve"
+	"github.com/projectdiscovery/tldfinder/pkg/utils"
 )
 
 // Runner is an instance of the domain enumeration
@@ -147,9 +148,9 @@ func (r *Runner) EnumerateMultipleQueriesWithCtx(ctx context.Context, reader io.
 	scanner := bufio.NewScanner(reader)
 	ip, _ := regexp.Compile(`^([0-9\.]+$)`)
 	for scanner.Scan() {
-		domain, err := normalizeLowercase(scanner.Text())
+		domain, err := utils.Sanitize(scanner.Text())
 		isIp := ip.MatchString(domain)
-		if errors.Is(err, ErrEmptyInput) || (r.options.ExcludeIps && isIp) {
+		if errors.Is(err, utils.ErrEmptyInput) || (r.options.ExcludeIps && isIp) {
 			continue
 		}
 

internal/runner/util.go

@@ -1,17 +1,12 @@
 package runner
 
 import (
-	"strings"
-
 	"github.com/pkg/errors"
 
+	"github.com/projectdiscovery/tldfinder/pkg/utils"
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
-var (
-	ErrEmptyInput = errors.New("empty data")
-)
-
 func loadFromFile(file string) ([]string, error) {
 	chanItems, err := fileutil.ReadFile(file)
 	if err != nil {
@@ -20,24 +15,11 @@ func loadFromFile(file string) ([]string, error) {
 	var items []string
 	for item := range chanItems {
 		var err error
-		item, err = sanitize(item)
-		if errors.Is(err, ErrEmptyInput) {
+		item, err = utils.Sanitize(item)
+		if errors.Is(err, utils.ErrEmptyInput) {
 			continue
 		}
 		items = append(items, item)
 	}
 	return items, nil
 }
-
-func sanitize(data string) (string, error) {
-	data = strings.Trim(data, "\n\t\"'` ")
-	if data == "" {
-		return "", ErrEmptyInput
-	}
-	return data, nil
-}
-
-func normalizeLowercase(s string) (string, error) {
-	data, err := sanitize(s)
-	return strings.ToLower(data), err
-}

pkg/registry/private_tlds.txt

@@ -0,0 +1,199 @@
+AAA  
+AARP  
+ABB  
+ABBOTT  
+ABBVIE  
+ABC  
+ABLE  
+ACCENTURE  
+ACO  
+ACTIVE  
+ADAC  
+AFL  
+AIG  
+ALLFINANZ  
+ALLSTATE  
+APPLE  
+AQUARELLE  
+ARTE  
+AUDI  
+AUSPOST  
+AWS  
+AXA  
+BARCLAYCARD  
+BARCLAYS  
+BBC  
+BBVA  
+BEATS  
+BENTLEY  
+BING  
+BLOOMBERG  
+BMW  
+BNPPARIBAS  
+BOSTIK  
+BRADESCO  
+BRIDGESTONE  
+BROTHER  
+BUGATTI  
+CANCERRESEARCH  
+CANON  
+CBA  
+CBS  
+CERN  
+CFA  
+CHANEL  
+CHASE  
+CHINTAI  
+CISCO  
+CITI  
+CITIC  
+CLUBMED  
+CROWN  
+CRS  
+CSC  
+CUISINELLA  
+DABUR  
+DELL  
+DELOITTE  
+DHL  
+DISCOVER  
+DNP  
+DUPONT  
+DVAG  
+EDEKA  
+EMERCK  
+ERNI  
+ESURANCE  
+EVERBANK  
+FAGE  
+FAIRWINDS  
+FERRERO  
+FIRMDALE  
+FORD  
+FOX  
+FROGANS  
+FUJITSU  
+GENTING  
+GLOBO  
+GMAIL  
+GMO  
+GOO  
+GOOG  
+GOOGLE  
+GUCCI  
+HISAMITSU  
+HITACHI  
+HONDA  
+HSBC  
+ICE  
+IEEE  
+IFM  
+IKANO  
+INTEL  
+IPIRANGA  
+ISELECT  
+ITV  
+JAGUAR  
+JAVA  
+JCB  
+JLL  
+JMP  
+JNJ  
+JPMORGAN  
+JPRS  
+KIA  
+KINDER  
+KOMATSU  
+KPMG  
+KPN  
+KRED  
+LAMBORGHINI  
+LANDROVER  
+LATROBE  
+LECLERC  
+LIDL  
+LILLY  
+LINDE  
+LIXIL  
+LOCUS  
+MAIF  
+MAN  
+MANGO  
+MICROSOFT  
+MINI  
+MIT  
+MLB  
+MMA  
+MONASH  
+MTN  
+NAB  
+NADEX  
+NATURA  
+NEC  
+NEUSTAR  
+NICO  
+NISSAN  
+NRA  
+NTT  
+OMEGA  
+ORACLE  
+ORANGE  
+OTSUKA  
+OVH  
+PHILIPS  
+PICTET  
+PIONEER  
+PRAXI  
+PRU  
+PRUDENTIAL  
+REDSTONE  
+REDUMBRELLA  
+RICOH  
+RMIT  
+ROCHER  
+SAMSUNG  
+SANDVIK  
+SANOFI  
+SAP  
+SAXO  
+SBI  
+SCB  
+SCHMIDT  
+SCHWARZ  
+SEAT  
+SENER  
+SHARP  
+SHELL  
+SHRIRAM  
+SKY  
+SKYPE  
+SMART  
+SNCF  
+SOHU  
+SONY  
+STATEFARM  
+STC  
+SUZUKI  
+SWATCH  
+TATAMOTORS  
+TEMASEK  
+TEVA  
+TORAY  
+TOTAL  
+TOYOTA  
+TRAVELERS  
+TRV  
+UNICOM  
+UOL  
+VANGUARD  
+VIG  
+WEBER  
+WEIR  
+WILLIAMHILL  
+WINDOWS  
+WOODSIDE  
+XBOX  
+YANDEX  
+YOUTUBE  
+ZAPPOS  
+ZARA

pkg/registry/tld_registry.go

@@ -0,0 +1,27 @@
+package registry
+
+import (
+	_ "embed"
+	"strings"
+
+	"github.com/projectdiscovery/tldfinder/pkg/utils"
+)
+
+//go:embed tlds.txt
+var tldData string
+
+//go:embed private_tlds.txt
+var privateTldData string
+
+var (
+	TLDs        = processTLDData(tldData)
+	PrivateTLDs = processTLDData(privateTldData)
+)
+
+func processTLDData(data string) []string {
+	lines := strings.Split(data, "\n")
+	for i, line := range lines {
+		lines[i], _ = utils.Sanitize(line)
+	}
+	return lines
+}