From c44eb9d91f85978250456076c49ee39ce191fe9e Mon Sep 17 00:00:00 2001 From: sinspred Date: Sat, 20 Jul 2024 15:53:57 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=B5=8B=E9=80=9F=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 + main.go | 509 +++++++++++++++++++++++++++++++--------------- mergeCSV.go | 13 +- processCIDR.go | 53 ++++- processFofaCSV.go | 36 +++- update.go | 40 +++- 7 files changed, 470 insertions(+), 184 deletions(-) diff --git a/go.mod b/go.mod index 925d773..f2532a8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/sinspired/CloudflareBestIP go 1.22.4 require ( + github.com/VividCortex/ewma v1.2.0 github.com/mattn/go-ieproxy v0.0.12 golang.org/x/sys v0.22.0 golang.org/x/text v0.16.0 diff --git a/go.sum b/go.sum index b7c5ccd..c317999 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M= github.com/mattn/go-ieproxy v0.0.12/go.mod h1:Vn+N61199DAnVeTgaF8eoB9PvLO8P3OBnG95ENh7B7c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= diff --git a/main.go b/main.go index 4a60500..0a7ee0d 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "bytes" "context" "encoding/csv" "encoding/json" @@ -22,6 +21,7 @@ import ( "sync/atomic" "time" + "github.com/VividCortex/ewma" "github.com/mattn/go-ieproxy" "golang.org/x/sys/windows/registry" "golang.org/x/text/cases" @@ -32,23 +32,25 @@ import ( // 定义终端命令行变量 var ( - File = flag.String("file", "txt.zip", "IP地址文件名称(*.txt或*.zip)") // IP地址文件名称 - outFile = flag.String("outfile", "result.csv", "输出文件名称(自动设置)") // 输出文件名称 - defaultPort = flag.Int("port", 443, "默认端口") // 端口 - maxThreads = flag.Int("max", 200, "并发请求最大协程数") // 最大协程数 - speedTestThreads = flag.Int("speedtest", 5, "下载测速协程数量,设为0禁用测速") // 下载测速协程数量 - speedLimit = flag.Float64("speedlimit", 5, "最低下载速度(MB/s)") // 最低下载速度 - speedTestURL = flag.String("url", "speed.cloudflare.com/__down?bytes=500000000", "测速文件地址") // 测速文件地址 - enableTLS = flag.Bool("tls", true, "是否启用TLS") // TLS是否启用 - multipleNum = flag.Float64("mulnum", 1, "多协程测速造成测速不准，可进行倍数补偿") // speedTest比较大时修改 - tcpLimit = flag.Int("tcplimit", 9999, "TCP最大延迟(ms)") // TCP最大延迟(ms) - httpLimit = flag.Int("httplimit", 9999, "HTTP最大延迟(ms)") // HTTP最大延迟(ms) - countryCodes = flag.String("countries", "", "国家代码(US,SG,JP,DE...)，以逗号分隔，留空时检测所有") // 国家代码数组 - DownloadipLab = flag.Bool("iplab", false, "为true时检查ip库中的文件并依次下载") // 自动下载一些知名的反代IP列表 - Domain = flag.String("domain", "", "上传地址，默认为空,用Text2KV项目建立的简易文件存储storage.example.com") - Token = flag.String("token", "", "上传地址的token，默认为空") - RequestNum = flag.Int("num", 6, "测速结果数量") // 测速结果数量 - DownloadTestAll = flag.Bool("dlall", false, "为true对有效满足延迟检测的有效ip测速，可能需要很长时间") + DirectIP = flag.String("ip", "", "直接检测ip地址") // IP地址名称 + File = flag.String("file", "txt.zip", "IP地址文件名称(*.txt或*.zip)") // IP地址文件名称 + outFile = flag.String("outfile", "result.csv", "输出文件名称(自动设置)") // 输出文件名称 + defaultPort = flag.Int("port", 443, "默认端口") // 端口 + maxThreads = flag.Int("max", 200, "并发请求最大协程数") // 最大协程数 + speedTestThreads = flag.Int("speedtest", 5, "下载测速协程数量,设为0禁用测速") // 下载测速协程数量 + speedLimit = flag.Float64("speedlimit", 5, "最低下载速度(MB/s)") // 最低下载速度 + speedTestURL = flag.String("url", "st.notime.icu/200000000", "测速文件地址") // 测速文件地址 + // speedTestURL = flag.String("url", "speed.cloudflare.com/__down?bytes=500000000", "测速文件地址") // 测速文件地址 + enableTLS = flag.Bool("tls", true, "是否启用TLS") // TLS是否启用 + multipleNum = flag.Float64("mulnum", 1, "多协程测速造成测速不准，可进行倍数补偿") // speedTest比较大时修改 + tcpLimit = flag.Int("tcplimit", 9999, "TCP最大延迟(ms)") // TCP最大延迟(ms) + httpLimit = flag.Int("httplimit", 9999, "HTTP最大延迟(ms)") // HTTP最大延迟(ms) + countryCodes = flag.String("country", "", "国家代码(US,SG,JP,DE...)，以逗号分隔，留空时检测所有") // 国家代码数组 + DownloadipLab = flag.Bool("iplab", false, "为true时检查ip库中的文件并依次下载") // 自动下载一些知名的反代IP列表 + Domain = flag.String("domain", "", "上传地址，默认为空,用Text2KV项目建立的简易文件存储storage.example.com") + Token = flag.String("token", "", "上传地址的token，默认为空") + RequestNum = flag.Int("num", 6, "测速结果数量") // 测速结果数量 + DownloadTestAll = flag.Bool("dlall", false, "为true对有效满足延迟检测的有效ip测速，可能需要很长时间") ) var ipLabs = map[string]string{ @@ -59,13 +61,18 @@ var ipLabs = map[string]string{ } const ( - timeout = 1 * time.Second // 超时时间 - maxDuration = 2 * time.Second // 最大持续时间 + bufferSize = 1024 + tcpTimeout = 1 * time.Second // 超时时间 + httpTimeout = 2 * time.Second // 超时时间 + maxDuration = 5 * time.Second // 最大延迟 + dlTimeout = 10 * time.Second // 下载超时 ) var ( - requestURL = "speed.cloudflare.com/cdn-cgi/trace" // 请求trace URL + // requestURL = "cf.xiu2.xyz/url" // 请求trace URLcf.xiu2.xyz/url + requestURL = "st.notime.icu/cdn-cgi/trace" // 请求trace URL locationsJsonUrl = "https://speed.cloudflare.com/locations" // location.json下载 URL + ) var ( @@ -216,8 +223,19 @@ func main() { } } - // 设置测速结果文件名 - resetOutFileName() + // 支持直接设置ip参数检测单个ip + if *DirectIP != "" { + *outFile = "result_TestIP.csv" + var DirectIPs []string + DirectIPs = strings.Split(*DirectIP, ",") + delayedDetectionIPs(DirectIPs, *enableTLS, *defaultPort) + pause() + downloadSpeedTest() + os.Exit(0) + } else { + // 设置测速结果文件名 + resetOutFileName(*File) + } if strings.HasSuffix(*File, ".zip") { // 获取压缩包文件名 @@ -247,6 +265,25 @@ func main() { downloadSpeedTest() } } else { + + if strings.ToLower(*File) == "ip_cfv4ipdb.txt" { + parsedFile := "ip_CFv4IPDB_Parsed.txt" + randomParseCIDR(*File, parsedFile) + *File = parsedFile + // 设置测速结果文件名 + resetOutFileName(*File) + *outFile = "result_Cfv4ipdb.csv" + } + if strings.Contains(strings.ToLower(*File), "cidr") { + pasedName := strings.Split(*File, ".")[0] + parsedFile := pasedName + "_Pased.txt" + randomParseCIDR(*File, parsedFile) + *File = parsedFile + // 设置测速结果文件名 + resetOutFileName(*File) + *outFile = "result_" + strings.Split(pasedName, "_")[1] + ".csv" + } + // 处理非 ZIP 文件 processIPListFile(*File) // 下载测速并保存结果 @@ -257,8 +294,9 @@ func main() { reader := bufio.NewReader(os.Stdin) switch { case *Domain != "" && *Token != "" && countQualified > 0: - switch strings.ToLower(*File) { - case "txt.zip", "ip_proxyipdb.txt", "ip_cfv4ipdb.txt", "ip_scanner.txt", "ip_selected.txt", "fofa.zip", "ip_proxyipdb", "ip_fofa.txt", "fofas.zip": + switch strings.ToLower(*outFile) { + case "result_baipiaoge.csv", "result_proxyipdb.csv", "result_cfv4ipdb.csv", "result_scanner.csv", "result_selected.csv", "result_fofa.csv", "result_fofas.csv": + // case "txt.zip", "ip_proxyipdb.txt", "ip_cfv4ipdb.txt", "ip_scanner.txt", "ip_selected.txt", "fofa.zip", "ip_proxyipdb", "ip_fofa.txt", "fofas.zip": dataUpdate(*outFile, *Domain, *Token) default: fmt.Printf("\n> 优质ip数量：\033[32m%d\033[0m ,是否要上传数据？(y/n):", countQualified) @@ -508,9 +546,9 @@ func downloadFile(url, fileName string) { } // 测速结果输出文件重命名函数 -func resetOutFileName() { - if strings.Contains(*File, "_") { - FileName := strings.Split(*File, ".")[0] // 去掉后缀名 +func resetOutFileName(File string) { + if strings.Contains(File, "_") { + FileName := strings.Split(File, ".")[0] // 去掉后缀名 resultName := strings.Split(FileName, "_")[1] // 分离名字字段 caser := cases.Title(language.English) // 使用English作为默认语言标签 resultName = caser.String(resultName) // 首字母大写 @@ -520,20 +558,20 @@ func resetOutFileName() { *outFile = "result_" + resultName + ".csv" } - } else if *File == "txt.zip" { + } else if File == "txt.zip" { // 默认反代IP列表 if *outFile == "result.csv" { *outFile = "result_Baipiaoge.csv" } - } else if *File == "ip.txt" { + } else if File == "ip.txt" { // 默认ip列表 if *outFile == "result.csv" { *outFile = "result_Test.csv" } } else { - FileName := strings.Split(*File, ".")[0] // 去掉后缀名 - caser := cases.Title(language.English) // 使用English作为默认语言标签 - FileName = caser.String(FileName) // 首字母大写 + FileName := strings.Split(File, ".")[0] // 去掉后缀名 + caser := cases.Title(language.English) // 使用English作为默认语言标签 + FileName = caser.String(FileName) // 首字母大写 if *outFile == "result.csv" { *outFile = "result_" + FileName + ".csv" @@ -640,14 +678,16 @@ func processIPListFile(fileName string) { // logWithProgress 显示进度条 func logWithProgress(countProcessed, total int) { - percentage := float64(countProcessed) / float64(total) * 100 - barLength := 28 // 进度条长度 - progress := int(percentage / 100 * float64(barLength)) - - // 构建进度条 - bar := fmt.Sprintf("\033[32m%s\033[0m%s", strings.Repeat("■", progress), strings.Repeat("▣", barLength-progress)) - // 并发检测进度显示 - fmt.Printf("\r进度%s\033[1;32m%6.2f%%\033[0m \033[90m%d\033[0m/\033[90m%d\033[0m ", bar, percentage, countProcessed, total) + if total != 0 { + percentage := float64(countProcessed) / float64(total) * 100 + barLength := 28 // 进度条长度 + progress := int(percentage / 100 * float64(barLength)) + + // 构建进度条 + bar := fmt.Sprintf("\033[32m%s\033[0m%s", strings.Repeat("■", progress), strings.Repeat("▣", barLength-progress)) + // 并发检测进度显示 + fmt.Printf("\r进度%s\033[1;32m%6.2f%%\033[0m \033[90m%d\033[0m/\033[90m%d\033[0m ", bar, percentage, countProcessed, total) + } } // delayedDetectionIPs 对给定的 IP 列表进行延迟检测，并返回存活 IP 的数量。 @@ -716,31 +756,40 @@ func delayedDetectionIPs(ips []string, enableTLS bool, port int) int { } } - // 进行tcp和http连接延迟检测 - dialer := &net.Dialer{ - Timeout: timeout, - KeepAlive: 0, + // 检测平均延迟和丢包率 + var ( + recv int + totalDelay time.Duration + PingTimes = 4 + ) + for i := 0; i < PingTimes; i++ { + if ok, delay := tcping(ip, port); ok { + recv++ + totalDelay += delay + } } - start := time.Now() + Sended := PingTimes + Received := recv + LossRate := float64((Sended - Received) / Sended) - // 使用 DialContext 函数，这里 context.Background() 提供了一个空的上下文 - conn, err := dialer.DialContext(context.Background(), "tcp", net.JoinHostPort(ip, strconv.Itoa(port))) - if err != nil { + if LossRate > 0.3 { return } - defer conn.Close() - - tcpDuration := time.Since(start) - start = time.Now() + tcpDuration := totalDelay / time.Duration(Received) + // httping 延迟检测 + start := time.Now() client := http.Client{ Transport: &http.Transport{ // 使用 DialContext 函数 DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return conn, nil + return (&net.Dialer{ + Timeout: httpTimeout, + KeepAlive: 0, + }).DialContext(ctx, network, net.JoinHostPort(ip, strconv.Itoa(port))) }, }, - Timeout: timeout, + Timeout: httpTimeout, } var protocol string @@ -750,33 +799,42 @@ func delayedDetectionIPs(ips []string, enableTLS bool, port int) int { protocol = "http://" } requestURL := protocol + requestURL - req, _ := http.NewRequest("GET", requestURL, nil) + req, _ := http.NewRequest(http.MethodGet, requestURL, nil) // 添加用户代理 - req.Header.Set("User-Agent", "Mozilla/5.0") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") req.Close = true resp, err := client.Do(req) if err != nil { + // fmt.Println("错误:",err) return } - - duration := time.Since(start) - if duration > maxDuration { + if resp.StatusCode != http.StatusOK { + fmt.Println(resp.StatusCode) return } - - // 将响应体内容复制到缓冲区 - var buf bytes.Buffer - _, err = io.Copy(&buf, resp.Body) - if err != nil { - // fmt.Printf("IP %s 读取响应体错误: %v\n", ip, err) + duration := time.Since(start) + if duration > maxDuration { + fmt.Printf("duration:%v,max:%v \033[31m超时\033[0m]\n", duration, maxDuration) return } - body := buf - - if strings.Contains(body.String(), "uag=Mozilla/5.0") { - if matches := regexp.MustCompile(`colo=([A-Z]+)`).FindStringSubmatch(body.String()); len(matches) > 1 { + body, err := io.ReadAll(resp.Body) + + // if resp.Header.Get("Server") == "cloudflare" { + // // str := resp.Header.Get("CF-RAY") // 示例 cf-ray: 7bd32409eda7b020-SJC + // // colo := regexp.MustCompile(`[A-Z]{3}`).FindString(str) + // // fmt.Printf("Cloudflare:%s\n", colo) + // } else { + // str := resp.Header.Get("x-amz-cf-pop") // 示例 X-Amz-Cf-Pop: SIN52-P1 + // colo := regexp.MustCompile(`[A-Z]{3}`).FindString(str) + // fmt.Printf("AWS:%s\n", colo) + // fmt.Println(string(body)) + // } + + // pause() + if strings.Contains(string(body), "uag=Mozilla/5.0") { + if matches := regexp.MustCompile(`colo=([A-Z]+)`).FindStringSubmatch(string(body)); len(matches) > 1 { dataCenter := matches[1] loc, ok := locationMap[dataCenter] @@ -786,7 +844,7 @@ func delayedDetectionIPs(ips []string, enableTLS bool, port int) int { if len(countries) == 0 || containsIgnoreCase(countries, loc.Cca2) { countAlive++ // 记录存活 IP 数量 if ok { - fmt.Printf("-IP %-15s 端口 %-5d 位置:%2s%12s 延迟 %3d ms \n", ip, port, loc.Cca2, loc.City, tcpDuration.Milliseconds()) + fmt.Printf("-IP %-15s 端口 %-5d 位置:%2s%12s 延迟 %3d ms 丢包 %.2f \n", ip, port, loc.Cca2, loc.City, tcpDuration.Milliseconds(), LossRate) resultChan <- latencyTestResult{ip, port, enableTLS, dataCenter, loc.Region, loc.Cca2, loc.City, fmt.Sprintf("%d", duration.Milliseconds()), fmt.Sprintf("%d", tcpDuration.Milliseconds())} } else { @@ -797,6 +855,8 @@ func delayedDetectionIPs(ips []string, enableTLS bool, port int) int { } } } + } else { + return } }(ip) } @@ -810,6 +870,7 @@ func delayedDetectionIPs(ips []string, enableTLS bool, port int) int { } // 并发检测执行完毕后输出信息 if int(atomic.LoadInt32(&countProcessedInt32)) == total { + time.Sleep(200 * time.Millisecond) countProcessedInt := int(atomic.LoadInt32(&countProcessedInt32)) logWithProgress(countProcessedInt, total) fmt.Printf("存活 IP: \033[1;32;5m%-3d\033[0m \r", countAlive) @@ -833,6 +894,9 @@ func readIPs(File string) ([]string, error) { scanner := bufio.NewScanner(file) for scanner.Scan() { ipAddr := scanner.Text() + if len(ipAddr) < 7 { + continue + } // 判断是否为 CIDR 格式的 IP 地址 if strings.Contains(ipAddr, "/") && strings.Count(ipAddr, ":") != 1 && strings.Count(ipAddr, "#") != 1 { ip, ipNet, err := net.ParseCIDR(ipAddr) @@ -855,7 +919,7 @@ func readIPs(File string) ([]string, error) { portStr := strings.Split(ipPort, ":")[1] port, err := strconv.Atoi(portStr) if err != nil { - fmt.Println("端口转换错误:", err) + fmt.Println("%s端口转换错误:", ipAddr, err) continue } // ipMap[ip] = struct{}{} @@ -865,7 +929,8 @@ func readIPs(File string) ([]string, error) { portStr := strings.Split(ipAddr, ":")[1] port, err := strconv.Atoi(portStr) if err != nil { - fmt.Println("端口转换错误:", err) + fmt.Println(ipAddr) + fmt.Println("%s端口转换错误:", ipAddr, err) continue } // ipMap[ip] = struct{}{} @@ -899,84 +964,127 @@ func inc(ip net.IP) { } // 下载测速函数 -func getDownloadSpeed(ip string, port int, enableTLS bool, latency string, tcplatency string) float64 { +func getDownloadSpeed(ip string, port int, enableTLS bool, latency string, tcplatency string, country string) float64 { var protocol string - if enableTLS { protocol = "https://" } else { protocol = "http://" } + fullURL := protocol + *speedTestURL - speedTestURL := protocol + *speedTestURL - // 创建请求 - req, _ := http.NewRequest("GET", speedTestURL, nil) - req.Header.Set("User-Agent", "Mozilla/5.0") - - // 创建TCP连接 - dialer := &net.Dialer{ - Timeout: timeout, - KeepAlive: 0, - } - conn, err := dialer.DialContext(context.Background(), "tcp", net.JoinHostPort(ip, strconv.Itoa(port))) + // Create request + req, err := http.NewRequest("GET", fullURL, nil) if err != nil { - return 0 + fmt.Printf("-IP %-15s 端口 %-5s 国家 %s \033[9;31m测速无效1\033[0m%s\n", ip, strconv.Itoa(port), country, strings.Repeat(" ", 15)) + return 0.0 } - defer conn.Close() + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36") + + // Mark start time + // startTime := time.Now() - // 标记时间点 - startTime_duration := time.Now() - // 创建HTTP客户端 - client := http.Client{ + // Create HTTP client + client := &http.Client{ Transport: &http.Transport{ - // 使用新的DialContext函数 DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return conn, nil + return (&net.Dialer{ + Timeout: tcpTimeout, + KeepAlive: 0, + }).DialContext(ctx, network, net.JoinHostPort(ip, strconv.Itoa(port))) }, }, - // 外网访问延迟较大，设置单个IP延迟最长时间为5秒 - Timeout: 10 * time.Second, + Timeout: dlTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 10 { + return http.ErrUseLastResponse + } + return nil + }, } - - // 发送请求 - req.Close = true - resp, err := client.Do(req) + // timeStart := time.Now() // 开始时间（当前） + // Send request + response, err := client.Do(req) if err != nil { - // fmt.Printf("-IP %-15s 端口 %-5s \033[9;31m测速无效\033[0m%s\n", ip, strconv.Itoa(port), strings.Repeat(" ", 15)) - return 0 + fmt.Printf("-IP %-15s 端口 %-5s 国家 %s \033[9;31m测速无效\033[0m%s\n", ip, strconv.Itoa(port), country, strings.Repeat(" ", 15)) + return 0.0 } - defer resp.Body.Close() + defer response.Body.Close() - // 复制响应体到/dev/null，并计算下载速度 - written, _ := io.Copy(io.Discard, resp.Body) - duration := time.Since(startTime_duration) + if response.StatusCode != http.StatusOK { + return 0.0 + } - speedOrignal := float64(written) / duration.Seconds() / (1024 * 1024) // 真实测速数据，如开多协程会有失真。单位MB/s + timeStart := time.Now() // 开始时间（当前） + timeEnd := timeStart.Add(dlTimeout) // 加上下载测速时间得到的结束时间 + + contentLength := response.ContentLength // 文件大小 + buffer := make([]byte, bufferSize) + + var ( + contentRead int64 = 0 + timeSlice = dlTimeout / 100 + timeCounter = 1 + lastContentRead int64 = 0 + ) + + nextTime := timeStart.Add(timeSlice * time.Duration(timeCounter)) + e := ewma.NewMovingAverage() + + // 循环计算，如果文件下载完了（两者相等），则退出循环（终止测速） + for contentLength != contentRead { + currentTime := time.Now() + if currentTime.After(nextTime) { + timeCounter++ + nextTime = timeStart.Add(timeSlice * time.Duration(timeCounter)) + e.Add(float64(contentRead - lastContentRead)) + lastContentRead = contentRead + } + // 如果超出下载测速时间，则退出循环（终止测速） + if currentTime.After(timeEnd) { + break + } + bufferRead, err := response.Body.Read(buffer) + if err != nil { + if err != io.EOF { // 如果文件下载过程中遇到报错（如 Timeout），且并不是因为文件下载完了，则退出循环（终止测速） + break + } else if contentLength == -1 { // 文件下载完成 且 文件大小未知，则退出循环（终止测速），例如：https://speed.cloudflare.com/__down?bytes=200000000 这样的，如果在 10 秒内就下载完成了，会导致测速结果明显偏低甚至显示为 0.00（下载速度太快时） + break + } + // 获取上个时间片 + last_time_slice := timeStart.Add(timeSlice * time.Duration(timeCounter-1)) + // 下载数据量 / (用当前时间 - 上个时间片/ 时间片) + e.Add(float64(contentRead-lastContentRead) / (float64(currentTime.Sub(last_time_slice)) / float64(timeSlice))) + } + contentRead += int64(bufferRead) + } + speed := e.Value() / (dlTimeout.Seconds() / 120) / (1024 * 1024) + // 输出结果 + // fmt.Printf("-IP %-15s 端口 %-5s 国家 %2s 延迟 %3s ms 速度 %2.1f MB/s%s\n", ip, strconv.Itoa(port), country, tcplatency, speed, strings.Repeat(" ", 12)) if *multipleNum == 1 || *speedTestThreads < 5 { - speed := float64(written) / duration.Seconds() / (1024 * 1024) // 输出结果 - if speed >= 1 { - fmt.Printf("-IP %-15s 端口 %-5s 延迟 %3s ms 速度 %2.1f MB/s%s\n", ip, strconv.Itoa(port), tcplatency, speed, strings.Repeat(" ", 12)) + if speed >= 0 { + fmt.Printf("-IP %-15s 端口 %-5s 国家 %2s 延迟 %3s ms 速度 %4.1f MB/s%s\n", ip, strconv.Itoa(port), country, tcplatency, speed, strings.Repeat(" ", 8)) } return speed } else { // 多协程测速会有速度损失，加以补偿 - speed := float64(written) / duration.Seconds() / (1024 * 1024) * (*multipleNum) - fmt.Printf("-IP %-15s 端口 %-5s 延迟 %3s ms 下载速度 %2.1f MB/s, 补偿系数 %.0f×%2.1f MB/s\n", ip, strconv.Itoa(port), tcplatency, speed, *multipleNum, speedOrignal) - return speed + xspeed := speed * (*multipleNum) + fmt.Printf("-IP %-15s 端口 %-5s 延迟 %3s ms 速度 %4.1f MB/s, %.0f×%2.1f%s\n", ip, strconv.Itoa(port), tcplatency, xspeed, *multipleNum, speed, strings.Repeat(" ", 1)) + return xspeed } } // 并发下载测速函数 func downloadSpeedTest() { var results []speedTestResult - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + var badresults []speedTestResult + if *speedTestThreads > 0 { fmt.Printf("\n\n开始下载测速\n") - if *speedTestThreads > 5 && *multipleNum == 1 { - fmt.Printf("\033[90m> 即将建立\033[0m \033[31m%d\033[0m \033[90m个并发协程，测速可能失真。建议增加\033[0m\033[4;33m %d \033[0m\033[90m倍补偿系数\033[0m\n", *speedTestThreads, *speedTestThreads/5) + if *speedTestThreads > 1 && *multipleNum == 1 { + fmt.Printf("\033[90m> 即将建立\033[0m \033[31m%d\033[0m \033[90m个并发协程，测速可能失真。建议增加\033[0m\033[4;33m %d \033[0m\033[90m倍补偿系数\033[0m\n", *speedTestThreads, *speedTestThreads) // 创建一个新的读数器 reader := bufio.NewReader(os.Stdin) @@ -1008,58 +1116,103 @@ func downloadSpeedTest() { fmt.Printf("\n\033[90m> 并发测速补偿系数已设置为\033[0m \033[32m%.1f\033[0m\n\n", *multipleNum) } - var wg sync.WaitGroup - var countSt int32 = 0 // 下载测速进度计数器 - var mu sync.Mutex // 同步锁 - - thread := make(chan struct{}, *speedTestThreads) - total := len(totalResultChan) - var resultCount int32 = 0 - // 根据ticp延迟排序 + // 根据ticp延迟排序tcp延迟结果 sort.Slice(totalResultChan, func(i, j int) bool { durationI, _ := strconv.ParseFloat(totalResultChan[i].tcpDuration, 64) durationJ, _ := strconv.ParseFloat(totalResultChan[j].tcpDuration, 64) return durationI < durationJ }) + // 创建并发协程 + var wg sync.WaitGroup + var countSt int32 = 0 // 下载测速进度计数器 + var mu sync.Mutex // 同步锁 + var resultCount int32 = 0 + ctx, cancel := context.WithCancel(context.Background()) + // stopChan := make(chan struct{}) // 停止信号通道 + netIDs := sync.Map{} // 使用 sync.Map 记录已存储结果的网络段 + thread := make(chan struct{}, *speedTestThreads) + total := len(totalResultChan) + for id, res := range totalResultChan { - wg.Add(1) - thread <- struct{}{} - go func(id int, res latencyTestResult) { - defer func() { - <-thread - wg.Done() - // 记录下载测速进度 - atomic.AddInt32(&countSt, 1) - - logWithProgress(int(countSt), total) - fmt.Printf(" 优选 IP：\033[1;32;5m%-2d\033[0m \r", atomic.LoadInt32(&resultCount)) - // 测速进程运行完成 - if atomic.LoadInt32(&countSt) == int32(total) { - fmt.Println("\n\n下载速度测试完成！") - } - }() - - select { - case <-ctx.Done(): - return - default: - downloadSpeed := getDownloadSpeed(res.ip, res.port, res.tls, res.latency, res.tcpDuration) - mu.Lock() - defer mu.Unlock() - results = append(results, speedTestResult{latencyTestResult: res, downloadSpeed: downloadSpeed}) - if downloadSpeed >= *speedLimit { - atomic.AddInt32(&resultCount, 1) - } - if atomic.LoadInt32(&resultCount) == int32(*RequestNum) && countAlive >= 200 && !*DownloadTestAll { - fmt.Println("\n4已获取到足够数量的ip，继续运行会消耗过多资源和时间！") - cancel() + select { + // case <-stopChan: + case <-ctx.Done(): + break + default: + wg.Add(1) + thread <- struct{}{} + go func(id int, res latencyTestResult) { + defer func() { + <-thread + wg.Done() + + // select { + // case <-ctx.Done(): + // return + // default: + // 记录下载测速进度 + atomic.AddInt32(&countSt, 1) + + // 输出测速进度 + countStInt := int(atomic.LoadInt32(&countSt)) + logWithProgress(countStInt, total) + fmt.Printf(" 优选 IP：\033[1;32;5m%-2d\033[0m \r", atomic.LoadInt32(&resultCount)) + // } + }() + + select { + case <-ctx.Done(): return + default: + var downloadSpeed float64 + // 获取ip的前三段即网络段net-id + netID := getNetworkSegment(res.ip) + + // 程序提前结束条件判断 + if atomic.LoadInt32(&resultCount) >= int32(*RequestNum) && countAlive >= 40 && !*DownloadTestAll { + // close(stopChan) // 关闭停止信号通道，通知所有协程停止 + cancel() + fmt.Println("\n\033[32m已获取足够数量满足条件的ip，提前结束测试！\033[0m") + return + } + // 每个ip网段仅保留一个主机，当有合格ip后不再对相同网段测速 + if _, exists := netIDs.Load(netID); !exists { + downloadSpeed = getDownloadSpeed(res.ip, res.port, res.tls, res.latency, res.tcpDuration, res.country) + } + + mu.Lock() // 同步锁 + defer mu.Unlock() + if downloadSpeed >= *speedLimit { + // 如果当前网络段不存在于networkSegments中，一个ip端仅收录一个达标ip + if _, exists := netIDs.Load(netID); !exists { + // 将当前网络段标记为已存储 + netIDs.Store(netID, true) + if atomic.LoadInt32(&resultCount) >= int32(*RequestNum) && !*DownloadTestAll { + cancel() + } + if atomic.LoadInt32(&resultCount) < int32(*RequestNum) && !*DownloadTestAll { + // 将当前测速结果添加到results中 + results = append(results, speedTestResult{latencyTestResult: res, downloadSpeed: downloadSpeed}) + // 如果下载速度大于等于速度限制，增加resultCount计数 + atomic.AddInt32(&resultCount, 1) + } else if *DownloadTestAll { + // 将当前测速结果添加到results中 + results = append(results, speedTestResult{latencyTestResult: res, downloadSpeed: downloadSpeed}) + // 如果下载速度大于等于速度限制，增加resultCount计数 + atomic.AddInt32(&resultCount, 1) + } + } + } + badresults = append(badresults, speedTestResult{latencyTestResult: res, downloadSpeed: downloadSpeed}) } - } - }(id, res) + }(id, res) + } } wg.Wait() + // 测速进程运行完成 + // time.Sleep(10 * time.Second) + fmt.Println("\n下载速度测试完成！") // pause() } else { for _, res := range totalResultChan { @@ -1070,6 +1223,9 @@ func downloadSpeedTest() { sort.Slice(results, func(i, j int) bool { return results[i].downloadSpeed > results[j].downloadSpeed }) + sort.Slice(badresults, func(i, j int) bool { + return badresults[i].downloadSpeed > badresults[j].downloadSpeed + }) } else { sort.Slice(results, func(i, j int) bool { return results[i].latencyTestResult.tcpDuration < results[j].latencyTestResult.tcpDuration @@ -1077,11 +1233,20 @@ func downloadSpeedTest() { } // 下载测速结果写入文件 - writeResults(results) + writeResults(results, badresults) +} + +// 辅助函数，提取网络段 +func getNetworkSegment(ip string) string { + parts := strings.Split(ip, ".") + if len(parts) < 3 { + return ip + } + return strings.Join(parts[:3], ".") } // writeResults 写入文件函数 -func writeResults(results []speedTestResult) { +func writeResults(results []speedTestResult, badresults []speedTestResult) { // 放到外面以便调用 countQualified = 0 // 下载速度达标ip 计数器 countSpeed := 0 // 下载速度大于0 计数器 @@ -1147,7 +1312,7 @@ func writeResults(results []speedTestResult) { writer.Write([]string{res.latencyTestResult.ip, strconv.Itoa(res.latencyTestResult.port), strconv.FormatBool(*enableTLS), res.latencyTestResult.dataCenter, res.latencyTestResult.region, res.latencyTestResult.country + "-" + suffixName, res.latencyTestResult.city, res.latencyTestResult.latency, res.tcpDuration, fmt.Sprintf("%.1f", res.downloadSpeed)}) // 终端输出优选结果 - fmt.Printf("%-15s:%-5d#%-2s-%2.1f MB/s tcp延迟 %-3sms\n", res.latencyTestResult.ip, res.latencyTestResult.port, res.latencyTestResult.country, res.downloadSpeed, res.tcpDuration) + fmt.Printf("%-15s:%-5d#%-2s-%2.1f MB/s 平均延迟 %-3sms\n", res.latencyTestResult.ip, res.latencyTestResult.port, res.latencyTestResult.country, res.downloadSpeed, res.tcpDuration) } else { writerUnqualified.Write([]string{res.latencyTestResult.ip, strconv.Itoa(res.latencyTestResult.port), strconv.FormatBool(*enableTLS), res.latencyTestResult.dataCenter, res.latencyTestResult.region, res.latencyTestResult.country, res.latencyTestResult.city, res.latencyTestResult.latency, res.tcpDuration, fmt.Sprintf("%.1f", res.downloadSpeed)}) @@ -1158,6 +1323,15 @@ func writeResults(results []speedTestResult) { } } + for _, res := range badresults { + if *speedTestThreads > 0 { + writerUnqualified.Write([]string{res.latencyTestResult.ip, strconv.Itoa(res.latencyTestResult.port), strconv.FormatBool(*enableTLS), res.latencyTestResult.dataCenter, res.latencyTestResult.region, res.latencyTestResult.country, res.latencyTestResult.city, res.latencyTestResult.latency, res.tcpDuration, fmt.Sprintf("%.1f", res.downloadSpeed)}) + } else { + writer.Write([]string{res.latencyTestResult.ip, strconv.Itoa(res.latencyTestResult.port), strconv.FormatBool(*enableTLS), res.latencyTestResult.dataCenter, res.latencyTestResult.region, res.latencyTestResult.country, res.latencyTestResult.city, res.latencyTestResult.latency}) + writerUnqualified.Write([]string{res.latencyTestResult.ip, strconv.Itoa(res.latencyTestResult.port), strconv.FormatBool(*enableTLS), res.latencyTestResult.dataCenter, res.latencyTestResult.region, res.latencyTestResult.country, res.latencyTestResult.city, res.latencyTestResult.latency}) + } + } + writer.Flush() writerUnqualified.Flush() // 清除输出内容 @@ -1181,3 +1355,16 @@ func containsIgnoreCase(slice []string, item string) bool { } return false } + +// tcping tcp延迟检测函数 +func tcping(ip string, port int) (bool, time.Duration) { + startTime := time.Now() + + conn, err := net.DialTimeout("tcp", net.JoinHostPort(ip, strconv.Itoa(port)), tcpTimeout) + if err != nil { + return false, 0 + } + defer conn.Close() + duration := time.Since(startTime) + return true, duration +} diff --git a/mergeCSV.go b/mergeCSV.go index bf7359d..aa23bad 100644 --- a/mergeCSV.go +++ b/mergeCSV.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strings" ) func main() { @@ -17,16 +18,24 @@ func main() { // 检查是否提供了文件夹路径 if len(args) < 1 { - fmt.Println("请提供文件夹路径（相对路径或绝对路径），例如：go run mergeCSV.go folderpath") + fmt.Println("请提供文件夹路径（相对路径或绝对路径）和结果文件名，例如：go run mergeCSV.go folderpath mergedTxtFile") return } // 文件夹路径 folderPath := args[0] + mergedTxtFile := args[1] + if mergedTxtFile == "" { + if strings.Contains(folderPath, "ipscanner") { + mergedTxtFile = "ip_Scanner.txt" + } else { + mergedTxtFile = "ip_Merged.txt" + } + } // 创建一个新的 TXT 文件来存储结果 // https://codeload.github.com/ip-scanner/cloudflare/zip/refs/heads/main - outputFile, err := os.Create("ip_Scanner.txt") + outputFile, err := os.Create(mergedTxtFile) if err != nil { fmt.Println("无法创建输出文件:", err) return diff --git a/processCIDR.go b/processCIDR.go index 11d7637..5c93abf 100644 --- a/processCIDR.go +++ b/processCIDR.go @@ -2,6 +2,9 @@ package main import ( "bufio" // 用于读取文件 + // "flag" + "fmt" + // "fmt" "log" // 用于日志记录 "math/rand" // 用于生成随机数 @@ -11,13 +14,16 @@ import ( "time" // 用于时间相关操作 ) -const defaultInputFile = "ip.txt" +const ( + defaultInputFile = "ip_CFv4IPDB.txt" + defaultOutputFile = "ip_CFv4IPDB_Parsed.txt" +) var ( // TestAll 表示是否测试所有IP TestAll = false // IPFile 是包含IP范围的文件名 - IPFile = defaultInputFile + IPFile string IPText string ) @@ -155,7 +161,7 @@ func (r *IPRanges) chooseIPv6() { } // loadIPRanges 从文件或字符串中加载IP范围 -func loadIPRanges() []*net.IPAddr { +func loadIPRanges(IPFile string) []*net.IPAddr { ranges := newIPRanges() if IPText != "" { // 从参数中获取IP段数据 IPs := strings.Split(IPText, ",") @@ -197,11 +203,14 @@ func loadIPRanges() []*net.IPAddr { return ranges.ips } -func main() { - ips := loadIPRanges() // 获取IP列表 +func randomParseCIDR(IPFile string, parsedIPFile string) { + ips := loadIPRanges(IPFile) // 获取IP列表 + if parsedIPFile == "" { + parsedIPFile = defaultOutputFile + } // 创建文件 - file, err := os.Create("ip_CFip.txt") + file, err := os.Create(parsedIPFile) if err != nil { log.Fatal(err) } @@ -215,4 +224,36 @@ func main() { log.Fatal(err) } } + fmt.Printf("\033[90mCIDR地址 %s 已解析并随机主机名,保存为 %s。\n如未取得优选结果，可修改参数并再次解析。\033[0m\n", IPFile, parsedIPFile) } + +// func main() { +// // 定义命令行参数 +// flag.Parse() +// args := flag.Args() + +// // 检查是否提供了文件路径 +// if len(args) < 1 { +// fmt.Println("请提供文件路径（相对路径或绝对路径）和解析结果文件名(如留空会自动生成)") +// return +// } + +// // 文件路径 +// IPFile := args[0] +// parsedIPFile := args[1] +// if IPFile == "" { +// IPFile = defaultInputFile +// } +// if parsedIPFile != "" { +// pasedName := strings.Split(parsedIPFile, ".")[0] +// parts := strings.Split(pasedName, "_") +// if len(parts) > 1 && parts[1] != "" { +// parsedIPFile = parts[1] + "_Pased.txt" +// } else { +// parsedIPFile = pasedName + "_Pased.txt" +// } + +// } + +// randomParseCIDR(IPFile, parsedIPFile) +// } diff --git a/processFofaCSV.go b/processFofaCSV.go index 8439142..4c45dd0 100644 --- a/processFofaCSV.go +++ b/processFofaCSV.go @@ -2,6 +2,7 @@ package main import ( "archive/zip" + "bufio" "encoding/csv" "fmt" "io" @@ -111,7 +112,6 @@ func processCSV(filePath, outputFolder string, zipFiles *[]string) { } cleanIPs := []string{} for i, ip := range ips { - if !strings.Contains(orgs[i], "Alibaba") && !strings.Contains(orgs[i], "Cloudflare") { cleanIPs = append(cleanIPs, ip) } @@ -135,7 +135,7 @@ func processCSV(filePath, outputFolder string, zipFiles *[]string) { } else { httpIPs, httpsIPs := []string{}, []string{} for i, ip := range ips { - if !strings.Contains(orgs[i], "Alibaba") && !strings.Contains(orgs[i], "Cloudflare") { + if !strings.Contains(orgs[i], "Alibaba") && !strings.Contains(orgs[i], "Cloudflare") && ip != "" { if protocols[i] == "http" { httpIPs = append(httpIPs, fmt.Sprintf("%s:%s", ip, ports[i])) } else { @@ -170,14 +170,40 @@ func unique(slice []string) []string { } func writeToFile(filePath string, data []string) { - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + // 创建一个 map 用于存储唯一的行 + lines := make(map[string]struct{}) + + // 如果文件存在，读取现有文件内容 + file, err := os.OpenFile(filePath, os.O_RDONLY, 0o644) + if err == nil { + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) // 去除行首尾的空白字符 + if line != "" { // 过滤空行 + lines[line] = struct{}{} + } + } + file.Close() // 关闭文件 + } + + // 将新数据添加到 map 中 + for _, line := range data { + line = strings.TrimSpace(line) // 去除行首尾的空白字符 + if line != "" { // 过滤空行 + lines[line] = struct{}{} + } + } + + // 以写模式打开文件（清空文件内容） + file, err = os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { fmt.Println("写入文件错误:", err) return } - defer file.Close() + defer file.Close() // 函数结束前关闭文件 - for _, line := range data { + // 将唯一的行写入文件 + for line := range lines { _, err := file.WriteString(line + "\n") if err != nil { fmt.Println("写入文件错误:", err) diff --git a/update.go b/update.go index d81a8c6..0a4f164 100644 --- a/update.go +++ b/update.go @@ -13,7 +13,7 @@ import ( ) // func main() { -// dataUpdate("result_test.csv") +// dataUpdate("result_test.csv", "example.com", "your_token") // } // 移除BOM（字节顺序标记） @@ -24,6 +24,22 @@ func removeBOM(data []byte) []byte { return data } +// 发送HTTP GET请求并处理重试逻辑 +func sendGetRequest(client *http.Client, urlStr string, maxRetries int) (*http.Response, error) { + var resp *http.Response + var err error + for i := 0; i < maxRetries; i++ { + resp, err = client.Get(urlStr) + if err == nil { + return resp, nil + } + // fmt.Printf("请求失败（重试 %d/%d 次）：%v\n", i+1, maxRetries, err) + fmt.Printf("请求失败（重试 %d/%d 次）\n", i+1, maxRetries) + time.Sleep(1 * time.Second) // 等待2秒后重试 + } + return nil, err +} + // 更新数据 func dataUpdate(fileName string, domain string, token string) { // 清除输出内容 @@ -55,17 +71,24 @@ func dataUpdate(fileName string, domain string, token string) { // 构造更新URL updateUrlStr := fmt.Sprintf("https://%s/%s?token=%s&b64=%s&v=%d", domain, url.PathEscape(fileName), token, url.QueryEscape(base64Text), time.Now().Unix()) - // 设置超时 + // 设置HTTP客户端，启用Keep-Alive和重试逻辑 client := &http.Client{ - Timeout: time.Second * 230, // 设置超时时间为30秒 + Transport: &http.Transport{ + MaxIdleConns: 10, // 最大空闲连接数 + IdleConnTimeout: 30 * time.Second, // 空闲连接的超时时间 + MaxIdleConnsPerHost: 2, // 每个主机的最大空闲连接数 + DisableKeepAlives: false, // 启用 Keep-Alive + // TLSHandshakeTimeout: 10 * time.Second, + }, + Timeout: time.Second * 30, // 设置单次请求超时时间为30秒 } + // 发送更新请求 - resp, err := client.Get(updateUrlStr) + resp, err := sendGetRequest(client, updateUrlStr, 5) // 尝试重试5次 if err != nil { - fmt.Printf("发送更新请求时出错,请手动上传文件或重新执行程序！\n %v\n", err) + fmt.Printf("自动更新失败,请手动上传文件或重新执行程序！\n %v\n", err) return } - defer resp.Body.Close() // 检查更新请求的状态码 @@ -75,14 +98,11 @@ func dataUpdate(fileName string, domain string, token string) { } fmt.Println("发起数据更新请求...........................................[\033[32mok\033[0m]") - // 等待一段时间以确保服务器处理更新请求 - // time.Sleep(2 * time.Second) - // 构造读取URL readUrlStr := fmt.Sprintf("https://%s/%s?token=%s&v=%d", domain, url.PathEscape(fileName), token, time.Now().Unix()) // 发送读取请求 - resp, err = http.Get(readUrlStr) + resp, err = sendGetRequest(client, readUrlStr, 5) // 尝试重试5次 if err != nil { fmt.Printf("发送读取请求时出错: %v\n", err) return