diff --git a/.travis.yml b/.travis.yml index cc8049f..2ae25f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,3 +7,8 @@ notifications: email: recipients: - team@onionscan.org + +script: + - go test -v ./... + - GOFMT=$(gofmt -d .) && echo "$GOFMT" + - test -z "$GOFMT" diff --git a/deanonymization/check_exif.go b/deanonymization/check_exif.go index 2aa3b4b..7b42441 100644 --- a/deanonymization/check_exif.go +++ b/deanonymization/check_exif.go @@ -1,6 +1,7 @@ package deanonymization import ( + "bytes" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/xiam/exif" @@ -16,7 +17,7 @@ func CheckExif(osreport *report.OnionScanReport, anonreport *report.AnonymityRep if crawlRecord.Page.Status == 200 && strings.Contains(crawlRecord.Page.Headers.Get("Content-Type"), "image/jpeg") { reader := exif.New() - _, err := io.Copy(reader, strings.NewReader(string(crawlRecord.Page.Snapshot))) + _, err := io.Copy(reader, bytes.NewReader(crawlRecord.Page.Raw)) // exif.FoundExifInData is a signal that the EXIF parser has all it needs, // it doesn't need to be given the whole image. diff --git a/deanonymization/private_key.go b/deanonymization/private_key.go new file mode 100644 index 0000000..3b29d63 --- /dev/null +++ b/deanonymization/private_key.go @@ -0,0 +1,57 @@ +package deanonymization + +import ( + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "encoding/asn1" + "encoding/base32" + "encoding/pem" + "fmt" + "github.com/s-rah/onionscan/config" + "github.com/s-rah/onionscan/report" + "net/url" + "regexp" + "strings" +) + +func PrivateKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig) { + for _, id := range osreport.Crawls { + crawlRecord, _ := osc.Database.GetCrawlRecord(id) + + uri, _ := url.Parse(crawlRecord.URL) + if crawlRecord.Page.Status == 200 && strings.HasSuffix(uri.Path, "/private_key") { + privateKeyRegex := regexp.MustCompile("-----BEGIN RSA PRIVATE KEY-----((?s).*)-----END RSA PRIVATE KEY-----") + foundPrivateKey := privateKeyRegex.FindAllString(crawlRecord.Page.Snapshot, -1) + for _, keyString := range foundPrivateKey { + osc.LogInfo(fmt.Sprintf("Found Potential Private Key")) + block, _ := pem.Decode([]byte(keyString)) + if block == nil || block.Type != "RSA PRIVATE KEY" { + osc.LogInfo("Could not parse privacy key: no valid PEM data found") + continue + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + osc.LogInfo("Could not parse private key") + continue + } + + // DER Encode the Public Key + publicKeyBytes, _ := asn1.Marshal(rsa.PublicKey{ + N: privateKey.PublicKey.N, + E: privateKey.PublicKey.E, + }) + + h := sha1.New() + h.Write(publicKeyBytes) + sha1bytes := h.Sum(nil) + + data := base32.StdEncoding.EncodeToString(sha1bytes) + hostname := strings.ToLower(data[0:16]) + osc.LogInfo(fmt.Sprintf("Found Private Key for Host %s.onion", hostname)) + report.PrivateKeyDetected = true + } + } + } +} diff --git a/deanonymization/process_report.go b/deanonymization/process_report.go index 44a8176..b580842 100644 --- a/deanonymization/process_report.go +++ b/deanonymization/process_report.go @@ -13,6 +13,7 @@ func ProcessReport(osreport *report.OnionScanReport, osc *config.OnionScanConfig PGPContentScan(osreport, anonreport, osc) MailtoScan(osreport, anonreport, osc) CheckExif(osreport, anonreport, osc) + PrivateKey(osreport, anonreport, osc) ExtractGoogleAnalyticsID(osreport, anonreport, osc) ExtractGooglePublisherID(osreport, anonreport, osc) ExtractBitcoinAddress(osreport, anonreport, osc) diff --git a/model/page.go b/model/page.go index 671cf7c..2e060ac 100644 --- a/model/page.go +++ b/model/page.go @@ -14,6 +14,7 @@ type Page struct { Links []Element Scripts []Element Snapshot string + Raw []byte Hash string } diff --git a/onionscan/onionscan.go b/onionscan/onionscan.go index 318d65c..9b03451 100644 --- a/onionscan/onionscan.go +++ b/onionscan/onionscan.go @@ -27,6 +27,9 @@ func (os *OnionScan) GetAllActions() []string { "vnc", "xmpp", "bitcoin", + "bitcoin_test", + "litecoin", + "dogecoin", } } @@ -62,8 +65,8 @@ func (os *OnionScan) PerformNextAction(report *report.OnionScanReport, nextActio case "xmpp": xmppps := new(protocol.XMPPProtocolScanner) xmppps.ScanProtocol(report.HiddenService, os.Config, report) - case "bitcoin": - bps := new(protocol.BitcoinProtocolScanner) + case "bitcoin", "bitcoin_test", "litecoin", "litecoin_test", "dogecoin", "dogecoin_test": + bps := protocol.NewBitcoinProtocolScanner(nextAction) bps.ScanProtocol(report.HiddenService, os.Config, report) case "none": return nil diff --git a/protocol/bitcoin_scanner.go b/protocol/bitcoin_scanner.go index f46f028..c6e0106 100644 --- a/protocol/bitcoin_scanner.go +++ b/protocol/bitcoin_scanner.go @@ -17,20 +17,17 @@ import ( ) type BitcoinProtocolScanner struct { + name string + port int + msgstart []byte } -// Message start of packets on mainnet -var MsgStartMainnet = []byte{0xf9, 0xbe, 0xb4, 0xd9} - // User agent to send to scanned nodes const user_agent = "/OnionScan:0.0.1/" // Protocol version to send to scanned nodes const protocol_version uint32 = 70014 -// Bitcoin protocol port -const PORT int = 8333 - // Maximum length of user agent const MAX_SUBVERSION_LENGTH = 256 @@ -120,9 +117,9 @@ func cstring(n []byte) string { } // Send P2P packet to connection -func SendPacket(conn net.Conn, pkt *Packet) error { +func SendPacket(conn net.Conn, msgstart []byte, pkt *Packet) error { hdr := make([]byte, 24, 24) - copy(hdr[0:4], MsgStartMainnet) + copy(hdr[0:4], msgstart) copy(hdr[4:16], pkt.msgtype) binary.LittleEndian.PutUint32(hdr[16:20], uint32(len(pkt.payload))) copy(hdr[20:24], Checksum(pkt.payload)) @@ -139,7 +136,7 @@ func SendPacket(conn net.Conn, pkt *Packet) error { } // Receive P2P packet from connection -func ReceivePacket(conn net.Conn) (*Packet, error) { +func ReceivePacket(conn net.Conn, msgstart []byte) (*Packet, error) { var pkt Packet hdr := make([]byte, 24, 24) _, err := io.ReadFull(conn, hdr) @@ -147,8 +144,8 @@ func ReceivePacket(conn net.Conn) (*Packet, error) { if err != nil { return nil, fmt.Errorf("Could not read P2P packet header: %s", err) } - if !bytes.Equal(hdr[0:4], MsgStartMainnet) { - return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], MsgStartMainnet) + if !bytes.Equal(hdr[0:4], msgstart) { + return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], msgstart) } pkt.msgtype = cstring(hdr[4:16]) length := binary.LittleEndian.Uint32(hdr[16:20]) @@ -195,7 +192,7 @@ func DecodeOnion(addr []byte) (string, error) { } // Build and send version message -func SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error { +func (rps *BitcoinProtocolScanner) SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error { // Most fields can be left at zero payload := make([]byte, 80, 80) // static part of payload tail := make([]byte, 5, 5) // last five bytes @@ -203,50 +200,50 @@ func SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService strin binary.LittleEndian.PutUint64(payload[12:20], uint64(time.Now().Unix())) theiraddr, err := EncodeOnion(hiddenService) - if err != nil { - return err + if err == nil { + // Only send their address if the target address can be parsed as onion address + copy(payload[28:28+16], theiraddr) + binary.BigEndian.PutUint16(payload[44:46], uint16(rps.port)) } - copy(payload[28:28+16], theiraddr) - binary.BigEndian.PutUint16(payload[44:46], uint16(PORT)) payload = append(payload, uint8(len(user_agent))) payload = append(payload, user_agent...) payload = append(payload, tail...) - return SendPacket(conn, &Packet{MSG_VERSION, payload}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_VERSION, payload}) } // Handle incoming version message, and parse message payload into report -func HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { - report.BitcoinProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4])) +func (rps *BitcoinProtocolScanner) HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { + report.ProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4])) user_agent_length, sizesize := ReadCompactSize(pkt.payload[80:]) if sizesize != 0 && user_agent_length < MAX_SUBVERSION_LENGTH { - report.BitcoinUserAgent = string(pkt.payload[81 : 81+user_agent_length]) + report.UserAgent = string(pkt.payload[80+sizesize : 80+sizesize+int(user_agent_length)]) } else { return fmt.Errorf("User agent string too long") } - osc.LogInfo(fmt.Sprintf("Found Bitcoin version: %s (%d)", report.BitcoinUserAgent, report.BitcoinProtocolVersion)) + osc.LogInfo(fmt.Sprintf("Found %s version: %s (%d)", rps.name, report.UserAgent, report.ProtocolVersion)) return nil } // Handle incoming verack message -func HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { // This message has no content. However when receiving this message the // version negotiation has been completed, and that other queries can be sent. osc.LogInfo(fmt.Sprintf("Sending getaddr message")) - return SendPacket(conn, &Packet{MSG_GETADDR, []byte{}}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_GETADDR, []byte{}}) } // Handle incoming ping message -func HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { if len(pkt.payload) >= 8 { // Ping message with nonce, peer expects a pong - return SendPacket(conn, &Packet{MSG_PONG, pkt.payload[0:8]}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_PONG, pkt.payload[0:8]}) } return nil } // Handle incoming addr message, and parse message payload into report -func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { numaddr, sizesize := ReadCompactSize(pkt.payload) if sizesize == 0 || numaddr > MAX_ADDR { return fmt.Errorf("Invalid number of addresses") @@ -263,7 +260,7 @@ func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.Onion port := binary.BigEndian.Uint16(pkt.payload[ptr+28 : ptr+30]) spec := fmt.Sprintf("%s:%d", onion, port) osc.LogInfo(fmt.Sprintf("Found onion peer: %s", spec)) - report.BitcoinOnionPeers = append(report.BitcoinOnionPeers, spec) + report.OnionPeers = append(report.OnionPeers, spec) } ptr += 30 } @@ -271,31 +268,31 @@ func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.Onion } // Receive messages and handle them -func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport) error { +func (rps *BitcoinProtocolScanner) MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService) error { addrCount := 0 for { - pkt, err := ReceivePacket(conn) + pkt, err := ReceivePacket(conn, rps.msgstart) if err != nil { return fmt.Errorf("Error receiving P2P packet: %s", err) } switch pkt.msgtype { case MSG_VERSION: - err = HandleVersion(conn, osc, report, pkt) + err = rps.HandleVersion(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling version message: %s", err) } case MSG_VERACK: - err = HandleVerAck(conn, osc, report, pkt) + err = rps.HandleVerAck(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling verack message: %s", err) } case MSG_PING: - err = HandlePing(conn, osc, report, pkt) + err = rps.HandlePing(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling ping message: %s", err) } case MSG_ADDR: - err = HandleAddr(conn, osc, report, pkt) + err = rps.HandleAddr(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling addr message: %s", err) } @@ -312,26 +309,61 @@ func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.Onio return nil } +func NewBitcoinProtocolScanner(protocolName string) *BitcoinProtocolScanner { + rps := new(BitcoinProtocolScanner) + rps.name = protocolName + switch protocolName { + case "bitcoin": + rps.port = 8333 + rps.msgstart = []byte{0xf9, 0xbe, 0xb4, 0xd9} + case "bitcoin_test": + rps.port = 18333 + rps.msgstart = []byte{0x0b, 0x11, 0x09, 0x07} + case "litecoin": + rps.port = 9333 + rps.msgstart = []byte{0xfb, 0xc0, 0xb6, 0xdb} + case "litecoin_test": + rps.port = 19333 + rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc} + case "dogecoin": + rps.port = 22556 + rps.msgstart = []byte{0xc0, 0xc0, 0xc0, 0xc0} + case "dogecoin_test": + rps.port = 44556 + rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc} + default: // Unknown protocol + return nil + } + return rps +} + func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionScanConfig, report *report.OnionScanReport) { - // Bitcoin - osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(%d)\n", hiddenService, PORT)) - conn, err := utils.GetNetworkConnection(hiddenService, PORT, osc.TorProxyAddress, osc.Timeout) + // Bitcoin and derived protocols + osc.LogInfo(fmt.Sprintf("Checking %s %s(%d)\n", hiddenService, rps.name, rps.port)) + var subreport = report.AddBitcoinService(rps.name) + conn, err := utils.GetNetworkConnection(hiddenService, rps.port, osc.TorProxyAddress, osc.Timeout) if err != nil { - osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", PORT)) - report.BitcoinDetected = false + osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", rps.port)) + if rps.name == "bitcoin" { + report.BitcoinDetected = false + } + subreport.Detected = false } else { - osc.LogInfo("Detected possible Bitcoin instance\n") - report.BitcoinDetected = true + osc.LogInfo(fmt.Sprintf("Detected possible %s instance\n", rps.name)) + if rps.name == "bitcoin" { + report.BitcoinDetected = true + } + subreport.Detected = true conn.SetDeadline(time.Now().Add(30 * time.Second)) // Allow it to take 30 seconds at most - err = SendVersion(conn, osc, hiddenService) + err = rps.SendVersion(conn, osc, hiddenService) if err == nil { - err = MessageLoop(conn, osc, report) + err = rps.MessageLoop(conn, osc, subreport) if err != nil { osc.LogInfo(fmt.Sprintf("Error in receive loop: %s", err)) } } else { - osc.LogInfo(fmt.Sprintf("Error sending to Bitcoin node: %s\n", err)) + osc.LogInfo(fmt.Sprintf("Error sending to %s node: %s\n", rps.name, err)) } } if conn != nil { diff --git a/protocol/http_scanner.go b/protocol/http_scanner.go index 3e9feac..fac29f0 100644 --- a/protocol/http_scanner.go +++ b/protocol/http_scanner.go @@ -19,12 +19,12 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.O // HTTP osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService)) conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout) + if conn != nil { + conn.Close() + } if err != nil { osc.LogInfo("Failed to connect to service on port 80\n") report.WebDetected = false - if conn != nil { - conn.Close() - } } else { osc.LogInfo("Found potential service on http(80)\n") report.WebDetected = true diff --git a/report/onionscanreport.go b/report/onionscanreport.go index c27e851..79e9b69 100644 --- a/report/onionscanreport.go +++ b/report/onionscanreport.go @@ -13,6 +13,13 @@ type PGPKey struct { FingerPrint string `json:"fingerprint"` } +type BitcoinService struct { + Detected bool `json:"detected"` + UserAgent string `json:"userAgent"` + ProtocolVersion int `json:"prototocolVersion"` + OnionPeers []string `json:"onionPeers"` +} + type OnionScanReport struct { HiddenService string `json:"hiddenService"` DateScanned time.Time `json:"dateScanned"` @@ -42,11 +49,9 @@ type OnionScanReport struct { // TLS Certificates []x509.Certificate `json:"certificates"` - //Bitcoin - BitcoinAddresses []string `json:"bitcoinAddresses"` - BitcoinUserAgent string `json:"bitcoinUserAgent"` - BitcoinProtocolVersion int `json:"bitcoinPrototocolVersion"` - BitcoinOnionPeers []string `json:"bitcoinOnionPeers"` + // Bitcoin + BitcoinAddresses []string `json:"bitcoinAddresses"` + BitcoinServices map[string]*BitcoinService `json:"bitcoinServices"` // SSH SSHKey string `json:"sshKey"` @@ -80,6 +85,7 @@ func NewOnionScanReport(hiddenService string) *OnionScanReport { report.DateScanned = time.Now() report.Crawls = make(map[string]int) report.PerformedScans = []string{} + report.BitcoinServices = make(map[string]*BitcoinService) return report } @@ -88,6 +94,12 @@ func (osr *OnionScanReport) AddPGPKey(armoredKey, identity, fingerprint string) //TODO map of fingerprint:PGPKeys? and utils.RemoveDuplicates(&osr.PGPKeys) } +func (osr *OnionScanReport) AddBitcoinService(name string) *BitcoinService { + var s = new(BitcoinService) + osr.BitcoinServices[name] = s + return s +} + func (osr *OnionScanReport) Serialize() (string, error) { report, err := json.Marshal(osr) if err != nil { diff --git a/report/report_generator.go b/report/report_generator.go index 16cfcbb..8a19a3c 100644 --- a/report/report_generator.go +++ b/report/report_generator.go @@ -143,6 +143,13 @@ func GenerateSimpleReport(reportFile string, report *AnonymityReport) { buffer.WriteString("\n") } + if report.PrivateKeyDetected { + buffer.WriteString("\033[091mCritical Risk:\033[0m Hidden service private key is accessible!\n") + buffer.WriteString("\t Why this is bad: This can be used to impersonate the service at any point in the future.\n") + buffer.WriteString("\t To fix, generate a new hidden service and make sure the private_key file is not reachable from\n") + buffer.WriteString("\t the web root\n") + } + if len(reportFile) > 0 { f, err := os.Create(reportFile) diff --git a/spider/onionspider.go b/spider/onionspider.go index 812ddb7..abfe267 100644 --- a/spider/onionspider.go +++ b/spider/onionspider.go @@ -20,11 +20,11 @@ type OnionSpider struct { func (os *OnionSpider) Crawl(hiddenservice string, osc *config.OnionScanConfig, report *report.OnionScanReport) { torDialer, err := proxy.SOCKS5("tcp", osc.TorProxyAddress, nil, proxy.Direct) - + if err != nil { - osc.LogError(err) + osc.LogError(err) } - + transportConfig := &http.Transport{ Dial: torDialer.Dial, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -112,12 +112,14 @@ func (os *OnionSpider) Crawl(hiddenservice string, osc *config.OnionScanConfig, // Grab Server Status if it Exists // We add it as a resource so we can pull any information out of it later. mod_status, _ := url.Parse("http://" + hiddenservice + "/server-status") + osc.LogInfo(fmt.Sprintf("Scanning URI: %s", mod_status.String())) id, err = os.GetPage(mod_status.String(), base, osc, true) addCrawl(mod_status.String(), id, err) // Grab Private Key if it Exists // This would be a major security fail private_key, _ := url.Parse("http://" + hiddenservice + "/private_key") + osc.LogInfo(fmt.Sprintf("Scanning URI: %s", private_key.String())) id, err = os.GetPage(private_key.String(), base, osc, true) addCrawl(private_key.String(), id, err) @@ -169,8 +171,8 @@ func (os *OnionSpider) GetPage(uri string, base *url.URL, osc *config.OnionScanC if strings.Contains(response.Header.Get("Content-Type"), "text/html") { page = ParsePage(response.Body, base, snapshot) } else if strings.Contains(response.Header.Get("Content-Type"), "image/jpeg") { - page = SnapshotResource(response.Body) - osc.LogInfo(fmt.Sprintf("Fetched %d byte image", len(page.Snapshot))) + page = SnapshotBinaryResource(response.Body) + osc.LogInfo(fmt.Sprintf("Fetched %d byte image", len(page.Raw))) } else if snapshot { page = SnapshotResource(response.Body) osc.LogInfo(fmt.Sprintf("Grabbed %d byte document", len(page.Snapshot))) diff --git a/spider/pageparser.go b/spider/pageparser.go index 2695235..1a9467e 100644 --- a/spider/pageparser.go +++ b/spider/pageparser.go @@ -26,6 +26,14 @@ func SnapshotResource(response io.Reader) model.Page { return page } +func SnapshotBinaryResource(response io.Reader) model.Page { + page := model.Page{} + buf := make([]byte, 1024*512) // Read Max 0.5 MB + n, _ := io.ReadFull(response, buf) + page.Raw = buf[0:n] + return page +} + func ParsePage(response io.Reader, base *url.URL, snapshot bool) model.Page { page := model.Page{}