From 00e6673e168253b0c6f296db99c375d341517ce4 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sun, 2 Oct 2016 01:54:15 +0200 Subject: [PATCH 1/7] Parametrize bitcoin scan to be able to scan for other coins --- onionscan/onionscan.go | 7 ++- protocol/bitcoin_scanner.go | 118 +++++++++++++++++++++++------------- report/onionscanreport.go | 22 +++++-- 3 files changed, 97 insertions(+), 50 deletions(-) diff --git a/onionscan/onionscan.go b/onionscan/onionscan.go index 318d65c..9b03451 100644 --- a/onionscan/onionscan.go +++ b/onionscan/onionscan.go @@ -27,6 +27,9 @@ func (os *OnionScan) GetAllActions() []string { "vnc", "xmpp", "bitcoin", + "bitcoin_test", + "litecoin", + "dogecoin", } } @@ -62,8 +65,8 @@ func (os *OnionScan) PerformNextAction(report *report.OnionScanReport, nextActio case "xmpp": xmppps := new(protocol.XMPPProtocolScanner) xmppps.ScanProtocol(report.HiddenService, os.Config, report) - case "bitcoin": - bps := new(protocol.BitcoinProtocolScanner) + case "bitcoin", "bitcoin_test", "litecoin", "litecoin_test", "dogecoin", "dogecoin_test": + bps := protocol.NewBitcoinProtocolScanner(nextAction) bps.ScanProtocol(report.HiddenService, os.Config, report) case "none": return nil diff --git a/protocol/bitcoin_scanner.go b/protocol/bitcoin_scanner.go index f46f028..c6e0106 100644 --- a/protocol/bitcoin_scanner.go +++ b/protocol/bitcoin_scanner.go @@ -17,20 +17,17 @@ import ( ) type BitcoinProtocolScanner struct { + name string + port int + msgstart []byte } -// Message start of packets on mainnet -var MsgStartMainnet = []byte{0xf9, 0xbe, 0xb4, 0xd9} - // User agent to send to scanned nodes const user_agent = "/OnionScan:0.0.1/" // Protocol version to send to scanned nodes const protocol_version uint32 = 70014 -// Bitcoin protocol port -const PORT int = 8333 - // Maximum length of user agent const MAX_SUBVERSION_LENGTH = 256 @@ -120,9 +117,9 @@ func cstring(n []byte) string { } // Send P2P packet to connection -func SendPacket(conn net.Conn, pkt *Packet) error { +func SendPacket(conn net.Conn, msgstart []byte, pkt *Packet) error { hdr := make([]byte, 24, 24) - copy(hdr[0:4], MsgStartMainnet) + copy(hdr[0:4], msgstart) copy(hdr[4:16], pkt.msgtype) binary.LittleEndian.PutUint32(hdr[16:20], uint32(len(pkt.payload))) copy(hdr[20:24], Checksum(pkt.payload)) @@ -139,7 +136,7 @@ func SendPacket(conn net.Conn, pkt *Packet) error { } // Receive P2P packet from connection -func ReceivePacket(conn net.Conn) (*Packet, error) { +func ReceivePacket(conn net.Conn, msgstart []byte) (*Packet, error) { var pkt Packet hdr := make([]byte, 24, 24) _, err := io.ReadFull(conn, hdr) @@ -147,8 +144,8 @@ func ReceivePacket(conn net.Conn) (*Packet, error) { if err != nil { return nil, fmt.Errorf("Could not read P2P packet header: %s", err) } - if !bytes.Equal(hdr[0:4], MsgStartMainnet) { - return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], MsgStartMainnet) + if !bytes.Equal(hdr[0:4], msgstart) { + return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], msgstart) } pkt.msgtype = cstring(hdr[4:16]) length := binary.LittleEndian.Uint32(hdr[16:20]) @@ -195,7 +192,7 @@ func DecodeOnion(addr []byte) (string, error) { } // Build and send version message -func SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error { +func (rps *BitcoinProtocolScanner) SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error { // Most fields can be left at zero payload := make([]byte, 80, 80) // static part of payload tail := make([]byte, 5, 5) // last five bytes @@ -203,50 +200,50 @@ func SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService strin binary.LittleEndian.PutUint64(payload[12:20], uint64(time.Now().Unix())) theiraddr, err := EncodeOnion(hiddenService) - if err != nil { - return err + if err == nil { + // Only send their address if the target address can be parsed as onion address + copy(payload[28:28+16], theiraddr) + binary.BigEndian.PutUint16(payload[44:46], uint16(rps.port)) } - copy(payload[28:28+16], theiraddr) - binary.BigEndian.PutUint16(payload[44:46], uint16(PORT)) payload = append(payload, uint8(len(user_agent))) payload = append(payload, user_agent...) payload = append(payload, tail...) - return SendPacket(conn, &Packet{MSG_VERSION, payload}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_VERSION, payload}) } // Handle incoming version message, and parse message payload into report -func HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { - report.BitcoinProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4])) +func (rps *BitcoinProtocolScanner) HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { + report.ProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4])) user_agent_length, sizesize := ReadCompactSize(pkt.payload[80:]) if sizesize != 0 && user_agent_length < MAX_SUBVERSION_LENGTH { - report.BitcoinUserAgent = string(pkt.payload[81 : 81+user_agent_length]) + report.UserAgent = string(pkt.payload[80+sizesize : 80+sizesize+int(user_agent_length)]) } else { return fmt.Errorf("User agent string too long") } - osc.LogInfo(fmt.Sprintf("Found Bitcoin version: %s (%d)", report.BitcoinUserAgent, report.BitcoinProtocolVersion)) + osc.LogInfo(fmt.Sprintf("Found %s version: %s (%d)", rps.name, report.UserAgent, report.ProtocolVersion)) return nil } // Handle incoming verack message -func HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { // This message has no content. However when receiving this message the // version negotiation has been completed, and that other queries can be sent. osc.LogInfo(fmt.Sprintf("Sending getaddr message")) - return SendPacket(conn, &Packet{MSG_GETADDR, []byte{}}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_GETADDR, []byte{}}) } // Handle incoming ping message -func HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { if len(pkt.payload) >= 8 { // Ping message with nonce, peer expects a pong - return SendPacket(conn, &Packet{MSG_PONG, pkt.payload[0:8]}) + return SendPacket(conn, rps.msgstart, &Packet{MSG_PONG, pkt.payload[0:8]}) } return nil } // Handle incoming addr message, and parse message payload into report -func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error { +func (rps *BitcoinProtocolScanner) HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error { numaddr, sizesize := ReadCompactSize(pkt.payload) if sizesize == 0 || numaddr > MAX_ADDR { return fmt.Errorf("Invalid number of addresses") @@ -263,7 +260,7 @@ func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.Onion port := binary.BigEndian.Uint16(pkt.payload[ptr+28 : ptr+30]) spec := fmt.Sprintf("%s:%d", onion, port) osc.LogInfo(fmt.Sprintf("Found onion peer: %s", spec)) - report.BitcoinOnionPeers = append(report.BitcoinOnionPeers, spec) + report.OnionPeers = append(report.OnionPeers, spec) } ptr += 30 } @@ -271,31 +268,31 @@ func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.Onion } // Receive messages and handle them -func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport) error { +func (rps *BitcoinProtocolScanner) MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService) error { addrCount := 0 for { - pkt, err := ReceivePacket(conn) + pkt, err := ReceivePacket(conn, rps.msgstart) if err != nil { return fmt.Errorf("Error receiving P2P packet: %s", err) } switch pkt.msgtype { case MSG_VERSION: - err = HandleVersion(conn, osc, report, pkt) + err = rps.HandleVersion(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling version message: %s", err) } case MSG_VERACK: - err = HandleVerAck(conn, osc, report, pkt) + err = rps.HandleVerAck(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling verack message: %s", err) } case MSG_PING: - err = HandlePing(conn, osc, report, pkt) + err = rps.HandlePing(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling ping message: %s", err) } case MSG_ADDR: - err = HandleAddr(conn, osc, report, pkt) + err = rps.HandleAddr(conn, osc, report, pkt) if err != nil { return fmt.Errorf("Error handling addr message: %s", err) } @@ -312,26 +309,61 @@ func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.Onio return nil } +func NewBitcoinProtocolScanner(protocolName string) *BitcoinProtocolScanner { + rps := new(BitcoinProtocolScanner) + rps.name = protocolName + switch protocolName { + case "bitcoin": + rps.port = 8333 + rps.msgstart = []byte{0xf9, 0xbe, 0xb4, 0xd9} + case "bitcoin_test": + rps.port = 18333 + rps.msgstart = []byte{0x0b, 0x11, 0x09, 0x07} + case "litecoin": + rps.port = 9333 + rps.msgstart = []byte{0xfb, 0xc0, 0xb6, 0xdb} + case "litecoin_test": + rps.port = 19333 + rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc} + case "dogecoin": + rps.port = 22556 + rps.msgstart = []byte{0xc0, 0xc0, 0xc0, 0xc0} + case "dogecoin_test": + rps.port = 44556 + rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc} + default: // Unknown protocol + return nil + } + return rps +} + func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionScanConfig, report *report.OnionScanReport) { - // Bitcoin - osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(%d)\n", hiddenService, PORT)) - conn, err := utils.GetNetworkConnection(hiddenService, PORT, osc.TorProxyAddress, osc.Timeout) + // Bitcoin and derived protocols + osc.LogInfo(fmt.Sprintf("Checking %s %s(%d)\n", hiddenService, rps.name, rps.port)) + var subreport = report.AddBitcoinService(rps.name) + conn, err := utils.GetNetworkConnection(hiddenService, rps.port, osc.TorProxyAddress, osc.Timeout) if err != nil { - osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", PORT)) - report.BitcoinDetected = false + osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", rps.port)) + if rps.name == "bitcoin" { + report.BitcoinDetected = false + } + subreport.Detected = false } else { - osc.LogInfo("Detected possible Bitcoin instance\n") - report.BitcoinDetected = true + osc.LogInfo(fmt.Sprintf("Detected possible %s instance\n", rps.name)) + if rps.name == "bitcoin" { + report.BitcoinDetected = true + } + subreport.Detected = true conn.SetDeadline(time.Now().Add(30 * time.Second)) // Allow it to take 30 seconds at most - err = SendVersion(conn, osc, hiddenService) + err = rps.SendVersion(conn, osc, hiddenService) if err == nil { - err = MessageLoop(conn, osc, report) + err = rps.MessageLoop(conn, osc, subreport) if err != nil { osc.LogInfo(fmt.Sprintf("Error in receive loop: %s", err)) } } else { - osc.LogInfo(fmt.Sprintf("Error sending to Bitcoin node: %s\n", err)) + osc.LogInfo(fmt.Sprintf("Error sending to %s node: %s\n", rps.name, err)) } } if conn != nil { diff --git a/report/onionscanreport.go b/report/onionscanreport.go index c27e851..79e9b69 100644 --- a/report/onionscanreport.go +++ b/report/onionscanreport.go @@ -13,6 +13,13 @@ type PGPKey struct { FingerPrint string `json:"fingerprint"` } +type BitcoinService struct { + Detected bool `json:"detected"` + UserAgent string `json:"userAgent"` + ProtocolVersion int `json:"prototocolVersion"` + OnionPeers []string `json:"onionPeers"` +} + type OnionScanReport struct { HiddenService string `json:"hiddenService"` DateScanned time.Time `json:"dateScanned"` @@ -42,11 +49,9 @@ type OnionScanReport struct { // TLS Certificates []x509.Certificate `json:"certificates"` - //Bitcoin - BitcoinAddresses []string `json:"bitcoinAddresses"` - BitcoinUserAgent string `json:"bitcoinUserAgent"` - BitcoinProtocolVersion int `json:"bitcoinPrototocolVersion"` - BitcoinOnionPeers []string `json:"bitcoinOnionPeers"` + // Bitcoin + BitcoinAddresses []string `json:"bitcoinAddresses"` + BitcoinServices map[string]*BitcoinService `json:"bitcoinServices"` // SSH SSHKey string `json:"sshKey"` @@ -80,6 +85,7 @@ func NewOnionScanReport(hiddenService string) *OnionScanReport { report.DateScanned = time.Now() report.Crawls = make(map[string]int) report.PerformedScans = []string{} + report.BitcoinServices = make(map[string]*BitcoinService) return report } @@ -88,6 +94,12 @@ func (osr *OnionScanReport) AddPGPKey(armoredKey, identity, fingerprint string) //TODO map of fingerprint:PGPKeys? and utils.RemoveDuplicates(&osr.PGPKeys) } +func (osr *OnionScanReport) AddBitcoinService(name string) *BitcoinService { + var s = new(BitcoinService) + osr.BitcoinServices[name] = s + return s +} + func (osr *OnionScanReport) Serialize() (string, error) { report, err := json.Marshal(osr) if err != nil { From 67108f5231d1ff8af7ec8c50d5330782c3aa079f Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sun, 2 Oct 2016 14:32:24 +0200 Subject: [PATCH 2/7] Fix EXIF parsing: avoid converting images to unicode strings This patch fixes EXIF parsing by storing images as bytes in crawldb instead of unicode strings. --- deanonymization/check_exif.go | 3 ++- model/page.go | 1 + spider/onionspider.go | 4 ++-- spider/pageparser.go | 8 ++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/deanonymization/check_exif.go b/deanonymization/check_exif.go index 2aa3b4b..7b42441 100644 --- a/deanonymization/check_exif.go +++ b/deanonymization/check_exif.go @@ -1,6 +1,7 @@ package deanonymization import ( + "bytes" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" "github.com/xiam/exif" @@ -16,7 +17,7 @@ func CheckExif(osreport *report.OnionScanReport, anonreport *report.AnonymityRep if crawlRecord.Page.Status == 200 && strings.Contains(crawlRecord.Page.Headers.Get("Content-Type"), "image/jpeg") { reader := exif.New() - _, err := io.Copy(reader, strings.NewReader(string(crawlRecord.Page.Snapshot))) + _, err := io.Copy(reader, bytes.NewReader(crawlRecord.Page.Raw)) // exif.FoundExifInData is a signal that the EXIF parser has all it needs, // it doesn't need to be given the whole image. diff --git a/model/page.go b/model/page.go index 671cf7c..2e060ac 100644 --- a/model/page.go +++ b/model/page.go @@ -14,6 +14,7 @@ type Page struct { Links []Element Scripts []Element Snapshot string + Raw []byte Hash string } diff --git a/spider/onionspider.go b/spider/onionspider.go index 812ddb7..11f567f 100644 --- a/spider/onionspider.go +++ b/spider/onionspider.go @@ -169,8 +169,8 @@ func (os *OnionSpider) GetPage(uri string, base *url.URL, osc *config.OnionScanC if strings.Contains(response.Header.Get("Content-Type"), "text/html") { page = ParsePage(response.Body, base, snapshot) } else if strings.Contains(response.Header.Get("Content-Type"), "image/jpeg") { - page = SnapshotResource(response.Body) - osc.LogInfo(fmt.Sprintf("Fetched %d byte image", len(page.Snapshot))) + page = SnapshotBinaryResource(response.Body) + osc.LogInfo(fmt.Sprintf("Fetched %d byte image", len(page.Raw))) } else if snapshot { page = SnapshotResource(response.Body) osc.LogInfo(fmt.Sprintf("Grabbed %d byte document", len(page.Snapshot))) diff --git a/spider/pageparser.go b/spider/pageparser.go index 2695235..1a9467e 100644 --- a/spider/pageparser.go +++ b/spider/pageparser.go @@ -26,6 +26,14 @@ func SnapshotResource(response io.Reader) model.Page { return page } +func SnapshotBinaryResource(response io.Reader) model.Page { + page := model.Page{} + buf := make([]byte, 1024*512) // Read Max 0.5 MB + n, _ := io.ReadFull(response, buf) + page.Raw = buf[0:n] + return page +} + func ParsePage(response io.Reader, base *url.URL, snapshot bool) model.Page { page := model.Page{} From 72d39daca856fca1a4e5f0eb72c9477a05b97078 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sun, 2 Oct 2016 17:36:08 +0200 Subject: [PATCH 3/7] http protocol: close probing connection immediately I was noticing strange behavior where an extra HTTP connection would be open for the whole time the spider ran, which did not get any request. This turned out to be the initial probing connection to see if port 80 is open. It is subsequentially never used until the handle goes out of scope. In thie patch I changed it to close the connection immediately. This saves some resources on the server side (as well as makes single-threaded test servers such as Python's SimpleServer not hang during probing). --- protocol/http_scanner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/protocol/http_scanner.go b/protocol/http_scanner.go index 3e9feac..fac29f0 100644 --- a/protocol/http_scanner.go +++ b/protocol/http_scanner.go @@ -19,12 +19,12 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.O // HTTP osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService)) conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout) + if conn != nil { + conn.Close() + } if err != nil { osc.LogInfo("Failed to connect to service on port 80\n") report.WebDetected = false - if conn != nil { - conn.Close() - } } else { osc.LogInfo("Found potential service on http(80)\n") report.WebDetected = true From eec93a0b19db3d48757a1c29d4e90becdbaa5d1c Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 3 Oct 2016 07:50:42 +0200 Subject: [PATCH 4/7] travis: add gofmt checking Print a diff and fail the check if any code is not gofmt-ed. `go test -v ./...` is the [default test script](https://docs.travis-ci.com/user/languages/go#Default-Test-Script), so that goes first. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index cc8049f..2ae25f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,3 +7,8 @@ notifications: email: recipients: - team@onionscan.org + +script: + - go test -v ./... + - GOFMT=$(gofmt -d .) && echo "$GOFMT" + - test -z "$GOFMT" From e6e507f08b5fb0cd822b39fb73eb69c50988d7db Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 3 Oct 2016 08:08:54 +0200 Subject: [PATCH 5/7] spider/onionspider.go: run gofmt --- spider/onionspider.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spider/onionspider.go b/spider/onionspider.go index 11f567f..c06728b 100644 --- a/spider/onionspider.go +++ b/spider/onionspider.go @@ -20,11 +20,11 @@ type OnionSpider struct { func (os *OnionSpider) Crawl(hiddenservice string, osc *config.OnionScanConfig, report *report.OnionScanReport) { torDialer, err := proxy.SOCKS5("tcp", osc.TorProxyAddress, nil, proxy.Direct) - + if err != nil { - osc.LogError(err) + osc.LogError(err) } - + transportConfig := &http.Transport{ Dial: torDialer.Dial, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, From a66a1a1607efaf31939d24c4009878b82c18256b Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 3 Oct 2016 19:16:24 +0200 Subject: [PATCH 6/7] Check private_key and raise an alarm if it is a hidden service key --- deanonymization/private_key.go | 43 +++++++++++++++++++++++++++++++ deanonymization/process_report.go | 1 + report/report_generator.go | 7 +++++ 3 files changed, 51 insertions(+) create mode 100644 deanonymization/private_key.go diff --git a/deanonymization/private_key.go b/deanonymization/private_key.go new file mode 100644 index 0000000..d8b0ec4 --- /dev/null +++ b/deanonymization/private_key.go @@ -0,0 +1,43 @@ +package deanonymization + +import ( + "net/url" + + "encoding/base64" + "github.com/s-rah/onionscan/config" + "github.com/s-rah/onionscan/report" + "strings" +) + +func ProcessKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig, key string) { + _, err := base64.StdEncoding.DecodeString(key) + if err == nil { // Parses as base64 - could further check data as DER key, but this seems enough + report.PrivateKeyDetected = true + } +} + +func PrivateKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig) { + for _, id := range osreport.Crawls { + crawlRecord, _ := osc.Database.GetCrawlRecord(id) + + uri, _ := url.Parse(crawlRecord.URL) + if crawlRecord.Page.Status == 200 && strings.HasSuffix(uri.Path, "/private_key") { + contents := crawlRecord.Page.Snapshot + + key := "" + inKey := false + for _, line := range strings.Split(contents, "\n") { + line := strings.TrimSpace(line) + if line == "-----BEGIN RSA PRIVATE KEY-----" { + inKey = true + key = "" + } else if line == "-----END RSA PRIVATE KEY-----" { + ProcessKey(osreport, report, osc, key) + inKey = false + } else if inKey { + key += line + } + } + } + } +} diff --git a/deanonymization/process_report.go b/deanonymization/process_report.go index 44a8176..b580842 100644 --- a/deanonymization/process_report.go +++ b/deanonymization/process_report.go @@ -13,6 +13,7 @@ func ProcessReport(osreport *report.OnionScanReport, osc *config.OnionScanConfig PGPContentScan(osreport, anonreport, osc) MailtoScan(osreport, anonreport, osc) CheckExif(osreport, anonreport, osc) + PrivateKey(osreport, anonreport, osc) ExtractGoogleAnalyticsID(osreport, anonreport, osc) ExtractGooglePublisherID(osreport, anonreport, osc) ExtractBitcoinAddress(osreport, anonreport, osc) diff --git a/report/report_generator.go b/report/report_generator.go index 16cfcbb..8a19a3c 100644 --- a/report/report_generator.go +++ b/report/report_generator.go @@ -143,6 +143,13 @@ func GenerateSimpleReport(reportFile string, report *AnonymityReport) { buffer.WriteString("\n") } + if report.PrivateKeyDetected { + buffer.WriteString("\033[091mCritical Risk:\033[0m Hidden service private key is accessible!\n") + buffer.WriteString("\t Why this is bad: This can be used to impersonate the service at any point in the future.\n") + buffer.WriteString("\t To fix, generate a new hidden service and make sure the private_key file is not reachable from\n") + buffer.WriteString("\t the web root\n") + } + if len(reportFile) > 0 { f, err := os.Create(reportFile) From 34934edc70bf5bc393e0fd7813753af79820e3b1 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Mon, 3 Oct 2016 18:34:19 -0700 Subject: [PATCH 7/7] Checking Validity of Private_Key --- deanonymization/private_key.go | 62 +++++++++++++++++++++------------- spider/onionspider.go | 2 ++ 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/deanonymization/private_key.go b/deanonymization/private_key.go index d8b0ec4..3d35be5 100644 --- a/deanonymization/private_key.go +++ b/deanonymization/private_key.go @@ -1,42 +1,56 @@ package deanonymization import ( - "net/url" - - "encoding/base64" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "encoding/asn1" + "encoding/base32" + "encoding/pem" + "fmt" "github.com/s-rah/onionscan/config" "github.com/s-rah/onionscan/report" + "net/url" "strings" + "regexp" ) -func ProcessKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig, key string) { - _, err := base64.StdEncoding.DecodeString(key) - if err == nil { // Parses as base64 - could further check data as DER key, but this seems enough - report.PrivateKeyDetected = true - } -} - func PrivateKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig) { for _, id := range osreport.Crawls { crawlRecord, _ := osc.Database.GetCrawlRecord(id) uri, _ := url.Parse(crawlRecord.URL) if crawlRecord.Page.Status == 200 && strings.HasSuffix(uri.Path, "/private_key") { - contents := crawlRecord.Page.Snapshot + privateKeyRegex := regexp.MustCompile("-----BEGIN RSA PRIVATE KEY-----((?s).*)-----END RSA PRIVATE KEY-----") + foundPrivateKey := privateKeyRegex.FindAllString(crawlRecord.Page.Snapshot, -1) + for _, keyString := range foundPrivateKey { + osc.LogInfo(fmt.Sprintf("Found Potential Private Key")) + block,_ := pem.Decode([]byte(keyString)) + if block == nil || block.Type != "RSA PRIVATE KEY" { + osc.LogInfo("Could not parse privacy key: no valid PEM data found") + continue + } + + privateKey,err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + osc.LogInfo("Could not parse private key") + continue + } + + // DER Encode the Public Key + publicKeyBytes,_ := asn1.Marshal(rsa.PublicKey{ + N: privateKey.PublicKey.N, + E: privateKey.PublicKey.E, + }) + + h := sha1.New() + h.Write(publicKeyBytes) + sha1bytes := h.Sum(nil) - key := "" - inKey := false - for _, line := range strings.Split(contents, "\n") { - line := strings.TrimSpace(line) - if line == "-----BEGIN RSA PRIVATE KEY-----" { - inKey = true - key = "" - } else if line == "-----END RSA PRIVATE KEY-----" { - ProcessKey(osreport, report, osc, key) - inKey = false - } else if inKey { - key += line - } + data := base32.StdEncoding.EncodeToString(sha1bytes) + hostname := strings.ToLower(data[0:16]) + osc.LogInfo(fmt.Sprintf("Found Private Key for Host %s.onion", hostname)) + report.PrivateKeyDetected = true } } } diff --git a/spider/onionspider.go b/spider/onionspider.go index c06728b..abfe267 100644 --- a/spider/onionspider.go +++ b/spider/onionspider.go @@ -112,12 +112,14 @@ func (os *OnionSpider) Crawl(hiddenservice string, osc *config.OnionScanConfig, // Grab Server Status if it Exists // We add it as a resource so we can pull any information out of it later. mod_status, _ := url.Parse("http://" + hiddenservice + "/server-status") + osc.LogInfo(fmt.Sprintf("Scanning URI: %s", mod_status.String())) id, err = os.GetPage(mod_status.String(), base, osc, true) addCrawl(mod_status.String(), id, err) // Grab Private Key if it Exists // This would be a major security fail private_key, _ := url.Parse("http://" + hiddenservice + "/private_key") + osc.LogInfo(fmt.Sprintf("Scanning URI: %s", private_key.String())) id, err = os.GetPage(private_key.String(), base, osc, true) addCrawl(private_key.String(), id, err)