Merge branch 'laanwj-2016_10_web_hidden_service_key' into onionscan-0.2

.travis.yml

@@ -7,3 +7,8 @@ notifications:
    email:
      recipients:
        - [email protected]
+
+script:
+    - go test -v ./...
+    - GOFMT=$(gofmt -d .) && echo "$GOFMT"
+    - test -z "$GOFMT"

deanonymization/check_exif.go

@@ -1,6 +1,7 @@
 package deanonymization
 
 import (
+	"bytes"
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/report"
 	"github.com/xiam/exif"
@@ -16,7 +17,7 @@ func CheckExif(osreport *report.OnionScanReport, anonreport *report.AnonymityRep
 
 		if crawlRecord.Page.Status == 200 && strings.Contains(crawlRecord.Page.Headers.Get("Content-Type"), "image/jpeg") {
 			reader := exif.New()
-			_, err := io.Copy(reader, strings.NewReader(string(crawlRecord.Page.Snapshot)))
+			_, err := io.Copy(reader, bytes.NewReader(crawlRecord.Page.Raw))
 
 			// exif.FoundExifInData is a signal that the EXIF parser has all it needs,
 			// it doesn't need to be given the whole image.

deanonymization/private_key.go

@@ -0,0 +1,57 @@
+package deanonymization
+
+import (
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
+	"encoding/asn1"
+	"encoding/base32"
+	"encoding/pem"
+	"fmt"
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/report"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+func PrivateKey(osreport *report.OnionScanReport, report *report.AnonymityReport, osc *config.OnionScanConfig) {
+	for _, id := range osreport.Crawls {
+		crawlRecord, _ := osc.Database.GetCrawlRecord(id)
+
+		uri, _ := url.Parse(crawlRecord.URL)
+		if crawlRecord.Page.Status == 200 && strings.HasSuffix(uri.Path, "/private_key") {
+			privateKeyRegex := regexp.MustCompile("-----BEGIN RSA PRIVATE KEY-----((?s).*)-----END RSA PRIVATE KEY-----")
+			foundPrivateKey := privateKeyRegex.FindAllString(crawlRecord.Page.Snapshot, -1)
+			for _, keyString := range foundPrivateKey {
+				osc.LogInfo(fmt.Sprintf("Found Potential Private Key"))
+				block, _ := pem.Decode([]byte(keyString))
+				if block == nil || block.Type != "RSA PRIVATE KEY" {
+					osc.LogInfo("Could not parse privacy key: no valid PEM data found")
+					continue
+				}
+
+				privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+				if err != nil {
+					osc.LogInfo("Could not parse private key")
+					continue
+				}
+
+				// DER Encode the Public Key
+				publicKeyBytes, _ := asn1.Marshal(rsa.PublicKey{
+					N: privateKey.PublicKey.N,
+					E: privateKey.PublicKey.E,
+				})
+
+				h := sha1.New()
+				h.Write(publicKeyBytes)
+				sha1bytes := h.Sum(nil)
+
+				data := base32.StdEncoding.EncodeToString(sha1bytes)
+				hostname := strings.ToLower(data[0:16])
+				osc.LogInfo(fmt.Sprintf("Found Private Key for Host %s.onion", hostname))
+				report.PrivateKeyDetected = true
+			}
+		}
+	}
+}

deanonymization/process_report.go

@@ -13,6 +13,7 @@ func ProcessReport(osreport *report.OnionScanReport, osc *config.OnionScanConfig
 	PGPContentScan(osreport, anonreport, osc)
 	MailtoScan(osreport, anonreport, osc)
 	CheckExif(osreport, anonreport, osc)
+	PrivateKey(osreport, anonreport, osc)
 	ExtractGoogleAnalyticsID(osreport, anonreport, osc)
 	ExtractGooglePublisherID(osreport, anonreport, osc)
 	ExtractBitcoinAddress(osreport, anonreport, osc)

model/page.go

@@ -14,6 +14,7 @@ type Page struct {
 	Links    []Element
 	Scripts  []Element
 	Snapshot string
+	Raw      []byte
 	Hash     string
 }
 

onionscan/onionscan.go

@@ -27,6 +27,9 @@ func (os *OnionScan) GetAllActions() []string {
 		"vnc",
 		"xmpp",
 		"bitcoin",
+		"bitcoin_test",
+		"litecoin",
+		"dogecoin",
 	}
 }
 
@@ -62,8 +65,8 @@ func (os *OnionScan) PerformNextAction(report *report.OnionScanReport, nextActio
 	case "xmpp":
 		xmppps := new(protocol.XMPPProtocolScanner)
 		xmppps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "bitcoin":
-		bps := new(protocol.BitcoinProtocolScanner)
+	case "bitcoin", "bitcoin_test", "litecoin", "litecoin_test", "dogecoin", "dogecoin_test":
+		bps := protocol.NewBitcoinProtocolScanner(nextAction)
 		bps.ScanProtocol(report.HiddenService, os.Config, report)
 	case "none":
 		return nil

protocol/bitcoin_scanner.go

@@ -17,20 +17,17 @@ import (
 )
 
 type BitcoinProtocolScanner struct {
+	name     string
+	port     int
+	msgstart []byte
 }
 
-// Message start of packets on mainnet
-var MsgStartMainnet = []byte{0xf9, 0xbe, 0xb4, 0xd9}
-
 // User agent to send to scanned nodes
 const user_agent = "/OnionScan:0.0.1/"
 
 // Protocol version to send to scanned nodes
 const protocol_version uint32 = 70014
 
-// Bitcoin protocol port
-const PORT int = 8333
-
 // Maximum length of user agent
 const MAX_SUBVERSION_LENGTH = 256
 
@@ -120,9 +117,9 @@ func cstring(n []byte) string {
 }
 
 // Send P2P packet to connection
-func SendPacket(conn net.Conn, pkt *Packet) error {
+func SendPacket(conn net.Conn, msgstart []byte, pkt *Packet) error {
 	hdr := make([]byte, 24, 24)
-	copy(hdr[0:4], MsgStartMainnet)
+	copy(hdr[0:4], msgstart)
 	copy(hdr[4:16], pkt.msgtype)
 	binary.LittleEndian.PutUint32(hdr[16:20], uint32(len(pkt.payload)))
 	copy(hdr[20:24], Checksum(pkt.payload))
@@ -139,16 +136,16 @@ func SendPacket(conn net.Conn, pkt *Packet) error {
 }
 
 // Receive P2P packet from connection
-func ReceivePacket(conn net.Conn) (*Packet, error) {
+func ReceivePacket(conn net.Conn, msgstart []byte) (*Packet, error) {
 	var pkt Packet
 	hdr := make([]byte, 24, 24)
 	_, err := io.ReadFull(conn, hdr)
 
 	if err != nil {
 		return nil, fmt.Errorf("Could not read P2P packet header: %s", err)
 	}
-	if !bytes.Equal(hdr[0:4], MsgStartMainnet) {
-		return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], MsgStartMainnet)
+	if !bytes.Equal(hdr[0:4], msgstart) {
+		return nil, fmt.Errorf("P2P packet started with %q instead of %q", hdr[0:4], msgstart)
 	}
 	pkt.msgtype = cstring(hdr[4:16])
 	length := binary.LittleEndian.Uint32(hdr[16:20])
@@ -195,58 +192,58 @@ func DecodeOnion(addr []byte) (string, error) {
 }
 
 // Build and send version message
-func SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error {
+func (rps *BitcoinProtocolScanner) SendVersion(conn net.Conn, osc *config.OnionScanConfig, hiddenService string) error {
 	// Most fields can be left at zero
 	payload := make([]byte, 80, 80) // static part of payload
 	tail := make([]byte, 5, 5)      // last five bytes
 	binary.LittleEndian.PutUint32(payload[0:4], protocol_version)
 	binary.LittleEndian.PutUint64(payload[12:20], uint64(time.Now().Unix()))
 
 	theiraddr, err := EncodeOnion(hiddenService)
-	if err != nil {
-		return err
+	if err == nil {
+		// Only send their address if the target address can be parsed as onion address
+		copy(payload[28:28+16], theiraddr)
+		binary.BigEndian.PutUint16(payload[44:46], uint16(rps.port))
 	}
-	copy(payload[28:28+16], theiraddr)
-	binary.BigEndian.PutUint16(payload[44:46], uint16(PORT))
 
 	payload = append(payload, uint8(len(user_agent)))
 	payload = append(payload, user_agent...)
 	payload = append(payload, tail...)
 
-	return SendPacket(conn, &Packet{MSG_VERSION, payload})
+	return SendPacket(conn, rps.msgstart, &Packet{MSG_VERSION, payload})
 }
 
 // Handle incoming version message, and parse message payload into report
-func HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error {
-	report.BitcoinProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4]))
+func (rps *BitcoinProtocolScanner) HandleVersion(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error {
+	report.ProtocolVersion = int(binary.LittleEndian.Uint32(pkt.payload[0:4]))
 	user_agent_length, sizesize := ReadCompactSize(pkt.payload[80:])
 	if sizesize != 0 && user_agent_length < MAX_SUBVERSION_LENGTH {
-		report.BitcoinUserAgent = string(pkt.payload[81 : 81+user_agent_length])
+		report.UserAgent = string(pkt.payload[80+sizesize : 80+sizesize+int(user_agent_length)])
 	} else {
 		return fmt.Errorf("User agent string too long")
 	}
-	osc.LogInfo(fmt.Sprintf("Found Bitcoin version: %s (%d)", report.BitcoinUserAgent, report.BitcoinProtocolVersion))
+	osc.LogInfo(fmt.Sprintf("Found %s version: %s (%d)", rps.name, report.UserAgent, report.ProtocolVersion))
 	return nil
 }
 
 // Handle incoming verack message
-func HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error {
+func (rps *BitcoinProtocolScanner) HandleVerAck(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error {
 	// This message has no content. However when receiving this message the
 	// version negotiation has been completed, and that other queries can be sent.
 	osc.LogInfo(fmt.Sprintf("Sending getaddr message"))
-	return SendPacket(conn, &Packet{MSG_GETADDR, []byte{}})
+	return SendPacket(conn, rps.msgstart, &Packet{MSG_GETADDR, []byte{}})
 }
 
 // Handle incoming ping message
-func HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error {
+func (rps *BitcoinProtocolScanner) HandlePing(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error {
 	if len(pkt.payload) >= 8 { // Ping message with nonce, peer expects a pong
-		return SendPacket(conn, &Packet{MSG_PONG, pkt.payload[0:8]})
+		return SendPacket(conn, rps.msgstart, &Packet{MSG_PONG, pkt.payload[0:8]})
 	}
 	return nil
 }
 
 // Handle incoming addr message, and parse message payload into report
-func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport, pkt *Packet) error {
+func (rps *BitcoinProtocolScanner) HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService, pkt *Packet) error {
 	numaddr, sizesize := ReadCompactSize(pkt.payload)
 	if sizesize == 0 || numaddr > MAX_ADDR {
 		return fmt.Errorf("Invalid number of addresses")
@@ -263,39 +260,39 @@ func HandleAddr(conn net.Conn, osc *config.OnionScanConfig, report *report.Onion
 			port := binary.BigEndian.Uint16(pkt.payload[ptr+28 : ptr+30])
 			spec := fmt.Sprintf("%s:%d", onion, port)
 			osc.LogInfo(fmt.Sprintf("Found onion peer: %s", spec))
-			report.BitcoinOnionPeers = append(report.BitcoinOnionPeers, spec)
+			report.OnionPeers = append(report.OnionPeers, spec)
 		}
 		ptr += 30
 	}
 	return nil
 }
 
 // Receive messages and handle them
-func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.OnionScanReport) error {
+func (rps *BitcoinProtocolScanner) MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.BitcoinService) error {
 	addrCount := 0
 	for {
-		pkt, err := ReceivePacket(conn)
+		pkt, err := ReceivePacket(conn, rps.msgstart)
 		if err != nil {
 			return fmt.Errorf("Error receiving P2P packet: %s", err)
 		}
 		switch pkt.msgtype {
 		case MSG_VERSION:
-			err = HandleVersion(conn, osc, report, pkt)
+			err = rps.HandleVersion(conn, osc, report, pkt)
 			if err != nil {
 				return fmt.Errorf("Error handling version message: %s", err)
 			}
 		case MSG_VERACK:
-			err = HandleVerAck(conn, osc, report, pkt)
+			err = rps.HandleVerAck(conn, osc, report, pkt)
 			if err != nil {
 				return fmt.Errorf("Error handling verack message: %s", err)
 			}
 		case MSG_PING:
-			err = HandlePing(conn, osc, report, pkt)
+			err = rps.HandlePing(conn, osc, report, pkt)
 			if err != nil {
 				return fmt.Errorf("Error handling ping message: %s", err)
 			}
 		case MSG_ADDR:
-			err = HandleAddr(conn, osc, report, pkt)
+			err = rps.HandleAddr(conn, osc, report, pkt)
 			if err != nil {
 				return fmt.Errorf("Error handling addr message: %s", err)
 			}
@@ -312,26 +309,61 @@ func MessageLoop(conn net.Conn, osc *config.OnionScanConfig, report *report.Onio
 	return nil
 }
 
+func NewBitcoinProtocolScanner(protocolName string) *BitcoinProtocolScanner {
+	rps := new(BitcoinProtocolScanner)
+	rps.name = protocolName
+	switch protocolName {
+	case "bitcoin":
+		rps.port = 8333
+		rps.msgstart = []byte{0xf9, 0xbe, 0xb4, 0xd9}
+	case "bitcoin_test":
+		rps.port = 18333
+		rps.msgstart = []byte{0x0b, 0x11, 0x09, 0x07}
+	case "litecoin":
+		rps.port = 9333
+		rps.msgstart = []byte{0xfb, 0xc0, 0xb6, 0xdb}
+	case "litecoin_test":
+		rps.port = 19333
+		rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc}
+	case "dogecoin":
+		rps.port = 22556
+		rps.msgstart = []byte{0xc0, 0xc0, 0xc0, 0xc0}
+	case "dogecoin_test":
+		rps.port = 44556
+		rps.msgstart = []byte{0xfc, 0xc1, 0xb7, 0xdc}
+	default: // Unknown protocol
+		return nil
+	}
+	return rps
+}
+
 func (rps *BitcoinProtocolScanner) ScanProtocol(hiddenService string, osc *config.OnionScanConfig, report *report.OnionScanReport) {
-	// Bitcoin
-	osc.LogInfo(fmt.Sprintf("Checking %s Bitcoin(%d)\n", hiddenService, PORT))
-	conn, err := utils.GetNetworkConnection(hiddenService, PORT, osc.TorProxyAddress, osc.Timeout)
+	// Bitcoin and derived protocols
+	osc.LogInfo(fmt.Sprintf("Checking %s %s(%d)\n", hiddenService, rps.name, rps.port))
+	var subreport = report.AddBitcoinService(rps.name)
+	conn, err := utils.GetNetworkConnection(hiddenService, rps.port, osc.TorProxyAddress, osc.Timeout)
 	if err != nil {
-		osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", PORT))
-		report.BitcoinDetected = false
+		osc.LogInfo(fmt.Sprintf("Failed to connect to service on port %d\n", rps.port))
+		if rps.name == "bitcoin" {
+			report.BitcoinDetected = false
+		}
+		subreport.Detected = false
 	} else {
-		osc.LogInfo("Detected possible Bitcoin instance\n")
-		report.BitcoinDetected = true
+		osc.LogInfo(fmt.Sprintf("Detected possible %s instance\n", rps.name))
+		if rps.name == "bitcoin" {
+			report.BitcoinDetected = true
+		}
+		subreport.Detected = true
 
 		conn.SetDeadline(time.Now().Add(30 * time.Second)) // Allow it to take 30 seconds at most
-		err = SendVersion(conn, osc, hiddenService)
+		err = rps.SendVersion(conn, osc, hiddenService)
 		if err == nil {
-			err = MessageLoop(conn, osc, report)
+			err = rps.MessageLoop(conn, osc, subreport)
 			if err != nil {
 				osc.LogInfo(fmt.Sprintf("Error in receive loop: %s", err))
 			}
 		} else {
-			osc.LogInfo(fmt.Sprintf("Error sending to Bitcoin node: %s\n", err))
+			osc.LogInfo(fmt.Sprintf("Error sending to %s node: %s\n", rps.name, err))
 		}
 	}
 	if conn != nil {

protocol/http_scanner.go

@@ -19,12 +19,12 @@ func (hps *HTTPProtocolScanner) ScanProtocol(hiddenService string, osc *config.O
 	// HTTP
 	osc.LogInfo(fmt.Sprintf("Checking %s http(80)\n", hiddenService))
 	conn, err := utils.GetNetworkConnection(hiddenService, 80, osc.TorProxyAddress, osc.Timeout)
+	if conn != nil {
+		conn.Close()
+	}
 	if err != nil {
 		osc.LogInfo("Failed to connect to service on port 80\n")
 		report.WebDetected = false
-		if conn != nil {
-			conn.Close()
-		}
 	} else {
 		osc.LogInfo("Found potential service on http(80)\n")
 		report.WebDetected = true