New

.travis.yml

@@ -1,14 +1,26 @@
 language: go
 go:
-  - 1.6.2
+  - 1.7
   - tip
 sudo: true
 notifications:
    email:
      recipients:
        - [email protected]
 
+install:
+    - go get golang.org/x/tools/cmd/cover
+    - go get github.com/mattn/goveralls
+    - go get github.com/HouzuoGuo/tiedot
+    - go get golang.org/x/crypto/openpgp
+    - go get golang.org/x/net/proxy
+    - go get golang.org/x/net/html
+    - go get github.com/rwcarlsen/goexif/exif
+    - go get github.com/rwcarlsen/goexif/tiff
+    - go get github.com/s-rah/go-ricochet
+
 script:
-    - go test -v ./...
-    - GOFMT=$(gofmt -d .) && echo "$GOFMT"
+
+    - cd $TRAVIS_BUILD_DIR && ./tests.sh
     - test -z "$GOFMT"
+    - goveralls -coverprofile=./coverage.out -service travis-ci

README.md

@@ -1,7 +1,4 @@
-# What is OnionScan?
-
-
-[![Build Status](https://travis-ci.org/s-rah/onionscan.svg?branch=onionscan-0.2)](https://travis-ci.org/s-rah/onionscan) [![Go Report Card](https://goreportcard.com/badge/github.com/s-rah/onionscan)](https://goreportcard.com/report/github.com/s-rah/onionscan)
+# What is OnionScan? [![Build Status](https://travis-ci.org/s-rah/onionscan.svg?branch=onionscan-0.3)](https://travis-ci.org/s-rah/onionscan) [![Go Report Card](https://goreportcard.com/badge/github.com/s-rah/onionscan)](https://goreportcard.com/report/github.com/s-rah/onionscan) [![Coverage Status](https://coveralls.io/repos/github/s-rah/onionscan/badge.svg?branch=onionscan-0.3)](https://coveralls.io/github/s-rah/onionscan?branch=onionscan-0.3)
 
 OnionScan is a free and open source tool for investigating the Dark Web. For all
 the amazing technological innovations in the anonymity and privacy space, there 
@@ -32,11 +29,12 @@ OnionScan has two primary goals:
 In order to install OnionScan you will need the following dependencies not 
 provided by the core go standard library:
 
-* golang.org/x/net/proxy - For the Tor SOCKS Proxy connection.
-* golang.org/x/net/crypto - For PGP parsing
-* golang.org/x/net/html - For HTML parsing
-* github.com/rwcarlsen/goexif - For EXIF data extraction.
 * github.com/HouzuoGuo/tiedot/db - For crawl database.
+* github.com/rwcarlsen/goexif/exif - For EXIF data extraction.
+* github.com/rwcarlsen/goexif/tiff - For EXIF data extraction.
+* golang.org/x/crypto/openpgp - For PGP parsing
+* golang.org/x/net/html - For HTML parsing
+* golang.org/x/net/proxy - For the Tor SOCKS Proxy connection.
 
 ### Grab with go get
 

deanonymization/check_bitcoin_addresses.go

@@ -10,57 +10,56 @@ import (
 	"strings"
 )
 
-// A25 is a type for a 25 byte (not base58 encoded) bitcoin address.
-type A25 [25]byte
-
-// Version extracts the version byte from a bitcoin address
-func (a *A25) Version() byte {
-	return a[0]
-}
-
-// EmbeddedChecksum returns the checksum of a bitcoin address
-func (a *A25) EmbeddedChecksum() (c [4]byte) {
-	copy(c[:], a[21:])
-	return
-}
+// Tmpl and Set58 are adapted from the C solution.
+// Go has big integers but this techinique seems better.
+var tmpl = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
 
-// DoubleSHA256 computes a double sha256 hash of the first 21 bytes of the
-// address.  This is the one function shared with the other bitcoin RC task.
-// Returned is the full 32 byte sha256 hash.  (The bitcoin checksum will be
-// the first four bytes of the slice.)
-func (a *A25) doubleSHA256() []byte {
-	h := sha256.New()
-	h.Write(a[:21])
-	d := h.Sum([]byte{})
-	h = sha256.New()
-	h.Write(d)
-	return h.Sum(d[:0])
+// ValidateA58 validates a base58 encoded bitcoin address.  An address is valid
+// if it can be decoded into a 25 byte address, the version number is 0
+// (P2PKH) or 5 (P2SH), and the checksum validates.  Return value ok will be
+// true for valid addresses.  If ok is false, the address is invalid and the
+// error value may indicate why.
+func ValidateA58(a58 []byte) (ok bool) {
+	a := make([]byte, 25)
+	if err := Set58(a58, a); err != nil {
+		return false
+	}
+	if Version(a) != 0 && Version(a) != 5 {
+		return false
+	}
+	return EmbeddedChecksum(a) == ComputeChecksum(a)
 }
 
-// ComputeChecksum returns a four byte checksum computed from the first 21
-// bytes of the address.  The embedded checksum is not updated.
-func (a *A25) ComputeChecksum() (c [4]byte) {
-	copy(c[:], a.doubleSHA256())
-	return
+// ValidateP58 validates a base58 encoded private key.  A private key is valid
+// if it can be decoded into a 37 byte private key, start with a 0x80 byte,
+// and the checksum validates.  Return value ok will be true for valid private keys.
+// If ok is false, the private key is invalid and the error value may indicate why.
+func ValidateP58(p58 []byte) (ok bool) {
+	p := make([]byte, 37)
+	if p58[0] != 'L' && p58[0] != 'K' {
+		return false
+	}
+	if err := Set58(p58[1:], p); err != nil {
+		return false
+	}
+	if Version(p) != 128 {
+		return false
+	}
+	return EmbeddedChecksum(p) == ComputeChecksum(p)
 }
 
-// Tmpl and Set58 are adapted from the C solution.
-// Go has big integers but this techinique seems better.
-var tmpl = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
-
-// Set58 takes a base58 encoded address and decodes it into the receiver.
-// Errors are returned if the argument is not valid base58 or if the decoded
-// value does not fit in the 25 byte address.  The address is not otherwise
-// checked for validity.
-func (a *A25) Set58(s []byte) error {
+// Set58 takes a base58 encoded string decodes it into a byte slice.
+// Errors are returned if the argument is not a valid base58 or if the decoded
+// value does not fit in the byte slice.
+func Set58(s []byte, b []byte) error {
 	for _, s1 := range s {
 		c := bytes.IndexByte(tmpl, s1)
 		if c < 0 {
 			return errors.New("bad char")
 		}
-		for j := 24; j >= 0; j-- {
-			c += 58 * int(a[j])
-			a[j] = byte(c % 256)
+		for j := len(b) - 1; j >= 0; j-- {
+			c += 58 * int(b[j])
+			b[j] = byte(c % 256)
 			c /= 256
 		}
 		if c > 0 {
@@ -70,35 +69,58 @@ func (a *A25) Set58(s []byte) error {
 	return nil
 }
 
-// ValidA58 validates a base58 encoded bitcoin address.  An address is valid
-// if it can be decoded into a 25 byte address, the version number is 0
-// (P2PKH) or 5 (P2SH), and the checksum validates.  Return value ok will be
-// true for valid addresses.  If ok is false, the address is invalid and the
-// error value may indicate why.
-func ValidA58(a58 []byte) (ok bool) {
-	var a A25
-	if err := a.Set58(a58); err != nil {
-		return false
-	}
-	if a.Version() != 0 && a.Version() != 5 {
-		return false
-	}
-	return a.EmbeddedChecksum() == a.ComputeChecksum()
+// Version extracts the version byte from the byte slice.
+func Version(b []byte) byte {
+	return b[0]
+}
+
+// ComputeChecksum returns a four byte checksum computed from bytes (except for
+// the last 4) of the slice.  The embedded checksum is not updated.
+func ComputeChecksum(b []byte) (c [4]byte) {
+	copy(c[:], doubleSHA256(b))
+	return
+}
+
+// EmbeddedChecksum returns the checksum of the byte slice.
+func EmbeddedChecksum(b []byte) (c [4]byte) {
+	copy(c[:], b[len(b)-4:])
+	return
+}
+
+// DoubleSHA256 computes a double sha256 hash of bytes (except for the last 4)
+// of the slice.  This is the one function shared with the other bitcoin RC task.
+// Returned is the full 32 byte sha256 hash.  (The checksum will be the first
+// four bytes of the slice.)
+func doubleSHA256(b []byte) []byte {
+	h := sha256.New()
+	h.Write(b[:len(b)-4])
+	d := h.Sum([]byte{})
+	h = sha256.New()
+	h.Write(d)
+	return h.Sum(d[:0])
 }
 
 // ExtractBitcoinAddress extracts any information related to bitcoin addresses from the current crawl.
 func ExtractBitcoinAddress(osreport *report.OnionScanReport, anonreport *report.AnonymityReport, osc *config.OnionScanConfig) {
 	bcaregex := regexp.MustCompile(`[13][a-km-zA-HJ-NP-Z1-9]{25,34}`)
+	pkregex := regexp.MustCompile(`[LK][a-km-zA-HJ-NP-Z1-9]{51}`)
 	for _, id := range osreport.Crawls {
 		crawlRecord, _ := osc.Database.GetCrawlRecord(id)
 		if strings.Contains(crawlRecord.Page.Headers.Get("Content-Type"), "text/html") {
 			foundBCID := bcaregex.FindAllString(crawlRecord.Page.Snapshot, -1)
 			for _, result := range foundBCID {
-				if ValidA58([]byte(result)) {
+				if ValidateA58([]byte(result)) {
 					anonreport.BitcoinAddresses = append(anonreport.BitcoinAddresses, result)
 					osc.Database.InsertRelationship(osreport.HiddenService, "snapshot", "bitcoin-address", result)
 				}
 			}
+			foundPKID := pkregex.FindAllString(crawlRecord.Page.Snapshot, -1)
+			for _, result := range foundPKID {
+				if ValidateP58([]byte(result)) {
+					anonreport.BitcoinPrivateKeys = append(anonreport.BitcoinPrivateKeys, result)
+					osc.Database.InsertRelationship(osreport.HiddenService, "snapshot", "bitcoin-private-key", result)
+				}
+			}
 		}
 	}
 }

deanonymization/check_bitcoin_addresses_test.go

@@ -42,7 +42,18 @@ func TestExtractBitcoinAddress(t *testing.T) {
 		t.Errorf("Unexpected bitcoin address found")
 	}
 
-	// Test 4: Multiple addresses
+	// Test 4: WIF bitcoin private key
+	ctx.CreatePage("/index.html", 200, "text/html", "<html><body>L5Kb8kLf9zgWQnogidDA76MzPL6TsZZY36hWXMssSzNydYXYB9KF</body></html>")
+	ExtractBitcoinAddress(ctx.osreport, ctx.report, ctx.osc)
+
+	if len(ctx.report.BitcoinPrivateKeys) != 1 {
+		t.Errorf("Should have detected a bitcoin private key")
+	}
+	if ctx.report.BitcoinPrivateKeys[0] != "L5Kb8kLf9zgWQnogidDA76MzPL6TsZZY36hWXMssSzNydYXYB9KF" {
+		t.Errorf("Unexpected bitcoin private key found")
+	}
+
+	// Test 5: Multiple addresses
 	ctx.report.BitcoinAddresses = []string{}
 	data, err := ioutil.ReadFile("testdata/bitcoin.html")
 	if err != nil {

main.go

@@ -56,7 +56,7 @@ func main() {
 	if *scans != "" {
 		scanslist = strings.Split(*scans, ",")
 	} else {
-		scanslist = onionScan.GetAllActions()
+		scanslist = onionScan.GetAllActions(true)
 	}
 
 	onionScan.Config = config.Configure(*torProxyAddress, *directoryDepth, *fingerprint, *timeout, *dbdir, scanslist, *crawlconfigdir, *cookiestring, *verbose)

onionscan/onionscan.go

@@ -5,6 +5,7 @@ import (
 	"github.com/s-rah/onionscan/config"
 	"github.com/s-rah/onionscan/protocol"
 	"github.com/s-rah/onionscan/report"
+	"sort"
 	"time"
 )
 
@@ -13,80 +14,59 @@ type OnionScan struct {
 	Config *config.OnionScanConfig
 }
 
-// GetAllActions returns a list of all possible protocol level  scans.
-func (os *OnionScan) GetAllActions() []string {
-	return []string{
-		"web",
-		"tls",
-		"ssh",
-		"irc",
-		"ricochet",
-		"ftp",
-		"smtp",
-		"mongodb",
-		"vnc",
-		"xmpp",
-		"bitcoin",
-		"bitcoin_test",
-		"litecoin",
-		"dogecoin",
-	}
+// Description record for a single scan type
+type scanDescription struct {
+	scanner      protocol.Scanner
+	runByDefault bool
+}
+
+// List of all scan types in onionscan
+var allScans = map[string]scanDescription{
+	"web":           {new(protocol.HTTPProtocolScanner), true},
+	"tls":           {new(protocol.TLSProtocolScanner), true},
+	"ssh":           {new(protocol.SSHProtocolScanner), true},
+	"irc":           {new(protocol.IRCProtocolScanner), true},
+	"ricochet":      {new(protocol.RicochetProtocolScanner), true},
+	"ftp":           {new(protocol.FTPProtocolScanner), true},
+	"smtp":          {new(protocol.SMTPProtocolScanner), true},
+	"mongodb":       {new(protocol.MongoDBProtocolScanner), true},
+	"vnc":           {new(protocol.VNCProtocolScanner), true},
+	"xmpp":          {new(protocol.XMPPProtocolScanner), true},
+	"bitcoin":       {protocol.NewBitcoinProtocolScanner("bitcoin"), true},
+	"bitcoin_test":  {protocol.NewBitcoinProtocolScanner("bitcoin_test"), true},
+	"litecoin":      {protocol.NewBitcoinProtocolScanner("litecoin"), true},
+	"litecoin_test": {protocol.NewBitcoinProtocolScanner("litecoin_test"), false},
+	"dogecoin":      {protocol.NewBitcoinProtocolScanner("dogecoin"), true},
+	"dogecoin_test": {protocol.NewBitcoinProtocolScanner("dogecoin_test"), false},
+	"none":          {nil, false},
 }
 
-// PerformNextAction  determined which scan to run next, and runs it.
-func (os *OnionScan) PerformNextAction(report *report.OnionScanReport, nextAction string) error {
-	switch nextAction {
-	case "web":
-		wps := new(protocol.HTTPProtocolScanner)
-		wps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "tls":
-		tps := new(protocol.TLSProtocolScanner)
-		tps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "ssh":
-		sps := new(protocol.SSHProtocolScanner)
-		sps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "irc":
-		ips := new(protocol.IRCProtocolScanner)
-		ips.ScanProtocol(report.HiddenService, os.Config, report)
-	case "ricochet":
-		rps := new(protocol.RicochetProtocolScanner)
-		rps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "ftp":
-		fps := new(protocol.FTPProtocolScanner)
-		fps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "smtp":
-		smps := new(protocol.SMTPProtocolScanner)
-		smps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "mongodb":
-		mdbps := new(protocol.MongoDBProtocolScanner)
-		mdbps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "vnc":
-		vncps := new(protocol.VNCProtocolScanner)
-		vncps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "xmpp":
-		xmppps := new(protocol.XMPPProtocolScanner)
-		xmppps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "bitcoin", "bitcoin_test", "litecoin", "litecoin_test", "dogecoin", "dogecoin_test":
-		bps := protocol.NewBitcoinProtocolScanner(nextAction)
-		bps.ScanProtocol(report.HiddenService, os.Config, report)
-	case "none":
-		return nil
-	default:
-		return fmt.Errorf("Unknown scanner %s", nextAction)
+// GetDefaultActions returns a list of all protocol level scans
+// (or optionally only those that should be enabled by default).
+func (os *OnionScan) GetAllActions(onlyDefault bool) []string {
+	var keys []string
+	for k := range allScans {
+		if !onlyDefault || allScans[k].runByDefault {
+			keys = append(keys, k)
+		}
 	}
-	return nil
+	sort.Strings(keys)
+	return keys
 }
 
 // Do performs all configured protocol level scans in this run.
 func (os *OnionScan) Do(osreport *report.OnionScanReport) error {
 
 	for _, nextAction := range os.Config.Scans {
-		err := os.PerformNextAction(osreport, nextAction)
-		if err != nil {
-			os.Config.LogInfo(fmt.Sprintf("Error: %s", err))
-		} else {
-			osreport.PerformedScans = append(osreport.PerformedScans, nextAction)
+		scan, ok := allScans[nextAction]
+		if scan.scanner == nil {
+			if !ok { // If key was not found, give error, otherwise this was the dummy scan "none"
+				os.Config.LogInfo(fmt.Sprintf("Unknown scanner %s", nextAction))
+			}
+			continue
 		}
+		scan.scanner.ScanProtocol(osreport.HiddenService, os.Config, osreport)
+		osreport.PerformedScans = append(osreport.PerformedScans, nextAction)
 		if time.Now().Sub(osreport.DateScanned).Seconds() > os.Config.Timeout.Seconds() {
 			osreport.TimedOut = true
 			break