Squashed 'src/github.com/getlantern/flashlight/' changes from f7b7af9…

…..97941f3

97941f3 Merge branch 'master' into devel
8aba67b Merge pull request #50 from getlantern/1606
f3b681a Merge pull request #48 from getlantern/channelwad
50d0ca7 Merge pull request #49 from getlantern/1752
05c6f5c Merged master
d084935 getlantern/lantern#1752 Removing use of gopkg.in
884e1e4 Added flashlight.yaml to .gitignore
2488040 Updated to latest waddell api
b1001ce Better handling of start/stop of nattywad
f3f81b8 Updated to latest nattwad and waddell apis
2c5b318 Merged
3c48853 Merge pull request #46 from getlantern/1946
87e581a Fixed unit test
7934de6 Took out future proofing comment on IpAddress
1a82d85 Code review updates
e75bcaf getlantern/lantern#1946 Made client.configure sensitive to changes in trusted CAs
0e6bc9d getlantern/lantern#1946 Applied global CAs change to natty branch
ee5f04f getlantern/lantern#1946 Applied global CAs change to natty branch
d590c3f Merged 1606 (natty) branch
f137163 getlantern/lantern#1946 Fixed unit test
76169df getlantern/lantern#1946 Added CommonName to trusted CAs list
57e59e9 getlantern/lantern#1946 Fixed unit test compilation issue
313621f getlantern/lantern#1946 Pinning global set of trusted masquerade certs instead of per-masquerade
b55a9be Updated usage in README
0f0d64d Merged from master
6c537fd Changed gox link to getlantern fork
51e1383 Made waddelcert configurable
4bb9ea9 Merged
ebe4c79 Made waddelcert configurable
7202144 Merge branch '1606' of https://github.com/getlantern/flashlight into 1606
6514def getlantern/lantern#1954 flashlight using nattywad with tls
e741fd6 getlantern/lantern#1954 Added support for tls, added integration test using production cloud waddell
4e42b8c copyexecutables now renames executables
56400cd Added tagandbuild.bash
38117da README update
dfbcb64 Merged
c90f9f2 Updated README
3e100b3 Switched to cross-compilation approach that supports cgo
80591a6 Merge branch '1606' of https://github.com/getlantern/flashlight into 1606
0f3b4c7 Added globals folder
1cec0ab Made dimension names lowercase since they get lowercased by statshub anyway
3baa7e0 Stats configuration updates and variable naming in server/server.go
dc50f34 Clearing currentReporter on stop
e105912 Only populating client defaults if role is client
7095bc3 Not checking statshubaddr inside statreporter.doConfigure()
2890963 getlantern/lantern#1943 Updated flashlight to allow updating of stats reporting parameters on the fly
0edc826 Merge branch 'master' into 1606
c917575 Encapsulated Country dim
3e27dc7 Got rid of unnecessary global state in statreporter
edfd768 Fixed log error in postStats
7623083 Merge branch '1606_merge' into 1606
c9d5732 Added tracelogging to statreporter.postUpdate
4a4aeab Merged master
cd234b7 Ensuring nattywad keepalive interval is shorter than idletimeout on enproxy
773da26 getlantern/lantern#1606 Moved peers ahead of masquerades for readability in yaml
892fe5f Added test for statreporter
0d96ab2 Finished merge
78febb6 Merging - fixed merge bug in client.Configure()
acd82eb Initial step of merging from 1913
3a7f05d getlantern/lantern#1606 Added dimension for tracking offerer+answerer country
f0bda77 Added back checking of command-line parameters
1c1ea2d Changed 'using server ...' message to trace level
bf31d0d Using keepalives on waddell connection
bda4058 getlantern/lantern#1606 Fixed nil pointer when doing NAT traversals without stat reporting
7a926c5 Logging updates
170a939 getlantern/lantern#1815 Further improvements to nattest
479f0f5 getlantern/lantern#1815 Further improvements to nattest
02b52c7 getlantern/lantern#1815 Made nattest server and client check contents of messages before considering connection successful
074e06f getlantern/lantern#1815 Logging change
b307ba3 Merge pull request #32 from getlantern/1606_ox
9f17845 getlantern/lantern#1815 More stat reporting fixes
e20c1ce getlantern/lantern#1815 More stat reporting fixes
a3731c2 getlantern/lantern#1815 More stat reporting fixes
32c3100 getlantern/lantern#1815 Stat reporting structural changes (split into Client and Server)
b9ee8dc getlantern/lantern#1815 Stat reporting tweaks
86da20b Merge pull request #33 from getlantern/1606_atavism
d70622b use write only channel since only stats reporter needs to read from it
b0fbb18 Made peers a map (so that we can identify them by JID)
97dbded Using yamlconf package for configuration, which adds support for an HTTP config server
e7ecae8 remove empty function declaration
24f84c4 remove empty function declaration
069d3a9 accumulate stats as counters
388b1ea report ints instead of bools
edf9571 Remove OperatingSystem config var
e3294b7 Add confirm connectivity message
4d3aba9 Add traversal stat reporting
cf8c686 Add traversal stat reporting
b76ce7d Updated to latest nattywad interface, showing how information for stats is exposed
945e104 Updated logging api
4317537 Client now proxies to waddell through enproxy
ebfbb2c getlantern/lantern#1606 Switched to using package golog
2277f99 Merged from 1606
4c06d53 getlantern/lantern#1606 Concurrency improvements, auto-reconnect to waddell, using external package nattywad
8654407 add peer ids array to outcome struct
e80e0ad remove extraneous return statement
8c94983 add call to stat reporter to post traversal outcomes
255687d seperate message into its own file
57d91cb add traversal outcomes map to track traversals
b5d52c3 getlantern/lantern#1606 Aliasing package nattraversal
7463b50 getlantern/lantern#1606 error should be last return parameter
24f4e06 rename PeerConn to more accurate PeerConfig
6bcff08 rename PeerConn to more accurate PeerConfig
9b25e5c add UDP code from go-natty
ccbcd82 use different waddell server for each peer in client mode
251881d use different waddell server for each peer in client mode
c333731 use different waddell server for each peer in client mode
320cc00 remove redundant active peers map
dfeadb7 remove redundant active peers map
889a48b remove redundant active peers map
72e68cc integrate remaining parts of go-natty
28c20a8 integrate remaining parts of go-natty
ed12b2d integrate remaining parts of go-natty
0ff7147 integrate remaining parts of go-natty
d89e8ee integrate remaining parts of go-natty
ae63193 initial integration
b118b51 getlantern/lantern#1606 Only applying command-line flags if specified
35f32a5 getlantern/lantern#1606 Only applying command-line flags if specified
6ec727a getlantern/lantern#1606 Only applying command-line flags if specified
b4c541f getlantern/lantern#1606 Server receiving messages from waddell
db6d5fd getlantern/lantern#1606 Server connecting to waddell
721ef3e getlantern/lantern#1606 Moved some server configuration into new server.ServerConfig

git-subtree-dir: src/github.com/getlantern/flashlight
git-subtree-split: 97941f3ce5674b18e364095133471f0ed2b5e728

.gitignore

@@ -6,3 +6,4 @@ flashlight.ex?
 *.swp
 *.pyc
 certs/errors.txt
+flashlight.yaml

README.md

@@ -32,11 +32,12 @@ Usage of flashlight:
  -addr="": ip:port on which to listen for requests. When running as a client proxy, we'll listen with http, when running as a server proxy we'll listen with https (required)
  -cloudconfig="": optional http(s) URL to a cloud-based source for configuration updates
  -cloudconfigca="": optional PEM encoded certificate used to verify TLS connections to fetch cloudconfig
+ -configaddr="": if specified, run an http-based configuration server at this address
  -configdir="": directory in which to store configuration, including flashlight.yaml (defaults to current directory)
  -country="xx": 2 digit country code under which to report stats. Defaults to xx.
  -cpuprofile="": write cpu profile to given file
  -help=false: Get usage help
- -instanceid="": instanceId under which to report stats to statshub
+ -instanceid="": instanceId under which to report stats to statshub. If not specified, no stats are reported.
  -memprofile="": write heap profile to given file
  -parentpid=0: the parent process's PID, used on Windows for killing flashlight when the parent disappears
  -portmap=0: try to map this port on the firewall to the port on which flashlight is listening, using UPnP or NAT-PMP. If mapping this port fails, flashlight will exit with status code 50
@@ -45,6 +46,8 @@ Usage of flashlight:
  -statsaddr="": host:port at which to make detailed stats available using server-sent events (optional)
  -statshub="pure-journey-3547.herokuapp.com": address of statshub server
  -statsperiod=0: time in seconds to wait between reporting stats. If not specified, stats are not reported. If specified, statshub, instanceid and statsaddr must also be specified.
+ -waddelladdr="": if specified, connect to this waddell server and process NAT traversal requests inbound from waddell
+ -waddellcert="": if specified, use this cert (PEM-encoded) to authenticate connections to waddell. Otherwise, a default certificate is used.
 ```
 
 Example Client:
@@ -77,7 +80,7 @@ Handling request for: http://www.google.com/humans.txt
 Flashlight requires [Go 1.3.x](http://golang.org/dl/).
 
 It is convenient to build flashlight for multiple platforms using
-[gox](https://github.com/mitchellh/gox).
+[gox](https://github.com/getlantern/gox).
 
 The typical cross-compilation setup doesn't work for anything that uses C code,
 which includes the DNS resolution code and some other things. See
@@ -118,7 +121,7 @@ git push --tags
 
 The script `tagandbuild.bash` tags and runs crosscompile.bash.
 
-`./tagandbuild.bash <tag<`
+`./tagandbuild.bash <tag>`
 
 Note - ./crosscompile.bash omits debug symbols to keep the build smaller.
 
@@ -147,7 +150,6 @@ note - Signing windows code requires that the
 installed. On OS X with homebrew, you can do this with
 `brew install osslsigncode`.
 
-
 ### Masquerade Host Management
 
 Masquerade host configuration is managed using utilities in the [`genconfig/`](genconfig/) subfolder.

client/client.go

@@ -1,15 +1,24 @@
 package client
 
 import (
+ "crypto/x509"
  "math/rand"
+ "net"
  "net/http"
  "reflect"
+ "runtime"
  "strconv"
  "sync"
  "time"
 
+ "github.com/getlantern/deepcopy"
  "github.com/getlantern/enproxy"
- "github.com/getlantern/flashlight/log"
+ "github.com/getlantern/flashlight/globals"
+ "github.com/getlantern/flashlight/nattest"
+ "github.com/getlantern/flashlight/statreporter"
+ "github.com/getlantern/golog"
+ "github.com/getlantern/nattywad"
+ "github.com/getlantern/waddell"
 )
 
 const (
@@ -18,6 +27,20 @@ const (
  REVERSE_PROXY_FLUSH_INTERVAL = 250 * time.Millisecond
 
  X_FLASHLIGHT_QOS = "X-Flashlight-QOS"
+
+ HighQOS = 10
+
+ // Cutoff for logging warnings about a dial having taken a long time.
+ longDialLimit = 10 * time.Second
+
+ // idleTimeout needs to be small enough that we stop using connections
+ // before the upstream server/CDN closes them itself.
+ // TODO: make this configurable.
+ idleTimeout = 10 * time.Second
+)
+
+var (
+ log = golog.LoggerFor("flashlight.client")
 )
 
 // Client is an HTTP proxy that accepts connections from local programs and
@@ -32,10 +55,12 @@ type Client struct {
  // WriteTimeout: (optional) timeout for write ops
  WriteTimeout time.Duration
 
- cfg *ClientConfig
+ priorCfg *ClientConfig
+ priorTrustedCAs *x509.CertPool
  cfgMutex sync.RWMutex
  servers []*server
  totalServerWeights int
+ nattywadClient *nattywad.Client
  verifiedSets map[string]*verifiedMasqueradeSet
 }
 
@@ -62,8 +87,9 @@ func (client *Client) Configure(cfg *ClientConfig, enproxyConfigs []*enproxy.Con
  defer client.cfgMutex.Unlock()
 
  log.Debug("Configure() called")
- if client.cfg != nil {
- if reflect.DeepEqual(client.cfg, cfg) {
+ if client.priorCfg != nil && client.priorTrustedCAs != nil {
+ if reflect.DeepEqual(client.priorCfg, cfg) &&
+ reflect.DeepEqual(client.priorTrustedCAs, globals.TrustedCAs) {
  log.Debugf("Client configuration unchanged")
  return
  } else {
@@ -73,7 +99,10 @@ func (client *Client) Configure(cfg *ClientConfig, enproxyConfigs []*enproxy.Con
  log.Debugf("Client configuration initialized")
  }
 
- client.cfg = cfg
+ // Make a copy of cfg for comparing later
+ client.priorCfg = &ClientConfig{}
+ deepcopy.Copy(client.priorCfg, cfg)
+ client.priorTrustedCAs = globals.TrustedCAs
 
  if client.verifiedSets != nil {
  // Stop old verifications
@@ -119,6 +148,40 @@ func (client *Client) Configure(cfg *ClientConfig, enproxyConfigs []*enproxy.Con
  for _, server := range client.servers {
  client.totalServerWeights = client.totalServerWeights + server.info.Weight
  }
+
+ if client.nattywadClient == nil {
+ client.nattywadClient = &nattywad.Client{
+ ClientMgr: &waddell.ClientMgr{
+ Dial: func(addr string) (net.Conn, error) {
+ // Clients always connect to waddell via a proxy to prevent the
+ // waddell connection from being blocked by censors.
+ server := client.randomServerForQOS(HighQOS)
+ return server.dialWithEnproxy("tcp", addr)
+ },
+ ServerCert: globals.WaddellCert,
+ },
+ OnSuccess: func(info *nattywad.TraversalInfo) {
+ log.Debugf("NAT traversal Succeeded: %s", info)
+ log.Tracef("Peer Country: %s", info.Peer.Extras["country"])
+ serverConnected := nattest.Ping(info.LocalAddr, info.RemoteAddr)
+ reportTraversalResult(info, true, serverConnected)
+ },
+ OnFailure: func(info *nattywad.TraversalInfo) {
+ log.Debugf("NAT traversal Failed: %s", info)
+ log.Tracef("Peer Country: %s", info.Peer.Extras["country"])
+ reportTraversalResult(info, false, false)
+ },
+ KeepAliveInterval: idleTimeout - 2*time.Second,
+ }
+ }
+
+ peers := make([]*nattywad.ServerPeer, len(cfg.Peers))
+ i = 0
+ for _, peer := range cfg.Peers {
+ peers[i] = peer
+ i = i + 1
+ }
+ go client.nattywadClient.Configure(peers)
 }
 
 // highestQos finds the server with the highest reported quality of service for
@@ -138,7 +201,7 @@ func (cfg *ClientConfig) highestQosServer(masqueradeSet string) *ServerInfo {
 // ServeHTTP implements the method from interface http.Handler
 func (client *Client) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
  server := client.randomServer(req)
- log.Debugf("Using server %s to handle request for %s", server.info.Host, req.RequestURI)
+ log.Tracef("Using server %s to handle request for %s", server.info.Host, req.RequestURI)
  if req.Method == CONNECT {
  server.enproxyConfig.Intercept(resp, req)
  } else {
@@ -152,8 +215,10 @@ func (client *Client) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 // value are considered for inclusion. However, if no servers meet the QOS
 // requirement, the last server in the list will be used by default.
 func (client *Client) randomServer(req *http.Request) *server {
- targetQOS := client.targetQOS(req)
+ return client.randomServerForQOS(targetQOS(req))
+}
 
+func (client *Client) randomServerForQOS(targetQOS int) *server {
  servers, totalServerWeights := client.getServers()
 
  // Pick a random server using a target value between 0 and the total server weights
@@ -182,7 +247,7 @@ func (client *Client) randomServer(req *http.Request) *server {
 
 // targetQOS determines the target quality of service given the X-Flashlight-QOS
 // header if available, else returns 0.
-func (client *Client) targetQOS(req *http.Request) int {
+func targetQOS(req *http.Request) int {
  requestedQOS := req.Header.Get(X_FLASHLIGHT_QOS)
  if requestedQOS != "" {
  rqos, err := strconv.Atoi(requestedQOS)
@@ -199,3 +264,34 @@ func (client *Client) getServers() ([]*server, int) {
  defer client.cfgMutex.RUnlock()
  return client.servers, client.totalServerWeights
 }
+
+func reportTraversalResult(info *nattywad.TraversalInfo, clientGotFiveTuple bool, connectionSucceeded bool) {
+ answererCountry := "xx"
+ if _, ok := info.Peer.Extras["country"]; ok {
+ answererCountry = info.Peer.Extras["country"].(string)
+ }
+
+ dims := statreporter.CountryDim().
+ And("answerercountry", answererCountry).
+ And("offereranswerercountries", globals.Country+"_"+answererCountry).
+ And("operatingsystem", runtime.GOOS)
+
+ dims.Increment("traversalAttempted").Add(1)
+
+ if info.ServerRespondedToSignaling {
+ dims.Increment("answererOnline").Add(1)
+ }
+ if info.ServerGotFiveTuple {
+ dims.Increment("answererGot5Tuple").Add(1)
+ }
+ if clientGotFiveTuple {
+ dims.Increment("offererGot5Tuple").Add(1)
+ }
+ if info.ServerGotFiveTuple && clientGotFiveTuple {
+ dims.Increment("traversalSucceeded").Add(1)
+ dims.Increment("durationOfSuccessfulTraversal").Add(int64(info.Duration.Seconds()))
+ }
+ if connectionSucceeded {
+ dims.Increment("connectionSucceeded").Add(1)
+ }
+}

client/config.go

@@ -6,12 +6,15 @@ import (
  "sort"
  "sync"
  "time"
+
+ "github.com/getlantern/nattywad"
 )
 
 // ClientConfig captures configuration information for a Client
 type ClientConfig struct {
  DumpHeaders bool // whether or not to dump headers of requests and responses
  Servers []*ServerInfo
+ Peers map[string]*nattywad.ServerPeer // keyed to peer id (e.g. XMPP JID)
  MasqueradeSets map[string][]*Masquerade
 }
 
@@ -54,23 +57,20 @@ type ServerInfo struct {
  tlsConfigsMutex sync.Mutex
 }
 
-type cachedConn struct {
- conn net.Conn
- dialed time.Time
-}
-
 // Masquerade contains the data for a single masquerade host, including
 // the domain and the root CA.
 type Masquerade struct {
  // Domain: the domain to use for domain fronting
  Domain string
 
  // IpAddress: pre-resolved ip address to use instead of Domain (if
- // available) - NOT YET IMPLEMENTED, JUST FUTURE-PROOFING
+ // available)
  IpAddress string
+}
 
- // RootCA: the root CA for the domain.
- RootCA string
+type cachedConn struct {
+ conn net.Conn
+ dialed time.Time
 }
 
 // SortHosts sorts the Servers array in place, ordered by host

client/http.go

@@ -5,12 +5,13 @@ import (
  "net/http"
 
  "github.com/getlantern/enproxy"
+ "github.com/getlantern/flashlight/globals"
 )
 
 // HttpClient creates a simple domain-fronted HTTP client using the specified
 // values for the upstream host to use and for the masquerade/domain fronted host.
 func HttpClient(serverInfo *ServerInfo, masquerade *Masquerade) *http.Client {
- if masquerade != nil && masquerade.RootCA == "" {
+ if masquerade != nil && globals.TrustedCAs == nil {
  serverInfo.InsecureSkipVerify = true
  }
 

client/masquerade.go

@@ -7,8 +7,6 @@ import (
  "net/http"
  "sync"
  "time"
-
- "github.com/getlantern/flashlight/log"
 )
 
 const (
@@ -48,6 +46,7 @@ func newVerifiedMasqueradeSet(testServer *ServerInfo, masquerades []*Masquerade)
  if MaxMasquerades < verifiedChSize {
  verifiedChSize = MaxMasquerades
  }
+
  vms := &verifiedMasqueradeSet{
  testServer: testServer,
  masquerades: masquerades,
@@ -82,21 +81,21 @@ func (vms *verifiedMasqueradeSet) feedCandidates() {
 func (vms *verifiedMasqueradeSet) feedCandidate(candidate *Masquerade) bool {
  select {
  case <-vms.stopCh:
- log.Debug("Received stop, not feeding any further")
+ log.Trace("Received stop, not feeding any further")
  return false
  case vms.candidatesCh <- candidate:
- log.Debug("Fed candidate")
+ log.Trace("Fed candidate")
  return true
  }
 }
 
 // stop stops the verification process
 func (vms *verifiedMasqueradeSet) stop() {
- log.Debug("Stop called")
+ log.Trace("Stop called")
  vms.stopCh <- nil
- log.Debug("Waiting for workers to finish")
+ log.Trace("Waiting for workers to finish")
  vms.wg.Wait()
- log.Debug("Stopped")
+ log.Trace("Stopped")
 }
 
 // verify checks masquerades obtained from candidatesCh to see if they work on
@@ -129,16 +128,16 @@ func (vms *verifiedMasqueradeSet) doVerify(masquerade *Masquerade) bool {
  req, _ := http.NewRequest("HEAD", "http://www.google.com/humans.txt", nil)
  resp, err := httpClient.Do(req)
  if err != nil {
- errCh <- fmt.Errorf("HTTP ERROR FOR MASQUERADE %v: %v", masquerade.Domain, err)
+ log.Debugf("Error verifying masquerade %v: %v", masquerade.Domain, err)
  return
  } else {
  body, err := ioutil.ReadAll(resp.Body)
  defer resp.Body.Close()
  if err != nil {
- errCh <- fmt.Errorf("HTTP Body Error: %s", body)
+ errCh <- fmt.Errorf("Error verifying masquerade %v: %v", masquerade.Domain, err)
  } else {
  delta := time.Now().Sub(start)
- log.Debugf("SUCCESSFUL CHECK FOR: %s IN %s, %s", masquerade.Domain, delta, body)
+ log.Tracef("Successful masquerade check for %s in %s, %s", masquerade.Domain, delta, body)
  errCh <- nil
  }
  }