diff --git a/.gitignore b/.gitignore index 7df3f1afe240f..11634b6e55741 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,3 @@ dogstatsd.yaml # Ignore debs from the root of the project. datadog-agent*_amd64.deb - -# Ignore pem created during the tests -*.pem diff --git a/pkg/config/config.go b/pkg/config/config.go index 7057c921f5aa5..a7749903ff4db 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -142,13 +142,6 @@ func init() { Datadog.SetDefault("kubernetes_http_kubelet_port", 10255) Datadog.SetDefault("kubernetes_https_kubelet_port", 10250) - Datadog.SetDefault("kubelet_tls_verify", true) - Datadog.SetDefault("kubelet_client_ca", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") - - Datadog.SetDefault("kubelet_auth_token_path", "") - Datadog.SetDefault("kubelet_client_crt", "") - Datadog.SetDefault("kubelet_client_key", "") - // Kube ApiServer Datadog.SetDefault("kubernetes_kubeconfig_path", "") @@ -198,8 +191,6 @@ func init() { Datadog.BindEnv("kubernetes_kubelet_host") Datadog.BindEnv("kubernetes_http_kubelet_port") Datadog.BindEnv("kubernetes_https_kubelet_port") - Datadog.BindEnv("kubelet_client_crt") - Datadog.BindEnv("kubelet_client_key") Datadog.BindEnv("forwarder_timeout") Datadog.BindEnv("forwarder_retry_queue_max_size") diff --git a/pkg/util/docker/docker_util.go b/pkg/util/docker/docker_util.go index a1dac0377881d..da933db6e43d9 100644 --- a/pkg/util/docker/docker_util.go +++ b/pkg/util/docker/docker_util.go @@ -39,9 +39,7 @@ func (d *DockerUtil) ContainerList(ctx context.Context, options types.ContainerL // DockerUtil wraps interactions with a local docker API. type DockerUtil struct { - // used to setup the DockerUtil - initRetry retry.Retrier - + retry.Retrier sync.Mutex cfg *Config cli *client.Client diff --git a/pkg/util/docker/global.go b/pkg/util/docker/global.go index ea772b7185cff..647e9a1e0cb46 100644 --- a/pkg/util/docker/global.go +++ b/pkg/util/docker/global.go @@ -37,7 +37,7 @@ var ( func GetDockerUtil() (*DockerUtil, error) { if globalDockerUtil == nil { globalDockerUtil = &DockerUtil{} - globalDockerUtil.initRetry.SetupRetrier(&retry.Config{ + globalDockerUtil.SetupRetrier(&retry.Config{ Name: "dockerutil", AttemptMethod: globalDockerUtil.init, Strategy: retry.RetryCount, @@ -45,7 +45,7 @@ func GetDockerUtil() (*DockerUtil, error) { RetryDelay: 30 * time.Second, }) } - err := globalDockerUtil.initRetry.TriggerRetry() + err := globalDockerUtil.TriggerRetry() if err != nil { log.Debugf("init error: %s", err) return nil, err @@ -58,7 +58,7 @@ func GetDockerUtil() (*DockerUtil, error) { // calls to the docker server will result in nil pointer exceptions. func EnableTestingMode() { globalDockerUtil = &DockerUtil{} - globalDockerUtil.initRetry.SetupRetrier(&retry.Config{ + globalDockerUtil.SetupRetrier(&retry.Config{ Name: "dockerutil", Strategy: retry.JustTesting, }) diff --git a/pkg/util/kubernetes/apiserver/apiserver.go b/pkg/util/kubernetes/apiserver/apiserver.go index c0a9d16254880..012fed7c951fc 100644 --- a/pkg/util/kubernetes/apiserver/apiserver.go +++ b/pkg/util/kubernetes/apiserver/apiserver.go @@ -37,9 +37,7 @@ const ( // ApiClient provides authenticated access to the // apiserver endpoints. Use the shared instance via GetApiClient. type APIClient struct { - // used to setup the APIClient - initRetry retry.Retrier - + retry.Retrier client *k8s.Client timeout time.Duration } @@ -51,7 +49,7 @@ func GetAPIClient() (*APIClient, error) { // TODO: make it configurable if requested timeout: 5 * time.Second, } - globalApiClient.initRetry.SetupRetrier(&retry.Config{ + globalApiClient.SetupRetrier(&retry.Config{ Name: "apiserver", AttemptMethod: globalApiClient.connect, Strategy: retry.RetryCount, @@ -59,7 +57,7 @@ func GetAPIClient() (*APIClient, error) { RetryDelay: 30 * time.Second, }) } - err := globalApiClient.initRetry.TriggerRetry() + err := globalApiClient.TriggerRetry() if err != nil { log.Debugf("init error: %s", err) return nil, err diff --git a/pkg/util/kubernetes/auth.go b/pkg/util/kubernetes/auth.go index 9b0cc44afb5da..13381e227a46d 100644 --- a/pkg/util/kubernetes/auth.go +++ b/pkg/util/kubernetes/auth.go @@ -6,54 +6,22 @@ package kubernetes import ( - "crypto/tls" - "crypto/x509" - "fmt" "io/ioutil" - "os" + + log "github.com/cihub/seelog" ) -// Kubernetes constants +// Kubelet constants const ( - ServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount" - ServiceAccountTokenPath = ServiceAccountPath + "/token" + AuthTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) -// IsServiceAccountTokenAvailable returns if a service account token is available on disk -func IsServiceAccountTokenAvailable() bool { - _, err := os.Stat(ServiceAccountTokenPath) - return err == nil -} - -// GetBearerToken reads the serviceaccount token -func GetBearerToken(authTokenPath string) (string, error) { - token, err := ioutil.ReadFile(authTokenPath) - if err != nil { - return "", fmt.Errorf("could not read token from %s: %s", authTokenPath, err) - } - return fmt.Sprintf("bearer %s", token), nil -} - -// GetCertificates loads the certificate and the private key -func GetCertificates(certFilePath, keyFilePath string) ([]tls.Certificate, error) { - var certs []tls.Certificate - cert, err := tls.LoadX509KeyPair(certFilePath, keyFilePath) - if err != nil { - return certs, err - } - return append(certs, cert), nil -} - -// GetCertificateAuthority loads the issuing certificate authority -func GetCertificateAuthority(certPath string) (*x509.CertPool, error) { - caCert, err := ioutil.ReadFile(certPath) +// GetAuthToken reads the serviceaccount token +func GetAuthToken() string { + token, err := ioutil.ReadFile(AuthTokenPath) if err != nil { - return nil, err - } - caCertPool := x509.NewCertPool() - ok := caCertPool.AppendCertsFromPEM(caCert) - if ok == false { - return caCertPool, fmt.Errorf("fail to load certificate authority: %s", certPath) + log.Errorf("Could not read token from %s: %s", AuthTokenPath, err) + return "" } - return caCertPool, nil + return string(token) } diff --git a/pkg/util/kubernetes/kubelet/init.go b/pkg/util/kubernetes/kubelet/init.go deleted file mode 100644 index a10911dfc54ab..0000000000000 --- a/pkg/util/kubernetes/kubelet/init.go +++ /dev/null @@ -1,51 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -// +build kubelet - -package kubelet - -import ( - "crypto/tls" - - log "github.com/cihub/seelog" - - "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes" -) - -func isCertificatesConfigured() bool { - return config.Datadog.GetString("kubelet_client_crt") != "" && config.Datadog.GetString("kubelet_client_key") != "" -} - -func isTokenPathConfigured() bool { - return config.Datadog.GetString("kubelet_auth_token_path") != "" -} - -func isConfiguredTLSVerify() bool { - return config.Datadog.GetBool("kubelet_tls_verify") -} - -func getTLSConfig() (*tls.Config, error) { - tlsConfig := &tls.Config{} - if isConfiguredTLSVerify() == false { - tlsConfig.InsecureSkipVerify = true - return tlsConfig, nil - } - - certPath := config.Datadog.GetString("kubelet_client_ca") - if certPath == "" { - log.Debugf("kubelet_client_ca isn't configured: certificate authority must be trusted") - return nil, nil - } - - caPool, err := kubernetes.GetCertificateAuthority(certPath) - if err != nil { - return tlsConfig, err - } - tlsConfig.RootCAs = caPool - tlsConfig.BuildNameToCertificate() - return tlsConfig, nil -} diff --git a/pkg/util/kubernetes/kubelet/kubelet.go b/pkg/util/kubernetes/kubelet/kubelet.go index c55dd74e6106b..f7feff8bf8bf2 100644 --- a/pkg/util/kubernetes/kubelet/kubelet.go +++ b/pkg/util/kubernetes/kubelet/kubelet.go @@ -8,13 +8,12 @@ package kubelet import ( - "crypto/tls" "encoding/json" "errors" "fmt" "io/ioutil" "net/http" - "net/url" + "strings" "time" log "github.com/cihub/seelog" @@ -25,9 +24,9 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/retry" ) +// Kubelet constants const ( - kubeletPodPath = "/pods" - authorizationHeaderKey = "Authorization" + KubeletHealthPath = "/healthz" ) var globalKubeUtil *KubeUtil @@ -35,55 +34,45 @@ var globalKubeUtil *KubeUtil // KubeUtil is a struct to hold the kubelet api url // Instantiate with GetKubeUtil type KubeUtil struct { - // used to setup the KubeUtil - initRetry retry.Retrier - - kubeletHost string // resolved hostname or IPAddress - kubeletApiEndpoint string // ${SCHEME}://${kubeletHost}:${PORT} - kubeletApiClient *http.Client - kubeletApiRequestHeaders *http.Header -} - -// ResetGlobalKubeUtil is a helper to remove the current KubeUtil global -// It is ONLY to be used for tests -func ResetGlobalKubeUtil() { - globalKubeUtil = nil -} - -func newKubeUtil() *KubeUtil { - ku := &KubeUtil{ - kubeletApiClient: &http.Client{Timeout: time.Second}, - kubeletApiRequestHeaders: &http.Header{}, - } - return ku + retry.Retrier + kubeletAPIURL string } // GetKubeUtil returns an instance of KubeUtil. func GetKubeUtil() (*KubeUtil, error) { if globalKubeUtil == nil { - globalKubeUtil = newKubeUtil() - globalKubeUtil.initRetry.SetupRetrier(&retry.Config{ + globalKubeUtil = &KubeUtil{} + globalKubeUtil.SetupRetrier(&retry.Config{ Name: "kubeutil", - AttemptMethod: globalKubeUtil.init, + AttemptMethod: globalKubeUtil.locateKubelet, Strategy: retry.RetryCount, RetryCount: 10, RetryDelay: 30 * time.Second, }) } - err := globalKubeUtil.initRetry.TriggerRetry() + err := globalKubeUtil.TriggerRetry() if err != nil { - log.Debugf("Init error: %s", err) + log.Debugf("init error: %s", err) return nil, err } return globalKubeUtil, nil } +func (ku *KubeUtil) locateKubelet() error { + url, err := locateKubelet() + if err == nil { + ku.kubeletAPIURL = url + return nil + } + return err +} + // GetNodeInfo returns the IP address and the hostname of the node where // this pod is running. func (ku *KubeUtil) GetNodeInfo() (ip, name string, err error) { pods, err := ku.GetLocalPodList() if err != nil { - return "", "", fmt.Errorf("error getting pod list from kubelet: %s", err) + return "", "", fmt.Errorf("Error getting pod list from kubelet: %s", err) } for _, pod := range pods { @@ -92,24 +81,22 @@ func (ku *KubeUtil) GetNodeInfo() (ip, name string, err error) { } } - return "", "", fmt.Errorf("failed to get node info") + return "", "", fmt.Errorf("Failed to get node info") } // GetLocalPodList returns the list of pods running on the node where this pod is running func (ku *KubeUtil) GetLocalPodList() ([]*Pod, error) { - data, code, err := ku.QueryKubelet(kubeletPodPath) + + data, err := PerformKubeletQuery(fmt.Sprintf("%s/pods", ku.kubeletAPIURL)) if err != nil { - return nil, fmt.Errorf("error performing kubelet query %s%s: %s", ku.kubeletApiEndpoint, kubeletPodPath, err) - } - if code != http.StatusOK { - return nil, fmt.Errorf("unexpected status code %d on %s%s: %s", code, ku.kubeletApiEndpoint, kubeletPodPath, string(data)) + return nil, fmt.Errorf("Error performing kubelet query: %s", err) } - v := &PodList{} - err = json.Unmarshal(data, v) - if err != nil { - return nil, err + v := new(PodList) + if err := json.Unmarshal(data, v); err != nil { + return nil, fmt.Errorf("Error unmarshalling json: %s", err) } + return v.Items, nil } @@ -138,146 +125,58 @@ func (ku *KubeUtil) searchPodForContainerID(podlist []*Pod, containerID string) return nil, fmt.Errorf("container %s not found in podlist", containerID) } -// setupKubeletApiClient will try to setup the http(s) client to query the kubelet -// with the following settings, in order: -// - Load Certificate Authority if needed -// - HTTPS w/ configured certificates -// - HTTPS w/ configured token -// - HTTPS w/ service account token -// - HTTP (unauthenticated) -func (ku *KubeUtil) setupKubeletApiClient() error { - tlsConfig, err := getTLSConfig() - if err != nil { - return err +// Try and find the hostname to query the kubelet +// TODO: Add TLS verification +func locateKubelet() (string, error) { + host := config.Datadog.GetString("kubernetes_kubelet_host") + if host == "" { + var err error + host, err = docker.HostnameProvider("") + if err != nil { + return "", fmt.Errorf("unable to get hostname from docker, please set the kubernetes_kubelet_host option: %s", err) + } } - transport := &http.Transport{TLSClientConfig: tlsConfig} - ku.kubeletApiClient.Transport = transport - - switch { - case isCertificatesConfigured(): - tlsConfig.Certificates, err = kubernetes.GetCertificates( - config.Datadog.GetString("kubelet_client_crt"), - config.Datadog.GetString("kubelet_client_key"), - ) - return err - case isTokenPathConfigured(): - return ku.setBearerToken(config.Datadog.GetString("kubelet_auth_token_path")) - - case kubernetes.IsServiceAccountTokenAvailable(): - return ku.setBearerToken(kubernetes.ServiceAccountTokenPath) - - default: - // Without Token and without certificates - return nil + port := config.Datadog.GetInt("kubernetes_http_kubelet_port") + url := fmt.Sprintf("http://%s:%d", host, port) + healthzURL := fmt.Sprintf("%s%s", url, KubeletHealthPath) + if _, err := PerformKubeletQuery(healthzURL); err == nil { + return url, nil } -} + log.Debugf("Couldn't query kubelet over HTTP, assuming it's not in no_auth mode.") -func (ku *KubeUtil) setBearerToken(tokenPath string) error { - token, err := kubernetes.GetBearerToken(tokenPath) - if err != nil { - return err + port = config.Datadog.GetInt("kubernetes_https_kubelet_port") + url = fmt.Sprintf("https://%s:%d", host, port) + healthzURL = fmt.Sprintf("%s%s", url, KubeletHealthPath) + if _, err := PerformKubeletQuery(healthzURL); err == nil { + return url, nil } - ku.kubeletApiRequestHeaders.Set("Authorization", token) - return nil -} -// QueryKubelet allows to query the KubeUtil registered kubelet API on the parameter path -// path commonly used are /healthz, /pods, /metrics -// return the content of the response, the response HTTP status code and an error in case of -func (ku *KubeUtil) QueryKubelet(path string) ([]byte, int, error) { - var err error + return "", fmt.Errorf("Could not find a method to connect to kubelet") +} - req := &http.Request{} - req.Header = *ku.kubeletApiRequestHeaders - req.URL, err = url.Parse(fmt.Sprintf("%s%s", ku.kubeletApiEndpoint, path)) +// PerformKubeletQuery performs a GET query against kubelet and return the response body +// Supports token-based auth +// TODO: TLS +func PerformKubeletQuery(url string) ([]byte, error) { + req, err := http.NewRequest("GET", url, nil) if err != nil { - log.Debugf("Fail to create the kubelet request: %s", err) - return nil, 0, err + return nil, fmt.Errorf("Could not create request: %s", err) } - response, err := ku.kubeletApiClient.Do(req) - if err != nil { - log.Debugf("Cannot request %s: %s", req.URL.String(), err) - return nil, 0, err + if strings.HasPrefix(url, "https") { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", kubernetes.GetAuthToken())) } - defer response.Body.Close() - b, err := ioutil.ReadAll(response.Body) + res, err := http.Get(url) if err != nil { - log.Debugf("Fail to read request %s body: %s", req.URL.String(), err) - return nil, 0, err - } - log.Tracef("Successfully connected to %s, status code: %d, body len: %d", req.URL.String(), response.StatusCode, len(b)) - return b, response.StatusCode, nil -} - -// GetKubeletApiEndpoint returns the current endpoint used to perform QueryKubelet -func (ku *KubeUtil) GetKubeletApiEndpoint() string { - return ku.kubeletApiEndpoint -} - -func (ku *KubeUtil) setupKubeletApiEndpoint() error { - // HTTPS - ku.kubeletApiEndpoint = fmt.Sprintf("https://%s:%d", ku.kubeletHost, config.Datadog.GetInt("kubernetes_https_kubelet_port")) - _, code, httpsUrlErr := ku.QueryKubelet(kubeletPodPath) - if httpsUrlErr == nil { - if code == http.StatusOK { - return nil - } - return fmt.Errorf("unexpected status code %d on endpoint %s%s", code, ku.kubeletApiEndpoint, kubeletPodPath) - } - log.Debugf("Cannot query %s%s: %s", ku.kubeletApiEndpoint, kubeletPodPath, httpsUrlErr) - - // We don't want to carry the token in open http communication - ku.kubeletApiRequestHeaders.Del(authorizationHeaderKey) - - // HTTP - ku.kubeletApiEndpoint = fmt.Sprintf("http://%s:%d", ku.kubeletHost, config.Datadog.GetInt("kubernetes_http_kubelet_port")) - _, code, httpUrlErr := ku.QueryKubelet(kubeletPodPath) - if httpUrlErr == nil { - if code == http.StatusOK { - return nil - } - return fmt.Errorf("unexpected status code %d on endpoint %s%s", code, ku.kubeletApiEndpoint, kubeletPodPath) - } - log.Debugf("Cannot query %s%s: %s", ku.kubeletApiEndpoint, kubeletPodPath, httpUrlErr) - - return fmt.Errorf("cannot connect: https: %q, http: %q", httpsUrlErr, httpUrlErr) -} - -func (ku *KubeUtil) init() error { - var err, errHTTPS, errHTTP error - - // setting the kubeletHost - ku.kubeletHost = config.Datadog.GetString("kubernetes_kubelet_host") - if ku.kubeletHost == "" { - ku.kubeletHost, err = docker.HostnameProvider("") - if err != nil { - return fmt.Errorf("unable to get hostname from docker, please set the kubernetes_kubelet_host option: %s", err) - } - } - - // trying connectivity insecurely with a dedicated client - c := http.Client{Timeout: time.Second} - c.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - - // HTTPS first - _, errHTTPS = c.Get(fmt.Sprintf("https://%s:%d/", ku.kubeletHost, config.Datadog.GetInt("kubernetes_https_kubelet_port"))) - if errHTTPS != nil { - log.Debugf("cannot connect: %s", errHTTPS) - // Only try the HTTP if HTTPS failed - _, errHTTP = c.Get(fmt.Sprintf("http://%s:%d/", ku.kubeletHost, config.Datadog.GetInt("kubernetes_http_kubelet_port"))) - } - - if errHTTP != nil { - log.Debugf("cannot connect: %s", errHTTP) - return fmt.Errorf("cannot connect: https: %q, http: %q", errHTTPS, errHTTP) + return nil, fmt.Errorf("Error executing request to %s: %s", url, err) } + defer res.Body.Close() - err = ku.setupKubeletApiClient() + body, err := ioutil.ReadAll(res.Body) if err != nil { - return err + return nil, fmt.Errorf("Error reading response from %s: %s", url, err) } - return ku.setupKubeletApiEndpoint() + return body, nil } diff --git a/pkg/util/kubernetes/kubelet/kubelet_test.go b/pkg/util/kubernetes/kubelet/kubelet_test.go index 33dba7d9d0a99..d388a7fe929df 100644 --- a/pkg/util/kubernetes/kubelet/kubelet_test.go +++ b/pkg/util/kubernetes/kubelet/kubelet_test.go @@ -8,53 +8,40 @@ package kubelet import ( - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "fmt" "io/ioutil" "net/http" "net/http/httptest" "net/url" - "os" "strconv" "testing" "time" log "github.com/cihub/seelog" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/DataDog/datadog-agent/pkg/config" ) -const ( - fakePath = "./testdata/invalidTokenFilePath" -) - // dummyKubelet allows tests to mock a kubelet's responses type dummyKubelet struct { Requests chan *http.Request PodsBody []byte - - testingCertificate string - testingPrivateKey string } func newDummyKubelet(podListJSONPath string) (*dummyKubelet, error) { + var kubelet *dummyKubelet if podListJSONPath == "" { - kubelet := &dummyKubelet{Requests: make(chan *http.Request, 3)} - return kubelet, nil - } - - podList, err := ioutil.ReadFile(podListJSONPath) - if err != nil { - return nil, err - } - kubelet := &dummyKubelet{ - Requests: make(chan *http.Request, 3), - PodsBody: podList, + kubelet = &dummyKubelet{Requests: make(chan *http.Request, 3)} + } else { + podlist, err := ioutil.ReadFile(podListJSONPath) + if err != nil { + return nil, err + } + kubelet = &dummyKubelet{ + Requests: make(chan *http.Request, 3), + PodsBody: podlist, + } } return kubelet, nil } @@ -65,21 +52,20 @@ func (d *dummyKubelet) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/healthz": w.Write([]byte("ok")) - case "/pods": - if d.PodsBody == nil { + if d.PodsBody != nil { + s, err := w.Write(d.PodsBody) + log.Debugf("dummyKubelet wrote %s bytes, err: %s", s, err) + } else { w.WriteHeader(http.StatusInternalServerError) - return } - s, err := w.Write(d.PodsBody) - log.Debugf("dummyKubelet wrote %d bytes, err: %v", s, err) - default: w.WriteHeader(http.StatusNotFound) } } -func (d *dummyKubelet) parsePort(ts *httptest.Server) (*httptest.Server, int, error) { +func (d *dummyKubelet) Start() (*httptest.Server, int, error) { + ts := httptest.NewServer(d) kubeletURL, err := url.Parse(ts.URL) if err != nil { return nil, 0, err @@ -91,95 +77,33 @@ func (d *dummyKubelet) parsePort(ts *httptest.Server) (*httptest.Server, int, er return ts, kubeletPort, nil } -func pemBlockForKey(privateKey interface{}) (*pem.Block, error) { - switch k := privateKey.(type) { - case *rsa.PrivateKey: - return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}, nil - - default: - return nil, fmt.Errorf("unrecognized format for privateKey") - } -} - -func (d *dummyKubelet) StartTLS() (*httptest.Server, int, error) { - ts := httptest.NewTLSServer(d) - cert := ts.TLS.Certificates - if len(ts.TLS.Certificates) != 1 { - return ts, 0, fmt.Errorf("unexpected number of testing certificates: 1 != %d", len(ts.TLS.Certificates)) - } - certOut, err := ioutil.TempFile("", "kubelet-test-cert-") - d.testingCertificate = certOut.Name() - if err != nil { - return ts, 0, err - } - keyOut, err := ioutil.TempFile("", "kubelet-test-key-") - d.testingPrivateKey = keyOut.Name() - if err != nil { - return ts, 0, err - } - for _, c := range cert { - for _, s := range c.Certificate { - pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: s}) - certOut.Close() - } - p, err := pemBlockForKey(c.PrivateKey) - if err != nil { - return ts, 0, err - } - err = pem.Encode(keyOut, p) - if err != nil { - return ts, 0, err - } - } - return d.parsePort(ts) -} - -func (d *dummyKubelet) Start() (*httptest.Server, int, error) { - ts := httptest.NewServer(d) - return d.parsePort(ts) -} - type KubeletTestSuite struct { suite.Suite } // Make sure globalKubeUtil is deleted before each test func (suite *KubeletTestSuite) SetupTest() { - ResetGlobalKubeUtil() - - config.Datadog.Set("kubelet_client_crt", "") - config.Datadog.Set("kubelet_client_key", "") - config.Datadog.Set("kubelet_client_ca", "") - config.Datadog.Set("kubelet_tls_verify", true) - config.Datadog.Set("kubelet_auth_token_path", "") - - config.Datadog.Set("kubernetes_kubelet_host", "") - config.Datadog.Set("kubernetes_http_kubelet_port", 10250) - config.Datadog.Set("kubernetes_https_kubelet_port", 10255) + globalKubeUtil = nil } func (suite *KubeletTestSuite) TestLocateKubeletHTTP() { - kubelet, err := newDummyKubelet("./testdata/podlist_1.6.json") + kubelet, err := newDummyKubelet("") require.Nil(suite.T(), err) ts, kubeletPort, err := kubelet.Start() defer ts.Close() require.Nil(suite.T(), err) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) - config.Datadog.Set("kubernetes_https_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubelet_auth_token_path", "") + config.Datadog.SetDefault("kubernetes_kubelet_host", "localhost") + config.Datadog.SetDefault("kubernetes_http_kubelet_port", kubeletPort) - ku := newKubeUtil() - err = ku.init() + kubeutil, err := GetKubeUtil() require.Nil(suite.T(), err) - require.NotNil(suite.T(), ku) + require.NotNil(suite.T(), kubeutil) select { case r := <-kubelet.Requests: - require.Equal(suite.T(), "GET", r.Method) - require.Equal(suite.T(), "/", r.URL.Path) + require.Equal(suite.T(), r.Method, "GET") + require.Equal(suite.T(), r.URL.Path, "/healthz") case <-time.After(2 * time.Second): require.FailNow(suite.T(), "Timeout on receive channel") } @@ -192,15 +116,13 @@ func (suite *KubeletTestSuite) TestGetLocalPodList() { defer ts.Close() require.Nil(suite.T(), err) - config.Datadog.Set("kubernetes_kubelet_host", "localhost") - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubelet_auth_token_path", "") + config.Datadog.SetDefault("kubernetes_kubelet_host", "localhost") + config.Datadog.SetDefault("kubernetes_http_kubelet_port", kubeletPort) kubeutil, err := GetKubeUtil() require.Nil(suite.T(), err) require.NotNil(suite.T(), kubeutil) - <-kubelet.Requests // Throwing away first / GET + <-kubelet.Requests // Throwing away /healthz GET pods, err := kubeutil.GetLocalPodList() require.Nil(suite.T(), err) @@ -223,15 +145,13 @@ func (suite *KubeletTestSuite) TestGetNodeInfo() { defer ts.Close() require.Nil(suite.T(), err) - config.Datadog.Set("kubernetes_kubelet_host", "localhost") - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubelet_auth_token_path", "") + config.Datadog.SetDefault("kubernetes_kubelet_host", "localhost") + config.Datadog.SetDefault("kubernetes_http_kubelet_port", kubeletPort) kubeutil, err := GetKubeUtil() require.Nil(suite.T(), err) require.NotNil(suite.T(), kubeutil) - <-kubelet.Requests // Throwing away first GET + <-kubelet.Requests // Throwing away /healthz GET ip, name, err := kubeutil.GetNodeInfo() require.Nil(suite.T(), err) @@ -254,13 +174,13 @@ func (suite *KubeletTestSuite) TestGetPodForContainerID() { defer ts.Close() require.Nil(suite.T(), err) - config.Datadog.Set("kubernetes_kubelet_host", "localhost") - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) + config.Datadog.SetDefault("kubernetes_kubelet_host", "localhost") + config.Datadog.SetDefault("kubernetes_http_kubelet_port", kubeletPort) kubeutil, err := GetKubeUtil() require.Nil(suite.T(), err) require.NotNil(suite.T(), kubeutil) - <-kubelet.Requests // Throwing away first GET + <-kubelet.Requests // Throwing away /healthz GET // Empty container ID pod, err := kubeutil.GetPodForContainerID("") @@ -284,153 +204,6 @@ func (suite *KubeletTestSuite) TestGetPodForContainerID() { require.Equal(suite.T(), pod.Metadata.Name, "kube-dns-1829567597-2xtct") } -func (suite *KubeletTestSuite) TestKubeletInitFailOnToken() { - // without token, with certs on HTTPS insecure - k, err := newDummyKubelet("./testdata/podlist_1.6.json") - require.Nil(suite.T(), err) - - s, kubeletPort, err := k.StartTLS() - defer os.Remove(k.testingCertificate) - defer os.Remove(k.testingPrivateKey) - require.Nil(suite.T(), err) - defer s.Close() - - config.Datadog.Set("kubernetes_https_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_auth_token_path", fakePath) - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku := newKubeUtil() - err = ku.init() - expectedErr := fmt.Errorf("could not read token from %s: open %s: no such file or directory", fakePath, fakePath) - assert.Equal(suite.T(), expectedErr, err, fmt.Sprintf("%v", err)) - assert.Equal(suite.T(), 0, len(ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.Certificates)) -} - -func (suite *KubeletTestSuite) TestKubeletInitTokenHttps() { - // with a token, without certs on HTTPS insecure - k, err := newDummyKubelet("./testdata/podlist_1.6.json") - require.Nil(suite.T(), err) - - s, kubeletPort, err := k.StartTLS() - defer os.Remove(k.testingCertificate) - defer os.Remove(k.testingPrivateKey) - require.Nil(suite.T(), err) - defer s.Close() - - config.Datadog.Set("kubernetes_https_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_auth_token_path", "./testdata/fakeBearerToken") - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku := newKubeUtil() - err = ku.init() - require.Nil(suite.T(), err) - <-k.Requests // Throwing away first GET - - assert.Equal(suite.T(), fmt.Sprintf("https://127.0.0.1:%d", kubeletPort), ku.kubeletApiEndpoint) - assert.Equal(suite.T(), "bearer fakeBearerToken", ku.kubeletApiRequestHeaders.Get("Authorization")) - assert.True(suite.T(), ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) - b, code, err := ku.QueryKubelet("/healthz") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "ok", string(b)) - assert.Equal(suite.T(), 200, code) - r := <-k.Requests - assert.Equal(suite.T(), "bearer fakeBearerToken", r.Header.Get(authorizationHeaderKey)) - assert.Equal(suite.T(), 0, len(ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.Certificates)) -} - -func (suite *KubeletTestSuite) TestKubeletInitHttpsCerts() { - // with a token, without certs on HTTPS insecure - k, err := newDummyKubelet("./testdata/podlist_1.6.json") - require.Nil(suite.T(), err) - - s, kubeletPort, err := k.StartTLS() - defer os.Remove(k.testingCertificate) - defer os.Remove(k.testingPrivateKey) - require.Nil(suite.T(), err) - defer s.Close() - - config.Datadog.Set("kubernetes_https_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", true) - config.Datadog.Set("kubelet_client_crt", k.testingCertificate) - config.Datadog.Set("kubelet_client_key", k.testingPrivateKey) - config.Datadog.Set("kubelet_client_ca", k.testingCertificate) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku := newKubeUtil() - err = ku.init() - require.Nil(suite.T(), err) - <-k.Requests // Throwing away first GET - - assert.Equal(suite.T(), fmt.Sprintf("https://127.0.0.1:%d", kubeletPort), ku.kubeletApiEndpoint) - assert.False(suite.T(), ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) - b, code, err := ku.QueryKubelet("/healthz") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "ok", string(b)) - assert.Equal(suite.T(), 200, code) - r := <-k.Requests - assert.Equal(suite.T(), "", r.Header.Get(authorizationHeaderKey)) - clientCerts := ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.Certificates - require.Equal(suite.T(), 1, len(clientCerts)) - assert.Equal(suite.T(), clientCerts, s.TLS.Certificates) -} - -func (suite *KubeletTestSuite) TestKubeletInitTokenHttp() { - // with an unused token, without certs on HTTP - k, err := newDummyKubelet("./testdata/podlist_1.6.json") - require.Nil(suite.T(), err) - - s, kubeletPort, err := k.Start() - require.Nil(suite.T(), err) - defer s.Close() - - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_auth_token_path", "./testdata/unusedBearerToken") - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku := newKubeUtil() - err = ku.init() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), fmt.Sprintf("http://127.0.0.1:%d", kubeletPort), ku.kubeletApiEndpoint) - assert.Equal(suite.T(), "", ku.kubeletApiRequestHeaders.Get(authorizationHeaderKey)) - assert.True(suite.T(), ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) - b, code, err := ku.QueryKubelet("/healthz") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "ok", string(b)) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), 0, len(ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.Certificates)) -} - -func (suite *KubeletTestSuite) TestKubeletInitHttp() { - // without token, without certs on HTTP - k, err := newDummyKubelet("./testdata/podlist_1.6.json") - require.Nil(suite.T(), err) - - s, kubeletPort, err := k.Start() - require.Nil(suite.T(), err) - defer s.Close() - - config.Datadog.Set("kubernetes_http_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku := newKubeUtil() - err = ku.init() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), fmt.Sprintf("http://127.0.0.1:%d", kubeletPort), ku.kubeletApiEndpoint) - assert.Equal(suite.T(), "", ku.kubeletApiRequestHeaders.Get("Authorization")) - assert.True(suite.T(), ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) - b, code, err := ku.QueryKubelet("/healthz") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "ok", string(b)) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), 0, len(ku.kubeletApiClient.Transport.(*http.Transport).TLSClientConfig.Certificates)) -} - func TestKubeletTestSuite(t *testing.T) { suite.Run(t, new(KubeletTestSuite)) } diff --git a/pkg/util/kubernetes/kubelet/podwatcher.go b/pkg/util/kubernetes/kubelet/podwatcher.go index 29e4a7226c9dc..46ea115b50ee5 100644 --- a/pkg/util/kubernetes/kubelet/podwatcher.go +++ b/pkg/util/kubernetes/kubelet/podwatcher.go @@ -38,20 +38,19 @@ func NewPodWatcher(expiryDuration time.Duration) (*PodWatcher, error) { return watcher, nil } -// PullChanges pulls a new podList from the kubelet and returns Pod objects for +// PullChanges pulls a new podlist from the kubelet and returns Pod objects for // new / updated pods. Updated pods will be sent entirely, user must replace // previous info for these pods. func (w *PodWatcher) PullChanges() ([]*Pod, error) { - var podList []*Pod - podList, err := w.kubeUtil.GetLocalPodList() + podlist, err := w.kubeUtil.GetLocalPodList() if err != nil { - return podList, err + return []*Pod{}, err } - return w.computeChanges(podList) + return w.computechanges(podlist) } -// computeChanges is used by PullChanges, split for testing -func (w *PodWatcher) computeChanges(podlist []*Pod) ([]*Pod, error) { +// computechanges is used by PullChanges, split for testing +func (w *PodWatcher) computechanges(podlist []*Pod) ([]*Pod, error) { now := time.Now() var updatedPods []*Pod @@ -70,7 +69,8 @@ func (w *PodWatcher) computeChanges(podlist []*Pod) ([]*Pod, error) { updatedPods = append(updatedPods, pod) } } - log.Debugf("found %d changed pods out of %d", len(updatedPods), len(podlist)) + log.Debugf("found %d changed pods out of %d", + len(updatedPods), len(podlist)) return updatedPods, nil } diff --git a/pkg/util/kubernetes/kubelet/podwatcher_test.go b/pkg/util/kubernetes/kubelet/podwatcher_test.go index 8fe3634340c2b..1a810fe2012b4 100644 --- a/pkg/util/kubernetes/kubelet/podwatcher_test.go +++ b/pkg/util/kubernetes/kubelet/podwatcher_test.go @@ -43,31 +43,31 @@ func (suite *PodwatcherTestSuite) TestPodWatcherComputeChanges() { expiryDuration: 5 * time.Minute, } - changes, err := watcher.computeChanges(threePods) + changes, err := watcher.computechanges(threePods) require.Nil(suite.T(), err) // The second pod is pending with no container require.Len(suite.T(), changes, 2) // Same list should detect no change - changes, err = watcher.computeChanges(threePods) + changes, err = watcher.computechanges(threePods) require.Nil(suite.T(), err) require.Len(suite.T(), changes, 0) // A pod with new containers should be sent - changes, err = watcher.computeChanges(fourthPod) + changes, err = watcher.computechanges(fourthPod) require.Nil(suite.T(), err) require.Len(suite.T(), changes, 1) require.Equal(suite.T(), changes[0].Metadata.UID, fourthPod[0].Metadata.UID) // A new container ID in an existing pod should trigger fourthPod[0].Status.Containers[0].ID = "testNewID" - changes, err = watcher.computeChanges(fourthPod) + changes, err = watcher.computechanges(fourthPod) require.Nil(suite.T(), err) require.Len(suite.T(), changes, 1) require.Equal(suite.T(), changes[0].Metadata.UID, fourthPod[0].Metadata.UID) // Sending the same pod again with no change - changes, err = watcher.computeChanges(fourthPod) + changes, err = watcher.computechanges(fourthPod) require.Nil(suite.T(), err) require.Len(suite.T(), changes, 0) } @@ -85,7 +85,7 @@ func (suite *PodwatcherTestSuite) TestPodWatcherExpireContainers() { expiryDuration: 5 * time.Minute, } - _, err = watcher.computeChanges(sourcePods) + _, err = watcher.computechanges(sourcePods) require.Nil(suite.T(), err) require.Len(suite.T(), watcher.lastSeen, 5) @@ -113,22 +113,21 @@ func (suite *PodwatcherTestSuite) TestPodWatcherExpireContainers() { func (suite *PodwatcherTestSuite) TestPullChanges() { kubelet, err := newDummyKubelet("./testdata/podlist_1.6.json") require.Nil(suite.T(), err) - ts, kubeletPort, err := kubelet.StartTLS() + ts, kubeletPort, err := kubelet.Start() defer ts.Close() require.Nil(suite.T(), err) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - config.Datadog.Set("kubernetes_https_kubelet_port", kubeletPort) - config.Datadog.Set("kubelet_tls_verify", false) + config.Datadog.SetDefault("kubernetes_kubelet_host", "localhost") + config.Datadog.SetDefault("kubernetes_http_kubelet_port", kubeletPort) watcher, err := NewPodWatcher(5 * time.Minute) require.Nil(suite.T(), err) require.NotNil(suite.T(), watcher) - <-kubelet.Requests // Throwing away the first /pods GET + <-kubelet.Requests // Throwing away /healthz GET pods, err := watcher.PullChanges() - require.Nil(suite.T(), err) <-kubelet.Requests // Throwing away /pods GET + require.Nil(suite.T(), err) // The second pod is pending with no container require.Len(suite.T(), pods, 3) } diff --git a/pkg/util/kubernetes/kubelet/testdata/fakeBearerToken b/pkg/util/kubernetes/kubelet/testdata/fakeBearerToken deleted file mode 100644 index d8bf317f93f2a..0000000000000 --- a/pkg/util/kubernetes/kubelet/testdata/fakeBearerToken +++ /dev/null @@ -1 +0,0 @@ -fakeBearerToken \ No newline at end of file diff --git a/pkg/util/kubernetes/kubelet/testdata/unusedBearerToken b/pkg/util/kubernetes/kubelet/testdata/unusedBearerToken deleted file mode 100644 index e743588312653..0000000000000 --- a/pkg/util/kubernetes/kubelet/testdata/unusedBearerToken +++ /dev/null @@ -1 +0,0 @@ -unusedBearerToken \ No newline at end of file diff --git a/releasenotes/notes/kubelet-tls-fa269e49b14554ba.yaml b/releasenotes/notes/kubelet-tls-fa269e49b14554ba.yaml deleted file mode 100644 index 247809d1397c3..0000000000000 --- a/releasenotes/notes/kubelet-tls-fa269e49b14554ba.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -security: - - | - The agent can try to setup the http client to query the kubelet's API with the following settings, in order: - - HTTPS w/ configured certificates - - HTTPS w/ configured token - - HTTPS w/ service account token - - HTTP diff --git a/tasks/agent.py b/tasks/agent.py index 8ab64e8c2da6e..28cefa07a7d62 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -190,7 +190,6 @@ def integration_tests(ctx, install_deps=False, race=False, remote_docker=False): "./test/integration/config_providers/...", "./test/integration/corechecks/...", "./test/integration/listeners/...", - "./test/integration/util/kubelet/...", ] for prefix in prefixes: diff --git a/test/integration/util/kube_apiserver/common.go b/test/integration/util/kube_apiserver/common.go index 78e9284bc9d16..c887b6d3201f8 100644 --- a/test/integration/util/kube_apiserver/common.go +++ b/test/integration/util/kube_apiserver/common.go @@ -3,44 +3,61 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2017 Datadog, Inc. +// +build docker // +build kubeapiserver package kubernetes import ( - apiv1 "github.com/ericchiang/k8s/api/v1" - metav1 "github.com/ericchiang/k8s/apis/meta/v1" + "fmt" - "github.com/DataDog/datadog-agent/test/integration/utils" + apiv1 "github.com/ericchiang/k8s/api/v1" + metav1 "github.com/ericchiang/k8s/apis/meta/v1" + + "github.com/DataDog/datadog-agent/pkg/util/docker" + "github.com/DataDog/datadog-agent/test/integration/utils" ) // initAPIServerCompose returns a ComposeConf ready to launch // with etcd and the apiserver running in the same network // namespace as the current process. func initAPIServerCompose() (*utils.ComposeConf, error) { - compose := &utils.ComposeConf{ - ProjectName: "kube_events", - FilePath: "testdata/apiserver-compose.yaml", - Variables: map[string]string{}, - } - return compose, nil + var networkMode string + du, err := docker.GetDockerUtil() + if err != nil { + return nil, err + } + + // Get container id if containerized and prepare compose + co, err := du.InspectSelf() + if err != nil { + networkMode = "host" + } else { + networkMode = fmt.Sprintf("container:%s", co.ID) + } + compose := &utils.ComposeConf{ + ProjectName: "kube_events", + FilePath: "testdata/apiserver.compose", + Variables: map[string]string{"network_mode": networkMode}, + } + return compose, nil } func createObjectReference(namespace, kind, name string) *apiv1.ObjectReference { - return &apiv1.ObjectReference{ - Namespace: &namespace, - Kind: &kind, - Name: &name, - } + return &apiv1.ObjectReference{ + Namespace: &namespace, + Kind: &kind, + Name: &name, + } } func createEvent(namespace, name, reason string, involvedObject *apiv1.ObjectReference) *apiv1.Event { - return &apiv1.Event{ - Metadata: &metav1.ObjectMeta{ - Namespace: &namespace, - Name: &name, - }, - InvolvedObject: involvedObject, - Reason: &reason, - } + return &apiv1.Event{ + Metadata: &metav1.ObjectMeta{ + Namespace: &namespace, + Name: &name, + }, + InvolvedObject: involvedObject, + Reason: &reason, + } } diff --git a/test/integration/util/kube_apiserver/testdata/apiserver-compose.yaml b/test/integration/util/kube_apiserver/testdata/apiserver.compose similarity index 65% rename from test/integration/util/kube_apiserver/testdata/apiserver-compose.yaml rename to test/integration/util/kube_apiserver/testdata/apiserver.compose index dddd524fe7a63..24c7f5846a5a5 100644 --- a/test/integration/util/kube_apiserver/testdata/apiserver-compose.yaml +++ b/test/integration/util/kube_apiserver/testdata/apiserver.compose @@ -3,35 +3,35 @@ services: etcd: image: "quay.io/coreos/etcd:latest" network_mode: ${network_mode} - environment: - - ETCDCTL_API=3 healthcheck: - test: ["CMD", "etcdctl", "--command-timeout=2s", "--dial-timeout=2s", "--endpoints", "http://127.0.0.1:2379", "endpoint", "health"] - interval: 5s + test: ["CMD", "etcdctl", "--endpoints", "http://127.0.0.1:2379", "cluster-health"] + interval: 1s timeout: 5s retries: 30 apiserver: - image: "gcr.io/google_containers/hyperkube:v1.8.3" - command: "/hyperkube - apiserver + image: "gcr.io/google_containers/kube-apiserver:v1.8.3" + command: "/usr/local/bin/kube-apiserver --apiserver-count=1 + --advertise-address=127.0.0.1 --insecure-bind-address=0.0.0.0 --insecure-port=8080 + --allow-privileged=true --service-cluster-ip-range=192.168.1.1/24 --admission-control=NamespaceLifecycle,LimitRanger,DefaultStorageClass,ResourceQuota + --kubelet-preferred-address-types=InternalIP,LegacyHostIP,ExternalDNS,InternalDNS,Hostname --authorization-mode=AlwaysAllow + --anonymous-auth=false --etcd-servers=http://127.0.0.1:2379" network_mode: ${network_mode} depends_on: etcd: condition: service_healthy healthcheck: - test: ["CMD", "/hyperkube", "kubectl", "get", "cs"] - interval: 5s + test: ["CMD", "sh", "-c", "wget -qO - localhost:8080/healthz | grep ok"] + interval: 1s timeout: 5s retries: 30 - pause: # # This pause container is here to wait until the apiserver diff --git a/test/integration/util/kubelet/common.go b/test/integration/util/kubelet/common.go deleted file mode 100644 index 8f0379283cc94..0000000000000 --- a/test/integration/util/kubelet/common.go +++ /dev/null @@ -1,72 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -// +build kubelet - -package kubernetes - -import ( - "fmt" - "os" - "path" - "time" - - "github.com/DataDog/datadog-agent/test/integration/utils" -) - -const ( - emptyPodList = `{"kind":"PodList","apiVersion":"v1","metadata":{},"items":null} -` -) - -// initInsecureKubelet create a standalone kubelet open to http and https calls -func initInsecureKubelet() (*utils.ComposeConf, error) { - compose := &utils.ComposeConf{ - ProjectName: "insecure_kubelet", - FilePath: "testdata/insecure-kubelet-compose.yaml", - Variables: map[string]string{}, - } - return compose, nil -} - -// initSecureKubelet create an etcd, kube-apiserver and kubelet to open https authNZ calls -// auth parameter allows to switch to secure + authenticated setup -func initSecureKubelet() (*utils.ComposeConf, *utils.CertificatesConfig, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, nil, err - } - - certsConfig := &utils.CertificatesConfig{ - Hosts: "127.0.0.1", - ValidFor: time.Duration(24 * time.Hour), - RsaBits: 1024, - EcdsaCurve: "", - CertFilePath: path.Join(cwd, "testdata/cert.pem"), - KeyFilePath: path.Join(cwd, "testdata/key.pem"), - } - err = utils.GenerateCertificates(certsConfig) - if err != nil { - return nil, nil, err - } - - projectName := "kubelet" - composeFile := "secure-kubelet-compose.yaml" - - compose := &utils.ComposeConf{ - ProjectName: projectName, - FilePath: fmt.Sprintf("testdata/%s", composeFile), - Variables: map[string]string{ - "certpem_path": certsConfig.CertFilePath, - "keypem_path": certsConfig.KeyFilePath, - }, - RemoveRebuildImages: true, - } - // try to remove any old staling resources, especially images - // this is because an old image can contain old certificates, key - // issued from a previous unTearDown build/test - compose.Stop() - return compose, certsConfig, nil -} diff --git a/test/integration/util/kubelet/insecurekubelet_test.go b/test/integration/util/kubelet/insecurekubelet_test.go deleted file mode 100644 index 52ef6399f1072..0000000000000 --- a/test/integration/util/kubelet/insecurekubelet_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -// +build kubelet - -package kubernetes - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet" -) - -type InsecureTestSuite struct { - suite.Suite -} - -// Make sure globalKubeUtil is deleted before each test -func (suite *InsecureTestSuite) SetupTest() { - kubelet.ResetGlobalKubeUtil() -} - -func (suite *InsecureTestSuite) TestHTTP() { - config.Datadog.Set("kubernetes_http_kubelet_port", 10255) - - // Giving 10255 http port to https setting will force an intended https discovery failure - // Then it forces the http usage - config.Datadog.Set("kubernetes_https_kubelet_port", 10255) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku, err := kubelet.GetKubeUtil() - require.Nil(suite.T(), err, fmt.Sprintf("%v", err)) - assert.Equal(suite.T(), "http://127.0.0.1:10255", ku.GetKubeletApiEndpoint()) - b, code, err := ku.QueryKubelet("/healthz") - require.Nil(suite.T(), err, fmt.Sprintf("%v", err)) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), "ok", string(b)) - - b, code, err = ku.QueryKubelet("/pods") - assert.Equal(suite.T(), 200, code) - require.Nil(suite.T(), err) - assert.Equal(suite.T(), emptyPodList, string(b)) - - podList, err := ku.GetLocalPodList() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 0, len(podList)) -} - -func (suite *InsecureTestSuite) TestInsecureHTTPS() { - config.Datadog.Set("kubernetes_http_kubelet_port", 10255) - config.Datadog.Set("kubernetes_https_kubelet_port", 10250) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", false) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku, err := kubelet.GetKubeUtil() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), "https://127.0.0.1:10250", ku.GetKubeletApiEndpoint()) - b, code, err := ku.QueryKubelet("/healthz") - assert.Equal(suite.T(), 200, code) - require.Nil(suite.T(), err) - assert.Equal(suite.T(), "ok", string(b)) - - b, code, err = ku.QueryKubelet("/pods") - assert.Equal(suite.T(), 200, code) - require.Nil(suite.T(), err) - assert.Equal(suite.T(), emptyPodList, string(b)) - - podList, err := ku.GetLocalPodList() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 0, len(podList)) -} - -func TestInsecureKubeletSuite(t *testing.T) { - compose, err := initInsecureKubelet() - require.Nil(t, err) - output, err := compose.Start() - defer compose.Stop() - require.Nil(t, err, string(output)) - - suite.Run(t, new(InsecureTestSuite)) -} diff --git a/test/integration/util/kubelet/securekubelet_test.go b/test/integration/util/kubelet/securekubelet_test.go deleted file mode 100644 index c131c68592757..0000000000000 --- a/test/integration/util/kubelet/securekubelet_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -// +build kubelet - -package kubernetes - -import ( - "fmt" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet" - "github.com/DataDog/datadog-agent/test/integration/utils" -) - -type SecureTestSuite struct { - suite.Suite - certsConfig *utils.CertificatesConfig -} - -// Make sure globalKubeUtil is deleted before each test -func (suite *SecureTestSuite) SetupTest() { - kubelet.ResetGlobalKubeUtil() -} - -// TestSecureHTTPSKubelet with: -// - https -// - tls_verify -// - cacert -func (suite *SecureTestSuite) TestWithTLSCA() { - config.Datadog.Set("kubernetes_https_kubelet_port", 10250) - config.Datadog.Set("kubernetes_http_kubelet_port", 10255) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", true) - config.Datadog.Set("kubelet_client_ca", suite.certsConfig.CertFilePath) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku, err := kubelet.GetKubeUtil() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), "https://127.0.0.1:10250", ku.GetKubeletApiEndpoint()) - b, code, err := ku.QueryKubelet("/healthz") - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), "ok", string(b)) - - b, code, err = ku.QueryKubelet("/pods") - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), emptyPodList, string(b)) - - podList, err := ku.GetLocalPodList() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 0, len(podList)) -} - -// TestSecureUnknownAuthHTTPSKubelet with: -// - https -// - tls_verify -// - WITHOUT cacert (expecting failure) -func (suite *SecureTestSuite) TestTLSWithoutCA() { - config.Datadog.Set("kubernetes_https_kubelet_port", 10250) - config.Datadog.Set("kubernetes_http_kubelet_port", 10255) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", true) - config.Datadog.Set("kubelet_client_ca", "") - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - _, err := kubelet.GetKubeUtil() - require.NotNil(suite.T(), err) - assert.True(suite.T(), strings.Contains(err.Error(), "Get https://127.0.0.1:10250/pods: x509: "), err.Error()) - assert.True(suite.T(), strings.Contains(err.Error(), "Get http://127.0.0.1:10255/pods: dial tcp 127.0.0.1:10255: getsockopt: connection refused"), err.Error()) -} - -// TestTLSWithCACertificate with: -// - https -// - tls_verify -// - cacert -// - certificate -func (suite *SecureTestSuite) TestTLSWithCACertificate() { - config.Datadog.Set("kubernetes_https_kubelet_port", 10250) - config.Datadog.Set("kubernetes_http_kubelet_port", 10255) - config.Datadog.Set("kubelet_auth_token_path", "") - config.Datadog.Set("kubelet_tls_verify", true) - config.Datadog.Set("kubelet_client_crt", suite.certsConfig.CertFilePath) - config.Datadog.Set("kubelet_client_ca", suite.certsConfig.CertFilePath) - config.Datadog.Set("kubernetes_kubelet_host", "127.0.0.1") - - ku, err := kubelet.GetKubeUtil() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), "https://127.0.0.1:10250", ku.GetKubeletApiEndpoint()) - b, code, err := ku.QueryKubelet("/healthz") - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), "ok", string(b)) - - b, code, err = ku.QueryKubelet("/pods") - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 200, code) - assert.Equal(suite.T(), emptyPodList, string(b)) - - podList, err := ku.GetLocalPodList() - require.Nil(suite.T(), err) - assert.Equal(suite.T(), 0, len(podList)) -} - -func TestSecureKubeletSuite(t *testing.T) { - compose, certsConfig, err := initSecureKubelet() - defer os.Remove(certsConfig.CertFilePath) - defer os.Remove(certsConfig.KeyFilePath) - require.Nil(t, err, fmt.Sprintf("%v", err)) - - output, err := compose.Start() - defer compose.Stop() - require.Nil(t, err, string(output)) - - sqt := &SecureTestSuite{ - certsConfig: certsConfig, - } - suite.Run(t, sqt) -} diff --git a/test/integration/util/kubelet/testdata/Dockerfile b/test/integration/util/kubelet/testdata/Dockerfile deleted file mode 100644 index a86ffdb37fa50..0000000000000 --- a/test/integration/util/kubelet/testdata/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM gcr.io/google_containers/hyperkube:v1.8.3 - -COPY cert.pem /etc/secrets/cert.pem -COPY key.pem /etc/secrets/key.pem diff --git a/test/integration/util/kubelet/testdata/insecure-kubelet-compose.yaml b/test/integration/util/kubelet/testdata/insecure-kubelet-compose.yaml deleted file mode 100644 index 197421ab0b18e..0000000000000 --- a/test/integration/util/kubelet/testdata/insecure-kubelet-compose.yaml +++ /dev/null @@ -1,31 +0,0 @@ -version: '2.3' -services: - kubelet: - image: "gcr.io/google_containers/hyperkube:v1.8.3" - command: "/hyperkube - kubelet - --cloud-provider='' - --fail-swap-on=false - --make-iptables-util-chains=false - --hairpin-mode=none - --pod-manifest-path=/opt - " - network_mode: ${network_mode} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: ["CMD", "/bin/ls", "/var/lib/kubelet/pki/kubelet.crt"] - interval: 1s - timeout: 1s - retries: 10 - - pause: - # - # This pause container is here to wait until the apiserver - # is healthy before returning. - # - image: "gcr.io/google_containers/pause" - depends_on: - kubelet: - condition: service_healthy - network_mode: none diff --git a/test/integration/util/kubelet/testdata/secure-kubelet-compose.yaml b/test/integration/util/kubelet/testdata/secure-kubelet-compose.yaml deleted file mode 100644 index d34bf1739939c..0000000000000 --- a/test/integration/util/kubelet/testdata/secure-kubelet-compose.yaml +++ /dev/null @@ -1,36 +0,0 @@ -version: '2.3' -services: - kubelet: - build: ./ - command: "/hyperkube - kubelet - --cloud-provider='' - --hostname-override=localhost - --fail-swap-on=false - --make-iptables-util-chains=false - --hairpin-mode=none - --read-only-port 0 - --client-ca-file=/etc/secrets/cert.pem - --tls-cert-file=/etc/secrets/cert.pem - --tls-private-key-file=/etc/secrets/key.pem - --pod-manifest-path=/opt - " - network_mode: ${network_mode} - healthcheck: - test: ["CMD", "/bin/ls", "/var/lib/kubelet"] - interval: 1s - timeout: 1s - retries: 10 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - pause: - # - # This pause container is here to wait until the apiserver - # is healthy before returning. - # - image: "gcr.io/google_containers/pause" - depends_on: - kubelet: - condition: service_healthy - network_mode: none diff --git a/test/integration/utils/certificates.go b/test/integration/utils/certificates.go deleted file mode 100644 index 9fde05573d6a5..0000000000000 --- a/test/integration/utils/certificates.go +++ /dev/null @@ -1,143 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -package utils - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" - "math/big" - "net" - "os" - "strings" - "time" -) - -type CertificatesConfig struct { - Hosts string - ValidFor time.Duration - RsaBits int - EcdsaCurve string - CertFilePath string - KeyFilePath string -} - -func publicKey(priv interface{}) interface{} { - switch k := priv.(type) { - case *rsa.PrivateKey: - return &k.PublicKey - case *ecdsa.PrivateKey: - return &k.PublicKey - default: - return nil - } -} - -func pemBlockForKey(privateKey interface{}) (*pem.Block, error) { - switch k := privateKey.(type) { - case *rsa.PrivateKey: - return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}, nil - case *ecdsa.PrivateKey: - b, err := x509.MarshalECPrivateKey(k) - if err != nil { - return nil, fmt.Errorf("unable to marshal ECDSA private key: %v", err) - } - return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}, nil - default: - return nil, fmt.Errorf("unrecognized format for privateKey") - } -} - -// GenerateCertificates create a self-signed certificate with a private key according the config in parameter -func GenerateCertificates(config *CertificatesConfig) error { - var privateKey interface{} - var err error - - switch config.EcdsaCurve { - case "": - privateKey, err = rsa.GenerateKey(rand.Reader, config.RsaBits) - case "P224": - privateKey, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader) - case "P256": - privateKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - case "P384": - privateKey, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader) - case "P521": - privateKey, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader) - default: - return fmt.Errorf("unrecognized elliptic curve: %q", config.EcdsaCurve) - } - - if err != nil { - return fmt.Errorf("failed to generate private key: %s", err) - } - - notBefore := time.Now() - notAfter := notBefore.Add(config.ValidFor) - - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - return fmt.Errorf("failed to generate serial number: %s", err) - } - - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"datadog"}, - }, - NotBefore: notBefore, - NotAfter: notAfter, - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - - hosts := strings.Split(config.Hosts, ",") - for _, h := range hosts { - ip := net.ParseIP(h) - if ip != nil { - template.IPAddresses = append(template.IPAddresses, ip) - continue - } - template.DNSNames = append(template.DNSNames, h) - } - - template.IsCA = true - template.KeyUsage = x509.KeyUsageCertSign - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(privateKey), privateKey) - if err != nil { - return fmt.Errorf("failed to create certificate: %s", err) - } - - certOut, err := os.Create(config.CertFilePath) - if err != nil { - return fmt.Errorf("failed to open %s for writing: %s", config.CertFilePath, err) - } - - pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - defer certOut.Close() - - keyOut, err := os.OpenFile(config.KeyFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - return fmt.Errorf("failed to open %s for writing: %s", config.KeyFilePath, err) - } - defer keyOut.Close() - - p, err := pemBlockForKey(privateKey) - if err != nil { - return err - } - pem.Encode(keyOut, p) - return nil -} diff --git a/test/integration/utils/certificates_test.go b/test/integration/utils/certificates_test.go deleted file mode 100644 index eeaf46ebc681e..0000000000000 --- a/test/integration/utils/certificates_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2018 Datadog, Inc. - -package utils - -import ( - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGenerateCertificates(t *testing.T) { - certsConfig := &CertificatesConfig{ - Hosts: "127.0.0.1,localhost", - ValidFor: time.Duration(24 * time.Hour), - RsaBits: 1024, - EcdsaCurve: "", - CertFilePath: "cert.pem", - KeyFilePath: "key.pem", - } - defer os.Remove(certsConfig.CertFilePath) - defer os.Remove(certsConfig.KeyFilePath) - - err := GenerateCertificates(certsConfig) - require.Nil(t, err) - _, err = os.Stat(certsConfig.CertFilePath) - assert.Nil(t, err) - _, err = os.Stat(certsConfig.KeyFilePath) - assert.Nil(t, err) -} diff --git a/test/integration/utils/compose.go b/test/integration/utils/compose.go index 8941d2a277788..3238786a6cc6d 100644 --- a/test/integration/utils/compose.go +++ b/test/integration/utils/compose.go @@ -3,8 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2018 Datadog, Inc. -// +build docker - package utils import ( @@ -12,67 +10,38 @@ import ( "os" "os/exec" "strings" - - "github.com/DataDog/datadog-agent/pkg/util/docker" ) type ComposeConf struct { - ProjectName string - FilePath string - Variables map[string]string - NetworkMode string // will provide $network_mode - RemoveRebuildImages bool + ProjectName string + FilePath string + Variables map[string]string } // Start runs a docker-compose configuration -// All environment variables are propagated to the compose as $variable -// $network_mode is automatically set if empty func (c *ComposeConf) Start() ([]byte, error) { - var err error - - if c.NetworkMode == "" { - c.NetworkMode, err = getNetworkMode() - if err != nil { - return nil, err - } - } - if len(c.Variables) == 0 { - // be sure we have an allocated map - c.Variables = map[string]string{} - } - - c.Variables["network_mode"] = c.NetworkMode - args := []string{ + runCmd := exec.Command( + "docker-compose", "--project-name", c.ProjectName, "--file", c.FilePath, - "up", - "-d", - } - if c.RemoveRebuildImages { - args = append(args, "--build") - } - runCmd := exec.Command("docker-compose", args...) - - customEnv := os.Environ() - for k, v := range c.Variables { - customEnv = append(customEnv, fmt.Sprintf("%s=%s", k, v)) + "up", "-d") + if len(c.Variables) > 0 { + customEnv := os.Environ() + for k, v := range c.Variables { + customEnv = append(customEnv, fmt.Sprintf("%s=%s", k, v)) + } + runCmd.Env = customEnv } - runCmd.Env = customEnv - return runCmd.CombinedOutput() } // Stop stops a running docker-compose configuration func (c *ComposeConf) Stop() ([]byte, error) { - args := []string{ + runCmd := exec.Command( + "docker-compose", "--project-name", c.ProjectName, "--file", c.FilePath, - "down", - } - if c.RemoveRebuildImages { - args = append(args, "--rmi", "all") - } - runCmd := exec.Command("docker-compose", args...) + "down") return runCmd.CombinedOutput() } @@ -98,18 +67,3 @@ func (c *ComposeConf) ListContainers() ([]string, error) { } return containerIDs, nil } - -// getNetworkMode provide a way to feed docker-compose network_mode -func getNetworkMode() (string, error) { - du, err := docker.GetDockerUtil() - if err != nil { - return "", err - } - - // Get container id if containerized - co, err := du.InspectSelf() - if err != nil { - return "host", nil - } - return fmt.Sprintf("container:%s", co.ID), nil -}