-
Notifications
You must be signed in to change notification settings - Fork 121
/
kubectl_wrappers.go
388 lines (327 loc) · 13.2 KB
/
kubectl_wrappers.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
package peirates
// kubectl_wrappers.go contains a bunch of helper functions for executing
// kubectl's codebase as if it were a separate executable. However, kubectl
// IS NOT BEING EXECUTED AS A SEPARATE EXECUTABLE! See the comments on
// kubectlAuthCanI for an example of how this can cause unexpected behavior.
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"strings"
"time"
kubectl "k8s.io/kubectl/pkg/cmd"
)
// runKubectl executes the kubectl library internally, allowing us to use the
// Kubernetes API and requiring no external binaries.
//
// runKubectl takes and io.Reader and two io.Writers, as well as a command to run in cmdArgs.
// The kubectl library will read from the io.Reader, representing stdin, and write its stdout and stderr via the corresponding io.Writers.
//
// runKubectl returns an error string, which indicates internal kubectl errors.
//
// NOTE: You should generally use runKubectlSimple(), which calls runKubectlWithConfig, which calls this.
func runKubectl(stdin io.Reader, stdout, stderr io.Writer, cmdArgs ...string) error {
var err error
// TODO: Can we run this with the KUBECONFIG set to empty?
cmd := exec.Cmd{
Path: "/proc/self/exe",
Args: append([]string{"kubectl"}, cmdArgs...),
Stdin: stdin,
Stdout: stdout,
Stderr: stdout,
}
err = cmd.Start()
if err != nil {
println("[-] Error with command: ", err)
}
// runKubectl has a timeout to deal with kubectl commands running forever.
// However, `kubectl exec` commands may take an arbitrary
// amount of time, so we disable the timeout when `exec` is found in the args.
// We also do the same for `kubectl delete` commands, as they can wait quite a long time.
longRunning := false
for _, arg := range cmdArgs {
if arg == "exec" || arg == "delete" {
longRunning = true
break
}
}
if !longRunning {
// Set up a function to handle the case where we've been running for over 10 seconds
// 10 seconds is an entirely arbitrary timeframe, adjust it if needed.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
// Since this always keeps cmdArgs alive in memory for at least 10 seconds, there is the
// potential for this to lead to excess memory usage if kubectl is run an astronimcal number
// of times within the timeout window. I don't expect this to be an issue, but if it is, I
// recommend a looping <sleeptime> iterations with a 1 second sleep between each iteration,
// allowing the routine to exit earlier when possible.
time.Sleep(10 * time.Second)
select {
case <-ctx.Done():
return
default:
log.Printf(
"\nKubectl took too long! This usually happens because the remote IP is wrong.\n"+
"Check that you've passed the right IP address with -i. If that doesn't help,\n"+
"and you're running in a test environment, try restarting the entire cluster.\n"+
"\n"+
"To help you debug, here are the arguments that were passed to peirates:\n"+
"\t%s\n"+
"\n"+
"And here are the arguments that were passed to the failing kubectl command:\n"+
"\t%s\n",
os.Args,
append([]string{"kubectl"}, cmdArgs...))
err = cmd.Process.Kill()
return
}
}()
}
return cmd.Wait()
}
// runKubectlWithConfig takes a server config and a list of arguments.
// It executes kubectl internally, setting authn secrets, certificate authority, and server based
// on the provided config, then appends the supplied arguments to the end of the command.
//
// NOTE: You should generally use runKubectlSimple() to call this.
func runKubectlWithConfig(cfg ServerInfo, stdin io.Reader, stdout, stderr io.Writer, cmdArgs ...string) error {
// Confirm that we have an API Server URL
if len(cfg.APIServer) == 0 {
return errors.New("api server not set")
}
// Confirm that we have a certificate authority path entry.
if len(cfg.CAPath) == 0 {
println("ERROR: certificate authority path not defined - will not communicate with api server")
return errors.New("certificate authority path not defined - will not communicate with api server")
}
connArgs := []string{
"--certificate-authority=" + cfg.CAPath,
"--server=" + cfg.APIServer,
}
// If cmdArgs contains "--all-namespaces" or ["-n","namespace"], make sure not to add a -n namespace to this.
appendNamespace := true
for _, arg := range cmdArgs {
if (arg == "--all-namespaces") || (arg == "-n") {
appendNamespace = false
}
}
if appendNamespace {
connArgs = append(connArgs, "-n", cfg.Namespace)
}
// If we are using token-based authentication, append that.
if len(cfg.Token) > 0 {
// Append the token to connArgs
connArgs = append(connArgs, "--token="+cfg.Token)
if Verbose {
fmt.Println("DEBUG: using token-based authentication")
}
}
// If we are using cert-based authentication, use that:
if len(cfg.ClientCertData) > 0 {
// TODO: How do we avoid writing temp files on every single kubectl command?
// Even better, can we use whatever library kubectl uses to parse kubeconfig files or just pass the file we found this cert in?
// One challenge - we might not always have access to the same filesystem where we found the cert?
if Verbose {
fmt.Println("DEBUG: using cert-based authentication")
}
// Create a temp file for the client cert
certTmpFile, err := os.CreateTemp("/tmp", "peirates-")
if err != nil {
println("ERROR: Could not create a temp file for the client cert requested")
return errors.New("could not create a temp file for the client cert requested")
}
if Verbose {
println("DEBUG: using cert-based auth with cert located at ", certTmpFile.Name())
}
_, err = io.WriteString(certTmpFile, cfg.ClientCertData)
if err != nil {
println("DEBUG: Could not write to temp file for the client cert requested")
return errors.New("could not write to temp file for the client cert requested")
}
err = certTmpFile.Sync()
if err != nil {
println("[-] Error with cert temp file: ", err)
}
// Create a temp file for the client key
keyTmpFile, err := os.CreateTemp("/tmp", "peirates-")
if err != nil {
println("DEBUG: Could not create a temp file for the client key requested")
return errors.New("could not create a temp file for the client key requested")
}
_, err = io.WriteString(keyTmpFile, cfg.ClientKeyData)
if err != nil {
println("DEBUG: Could not write to temp file for the client key requested")
return errors.New("could not write to temp file for the client key requested")
}
err = keyTmpFile.Sync()
if err != nil {
println("[-] Error with key temp file: ", err)
}
connArgs = append(connArgs, "--client-certificate="+certTmpFile.Name())
connArgs = append(connArgs, "--client-key="+keyTmpFile.Name())
}
if Verbose {
println("DEBUG: Running kubectl with the following arguments: ")
for _, arg := range connArgs {
println("DEBUG: " + arg)
}
}
return runKubectl(stdin, stdout, stderr, append(connArgs, cmdArgs...)...)
}
// runKubectlSimple executes runKubectlWithConfig, but supplies nothing for stdin, and aggregates
// the stdout and stderr streams into byte slices. It returns (stdout, stderr, execution error).
//
// NOTE: This function is what you want to use most of the time, rather than runKubectl() and runKubectlWithConfig().
func runKubectlSimple(cfg ServerInfo, cmdArgs ...string) ([]byte, []byte, error) {
stdin := strings.NewReader("")
stdout := bytes.Buffer{}
stderr := bytes.Buffer{}
err := runKubectlWithConfig(cfg, stdin, &stdout, &stderr, cmdArgs...)
return stdout.Bytes(), stderr.Bytes(), err
}
// Try this kubectl command as every single service account, with option to stop when we find one that works.
func attemptEveryAccount(stopOnFirstSuccess bool, connectionStringPointer *ServerInfo, serviceAccounts *[]ServiceAccount, clientCertificates *[]ClientCertificateKeyPair, logToFile bool, outputFileName string, cmdArgs ...string) ([]byte, []byte, error) {
// Try all service accounts first.
// Store the current service account or client certificate auth method.
// func assignServiceAccountToConnection(account ServiceAccount, info *ServerInfo) {
backupAuthContext := *connectionStringPointer
var successes int
if stopOnFirstSuccess {
println("Trying the command as every service account until we find one that works.")
} else {
println("Trying the command as every service account.")
}
for _, sa := range *serviceAccounts {
println("Trying " + sa.Name)
assignServiceAccountToConnection(sa, connectionStringPointer)
kubectlOutput, stderr, err := runKubectlSimple(*connectionStringPointer, cmdArgs...)
// If the command is successful...
if err == nil {
// ...tally another success...
successes += 1
// ...display the output...
outputToUser(string(kubectlOutput), logToFile, outputFileName)
println(string(stderr))
// ...and stop if we were told to stop on first success.
if stopOnFirstSuccess {
*connectionStringPointer = backupAuthContext
return kubectlOutput, stderr, err
}
}
}
// Now try all client certificates.
// clientCertificates
if stopOnFirstSuccess {
println("Trying the command as every client cert until we find one that works.")
} else {
println("Trying the command as every client cert.")
}
for _, cert := range *clientCertificates {
println("Trying " + cert.Name)
assignAuthenticationCertificateAndKeyToConnection(cert, connectionStringPointer)
kubectlOutput, stderr, err := runKubectlSimple(*connectionStringPointer, cmdArgs...)
// If the command is successful...
if err == nil {
// ...tally another success...
successes += 1
// ...display the output...
outputToUser(string(kubectlOutput), logToFile, outputFileName)
println(string(stderr))
// ...and stop if we were told to stop on first success.
if stopOnFirstSuccess {
*connectionStringPointer = backupAuthContext
return kubectlOutput, stderr, err
}
// This logic is repeated -- can we combine these two for loops?
}
}
// Restore the auth context
*connectionStringPointer = backupAuthContext
// Choose a return
if successes == 0 {
return nil, nil, errors.New("no principals worked")
} else {
fmt.Printf("%d principals were successful in running the command.\n", successes)
return nil, nil, nil
}
}
// runKubectlWithByteSliceForStdin is runKubectlSimple but you can pass in some bytes for stdin. Conven
// This function is unused and thus commented out for now.
// func runKubectlWithByteSliceForStdin(cfg ServerInfo, stdinBytes []byte, cmdArgs ...string) ([]byte, []byte, error) {
// stdin := bytes.NewReader(append(stdinBytes, '\n'))
// stdout := bytes.Buffer{}
// stderr := bytes.Buffer{}
// err := runKubectlWithConfig(cfg, stdin, &stdout, &stderr, cmdArgs...)
// return stdout.Bytes(), stderr.Bytes(), err
// }
// kubectlAuthCanI now has a history... We can't use the built in
// `kubectl auth can-i <args...>`, because when the response to the auth check
// is "no", kubectl exits with exit code 1. This has the unfortunate side
// effect of exiting peirates too, since we aren't running kubectl as a
// subprocess.
//
// The takeaway here is that we have to do it another way. See https://kubernetes.io/docs/reference/access-authn-authz/authorization/#checking-api-access
// for more details.
func kubectlAuthCanI(cfg ServerInfo, verb, resource string) bool {
type SelfSubjectAccessReviewResourceAttributes struct {
Group string `json:"group,omitempty"`
Resource string `json:"resource"`
Verb string `json:"verb"`
Namespace string `json:"namespace,omitempty"`
}
type SelfSubjectAccessReviewSpec struct {
ResourceAttributes SelfSubjectAccessReviewResourceAttributes `json:"resourceAttributes"`
}
type SelfSubjectAccessReviewQuery struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Spec SelfSubjectAccessReviewSpec `json:"spec"`
}
type SelfSubjectAccessReviewResponse struct {
Status struct {
Allowed bool `json:"allowed"`
} `json:"status"`
}
if !UseAuthCanI {
return true
}
// This doesn't work for certificate authentication yet.
if len(cfg.ClientCertData) > 0 {
return true
}
query := SelfSubjectAccessReviewQuery{
APIVersion: "authorization.k8s.io/v1",
Kind: "SelfSubjectAccessReview",
Spec: SelfSubjectAccessReviewSpec{
ResourceAttributes: SelfSubjectAccessReviewResourceAttributes{
Group: "",
Resource: resource,
Verb: verb,
Namespace: cfg.Namespace,
},
},
}
var response SelfSubjectAccessReviewResponse
err := DoKubernetesAPIRequest(cfg, "POST", "apis/authorization.k8s.io/v1/selfsubjectaccessreviews", query, &response)
if err != nil {
fmt.Printf("[-] kubectlAuthCanI failed to perform SelfSubjectAccessReview api requests with error %s: assuming you don't have permissions.\n", err.Error())
return false
}
return response.Status.Allowed
}
// ExecKubectlAndExit runs the internally compiled `kubectl` code as if this was the `kubectl` binary. stdin/stdout/stderr are process streams. args are process args.
func ExecKubectlAndExit() {
// Based on code from https://github.com/kubernetes/kubernetes/blob/2e0e1681a6ca7fe795f3bd5ec8696fb14687b9aa/cmd/kubectl/kubectl.go#L44
cmd := kubectl.NewKubectlCommand(os.Stdin, os.Stdout, os.Stderr)
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
os.Exit(0)
}