Skip to content

Commit

Permalink
Add plan and node-token commands
Browse files Browse the repository at this point in the history 
* plan can be used to generate a bash script to install a HA
cluster - a minimum set of options are included, PRs are welcome
to add anything else you may need
* node-token obtains a node token for k3sup join, which is
required when creating large clusters via automation, so that
there are not too many SSH connections created when it's not
required
* a helper method was added to obtain an SSH connection,
rather than repeating it in each command

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <[email protected]>
  • Loading branch information
@alexellis
alexellis committed Oct 18, 2023
1 parent 3a02867 commit 35e3a3a
Show file tree
Hide file tree
Showing 7 changed files with 433 additions and 115 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,6 @@ kubeconfig
.idea/
mc
/install.sh
/*.json
*.txt
/bootstrap.sh
120 changes: 75 additions & 45 deletions cmd/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/crypto/ssh/terminal"
"golang.org/x/term"
)

var kubeconfig []byte
Expand Down Expand Up @@ -117,6 +117,7 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som
command.Flags().String("tls-san", "", "Use an additional IP or hostname for the API server")

command.PreRunE = func(command *cobra.Command, args []string) error {

local, err := command.Flags().GetBool("local")
if err != nil {
return err
Expand Down Expand Up @@ -286,62 +287,25 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som
return nil
}

port, _ := command.Flags().GetInt("ssh-port")

fmt.Println("Public IP: " + host)

port, _ := command.Flags().GetInt("ssh-port")
user, _ := command.Flags().GetString("user")
sshKey, _ := command.Flags().GetString("ssh-key")

sshKeyPath := expandPath(sshKey)
address := fmt.Sprintf("%s:%d", host, port)

var sshOperator *operator.SSHOperator
var initialSSHErr error
if runtime.GOOS != "windows" {

var sshAgentAuthMethod ssh.AuthMethod
sshAgentAuthMethod, initialSSHErr = sshAgentOnly()
if initialSSHErr == nil {
// Try SSH agent without parsing key files, will succeed if the user
// has already added a key to the SSH Agent, or if using a configured
// smartcard
config := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{sshAgentAuthMethod},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}

sshOperator, initialSSHErr = operator.NewSSHOperator(address, config)
}
} else {
initialSSHErr = errors.New("ssh-agent unsupported on windows")
sshOperator, sshOperatorDone, errored, err := connectOperator(user, address, sshKeyPath)
if errored {
return err
}

// If the initial connection attempt fails fall through to the using
// the supplied/default private key file
if initialSSHErr != nil {
publicKeyFileAuth, closeSSHAgent, err := loadPublickey(sshKeyPath)
if err != nil {
return fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err)
}

defer closeSSHAgent()
if sshOperatorDone != nil {

config := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{publicKeyFileAuth},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}

sshOperator, err = operator.NewSSHOperator(address, config)
if err != nil {
return fmt.Errorf("unable to connect to %s over ssh: %w", address, err)
}
defer sshOperatorDone()
}

defer sshOperator.Close()

if !skipInstall {

if printCommand {
Expand All @@ -360,6 +324,7 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som
if printCommand {
fmt.Printf("ssh: %s\n", getConfigcommand)
}

if err = obtainKubeconfig(sshOperator, getConfigcommand, host, context, localKubeconfig, merge, printConfig); err != nil {
return err
}
Expand All @@ -370,6 +335,71 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som
return command
}

type DoneFunc func()

// connectOperator
//
// Try SSH agent without parsing key files, will succeed if the user
// has already added a key to the SSH Agent, or if using a configured
// smartcard.
//
// If the initial connection attempt fails fall through to the using
// the supplied/default private key file
// DoneFunc should be called by the caller to close the SSH connection when done
func connectOperator(user string, address string, sshKeyPath string) (*operator.SSHOperator, DoneFunc, bool, error) {
var sshOperator *operator.SSHOperator
var initialSSHErr error
var closeSSHAgentFunc func() error

doneFunc := func() {
if sshOperator != nil {
sshOperator.Close()
}
if closeSSHAgentFunc != nil {
closeSSHAgentFunc()
}
}

if runtime.GOOS != "windows" {
var sshAgentAuthMethod ssh.AuthMethod
sshAgentAuthMethod, initialSSHErr = sshAgentOnly()
if initialSSHErr == nil {

config := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{sshAgentAuthMethod},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}

sshOperator, initialSSHErr = operator.NewSSHOperator(address, config)
}
} else {
initialSSHErr = errors.New("ssh-agent unsupported on windows")
}

if initialSSHErr != nil {
publicKeyFileAuth, closeSSHAgent, err := loadPublickey(sshKeyPath)
if err != nil {
return nil, nil, true, fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err)
}

defer closeSSHAgent()

config := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{publicKeyFileAuth},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}

sshOperator, err = operator.NewSSHOperator(address, config)
if err != nil {
return nil, nil, true, fmt.Errorf("unable to connect to %s over ssh: %w", address, err)
}
}

return sshOperator, doneFunc, false, nil
}

func sshAgentOnly() (ssh.AuthMethod, error) {
sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
if err != nil {
Expand Down Expand Up @@ -540,7 +570,7 @@ func loadPublickey(path string) (ssh.AuthMethod, func() error, error) {

fmt.Printf("Enter passphrase for '%s': ", path)
STDIN := int(os.Stdin.Fd())
bytePassword, _ := terminal.ReadPassword(STDIN)
bytePassword, _ := term.ReadPassword(STDIN)

// Ignore any error from reading stdin to retain existing behaviour for unit test in
// install_test.go
Expand Down
112 changes: 43 additions & 69 deletions cmd/join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"net"
"os"
"path"
"runtime"
"strings"
Expand Down Expand Up @@ -65,6 +66,7 @@ func MakeJoin() *cobra.Command {

command.Flags().Bool("server", false, "Join the cluster as a server rather than as an agent for the embedded etcd mode")
command.Flags().Bool("print-command", false, "Print a command that you can use with SSH to manually recover from an error")
command.Flags().String("node-token-path", "", "prefetched token used by nodes to join the cluster")

command.Flags().String("k3s-extra-args", "", "Additional arguments to pass to k3s installer, wrapped in quotes (e.g. --k3s-extra-args '--node-taint key=value:NoExecute')")
command.Flags().String("k3s-version", "", "Set a version to install, overrides k3s-channel")
Expand All @@ -82,6 +84,18 @@ func MakeJoin() *cobra.Command {
return err
}

var nodeToken string

nodeTokenPath, _ := command.Flags().GetString("node-token-path")
if len(nodeTokenPath) > 0 {
data, err := os.ReadFile(nodeTokenPath)
if err != nil {
return err
}

nodeToken = strings.TrimSpace(string(data))
}

host, err := command.Flags().GetString("host")
if err != nil {
return err
Expand All @@ -94,6 +108,7 @@ func MakeJoin() *cobra.Command {
if err != nil {
return err
}

if len(dataDir) == 0 {
return fmt.Errorf("--server-data-dir must be set")
}
Expand Down Expand Up @@ -174,94 +189,53 @@ func MakeJoin() *cobra.Command {
if useSudo {
sudoPrefix = "sudo "
}

sshKeyPath := expandPath(sshKey)
address := fmt.Sprintf("%s:%d", serverHost, serverPort)

var sshOperator *operator.SSHOperator
var initialSSHErr error
if runtime.GOOS != "windows" {

var sshAgentAuthMethod ssh.AuthMethod
sshAgentAuthMethod, initialSSHErr = sshAgentOnly()
if initialSSHErr == nil {
// Try SSH agent without parsing key files, will succeed if the user
// has already added a key to the SSH Agent, or if using a configured
// smartcard
config := &ssh.ClientConfig{
User: serverUser,
Auth: []ssh.AuthMethod{sshAgentAuthMethod},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}

sshOperator, initialSSHErr = operator.NewSSHOperator(address, config)
}
} else {
initialSSHErr = fmt.Errorf("ssh-agent unsupported on windows")
}

// If the initial connection attempt fails fall through to the using
// the supplied/default private key file
var publicKeyFileAuth ssh.AuthMethod
var closeSSHAgent func() error
if initialSSHErr != nil {
var err error
publicKeyFileAuth, closeSSHAgent, err = loadPublickey(sshKeyPath)
if err != nil {
return fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err)
if len(nodeToken) == 0 {
address := fmt.Sprintf("%s:%d", serverHost, serverPort)

sshOperator, sshOperatorDone, errored, err := connectOperator(serverUser, address, sshKeyPath)
if errored {
return err
}

defer closeSSHAgent()
if sshOperatorDone != nil {
defer sshOperatorDone()
}

config := &ssh.ClientConfig{
User: serverUser,
Auth: []ssh.AuthMethod{
publicKeyFileAuth,
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
getTokenCommand := fmt.Sprintf("%scat %s\n", sudoPrefix, path.Join(dataDir, "/server/node-token"))
if printCommand {
fmt.Printf("ssh: %s\n", getTokenCommand)
}

sshOperator, err = operator.NewSSHOperator(address, config)
streamToStdio := false
res, err := sshOperator.ExecuteStdio(getTokenCommand, streamToStdio)

if err != nil {
return fmt.Errorf("unable to connect to (server) %s over ssh: %w", address, err)
return fmt.Errorf("unable to get join-token from server: %w", err)
}
}

defer sshOperator.Close()

getTokenCommand := fmt.Sprintf("%scat %s\n", sudoPrefix, path.Join(dataDir, "/server/node-token"))
if printCommand {
fmt.Printf("ssh: %s\n", getTokenCommand)
}

streamToStdio := false
res, err := sshOperator.ExecuteStdio(getTokenCommand, streamToStdio)

if err != nil {
return fmt.Errorf("unable to get join-token from server: %w", err)
}
if len(res.StdErr) > 0 {
fmt.Printf("Error or warning getting node-token: %s\n", res.StdErr)
} else {
fmt.Printf("Received node-token from %s.. ok.\n", serverHost)
}

if len(res.StdErr) > 0 {
fmt.Printf("Error or warning getting node-token: %s\n", res.StdErr)
} else {
fmt.Printf("Received node-token from %s.. ok.\n", serverHost)
}
// Explicit close of the SSH connection as early as possible
// which complements the defer
if sshOperatorDone != nil {
sshOperatorDone()
}

if closeSSHAgent != nil {
closeSSHAgent()
nodeToken = strings.TrimSpace(string(res.StdOut))
}

sshOperator.Close()

joinToken := strings.TrimSpace(string(res.StdOut))

if server {
tlsSan, _ := command.Flags().GetString("tls-san")

err = setupAdditionalServer(serverHost, host, port, user, sshKeyPath, joinToken, k3sExtraArgs, k3sVersion, k3sChannel, tlsSan, printCommand, serverURL)
err = setupAdditionalServer(serverHost, host, port, user, sshKeyPath, nodeToken, k3sExtraArgs, k3sVersion, k3sChannel, tlsSan, printCommand, serverURL)
} else {
err = setupAgent(serverHost, host, port, user, sshKeyPath, joinToken, k3sExtraArgs, k3sVersion, k3sChannel, printCommand, serverURL)
err = setupAgent(serverHost, host, port, user, sshKeyPath, nodeToken, k3sExtraArgs, k3sVersion, k3sChannel, printCommand, serverURL)
}

if err == nil {
Expand Down
Loading

0 comments on commit 35e3a3a

Please sign in to comment.