Merge pull request #195 from ibuildthecloud/rootless

rootless

pkg/agent/containerd/containerd.go

@@ -15,6 +15,7 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/natefinch/lumberjack"
+	"github.com/opencontainers/runc/libcontainer/system"
 	util2 "github.com/rancher/k3s/pkg/agent/util"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
@@ -27,10 +28,15 @@ const (
 	maxMsgSize = 1024 * 1024 * 16
 	configToml = `
 [plugins.opt]
-	path = "%OPT%"
+  path = "%OPT%"
 [plugins.cri]
   stream_server_address = "%NODE%"
   stream_server_port = "10010"
+`
+	configUserNSToml = `
+  disable_cgroup = true
+  disable_apparmor = true
+  restrict_oom_score_adj = true
 `
 	configCNIToml = `
   [plugins.cri.cni]
@@ -49,6 +55,9 @@ func Run(ctx context.Context, cfg *config.Node) error {
 	}
 
 	template := configToml
+	if system.RunningInUserNS() {
+		template += configUserNSToml
+	}
 	if !cfg.NoFlannel {
 		template += configCNIToml
 	}

pkg/agent/run.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/k3s/pkg/agent/tunnel"
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/daemons/agent"
+	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/rancher/norman/pkg/clientaccess"
 	"github.com/sirupsen/logrus"
 )
@@ -69,6 +70,12 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 		return err
 	}
 
+	if cfg.Rootless {
+		if err := rootless.Rootless(cfg.DataDir); err != nil {
+			return err
+		}
+	}
+
 	cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
 
 	if cfg.ClusterSecret != "" {

pkg/cli/agent/agent.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/rancher/k3s/pkg/agent"
 	"github.com/rancher/k3s/pkg/cli/cmds"
-	"github.com/rancher/norman/pkg/resolvehome"
+	"github.com/rancher/k3s/pkg/datadir"
 	"github.com/rancher/norman/signal"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -57,7 +57,7 @@ func Run(ctx *cli.Context) error {
 
 	logrus.Infof("Starting k3s agent %s", ctx.App.Version)
 
-	dataDir, err := resolvehome.Resolve(cmds.AgentConfig.DataDir)
+	dataDir, err := datadir.LocalHome(cmds.AgentConfig.DataDir, cmds.AgentConfig.Rootless)
 	if err != nil {
 		return err
 	}

pkg/cli/cmds/agent.go

@@ -20,6 +20,7 @@ type Agent struct {
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
 	Debug                    bool
+	Rootless                 bool
 	AgentShared
 	ExtraKubeletArgs   cli.StringSlice
 	ExtraKubeProxyArgs cli.StringSlice
@@ -113,6 +114,11 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 				Destination: &AgentConfig.ClusterSecret,
 				EnvVar:      "K3S_CLUSTER_SECRET",
 			},
+			cli.BoolFlag{
+				Name:        "rootless",
+				Usage:       "(experimental) Run rootless",
+				Destination: &AgentConfig.Rootless,
+			},
 			DockerFlag,
 			FlannelFlag,
 			NodeNameFlag,

pkg/cli/cmds/server.go

@@ -21,6 +21,7 @@ type Server struct {
 	ExtraAPIArgs        cli.StringSlice
 	ExtraSchedulerArgs  cli.StringSlice
 	ExtraControllerArgs cli.StringSlice
+	Rootless            bool
 }
 
 var ServerConfig Server
@@ -124,6 +125,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage: "Customized flag for kube-controller-manager process",
 				Value: &ServerConfig.ExtraControllerArgs,
 			},
+			cli.BoolFlag{
+				Name:        "rootless",
+				Usage:       "(experimental) Run rootless",
+				Destination: &ServerConfig.Rootless,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,

pkg/cli/server/server.go

@@ -15,12 +15,14 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/agent"
 	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/datadir"
+	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/rancher/k3s/pkg/server"
 	"github.com/rancher/norman/signal"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/csi"
 
 	_ "github.com/mattn/go-sqlite3" // ensure we have sqlite
 )
@@ -67,18 +69,30 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 
 	setupLogging(app)
 
-	if !cfg.DisableAgent && os.Getuid() != 0 {
+	if !cfg.DisableAgent && os.Getuid() != 0 && !cfg.Rootless {
 		return fmt.Errorf("must run as root unless --disable-agent is specified")
 	}
 
+	if cfg.Rootless {
+		dataDir, err := datadir.LocalHome(cfg.DataDir, true)
+		if err != nil {
+			return err
+		}
+		cfg.DataDir = dataDir
+		if err := rootless.Rootless(dataDir); err != nil {
+			return err
+		}
+	}
+
 	// If running agent in server, set this so that CSI initializes properly
-	volume.WaitForValidHost = !cfg.DisableAgent
+	csi.WaitForValidHostName = !cfg.DisableAgent
 
 	serverConfig := server.Config{}
 	serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret
 	serverConfig.ControlConfig.DataDir = cfg.DataDir
 	serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
 	serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
+	serverConfig.Rootless = cfg.Rootless
 	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
 	serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)

pkg/daemons/agent/agent.go

@@ -9,12 +9,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/component-base/logs"
 	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
 	"k8s.io/kubernetes/cmd/kubelet/app"
+
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
 	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
 )
@@ -107,8 +109,11 @@ func kubelet(cfg *config.Agent) {
 		argsMap["runtime-cgroups"] = root
 		argsMap["kubelet-cgroups"] = root
 	}
-	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
+	if system.RunningInUserNS() {
+		argsMap["feature-gates"] = "DevicePlugins=false"
+	}
 
+	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
 	command.SetArgs(args)
 
 	go func() {

pkg/datadir/datadir.go

@@ -15,8 +15,12 @@ const (
 )
 
 func Resolve(dataDir string) (string, error) {
+	return LocalHome(dataDir, false)
+}
+
+func LocalHome(dataDir string, forceLocal bool) (string, error) {
 	if dataDir == "" {
-		if os.Getuid() == 0 {
+		if os.Getuid() == 0 && !forceLocal {
 			dataDir = DefaultDataDir
 		} else {
 			dataDir = DefaultHomeDataDir

pkg/kubectl/main.go

@@ -17,7 +17,7 @@ import (
 func Main() {
 	kubenv := os.Getenv("KUBECONFIG")
 	if kubenv == "" {
-		config, err := server.HomeKubeConfig(false)
+		config, err := server.HomeKubeConfig(false, false)
 		if _, serr := os.Stat(config); err == nil && serr == nil {
 			os.Setenv("KUBECONFIG", config)
 		}

pkg/rootless/mounts.go

@@ -0,0 +1,64 @@
+package rootless
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+func setupMounts(stateDir string) error {
+	mountMap := [][]string{
+		{"/run", ""},
+		{"/var/run", ""},
+		{"/var/log", filepath.Join(stateDir, "logs")},
+		{"/var/lib/cni", filepath.Join(stateDir, "cni")},
+	}
+
+	for _, v := range mountMap {
+		if err := setupMount(v[0], v[1]); err != nil {
+			return errors.Wrapf(err, "failed to setup mount %s => %s", v[0], v[1])
+		}
+	}
+
+	return nil
+}
+
+func setupMount(target, dir string) error {
+	toCreate := target
+	for {
+		if toCreate == "/" {
+			return fmt.Errorf("missing /%s on the root filesystem", strings.Split(target, "/")[0])
+		}
+
+		if err := os.MkdirAll(toCreate, 0700); err == nil {
+			break
+		}
+
+		toCreate = filepath.Base(toCreate)
+	}
+
+	logrus.Debug("Mounting none ", toCreate, " tmpfs")
+	if err := unix.Mount("none", toCreate, "tmpfs", 0, ""); err != nil {
+		return errors.Wrapf(err, "failed to mount tmpfs to %s", toCreate)
+	}
+
+	if err := os.MkdirAll(target, 0700); err != nil {
+		return errors.Wrapf(err, "failed to create directory %s")
+	}
+
+	if dir == "" {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return errors.Wrapf(err, "failed to create directory %s")
+	}
+
+	logrus.Debug("Mounting ", dir, target, " none bind")
+	return unix.Mount(dir, target, "none", unix.MS_BIND, "")
+}