Add probe based mechanism for kubelet plugin discovery

hack/.golint_failures

@@ -166,6 +166,7 @@ pkg/kubelet/apis/deviceplugin/v1alpha
 pkg/kubelet/apis/deviceplugin/v1beta1
 pkg/kubelet/apis/kubeletconfig
 pkg/kubelet/apis/kubeletconfig/v1beta1
+pkg/kubelet/apis/pluginregistration/v1alpha1
 pkg/kubelet/cadvisor
 pkg/kubelet/cadvisor/testing
 pkg/kubelet/checkpoint
@@ -217,6 +218,9 @@ pkg/kubelet/sysctl
 pkg/kubelet/types
 pkg/kubelet/util
 pkg/kubelet/util/cache
+pkg/kubelet/util/pluginwatcher
+pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1
+pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2
 pkg/kubelet/util/queue
 pkg/kubelet/util/sliceutils
 pkg/kubemark

hack/update-generated-kubelet-plugin-registration-dockerized.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../" && pwd -P)"
+KUBELET_PLUGIN_REGISTRATION_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+KUBELET_EXAMPLE_PLUGIN_V1BETA1="${KUBE_ROOT}/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1/"
+KUBELET_EXAMPLE_PLUGIN_V1BETA2="${KUBE_ROOT}/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2/"
+
+source "${KUBE_ROOT}/hack/lib/protoc.sh"
+kube::protoc::generate_proto ${KUBELET_PLUGIN_REGISTRATION_ROOT}
+kube::protoc::generate_proto ${KUBELET_EXAMPLE_PLUGIN_V1BETA1}
+kube::protoc::generate_proto ${KUBELET_EXAMPLE_PLUGIN_V1BETA2}

hack/update-generated-kubelet-plugin-registration.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+# NOTE: All output from this script needs to be copied back to the calling
+# source tree.  This is managed in kube::build::copy_output in build/common.sh.
+# If the output set is changed update that function.
+
+${KUBE_ROOT}/build/run.sh hack/update-generated-kubelet-plugin-registration-dockerized.sh "$@"

hack/verify-generated-kubelet-plugin-registration.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+ERROR="Kubelet Plugin Registration api is out of date. Please run hack/update-generated-kubelet-plugin-registration.sh"
+KUBELET_PLUGIN_REGISTRATION_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+
+source "${KUBE_ROOT}/hack/lib/protoc.sh"
+kube::golang::setup_env
+
+function cleanup {
+	rm -rf ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/
+}
+
+trap cleanup EXIT
+
+mkdir -p ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp
+cp ${KUBELET_PLUGIN_REGISTRATION_ROOT}/api.pb.go ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/
+
+KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-kubelet-plugin-registration.sh"
+kube::protoc::diff "${KUBELET_PLUGIN_REGISTRATION_ROOT}/api.pb.go" "${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/api.pb.go" ${ERROR}
+echo "Generated Kubelet Plugin Registration api is up to date."

pkg/kubelet/BUILD

@@ -84,6 +84,7 @@ go_library(
         "//pkg/kubelet/util:go_default_library",
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/kubelet/util/manager:go_default_library",
+        "//pkg/kubelet/util/pluginwatcher:go_default_library",
         "//pkg/kubelet/util/queue:go_default_library",
         "//pkg/kubelet/util/sliceutils:go_default_library",
         "//pkg/kubelet/volumemanager:go_default_library",

pkg/kubelet/apis/BUILD

@@ -41,6 +41,7 @@ filegroup(
         "//pkg/kubelet/apis/deviceplugin/v1alpha:all-srcs",
         "//pkg/kubelet/apis/deviceplugin/v1beta1:all-srcs",
         "//pkg/kubelet/apis/kubeletconfig:all-srcs",
+        "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],
     tags = ["automanaged"],

pkg/kubelet/apis/pluginregistration/v1alpha1/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.pb.go",
+        "constants.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1alpha1",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "go_default_library_protos",
+    srcs = ["api.proto"],
+    visibility = ["//visibility:public"],
+)

pkg/kubelet/apis/pluginregistration/v1alpha1/api.proto

@@ -0,0 +1,60 @@
+// To regenerate api.pb.go run hack/update-generated-kubelet-plugin-registration.sh
+syntax = 'proto3';
+
+package pluginregistration;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+option (gogoproto.goproto_getters_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unrecognized_all) = false;
+
+// PluginInfo is the message sent from a plugin to the Kubelet pluginwatcher for plugin registration
+message PluginInfo {
+	// Type of the Plugin. CSIPlugin or DevicePlugin
+	string type = 1;
+	// Plugin name that uniquely identifies the plugin for the given plugin type.
+	// For DevicePlugin, this is the resource name that the plugin manages and
+	// should follow the extended resource name convention.
+	// For CSI, this is the CSI driver registrar name.
+	string name = 2;
+        // Optional endpoint location. If found set by Kubelet component,
+        // Kubelet component will use this endpoint for specific requests.
+        // This allows the plugin to register using one endpoint and possibly use
+        // a different socket for control operations. CSI uses this model to delegate
+        // its registration external from the plugin.
+        string endpoint = 3;
+	// Plugin service API versions the plugin supports.
+	// For DevicePlugin, this maps to the deviceplugin API versions the
+	// plugin supports at the given socket.
+	// The Kubelet component communicating with the plugin should be able
+	// to choose any preferred version from this list, or returns an error
+	// if none of the listed versions is supported.
+	repeated string supported_versions = 4;
+}
+
+// RegistrationStatus is the message sent from Kubelet pluginwatcher to the plugin for notification on registration status
+message RegistrationStatus {
+	// True if plugin gets registered successfully at Kubelet
+	bool plugin_registered  = 1;
+	// Error message in case plugin fails to register, empty string otherwise
+	string error  = 2;
+}
+
+// RegistrationStatusResponse is sent by plugin to kubelet in response to RegistrationStatus RPC
+message RegistrationStatusResponse {
+}
+
+// InfoRequest is the empty request message from Kubelet
+message InfoRequest {
+}
+
+// Registration is the service advertised by the Plugins.
+service Registration {
+	rpc GetInfo(InfoRequest) returns (PluginInfo) {}
+	rpc NotifyRegistrationStatus(RegistrationStatus) returns (RegistrationStatusResponse) {}
+}

pkg/kubelet/apis/pluginregistration/v1alpha1/constants.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pluginregistration
+
+const (
+	CSIPlugin    = "CSIPlugin"
+	DevicePlugin = "DevicePlugin"
+)

pkg/kubelet/kubelet.go

@@ -93,6 +93,7 @@ import (
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/kubelet/util/manager"
+	"k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher"
 	"k8s.io/kubernetes/pkg/kubelet/util/queue"
 	"k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
 	"k8s.io/kubernetes/pkg/kubelet/volumemanager"
@@ -775,6 +776,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	if err != nil {
 		return nil, err
 	}
+	klet.pluginWatcher = pluginwatcher.NewWatcher(klet.getPluginsDir())
 
 	// If the experimentalMounterPathFlag is set, we do not want to
 	// check node capabilities since the mount path is not the default
@@ -1150,6 +1152,11 @@ type Kubelet struct {
 	// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
 	// This can be useful for debugging volume related issues.
 	keepTerminatedPodVolumes bool // DEPRECATED
+
+	// pluginwatcher is a utility for Kubelet to register different types of node-level plugins
+	// such as device plugins or CSI plugins. It discovers plugins by monitoring inotify events under the
+	// directory returned by kubelet.getPluginsDir()
+	pluginWatcher pluginwatcher.Watcher
 }
 
 func allGlobalUnicastIPs() ([]net.IP, error) {
@@ -1264,6 +1271,11 @@ func (kl *Kubelet) initializeModules() error {
 		}
 	}
 
+	// Start the plugin watcher
+	if err := kl.pluginWatcher.Start(); err != nil {
+		return fmt.Errorf("failed to start Plugin Watcher. err: %v", err)
+	}
+
 	// Start the image manager.
 	kl.imageManager.Start()
 

pkg/kubelet/util/BUILD

@@ -93,9 +93,11 @@ filegroup(
         "//pkg/kubelet/util/format:all-srcs",
         "//pkg/kubelet/util/ioutils:all-srcs",
         "//pkg/kubelet/util/manager:all-srcs",
+        "//pkg/kubelet/util/pluginwatcher:all-srcs",
         "//pkg/kubelet/util/queue:all-srcs",
         "//pkg/kubelet/util/sliceutils:all-srcs",
         "//pkg/kubelet/util/store:all-srcs",
     ],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

pkg/kubelet/util/pluginwatcher/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "example_plugin.go",
+        "plugin_watcher.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher",
+    deps = [
+        "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2:go_default_library",
+        "//pkg/util/filesystem:go_default_library",
+        "//vendor/github.com/fsnotify/fsnotify:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:all-srcs",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["plugin_watcher_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1:go_default_library",
+        "//pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)

pkg/kubelet/util/pluginwatcher/README

@@ -0,0 +1,29 @@
+This folder contains a utility, pluginwatcher, for Kubelet to register
+different types of node-level plugins such as device plugins or CSI plugins.
+It discovers plugins by monitoring inotify events under the directory returned by
+kubelet.getPluginsDir(). Lets refer this directory as PluginsSockDir.
+For any discovered plugin, pluginwatcher issues Registration.GetInfo grpc call
+to get plugin type, name and supported service API versions. For any registered plugin type,
+pluginwatcher calls the registered callback function with the received plugin
+name, supported service API versions, and the full socket path. The Kubelet
+component that receives this callback can acknowledge or reject the plugin
+according to its own logic, and use the socket path to establish its service
+communication with any API version supported by the plugin.
+
+Here are the general rules that Kubelet plugin developers should follow:
+- Run as 'root' user. Currently creating socket under PluginsSockDir, a root owned directory, requires
+  plugin process to be running as 'root'.
+- Implements the Registration service specified in
+  pkg/kubelet/apis/pluginregistration/v*/api.proto.
+- The plugin name sent during Registration.GetInfo grpc should be unique
+  for the given plugin type (CSIPlugin or DevicePlugin).
+- The socket path needs to be unique and doesn't conflict with the path chosen
+  by any other potential plugins. Currently we only support flat fs namespace
+  under PluginsSockDir but will soon support recursive inotify watch for
+  hierarchical socket paths.
+- A plugin should clean up its own socket upon exiting or when a new instance
+  comes up. A plugin should NOT remove any sockets belonging to other plugins.
+- A plugin should make sure it has service ready for any supported service API
+  version listed in the PluginInfo.
+- For an example plugin implementation, take a look at example_plugin.go
+  included in this directory.