Merge pull request #712 from Mirantis/ivan4th/diag

Virtlet diagnostics

cmd/virtlet/virtlet.go

@@ -30,6 +30,7 @@ import (
  "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
  "github.com/Mirantis/virtlet/pkg/cni"
  "github.com/Mirantis/virtlet/pkg/config"
+ "github.com/Mirantis/virtlet/pkg/diag"
  "github.com/Mirantis/virtlet/pkg/libvirttools"
  "github.com/Mirantis/virtlet/pkg/manager"
  "github.com/Mirantis/virtlet/pkg/nsfix"
@@ -39,12 +40,17 @@ import (
 )
 
 const (
- wantTapManagerEnv = "WANT_TAP_MANAGER"
- nodeNameEnv = "KUBE_NODE_NAME"
+ wantTapManagerEnv = "WANT_TAP_MANAGER"
+ nodeNameEnv = "KUBE_NODE_NAME"
+ diagSocket = "/run/virtlet-diag.sock"
+ netnsDiagCommand = `if [ -d /var/run/netns ]; then cd /var/run/netns; for ns in *; do echo "*** ${ns} ***"; ip netns exec "${ns}" ip a; ip netns exec "${ns}" ip r; echo; done; fi`
+ criproxyLogCommand = `nsenter -t 1 -m -u -i journalctl -xe -u criproxy -n 20000 --no-pager || true`
+ qemuLogDir = "/var/log/libvirt/qemu"
 )
 
 var (
  dumpConfig = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
+ dumpDiag = flag.Bool("diag", false, "Dump diagnostics as JSON and exit")
  displayVersion = flag.Bool("version", false, "Display version and exit")
  versionFormat = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
 )
@@ -55,8 +61,8 @@ func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
  return r
 }
 
-func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig) {
- manager := manager.NewVirtletManager(config, nil, clientCfg)
+func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig, diagSet *diag.Set) {
+ manager := manager.NewVirtletManager(config, nil, clientCfg, diagSet)
  if err := manager.Run(); err != nil {
  glog.Errorf("Error: %v", err)
  os.Exit(1)
@@ -106,6 +112,32 @@ func setLogLevel(config *v1.VirtletConfig) {
  })
 }
 
+func runDiagServer() *diag.Set {
+ diagSet := diag.NewDiagSet()
+ diagSet.RegisterDiagSource("ip-a", diag.NewCommandSource("txt", []string{"ip", "a"}))
+ diagSet.RegisterDiagSource("ip-r", diag.NewCommandSource("txt", []string{"ip", "r"}))
+ diagSet.RegisterDiagSource("psaux", diag.NewCommandSource("txt", []string{"ps", "aux"}))
+ diagSet.RegisterDiagSource("netns", diag.NewCommandSource("txt", []string{"/bin/bash", "-c", netnsDiagCommand}))
+ diagSet.RegisterDiagSource("criproxy", diag.NewCommandSource("log", []string{"/bin/bash", "-c", criproxyLogCommand}))
+ diagSet.RegisterDiagSource("libvirt-logs", diag.NewLogDirSource(qemuLogDir))
+ diagSet.RegisterDiagSource("stack", diag.StackDumpSource)
+ server := diag.NewServer(diagSet)
+ go func() {
+ err := server.Serve(diagSocket, nil)
+ glog.V(1).Infof("Diag server returned: %v", err)
+ }()
+ return diagSet
+}
+
+func doDiag() {
+ dr, err := diag.RetrieveDiagnostics(diagSocket)
+ if err != nil {
+ glog.Errorf("Failed to retrieve diagnostics: %v", err)
+ os.Exit(1)
+ }
+ os.Stdout.Write(dr.ToJSON())
+}
+
 func main() {
  nsfix.HandleReexec()
  clientCfg := utils.BindFlags(flag.CommandLine)
@@ -131,9 +163,12 @@ func main() {
  glog.Errorf("Error writing config: %v", err)
  os.Exit(1)
  }
+ case *dumpDiag:
+ doDiag()
  default:
  localConfig = configWithDefaults(localConfig)
  go runTapManager(localConfig)
- runVirtlet(localConfig, clientCfg)
+ diagSet := runDiagServer()
+ runVirtlet(localConfig, clientCfg, diagSet)
  }
 }

cmd/virtletctl/virtletctl.go

@@ -43,14 +43,14 @@ func newRootCmd() *cobra.Command {
  flag.CommandLine.Parse([]string{})
 
  client := tools.NewRealKubeClient(clientCfg)
- cmd.AddCommand(tools.NewDumpMetadataCmd(client))
  cmd.AddCommand(tools.NewVirshCmd(client, os.Stdout))
  cmd.AddCommand(tools.NewSSHCmd(client, os.Stdout, ""))
  cmd.AddCommand(tools.NewVNCCmd(client, os.Stdout, true))
  cmd.AddCommand(tools.NewInstallCmd(cmd, "", ""))
  cmd.AddCommand(tools.NewGenDocCmd(cmd, os.Stdout))
  cmd.AddCommand(tools.NewGenCmd(os.Stdout))
  cmd.AddCommand(tools.NewVersionCommand(client, os.Stdout, nil))
+ cmd.AddCommand(tools.NewDiagCommand(client, os.Stdin, os.Stdout))
 
  for _, c := range cmd.Commands() {
  c.PreRunE = func(*cobra.Command, []string) error {

deploy/data/virtlet-ds.yaml

@@ -198,6 +198,9 @@ spec:
  name: image-name-translations
  - name: pods-log
  mountPath: /var/log/pods
+ # needed for diagnostic purposes
+ - name: libvirt-log
+ mountPath: /var/log/libvirt
  securityContext:
  privileged: true
  readinessProbe:
@@ -275,9 +278,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

docs/README.md

@@ -1,4 +1,4 @@
-This directory holds files containing documentation.
+This directory contains the Virtlet documentation.
 
 * For a basic example of VM pod definition, see [examples/cirros-vm.yaml](../examples/cirros-vm.yaml)
 * [Cloud-init data generation](cloud-init-data-generation.md)
@@ -11,4 +11,5 @@ This directory holds files containing documentation.
  * [Environment variables](environment-variables.md) support
  * [Image Handling](images.md)
  * [Image Name Translation](image-name-translation.md)
+* [Diagnostics](diagnostics.md)
 * [Update notes](update-notes.md)

docs/diagnostics.md

@@ -0,0 +1,97 @@
+# Diagnostics
+
+Virtlet provides a set of
+[virtletctl diag](virtletctl/virtletctl_diag.md) commands that can
+help with troubleshooting. The diagnostics can be invoked either
+directly or by means of a
+[Sonobuoy](https://github.com/heptio/sonobuoy) plugin.
+
+## Direct invocation
+
+The most basic diagnostics command is [virtletctl diag dump](virtletctl/virtletctl_diag_dump.md):
+```
+$ virtletctl diag out/
+$ ls -lR out
+total 0
+drwxr-xr-x 3 user wheel 96 Jul 11 01:56 nodes
+
+out/nodes:
+total 0
+drwxr-xr-x 12 user wheel 384 Jul 11 01:56 kube-node-1
+
+out/nodes/kube-node-1:
+total 5352
+-rwxr-xr-x 1 user wheel 1276000 Jul 11 01:56 criproxy.log
+-rwxr-xr-x 1 user wheel 1787 Jul 11 01:56 ip-a.txt
+-rwxr-xr-x 1 user wheel 322 Jul 11 01:56 ip-r.txt
+drwxr-xr-x 3 user wheel 96 Jul 11 01:56 libvirt-logs
+drwxr-xr-x 5 user wheel 160 Jul 11 01:56 libvirt-xml
+-rwxr-xr-x 1 user wheel 9964 Jul 11 01:56 metadata.txt
+-rwxr-xr-x 1 user wheel 1443 Jul 11 01:56 netns.txt
+-rwxr-xr-x 1 user wheel 9217 Jul 11 02:56 psaux.txt
+-rwxr-xr-x 1 user wheel 18214 Jul 11 01:56 stack.log
+-rwxr-xr-x 1 user wheel 64314 Jul 11 01:56 virtlet-pod-libvirt.log
+-rwxr-xr-x 1 user wheel 1349763 Jul 11 01:56 virtlet-pod-virtlet.log
+
+out/nodes/kube-node-1/libvirt-logs:
+total 8
+-rwxr-xr-x 1 user wheel 2172 Jul 11 01:56 virtlet-1b2261ca-7ed6-cirros-vm.log
+
+out/nodes/kube-node-1/libvirt-xml:
+total 24
+-rwxr-xr-x 1 user wheel 3511 Jul 11 01:56 domain-virtlet-1b2261ca-7ed6-cirros-vm.xml
+-rwxr-xr-x 1 user wheel 445 Jul 11 01:56 pool-volumes.xml
+-rwxr-xr-x 1 user wheel 1041 Jul 11 01:56 volume-virtlet_root_1b2261ca-7ed6-58e7-58de-0eef2c9d5320.xml
+```
+
+The following files and directories are produced for each Kubernetes
+node that runs Virtlet:
+* `criproxy.log` - the logs of CRI Proxy's systemd unit
+* `ip-a.txt` - the output of `ip a` on the node
+* `ip-r.txt` - the output of `ip r` on the node
+* `metadata.txt` - the contents of Virtlet's internal metadata db in a text form
+* `netns.txt` - the output of `ip a` and `ip r` for each network
+ namespace that's managed by Virtlet
+* `psaux.txt` - the output of `ps aux` command on the node
+* `stack.log` - the dump of Go stack of Virtlet process
+* `virtlet-pod-libvirt.log` - the log of Virtlet pod's libvirt container
+* `virtlet-pod-virtlet.log` - the log of Virtlet pod's virtlet container
+* `livirt-logs` - a directory with libvirt/QEMU logs for each domain
+* `libvirt-xml` - the dumps of all the domains, storage pools and storage volumes in libvirt
+
+It's also possible to dump Virtlet diagnostics as JSON to stdout using
+`virtletctl diag dump --json`. The JSON file can be subsequently
+unpacked into the aforementioned directory structure using
+[virtletctl diag unpack](virtletctl/virtletctl_diag_unpack.md).
+
+## Sonobuoy
+
+Virtlet diagnostics can be run as a
+[Sonobuoy](https://github.com/heptio/sonobuoy) plugin. Unfortunately,
+right now Sonobuoy's plugin support is
+[somewhat limited](https://github.com/heptio/sonobuoy/issues/405). Because
+of that problem, Sonobuoy run must be done in two phases, first
+generating YAML and then using `virtletctl` to patch it (inject
+Virtlet sonobuoy plugin):
+```
+$ cat sonobuoy.json
+{
+ "plugins": [ { "name": "virtlet" } ]
+}
+$ sonobuoy gen --config sonobuoy.json --e2e-focus nosuchtest |
+ virtletctl diag sonobuoy |
+ kubectl apply -f -
+$ # wait till sonobuoy run is complete
+$ sonobuoy status
+PLUGIN STATUS COUNT
+virtlet complete 1
+
+Sonobuoy has completed. Use `sonobuoy retrieve` to get results.
+$ sonobuoy retrieve
+```
+
+The diagnostics results are placed under `plugins/virtlet/results` and
+can be unpacked using [virtletctl diag unpack](virtletctl/virtletctl_diag_unpack.md):
+```
+$ virtletctl diag unpack out/ <sonobuoy_output_dir/plugins/virtlet/results
+```

docs/virtletctl/virtletctl.md

@@ -40,7 +40,7 @@ Kubernetes cluster.
 
 ### SEE ALSO
 
-* [virtletctl dump-metadata](virtletctl_dump-metadata.md) - Dump Virtlet metadata db
+* [virtletctl diag](virtletctl_diag.md) - Virtlet diagnostics
 * [virtletctl gen](virtletctl_gen.md) - Generate Kubernetes YAML for Virtlet deployment
 * [virtletctl gendoc](virtletctl_gendoc.md) - Generate Markdown documentation for the commands
 * [virtletctl install](virtletctl_install.md) - Install virtletctl as a kubectl plugin
@@ -49,4 +49,4 @@ Kubernetes cluster.
 * [virtletctl virsh](virtletctl_virsh.md) - Execute a virsh command
 * [virtletctl vnc](virtletctl_vnc.md) - Provide access to the VNC console of a VM pod
 
-###### Auto generated by spf13/cobra on 11-Jun-2018
+###### Auto generated by spf13/cobra on 11-Jul-2018

docs/virtletctl/virtletctl_diag.md

@@ -0,0 +1,51 @@
+## virtletctl diag
+
+Virtlet diagnostics
+
+### Synopsis
+
+Retrieve and unpack Virtlet diagnostics information
+
+### Options
+
+```
+ -h, --help help for diag
+```
+
+### Options inherited from parent commands
+
+```
+ --alsologtostderr log to standard error as well as files
+ --as string Username to impersonate for the operation
+ --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
+ --certificate-authority string Path to a cert file for the certificate authority
+ --client-certificate string Path to a client certificate file for TLS
+ --client-key string Path to a client key file for TLS
+ --cluster string The name of the kubeconfig cluster to use
+ --context string The name of the kubeconfig context to use
+ --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
+ --kubeconfig string Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0)
+ --log-dir string If non-empty, write log files in this directory
+ --logtostderr log to standard error instead of files
+ -n, --namespace string If present, the namespace scope for this CLI request
+ --password string Password for basic authentication to the API server
+ --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
+ -s, --server string The address and port of the Kubernetes API server
+ --stderrthreshold severity logs at or above this threshold go to stderr (default 2)
+ --token string Bearer token for authentication to the API server
+ --user string The name of the kubeconfig user to use
+ --username string Username for basic authentication to the API server
+ -v, --v Level log level for V logs
+ --virtlet-runtime string the name of virtlet runtime used in kubernetes.io/target-runtime annotation (default "virtlet.cloud")
+ --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
+```
+
+### SEE ALSO
+
+* [virtletctl](virtletctl.md) - Virtlet control tool
+* [virtletctl diag dump](virtletctl_diag_dump.md) - Dump Virtlet diagnostics information
+* [virtletctl diag sonobuoy](virtletctl_diag_sonobuoy.md) - Add Virtlet sonobuoy plugin to the sonobuoy output
+* [virtletctl diag unpack](virtletctl_diag_unpack.md) - Unpack Virtlet diagnostics information
+
+###### Auto generated by spf13/cobra on 11-Jul-2018

docs/virtletctl/virtletctl_diag_dump.md

@@ -0,0 +1,53 @@
+## virtletctl diag dump
+
+Dump Virtlet diagnostics information
+
+### Synopsis
+
+Pull Virtlet diagnostics information from the nodes and dump it as a directory tree or JSON
+
+```
+virtletctl diag dump output_dir [flags]
+```
+
+### Options
+
+```
+ -h, --help help for dump
+ --json Use JSON output
+```
+
+### Options inherited from parent commands
+
+```
+ --alsologtostderr log to standard error as well as files
+ --as string Username to impersonate for the operation
+ --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
+ --certificate-authority string Path to a cert file for the certificate authority
+ --client-certificate string Path to a client certificate file for TLS
+ --client-key string Path to a client key file for TLS
+ --cluster string The name of the kubeconfig cluster to use
+ --context string The name of the kubeconfig context to use
+ --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
+ --kubeconfig string Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0)
+ --log-dir string If non-empty, write log files in this directory
+ --logtostderr log to standard error instead of files
+ -n, --namespace string If present, the namespace scope for this CLI request
+ --password string Password for basic authentication to the API server
+ --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
+ -s, --server string The address and port of the Kubernetes API server
+ --stderrthreshold severity logs at or above this threshold go to stderr (default 2)
+ --token string Bearer token for authentication to the API server
+ --user string The name of the kubeconfig user to use
+ --username string Username for basic authentication to the API server
+ -v, --v Level log level for V logs
+ --virtlet-runtime string the name of virtlet runtime used in kubernetes.io/target-runtime annotation (default "virtlet.cloud")
+ --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
+```
+
+### SEE ALSO
+
+* [virtletctl diag](virtletctl_diag.md) - Virtlet diagnostics
+
+###### Auto generated by spf13/cobra on 11-Jul-2018