Skip to content

Commit

Permalink
Merge pull request #540 from Mirantis/ivan4th/tapfdsource-test
Browse files Browse the repository at this point in the history
Add TapFDSource test
  • Loading branch information
jellonek authored Dec 27, 2017
2 parents 0fbcc6c + 1620d75 commit af7336e
Show file tree
Hide file tree
Showing 10 changed files with 1,020 additions and 394 deletions.
8 changes: 7 additions & 1 deletion cmd/virtlet/virtlet.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (

"github.com/golang/glog"

"github.com/Mirantis/virtlet/pkg/cni"
"github.com/Mirantis/virtlet/pkg/libvirttools"
"github.com/Mirantis/virtlet/pkg/manager"
"github.com/Mirantis/virtlet/pkg/metadata"
Expand Down Expand Up @@ -115,7 +116,12 @@ func runVirtlet() {
}

func runTapManager() {
src, err := tapmanager.NewTapFDSource(*cniPluginsDir, *cniConfigsDir)
cniClient, err := cni.NewClient(*cniPluginsDir, *cniConfigsDir)
if err != nil {
glog.Errorf("Error initializing CNI client: %v", err)
os.Exit(1)
}
src, err := tapmanager.NewTapFDSource(cniClient)
if err != nil {
glog.Errorf("Error creating tap fd source: %v", err)
os.Exit(1)
Expand Down
20 changes: 16 additions & 4 deletions pkg/cni/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,24 @@ import (
"github.com/Mirantis/virtlet/pkg/utils"
)

// CNIClient provides an interface to CNI
type CNIClient interface {
// AddSandboxToNetwork adds a pod sandbox to the CNI network
AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error)
// RemoveSandboxFromNetwork removes a pod sandbox from the CNI network
RemoveSandboxFromNetwork(podId, podName, podNs string) error
// GetDummyNetwork creates a dummy network using CNI plugin.
// It's used for making a dummy gateway for Calico CNI plugin
GetDummyNetwork() (*cnicurrent.Result, string, error)
}

type Client struct {
cniConfig *libcni.CNIConfig
netConfigList *libcni.NetworkConfigList
}

var _ CNIClient = &Client{}

func NewClient(pluginsDir, configsDir string) (*Client, error) {
netConfigList, err := ReadConfiguration(configsDir)
glog.V(3).Infof("CNI config: name: %q type: %q", netConfigList.Plugins[0].Network.Name, netConfigList.Plugins[0].Network.Type)
Expand All @@ -45,8 +58,6 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) {
}, nil
}

func (c *Client) Type() string { return c.netConfigList.Plugins[0].Network.Type }

func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf {
r := &libcni.RuntimeConf{
ContainerID: podId,
Expand All @@ -64,8 +75,7 @@ func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeCon
return r
}

// GetDummyNetwork creates a dummy network using CNI plugin.
// It's used for making a dummy gateway for Calico CNI plugin.
// GetDummyNetwork implements GetDummyNetwork method of CNIClient interface
func (c *Client) GetDummyNetwork() (*cnicurrent.Result, string, error) {
// TODO: virtlet pod restarts should not grab another address for
// the gateway. That's not a big problem usually though
Expand All @@ -82,6 +92,7 @@ func (c *Client) GetDummyNetwork() (*cnicurrent.Result, string, error) {
return r, PodNetNSPath(podId), nil
}

// AddSandboxToNetwork implements AddSandboxToNetwork method of CNIClient interface
func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error) {
rtConf := c.cniRuntimeConf(podId, podName, podNs)
// NOTE: this annotation is only need by CNI Genie
Expand All @@ -106,6 +117,7 @@ func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.
return r, err
}

// RemoveSandboxFromNetwork implements RemoveSandboxFromNetwork method of CNIClient interface
func (c *Client) RemoveSandboxFromNetwork(podId, podName, podNs string) error {
glog.V(3).Infof("RemoveSandboxFromNetwork: podId %q, podName %q, podNs %q", podId, podName, podNs)
err := c.cniConfig.DelNetworkList(c.netConfigList, c.cniRuntimeConf(podId, podName, podNs))
Expand Down
61 changes: 13 additions & 48 deletions pkg/nettools/nettools.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ import (
"path/filepath"
"strconv"
"syscall"
"unsafe"

"github.com/containernetworking/cni/pkg/ns"
cnitypes "github.com/containernetworking/cni/pkg/types"
Expand Down Expand Up @@ -101,29 +100,6 @@ type ContainerNetwork struct {
DhcpNS ns.NetNS
}

func OpenTAP(devName string) (*os.File, error) {
tapFile, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
if err != nil {
return nil, err
}

var req ifReq

// set IFF_NO_PI to not provide packet information
// If flag IFF_NO_PI is not set each frame format is:
// Flags [2 bytes]
// Proto [2 bytes]
// Raw protocol ethernet frame.
// This extra 4-byte header breaks connectivity as in this case kernel truncates initial package
req.Flags = uint16(syscall.IFF_TAP | syscall.IFF_NO_PI | syscall.IFF_ONE_QUEUE)
copy(req.Name[:15], devName)
_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, tapFile.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&req)))
if errno != 0 {
return nil, fmt.Errorf("tuntap IOCTL TUNSETIFF failed, errno %v", errno)
}
return tapFile, nil
}

func makeVethPair(name, peer string, mtu int) (netlink.Link, error) {
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{
Expand Down Expand Up @@ -745,20 +721,9 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
}

tapInterfaceName := fmt.Sprintf(tapInterfaceNameTemplate, i)
tap := &netlink.Tuntap{
LinkAttrs: netlink.LinkAttrs{
Name: tapInterfaceName,
Flags: net.FlagUp,
MTU: link.Attrs().MTU,
},
Mode: netlink.TUNTAP_MODE_TAP,
}
if err := netlink.LinkAdd(tap); err != nil {
return nil, fmt.Errorf("failed to create tap interface: %v", err)
}

if err := netlink.LinkSetUp(tap); err != nil {
return nil, fmt.Errorf("failed to set %q up: %v", tapInterfaceName, err)
tap, err := CreateTAP(tapInterfaceName, link.Attrs().MTU)
if err != nil {
return nil, err
}

containerBridgeName := fmt.Sprintf(containerBridgeNameTemplate, i)
Expand Down Expand Up @@ -871,41 +836,41 @@ func TeardownBridge(bridge netlink.Link, links []netlink.Link) error {
return netlink.LinkSetDown(bridge)
}

// ConfigureLink adds to link ip address and routes based on info.
// ConfigureLink configures a link according to the CNI result
func ConfigureLink(link netlink.Link, info *cnicurrent.Result) error {
linkNo := -1
ifaceNo := -1
linkMAC := link.Attrs().HardwareAddr.String()
for i, iface := range info.Interfaces {
if iface.Mac == linkMAC {
linkNo = i
ifaceNo = i
break
}
}
if linkNo == -1 {
if ifaceNo == -1 {
return fmt.Errorf("can't find link with MAC %q in saved cni result: %s", linkMAC, spew.Sdump(info))
}

for _, addr := range info.IPs {
if addr.Interface == linkNo {
addr := &netlink.Addr{IPNet: &addr.Address}
if err := netlink.AddrAdd(link, addr); err != nil {
return err
if addr.Interface == ifaceNo {
linkAddr := &netlink.Addr{IPNet: &addr.Address}
if err := netlink.AddrAdd(link, linkAddr); err != nil {
return fmt.Errorf("error adding address %v to link %q: %v", addr.Address, link.Attrs().Name, err)
}

for _, route := range info.Routes {
// TODO: that's too naive - if there are more than one interfaces which have this gw address
// in their subnet - same gw will be added on both of them
// in theory this should be ok, but there is can lead to configuration other than prepared
// by cni plugins
if addr.Contains(route.GW) {
if linkAddr.Contains(route.GW) {
err := netlink.RouteAdd(&netlink.Route{
LinkIndex: link.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: &route.Dst,
Gw: route.GW,
})
if err != nil {
return err
return fmt.Errorf("error adding route (dst %v gw %v): %v", route.Dst, route.GW, err)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/nettools/nettools_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ func addTestRoute(t *testing.T, route *netlink.Route) {
func setupLink(hwAddrAsText string, link netlink.Link) netlink.Link {
hwAddr, err := net.ParseMAC(hwAddrAsText)
if err != nil {
log.Panicf("Error parsing hwaddr %q: %v", hwAddr, err)
log.Panicf("Error parsing hwaddr %q: %v", hwAddrAsText, err)
}
if err := SetHardwareAddr(link, hwAddr); err != nil {
log.Panicf("Error setting hardware address: %v", err)
Expand Down
73 changes: 73 additions & 0 deletions pkg/nettools/tap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
Copyright 2017 Mirantis
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nettools

import (
"fmt"
"net"
"os"
"syscall"
"unsafe"

"github.com/vishvananda/netlink"
)

// OpenTAP opens a tap device and returns an os.File for it
func OpenTAP(devName string) (*os.File, error) {
tapFile, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
if err != nil {
return nil, err
}

var req ifReq

// set IFF_NO_PI to not provide packet information
// If flag IFF_NO_PI is not set each frame format is:
// Flags [2 bytes]
// Proto [2 bytes]
// Raw protocol ethernet frame.
// This extra 4-byte header breaks connectivity as in this case kernel truncates initial package
req.Flags = uint16(syscall.IFF_TAP | syscall.IFF_NO_PI | syscall.IFF_ONE_QUEUE)
copy(req.Name[:15], devName)
_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, tapFile.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&req)))
if errno != 0 {
return nil, fmt.Errorf("tuntap IOCTL TUNSETIFF failed, errno %v", errno)
}
return tapFile, nil
}

// CreateTAP sets up a tap link and brings it up
func CreateTAP(devName string, mtu int) (netlink.Link, error) {
tap := &netlink.Tuntap{
LinkAttrs: netlink.LinkAttrs{
Name: devName,
Flags: net.FlagUp,
MTU: mtu,
},
Mode: netlink.TUNTAP_MODE_TAP,
}

if err := netlink.LinkAdd(tap); err != nil {
return nil, fmt.Errorf("failed to create tap interface: %v", err)
}

if err := netlink.LinkSetUp(tap); err != nil {
return nil, fmt.Errorf("failed to set %q up: %v", devName, err)
}

return tap, nil
}
19 changes: 5 additions & 14 deletions pkg/tapmanager/tapfdsource.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,10 @@ import (
"github.com/Mirantis/virtlet/pkg/nettools"
)

// InterfaceType presents type of network interface instance
type InterfaceType int

const (
InterfaceTypeTap InterfaceType = iota
calicoNetType = "calico"
calicoDefaultSubnet = 24
calicoSubnetVar = "VIRTLET_CALICO_SUBNET"
calicoNetType = "calico"
calicoDefaultSubnet = 24
calicoSubnetVar = "VIRTLET_CALICO_SUBNET"
)

// InterfaceDescription contains interface type with additional data
Expand Down Expand Up @@ -92,7 +88,7 @@ type podNetwork struct {
type TapFDSource struct {
sync.Mutex

cniClient *cni.Client
cniClient cni.CNIClient
dummyNetwork *cnicurrent.Result
dummyNetworkNsPath string
fdMap map[string]*podNetwork
Expand All @@ -102,12 +98,7 @@ var _ FDSource = &TapFDSource{}

// NewTapFDSource returns a TapFDSource for the specified CNI plugin &
// config dir
func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) {
cniClient, err := cni.NewClient(cniPluginsDir, cniConfigsDir)
if err != nil {
return nil, err
}

func NewTapFDSource(cniClient cni.CNIClient) (*TapFDSource, error) {
s := &TapFDSource{
cniClient: cniClient,
fdMap: make(map[string]*podNetwork),
Expand Down
12 changes: 7 additions & 5 deletions tests/network/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
This directory contains network tests that make use of network
namespaces and should not be mixed with non-network-namespace-aware
tests.
This directory contains the tests that examine Virtlet networking
by running a DHCP client and sending actual network packets.

For more info, see containernetworking/cni#262, vishvananda/netns#17
etc.
Note that before we switch to Go 1.10, there may be test flakes
because some goroutines may inherit a network namespace from other
goroutines unintentionally. Running at least some of the network tests
separately may help reduce the impact of this problem. For more info,
see containernetworking/cni#262, vishvananda/netns#17, golang/go#20676
Loading

0 comments on commit af7336e

Please sign in to comment.