Skip to content

Commit

Permalink
Add different wireguard tunnel mode options
Browse files Browse the repository at this point in the history 
Allow users to configure a wireguard backend mode which determines how
the tunnel(s) are setup. The current method (a tunnel per address
family) is kept as a "seperate" mode on top of that three modes are
added that use a single wireguard tunnel overe which traffic for both
network families travels. Specifically the following new modes are
added all using a single tunnel:

* ipv4: Use the remote nodes ipv4 address as wireguard pper
* ipv6: Use the remote nodes ipv6 address as wireguard pper
* auto: automatically determine which remote node address (v4 or v6) to
  use. (default)

The main benefit of these new modes is that all combinations of nodes
only need to have a single viable network path between them to setup the
full mesh. For example if in a dual-stack setup (some) nodes have no viable
ipv4 path the mesh can simply run over ipv6. This also simplifies
firewalling as it only requires a single port to be open. And finally it
avoids having odd issues where node pods can connect over one address family
but not the other due to only one tunnel having connectivity.

Signed-off-by: Sjoerd Simons <[email protected]>
  • Loading branch information
@sjoerdsimons
sjoerdsimons committed Jan 23, 2022
1 parent 875c202 commit b5565a0
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 65 deletions.
7 changes: 7 additions & 0 deletions Documentation/backends.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,13 @@ Type:
* `PSK` (string): Optional. The pre shared key to use. Use `wg genpsk` to generate a key.
* `ListenPort` (int): Optional. The udp port to listen on. Default is `51820`.
* `ListenPortV6` (int): Optional. The udp port to listen on for ipv6. Default is `51821`.
* `Mode` (string): Optional.
* auto - Single wireguard tunnel for both address families; autodetermine the preferred peer address (default)
* ipv4 - Single wireguard tunnel for both address families; use ipv4 for
the peer addresses
* ipv6 - Single wireguard tunnel for both address families; use ipv6 for
the peer addresses
* seperate - Use seperate wireguard tunnels for ipv4 and ipv6
* `PersistentKeepaliveInterval` (int): Optional. Default is 0 (disabled).

If no private key was generated before the private key is written to `/run/flannel/wgkey`. You can use environment `WIREGUARD_KEY_FILE` to change this path.
Expand Down
6 changes: 2 additions & 4 deletions backend/wireguard/device.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func (dev *wgDevice) ConfigureV6(devIP *ip.IP6, flannelnet ip.IP6Net) error {
return nil
}

func (dev *wgDevice) addPeer(publicEndpoint string, peerPublicKeyRaw string, peerSubnet *net.IPNet) error {
func (dev *wgDevice) addPeer(publicEndpoint string, peerPublicKeyRaw string, peerSubnets []net.IPNet) error {
udpEndpoint, err := net.ResolveUDPAddr("udp", publicEndpoint)
if err != nil {
return fmt.Errorf("failed to resolve UDP address: %w", err)
Expand All @@ -279,9 +279,7 @@ func (dev *wgDevice) addPeer(publicEndpoint string, peerPublicKeyRaw string, pee
PersistentKeepaliveInterval: dev.attrs.keepalive,
Endpoint: udpEndpoint,
ReplaceAllowedIPs: true,
AllowedIPs: []net.IPNet{
*peerSubnet,
},
AllowedIPs: peerSubnets,
},
}}

Expand Down
93 changes: 58 additions & 35 deletions backend/wireguard/wireguard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ import (
"golang.org/x/net/context"
)

type Mode string

const (
Seperate Mode = "seperate"
Auto = "auto"
Ipv4 = "ipv4"
Ipv6 = "ipv6"
)

func init() {
backend.Register("wireguard", New)
}
Expand All @@ -46,7 +55,7 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
return be, nil
}

func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, dev, v6Dev *wgDevice, publicKey string) (*subnet.LeaseAttrs, error) {
func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, enableIPv4, enableIPv6 bool, publicKey string) (*subnet.LeaseAttrs, error) {
data, err := json.Marshal(&wireguardLeaseAttrs{
PublicKey: publicKey,
})
Expand All @@ -58,30 +67,51 @@ func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, dev, v6Dev *wgDevice, pu
BackendType: "wireguard",
}

if publicIP != nil && dev != nil {
if publicIP != nil {
leaseAttrs.PublicIP = ip.FromIP(publicIP)
}

if enableIPv4 {
leaseAttrs.BackendData = json.RawMessage(data)
}

if publicIPv6 != nil && v6Dev != nil {
if publicIPv6 != nil {
leaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)
}

if enableIPv6 {
leaseAttrs.BackendV6Data = json.RawMessage(data)
}

return leaseAttrs, nil
}

func createWGDev(ctx context.Context, wg *sync.WaitGroup, name string, psk string, keepalive *time.Duration, listenPort int) (*wgDevice, error) {
devAttrs := wgDeviceAttrs{
keepalive: keepalive,
listenPort: listenPort,
name: name,
}
err := devAttrs.setupKeys(psk)
if err != nil {
return nil, err
}
return newWGDevice(&devAttrs, ctx, wg)
}

func (be *WireguardBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
// Parse out configuration
cfg := struct {
ListenPort int
ListenPortV6 int
PSK string
PersistentKeepaliveInterval time.Duration
Mode Mode
}{
ListenPort: 51820,
ListenPortV6: 51821,
PersistentKeepaliveInterval: 0,
Mode: Auto,
}

if len(config.Backend) > 0 {
Expand All @@ -95,43 +125,32 @@ func (be *WireguardBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGr
var err error
var dev, v6Dev *wgDevice
var publicKey string
if config.EnableIPv4 {
devAttrs := wgDeviceAttrs{
keepalive: &keepalive,
listenPort: cfg.ListenPort,
name: "flannel-wg",
if cfg.Mode == Seperate {
if config.EnableIPv4 {
dev, err = createWGDev(ctx, wg, "flannel-wg", cfg.PSK, &keepalive, cfg.ListenPort)
if err != nil {
return nil, err
}
publicKey = dev.attrs.publicKey.String()
}
err := devAttrs.setupKeys(cfg.PSK)
if err != nil {
return nil, err
if config.EnableIPv6 {
v6Dev, err = createWGDev(ctx, wg, "flannel-wg-v6", cfg.PSK, &keepalive, cfg.ListenPortV6)
if err != nil {
return nil, err
}
publicKey = dev.attrs.publicKey.String()
}
dev, err = newWGDevice(&devAttrs, ctx, wg)
} else if cfg.Mode == Auto || cfg.Mode == Ipv4 || cfg.Mode == Ipv6 {
dev, err = createWGDev(ctx, wg, "flannel-wg", cfg.PSK, &keepalive, cfg.ListenPort)
if err != nil {
return nil, err
}
publicKey = devAttrs.publicKey.String()
publicKey = dev.attrs.publicKey.String()
} else {
return nil, fmt.Errorf("No valid Mode configured")
}

// We create a second network device for IPv6 to ensure the inter-host communication is based on IPv6.
// We are required to do this because wireguard does not allow to have a peer with multiple endpoints.
if config.EnableIPv6 {
v6DevAttrs := wgDeviceAttrs{
keepalive: &keepalive,
listenPort: cfg.ListenPortV6,
name: "flannel-wg-v6",
}
err := v6DevAttrs.setupKeys(cfg.PSK)
if err != nil {
return nil, err
}
v6Dev, err = newWGDevice(&v6DevAttrs, ctx, wg)
if err != nil {
return nil, err
}
publicKey = v6DevAttrs.publicKey.String()
}

subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, dev, v6Dev, publicKey)
subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, config.EnableIPv4, config.EnableIPv6, publicKey)
if err != nil {
return nil, err
}
Expand All @@ -154,11 +173,15 @@ func (be *WireguardBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGr
}

if config.EnableIPv6 {
err = v6Dev.ConfigureV6(lease.IPv6Subnet.IP, config.IPv6Network)
if cfg.Mode == Seperate {
err = v6Dev.ConfigureV6(lease.IPv6Subnet.IP, config.IPv6Network)
} else {
err = dev.ConfigureV6(lease.IPv6Subnet.IP, config.IPv6Network)
}
if err != nil {
return nil, err
}
}

return newNetwork(be.sm, be.extIface, dev, v6Dev, lease)
return newNetwork(be.sm, be.extIface, dev, v6Dev, cfg.Mode, lease)
}
124 changes: 98 additions & 26 deletions backend/wireguard/wireguard_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ package wireguard
import (
"encoding/json"
"fmt"
"net"
"sync"

"github.com/flannel-io/flannel/backend"
"github.com/flannel-io/flannel/pkg/ip"
"github.com/flannel-io/flannel/subnet"
"golang.org/x/net/context"
log "k8s.io/klog"
Expand All @@ -43,15 +45,17 @@ type network struct {
dev *wgDevice
v6Dev *wgDevice
extIface *backend.ExternalInterface
mode Mode
lease *subnet.Lease
sm subnet.Manager
}

func newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface, dev, v6Dev *wgDevice, lease *subnet.Lease) (*network, error) {
func newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface, dev, v6Dev *wgDevice, mode Mode, lease *subnet.Lease) (*network, error) {
n := &network{
dev: dev,
v6Dev: v6Dev,
extIface: extIface,
mode: mode,
lease: lease,
sm: sm,
}
Expand Down Expand Up @@ -95,6 +99,38 @@ type wireguardLeaseAttrs struct {
PublicKey string
}

// Select the endpoint address that is most likely to allow for a successful
// connection.
// If both ipv4 and ipv6 addresses are provided:
// * Prefer ipv4 if the remote endpoint has a public ipv4 address
// and the external iface has an ipv4 address as well. Anything with
// an ipv4 address can likely connect to the public internet.
// * Use ipv6 if the remote endpoint has a publc address and the local
// interface has a public address as well. In which case it's likely that
// a connection can be made. The local interface having just an link-local
// address will only have a small chance of succeeding (ipv6 masquarading is
// very rare)
// * If neither is true default to ipv4 and cross fingers.
func (n *network) selectPublicEndpoint(ip4 *ip.IP4, ip6 *ip.IP6) string {
if ip4 != nil && ip6 == nil {
return ip4.String()
}

if ip4 == nil && ip6 != nil {
return fmt.Sprintf("[%s]", ip6.String())
}

if !ip4.IsPrivate() && n.extIface.ExtAddr != nil {
return ip4.String()
}

if !ip6.IsPrivate() && n.extIface.ExtV6Addr != nil && !ip.FromIP6(n.extIface.ExtV6Addr).IsPrivate() {
return fmt.Sprintf("[%s]", ip6.String())
}

return ip4.String()
}

func (n *network) handleSubnetEvents(batch []subnet.Event) {
for _, event := range batch {
switch event.Type {
Expand All @@ -105,42 +141,74 @@ func (n *network) handleSubnetEvents(batch []subnet.Event) {
continue
}

var wireguardAttrs wireguardLeaseAttrs
if event.Lease.EnableIPv4 && n.dev != nil {
log.Infof("Subnet added: %v via %v", event.Lease.Subnet, event.Lease.Attrs.PublicIP)

var v4wireguardAttrs, v6wireguardAttrs, wireguardAttrs wireguardLeaseAttrs
var subnets []*net.IPNet
if event.Lease.EnableIPv4 {
if len(event.Lease.Attrs.BackendData) > 0 {
if err := json.Unmarshal(event.Lease.Attrs.BackendData, &wireguardAttrs); err != nil {
if err := json.Unmarshal(event.Lease.Attrs.BackendData, &v4wireguardAttrs); err != nil {
log.Errorf("failed to unmarshal BackendData: %w", err)
continue
}
}

publicEndpoint := fmt.Sprintf("%s:%d", event.Lease.Attrs.PublicIP.String(), n.dev.attrs.listenPort)
if err := n.dev.addPeer(
publicEndpoint,
wireguardAttrs.PublicKey,
event.Lease.Subnet.ToIPNet()); err != nil {
log.Errorf("failed to setup ipv4 peer (%s): %v", wireguardAttrs.PublicKey, err)
}
wireguardAttrs = v4wireguardAttrs
subnets = append(subnets, event.Lease.Subnet.ToIPNet())
}

if event.Lease.EnableIPv6 && n.v6Dev != nil {
log.Infof("Subnet added: %v via %v", event.Lease.IPv6Subnet, event.Lease.Attrs.PublicIPv6)

if event.Lease.EnableIPv6 {
if len(event.Lease.Attrs.BackendV6Data) > 0 {
if err := json.Unmarshal(event.Lease.Attrs.BackendV6Data, &wireguardAttrs); err != nil {
if err := json.Unmarshal(event.Lease.Attrs.BackendV6Data, &v6wireguardAttrs); err != nil {
log.Errorf("failed to unmarshal BackendData: %w", err)
continue
}
}
wireguardAttrs = v6wireguardAttrs
subnets = append(subnets, event.Lease.IPv6Subnet.ToIPNet())
}

publicEndpoint := fmt.Sprintf("[%s]:%d", event.Lease.Attrs.PublicIPv6.String(), n.v6Dev.attrs.listenPort)
if err := n.v6Dev.addPeer(
if n.mode == Seperate {
if event.Lease.EnableIPv4 {
publicEndpoint := fmt.Sprintf("%s:%d", event.Lease.Attrs.PublicIP.String(), n.dev.attrs.listenPort)
log.Infof("Subnet added: %v via %v", event.Lease.Subnet, publicEndpoint)
if err := n.dev.addPeer(
publicEndpoint,
v4wireguardAttrs.PublicKey,
[]net.IPNet{*event.Lease.Subnet.ToIPNet()}); err != nil {
log.Errorf("failed to setup ipv4 peer (%s): %v", v4wireguardAttrs.PublicKey, err)
}
}

if event.Lease.EnableIPv6 {
publicEndpoint := fmt.Sprintf("[%s]:%d", event.Lease.Attrs.PublicIPv6.String(), n.v6Dev.attrs.listenPort)
log.Infof("Subnet added: %v via %v", event.Lease.IPv6Subnet, publicEndpoint)
if err := n.v6Dev.addPeer(
publicEndpoint,
v6wireguardAttrs.PublicKey,
[]net.IPNet{*event.Lease.IPv6Subnet.ToIPNet()}); err != nil {
log.Errorf("failed to setup ipv6 peer (%s): %w", v6wireguardAttrs.PublicKey, err)
}
}
} else {
var publicEndpoint string
if n.mode == Ipv4 {
publicEndpoint = fmt.Sprintf("%s:%d", event.Lease.Attrs.PublicIP.String(), n.dev.attrs.listenPort)
} else if n.mode == Ipv6 {
publicEndpoint = fmt.Sprintf("[%s]:%d", event.Lease.Attrs.PublicIPv6.String(), n.dev.attrs.listenPort)
} else { // Auto mode
publicEndpoint = fmt.Sprintf("%s:%d",
n.selectPublicEndpoint(&event.Lease.Attrs.PublicIP, event.Lease.Attrs.PublicIPv6),
n.dev.attrs.listenPort)
}

log.Infof("Subnet(s) added: %v via %v", subnets, publicEndpoint)
var peers []net.IPNet
for _, v := range subnets {
peers = append(peers, *v)
}
if err := n.dev.addPeer(
publicEndpoint,
wireguardAttrs.PublicKey,
event.Lease.IPv6Subnet.ToIPNet()); err != nil {
log.Errorf("failed to setup ipv6 peer (%s): %w", wireguardAttrs.PublicKey, err)
peers); err != nil {
log.Errorf("failed to setup peer (%s): %v", v4wireguardAttrs.PublicKey, err)
}
}

Expand Down Expand Up @@ -168,7 +236,7 @@ func (n *network) handleSubnetEvents(batch []subnet.Event) {
}
}

if event.Lease.EnableIPv6 && n.v6Dev != nil {
if event.Lease.EnableIPv6 {
log.Info("Subnet removed: ", event.Lease.IPv6Subnet)
if len(event.Lease.Attrs.BackendV6Data) > 0 {
if err := json.Unmarshal(event.Lease.Attrs.BackendV6Data, &wireguardAttrs); err != nil {
Expand All @@ -177,9 +245,13 @@ func (n *network) handleSubnetEvents(batch []subnet.Event) {
}
}

if err := n.v6Dev.removePeer(
wireguardAttrs.PublicKey,
); err != nil {
var err error
if n.mode == Seperate && n.v6Dev != nil {
err = n.v6Dev.removePeer(wireguardAttrs.PublicKey)
} else {
err = n.dev.removePeer(wireguardAttrs.PublicKey)
}
if err != nil {
log.Errorf("failed to remove ipv6 peer (%s): %w", wireguardAttrs.PublicKey, err)
}
}
Expand Down

0 comments on commit b5565a0

Please sign in to comment.