Skip to content

Commit 23da327

Browse files
authored
Allow use of direct gateway endpoint as wireguard endpoint (#5378)
Uses direct gateway ip as wireguard endpoint so packets are sent over udp Signed-off-by: Somtochi Onyekwere <somtochionyekwere@gmail.com>
1 parent 16c6d08 commit 23da327

4 files changed

Lines changed: 206 additions & 20 deletions

File tree

‎agent/server/server.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -277,11 +277,13 @@ func (s *server) buildTunnel(ctx context.Context, org *fly.Organization, reestab
277277
return
278278
}
279279

280+
websocketsOnly := env.IsCI() || os.Getenv("WSWG") != "" || s.ConfigWebsockets
281+
280282
var transport string
281283
if s.tokenModeFor(ctx, client) {
282284
transport = "token"
283285

284-
if tunnel, err = s.buildTokenTunnel(ctx, org, network, sessionToks); err != nil {
286+
if tunnel, err = s.buildTokenTunnel(ctx, org, network, sessionToks, !websocketsOnly); err != nil {
285287
s.reportTokenFailure(tk, err, nil)
286288

287289
return
@@ -293,7 +295,7 @@ func (s *server) buildTunnel(ctx context.Context, org *fly.Organization, reestab
293295
}
294296

295297
transport = "udp"
296-
if env.IsCI() || os.Getenv("WSWG") != "" || s.ConfigWebsockets {
298+
if websocketsOnly {
297299
transport = "websocket"
298300

299301
if tunnel, err = wg.ConnectWS(ctx, state); err != nil {
@@ -330,8 +332,9 @@ func (s *server) buildTunnel(ctx context.Context, org *fly.Organization, reestab
330332
// buildTokenTunnel connects to the token gateway with a fresh keypair. The
331333
// gateway allocates the peer inline and keeps it only as long as the token
332334
// verifies, so nothing is registered with the API or written to the config
333-
// file. Callers must hold s.mu.
334-
func (s *server) buildTokenTunnel(ctx context.Context, org *fly.Organization, network string, sessionToks *tokens.Tokens) (*wg.Tunnel, error) {
335+
// file. With udp, WireGuard goes to the gateway's direct IP over UDP when it
336+
// has one, rather than over the websocket. Callers must hold s.mu.
337+
func (s *server) buildTokenTunnel(ctx context.Context, org *fly.Organization, network string, sessionToks *tokens.Tokens, udp bool) (*wg.Tunnel, error) {
335338
if s.tokensFor(sessionToks).MacaroonsOnly().Empty() {
336339
return nil, errors.New("access token is too old, please reauthenticate")
337340
}
@@ -359,7 +362,7 @@ func (s *server) buildTokenTunnel(ctx context.Context, org *fly.Organization, ne
359362
Token: func() string { return s.tokensFor(sessionToks).MacaroonsOnly().FlapsHeader() },
360363
OrgSlug: slug,
361364
NetworkName: network,
362-
})
365+
}, udp)
363366
if err != nil {
364367
return nil, fmt.Errorf("token-mode WireGuard via %s: %w", s.TokenGateway, err)
365368
}

‎wg/token_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package wg
22

33
import (
4+
"context"
45
"encoding/base64"
56
"errors"
67
"net"
8+
"strconv"
79
"testing"
810
"time"
911

@@ -208,3 +210,138 @@ func TestTokenPeerName(t *testing.T) {
208210
assert.Equal(t, "token-01020304", TokenPeerName(base64.StdEncoding.EncodeToString(raw)))
209211
assert.Equal(t, "token-peer", TokenPeerName("not base64!"))
210212
}
213+
214+
// sendingTo returns where the tunnel's WireGuard device currently sends.
215+
func (t *Tunnel) sendingTo() string {
216+
t.mu.RLock()
217+
defer t.mu.RUnlock()
218+
219+
return t.endpointAddr
220+
}
221+
222+
func directUDP(ip string) string {
223+
return net.JoinHostPort(ip, strconv.Itoa(tokenGatewayUDPPort))
224+
}
225+
226+
func TestConnectTokenEndpointChoice(t *testing.T) {
227+
for _, tc := range []struct {
228+
name string
229+
udp bool
230+
direct string
231+
want string // "" means the websocket proxy
232+
}{
233+
{name: "udp with direct ip", udp: true, direct: "192.0.2.10", want: directUDP("192.0.2.10")},
234+
{name: "udp without direct ip", udp: true},
235+
{name: "websockets only", udp: false, direct: "192.0.2.10"},
236+
} {
237+
t.Run(tc.name, func(t *testing.T) {
238+
gw, _ := testKeypair(t)
239+
gateway := newFakeTokenGateway(t, []string{gw}, []tokenResult{
240+
{Type: "ok", PeerIP: "fdaa:0:18:a7b:1:1111:2222:3302", ExpiresAt: time.Now().Add(time.Hour).Unix(), Direct: tc.direct},
241+
})
242+
243+
pub, priv := testKeypair(t)
244+
state := &WireGuardState{Org: "test-org", LocalPublic: pub, LocalPrivate: priv}
245+
246+
tunnel, err := ConnectToken(context.Background(), state, gateway.endpoint(), &TokenAuth{
247+
Token: func() string { return "FlyV1 fm2_test" },
248+
OrgSlug: "test-org",
249+
}, tc.udp)
250+
require.NoError(t, err)
251+
defer tunnel.Close()
252+
253+
<-gateway.ready
254+
255+
want := tc.want
256+
if want == "" {
257+
want = tunnel.wsProxyAddr
258+
}
259+
assert.Equal(t, want, tunnel.sendingTo())
260+
})
261+
}
262+
}
263+
264+
func TestConnectTokenUDPSendsWireGuardToDirectIP(t *testing.T) {
265+
gw, _ := testKeypair(t)
266+
gateway := newFakeTokenGateway(t, []string{gw}, []tokenResult{
267+
{Type: "ok", PeerIP: "fdaa:0:18:a7b:1:1111:2222:3302", DNS: "fdaa:0:18::3", ExpiresAt: time.Now().Add(time.Hour).Unix(), Direct: "127.0.0.1"},
268+
})
269+
270+
pub, priv := testKeypair(t)
271+
state := &WireGuardState{Org: "test-org", LocalPublic: pub, LocalPrivate: priv}
272+
273+
tunnel, err := ConnectToken(context.Background(), state, gateway.endpoint(), &TokenAuth{
274+
Token: func() string { return "FlyV1 fm2_test" },
275+
OrgSlug: "test-org",
276+
}, true)
277+
require.NoError(t, err)
278+
defer tunnel.Close()
279+
280+
<-gateway.ready
281+
282+
// traffic into the tunnel makes the device start a handshake, which
283+
// should arrive at the gateway's direct IP over UDP
284+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
285+
defer cancel()
286+
go func() {
287+
if c, err := tunnel.DialContext(ctx, "tcp", "[fdaa:0:18::3]:53"); err == nil {
288+
c.Close()
289+
}
290+
}()
291+
292+
buf := make([]byte, 2048)
293+
require.NoError(t, udpGateway.SetReadDeadline(time.Now().Add(5*time.Second)))
294+
n, _, err := udpGateway.ReadFromUDP(buf)
295+
require.NoError(t, err, "no WireGuard packet at the direct IP")
296+
297+
// a handshake initiation: message type 1, 148 bytes
298+
assert.Equal(t, 148, n)
299+
assert.Equal(t, byte(1), buf[0])
300+
}
301+
302+
func TestConnectTokenUDPFollowsReprovision(t *testing.T) {
303+
gw1, _ := testKeypair(t)
304+
gw2, _ := testKeypair(t)
305+
306+
const (
307+
peerA = "fdaa:0:18:a7b:1:1111:2222:3302"
308+
peerB = "fdaa:0:18:c9d:2:3333:4444:5502"
309+
)
310+
311+
// the first gateway's direct IP isn't routed, so the reconnect falls
312+
// back to anycast, which lands on a second gateway with its own
313+
gateway := newFakeTokenGateway(t, []string{gw1, gw2}, []tokenResult{
314+
{Type: "ok", PeerIP: peerA, DNS: "fdaa:0:18::3", ExpiresAt: time.Now().Add(time.Hour).Unix(), Direct: "192.0.2.10"},
315+
{Type: "ok", PeerIP: peerB, DNS: "fdaa:0:18::3", ExpiresAt: time.Now().Add(time.Hour).Unix(), Direct: "192.0.2.20"},
316+
})
317+
318+
pub, priv := testKeypair(t)
319+
state := &WireGuardState{Org: "test-org", LocalPublic: pub, LocalPrivate: priv}
320+
321+
tunnel, err := ConnectToken(context.Background(), state, gateway.endpoint(), &TokenAuth{
322+
Token: func() string { return "FlyV1 fm2_test" },
323+
OrgSlug: "test-org",
324+
}, true)
325+
require.NoError(t, err)
326+
defer tunnel.Close()
327+
328+
<-gateway.ready
329+
assert.Equal(t, directUDP("192.0.2.10"), tunnel.sendingTo())
330+
331+
gateway.drop(0)
332+
<-gateway.ready
333+
334+
waitFor(t, "tunnel rebuild", func() bool {
335+
st, _ := tunnel.StateAndConfig()
336+
337+
return st.Peer.Peerip == peerB
338+
})
339+
340+
assert.Equal(t, directUDP("192.0.2.20"), tunnel.sendingTo())
341+
342+
select {
343+
case <-tunnel.Done():
344+
t.Fatalf("tunnel died: %v", tunnel.Err())
345+
default:
346+
}
347+
}

‎wg/tunnel.go‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"math/rand"
1111
"net"
1212
"net/netip"
13+
"strconv"
1314
"sync"
1415

1516
"github.com/miekg/dns"
@@ -22,6 +23,10 @@ import (
2223

2324
const DefaultTokenGateway = "gateway.machines.dev"
2425

26+
// tokenGatewayUDPPort is where a token gateway's direct IP takes WireGuard.
27+
// A variable so tests can point it at a local listener.
28+
var tokenGatewayUDPPort = 51820
29+
2530
type Tunnel struct {
2631
// mu guards everything below it. Token-mode tunnels replace their
2732
// device, netstack and addresses when an anycast reconnect lands on a
@@ -51,6 +56,11 @@ type Tunnel struct {
5156
done chan struct{}
5257
doneOnce sync.Once
5358
err error
59+
// wsProxyAddr is the local websocket proxy (127.0.0.1:port), which relays
60+
// WireGuard over the websocket to the gateway
61+
wsProxyAddr string
62+
// use gateway's direct IP as endpoint if it has one.
63+
udp bool
5464
}
5565

5666
func Connect(ctx context.Context, state *WireGuardState) (*Tunnel, error) {
@@ -105,7 +115,12 @@ func doConnect(ctx context.Context, state *WireGuardState, wswg bool) (*Tunnel,
105115
// packet are presented over the websocket and the gateway allocates our
106116
// address inline. The state's Peer and DNS are filled in from the gateway's
107117
// answer (and updated again if an anycast reconnect re-provisions us).
108-
func ConnectToken(ctx context.Context, state *WireGuardState, endpoint string, auth *TokenAuth) (*Tunnel, error) {
118+
//
119+
// With udp, WireGuard goes straight to the gateway's direct IP over UDP when
120+
// the gateway gives one, and the websocket only carries the token exchange,
121+
// refreshes and reconnects. Otherwise, or without a direct IP, WireGuard is
122+
// relayed over the websocket.
123+
func ConnectToken(ctx context.Context, state *WireGuardState, endpoint string, auth *TokenAuth, udp bool) (*Tunnel, error) {
109124
auth.Pubkey = state.LocalPublic
110125
if state.Name == "" {
111126
state.Name = TokenPeerName(state.LocalPublic)
@@ -133,7 +148,11 @@ func ConnectToken(ctx context.Context, state *WireGuardState, endpoint string, a
133148
prov := wswg.prov
134149
applyProvision(state, endpoint, prov)
135150

136-
tunnel, err := newTunnel(state, tokenTunnelConfig(state, prov), fmt.Sprintf("127.0.0.1:%d", port), wscancel)
151+
wsProxyAddr := fmt.Sprintf("127.0.0.1:%d", port)
152+
endpointAddr := tokenEndpoint(udp, wsProxyAddr, prov)
153+
log.Printf("sending WireGuard for peer %s to %s", prov.PeerIP, endpointAddr)
154+
155+
tunnel, err := newTunnel(state, tokenTunnelConfig(state, prov), endpointAddr, wscancel)
137156
if err != nil {
138157
wscancel()
139158
wswg.close()
@@ -143,13 +162,26 @@ func ConnectToken(ctx context.Context, state *WireGuardState, endpoint string, a
143162

144163
tunnel.ephemeral = true
145164
tunnel.wswg = wswg
165+
tunnel.wsProxyAddr = wsProxyAddr
166+
tunnel.udp = udp
146167

147168
wswg.setCallbacks(tunnel.reprovision, tunnel.fail)
148169
wswg.start(lifetimeCtx, endpoint)
149170

150171
return tunnel, nil
151172
}
152173

174+
// tokenEndpoint is where a token-mode device sends WireGuard: the gateway's
175+
// direct IP when udp is on and the gateway gave one, else the websocket
176+
// proxy (wsProxyAddr).
177+
func tokenEndpoint(udp bool, wsProxyAddr string, prov *TokenProvision) string {
178+
if udp && prov.Direct != "" {
179+
return net.JoinHostPort(prov.Direct, strconv.Itoa(tokenGatewayUDPPort))
180+
}
181+
182+
return wsProxyAddr
183+
}
184+
153185
func applyProvision(state *WireGuardState, endpoint string, prov *TokenProvision) {
154186
state.Peer = fly.CreatedWireGuardPeer{
155187
Peerip: prov.PeerIP,
@@ -293,7 +325,10 @@ func (t *Tunnel) reprovision(prov *TokenProvision) {
293325
applyProvision(&state, state.Peer.Endpointip, prov)
294326
cfg := tokenTunnelConfig(&state, prov)
295327

296-
dev, tunDev, gNet, err := startDevice(cfg, t.endpointAddr)
328+
// another gateway: over UDP, that's its own direct IP
329+
endpointAddr := tokenEndpoint(t.udp, t.wsProxyAddr, prov)
330+
331+
dev, tunDev, gNet, err := startDevice(cfg, endpointAddr)
297332
if err != nil {
298333
t.failLocked(&RebuildError{PeerIP: prov.PeerIP, Err: err})
299334

@@ -303,11 +338,12 @@ func (t *Tunnel) reprovision(prov *TokenProvision) {
303338
old := t.dev
304339
t.dev, t.tun, t.net, t.dnsIP = dev, tunDev, gNet, cfg.DNS
305340
t.State, t.Config = &state, cfg
341+
t.endpointAddr = endpointAddr
306342

307343
old.Close()
308344
t.closeConnsLocked()
309345

310-
log.Printf("rebuilt tunnel for %s as %s (gateway key %s)", state.Org, prov.PeerIP, prov.GatewayPubkey)
346+
log.Printf("rebuilt tunnel for %s as %s (gateway key %s, sending to %s)", state.Org, prov.PeerIP, prov.GatewayPubkey, endpointAddr)
311347
}
312348

313349
// fail marks a token-mode tunnel as permanently dead and closes it.

0 commit comments

Comments
 (0)