cmd/k8s-proxy,k8s-operator: fix serve config for userspace mode (#16919)

The serve code leaves it up to the system's DNS resolver and netstack to
figure out how to reach the proxy destination. Combined with k8s-proxy
running in userspace mode, this means we can't rely on MagicDNS being
available or tailnet IPs being routable. I'd like to implement that as a
feature for serve in userspace mode, but for now the safer fix to get
kube-apiserver ProxyGroups consistently working in all environments is to
switch to using localhost as the proxy target instead.

This has a small knock-on in the code that does WhoIs lookups, which now
needs to check the X-Forwarded-For header that serve populates to get
the correct tailnet IP to look up, because the request's remote address
will be loopback.

Fixes #16920

Change-Id: I869ddcaf93102da50e66071bb00114cc1acc1288

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
2025-08-22 16:07:05 +01:00
committed by GitHub
parent c85cdabdfc
commit 3eeecb4c7f
2 changed files with 24 additions and 8 deletions

View File

@@ -123,11 +123,11 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
if ap.authMode {
mode = "auth"
}
var tsLn net.Listener
var proxyLn net.Listener
var serve func(ln net.Listener) error
if ap.https {
var err error
tsLn, err = ap.ts.Listen("tcp", ":443")
proxyLn, err = ap.ts.Listen("tcp", ":443")
if err != nil {
return fmt.Errorf("could not listen on :443: %w", err)
}
@@ -143,7 +143,7 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
}
} else {
var err error
tsLn, err = ap.ts.Listen("tcp", ":80")
proxyLn, err = net.Listen("tcp", "localhost:80")
if err != nil {
return fmt.Errorf("could not listen on :80: %w", err)
}
@@ -152,8 +152,8 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
errs := make(chan error)
go func() {
ap.log.Infof("API server proxy in %s mode is listening on tailnet addresses %s", mode, tsLn.Addr())
if err := serve(tsLn); err != nil && err != http.ErrServerClosed {
ap.log.Infof("API server proxy in %s mode is listening on %s", mode, proxyLn.Addr())
if err := serve(proxyLn); err != nil && err != http.ErrServerClosed {
errs <- fmt.Errorf("error serving: %w", err)
}
}()
@@ -179,7 +179,7 @@ type APIServerProxy struct {
rp *httputil.ReverseProxy
authMode bool // Whether to run with impersonation using caller's tailnet identity.
https bool // Whether to serve on https for the device hostname; true for k8s-operator, false for k8s-proxy.
https bool // Whether to serve on https for the device hostname; true for k8s-operator, false (and localhost) for k8s-proxy.
ts *tsnet.Server
hs *http.Server
upstreamURL *url.URL
@@ -317,7 +317,23 @@ func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) {
}
func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
who, remoteErr := ap.lc.WhoIs(r.Context(), r.RemoteAddr)
if remoteErr == nil {
ap.log.Debugf("WhoIs from remote addr: %s", r.RemoteAddr)
return who, nil
}
var fwdErr error
fwdFor := r.Header.Get("X-Forwarded-For")
if fwdFor != "" && !ap.https {
who, fwdErr = ap.lc.WhoIs(r.Context(), fwdFor)
if fwdErr == nil {
ap.log.Debugf("WhoIs from X-Forwarded-For header: %s", fwdFor)
return who, nil
}
}
return nil, errors.Join(remoteErr, fwdErr)
}
func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) {