Revert "cmd/{k8s-nameserver,k8s-operator},k8s-operator: add a kube nameserver, make operator deploy it (#11017)" (#11669)

Temporarily reverting this PR to avoid releasing
half finished featue.

This reverts commit 9e2f58f846.

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina
2024-04-08 21:31:52 +01:00
committed by GitHub
parent 0001237253
commit 231e44e742
24 changed files with 12 additions and 1849 deletions

View File

@@ -1,348 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// k8s-nameserver is a simple nameserver implementation meant to be used with
// k8s-operator to allow to resolve magicDNS names associated with tailnet
// proxies in cluster.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net"
"os"
"os/signal"
"path/filepath"
"sync"
"syscall"
"github.com/fsnotify/fsnotify"
"github.com/miekg/dns"
operatorutils "tailscale.com/k8s-operator"
"tailscale.com/util/dnsname"
)
const (
// tsNetDomain is the domain that this DNS nameserver has registered a handler for.
tsNetDomain = "ts.net"
// addr is the the address that the UDP and TCP listeners will listen on.
addr = ":1053"
// The following constants are specific to the nameserver configuration
// provided by a mounted Kubernetes Configmap. The Configmap mounted at
// /config is the only supported way for configuring this nameserver.
defaultDNSConfigDir = "/config"
defaultDNSFile = "dns.json"
kubeletMountedConfigLn = "..data"
)
// nameserver is a simple nameserver that responds to DNS queries for A records
// for ts.net domain names over UDP or TCP. It serves DNS responses from
// in-memory IPv4 host records. It is intended to be deployed on Kubernetes with
// a ConfigMap mounted at /config that should contain the host records. It
// dynamically reconfigures its in-memory mappings as the contents of the
// mounted ConfigMap changes.
type nameserver struct {
// configReader returns the latest desired configuration (host records)
// for the nameserver. By default it gets set to a reader that reads
// from a Kubernetes ConfigMap mounted at /config, but this can be
// overridden in tests.
configReader configReaderFunc
// configWatcher is a watcher that returns an event when the desired
// configuration has changed and the nameserver should update the
// in-memory records.
configWatcher <-chan string
mu sync.Mutex // protects following
// ip4 are the in-memory hostname -> IP4 mappings that the nameserver
// uses to respond to A record queries.
ip4 map[dnsname.FQDN][]net.IP
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
// Ensure that we watch the kube Configmap mounted at /config for
// nameserver configuration updates and send events when updates happen.
c := ensureWatcherForKubeConfigMap(ctx)
ns := &nameserver{
configReader: configMapConfigReader,
configWatcher: c,
}
// Ensure that in-memory records get set up to date now and will get
// reset when the configuration changes.
ns.runRecordsReconciler(ctx)
// Register a DNS server handle for ts.net domain names. Not having a
// handle registered for any other domain names is how we enforce that
// this nameserver can only be used for ts.net domains - querying any
// other domain names returns Rcode Refused.
dns.HandleFunc(tsNetDomain, ns.handleFunc())
// Listen for DNS queries over UDP and TCP.
udpSig := make(chan os.Signal)
tcpSig := make(chan os.Signal)
go listenAndServe("udp", addr, udpSig)
go listenAndServe("tcp", addr, tcpSig)
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
s := <-sig
log.Printf("OS signal (%s) received, shutting down\n", s)
cancel() // exit the records reconciler and configmap watcher goroutines
udpSig <- s // stop the UDP listener
tcpSig <- s // stop the TCP listener
}
// handleFunc is a DNS query handler that can respond to A record queries from
// the nameserver's in-memory records.
// - If an A record query is received and the
// nameserver's in-memory records contain records for the queried domain name,
// return a success response.
// - If an A record query is received, but the
// nameserver's in-memory records do not contain records for the queried domain name,
// return NXDOMAIN.
// - If an A record query is received, but the queried domain name is not valid, return Format Error.
// - If a query is received for any other record type than A, return Not Implemented.
func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) {
h := func(w dns.ResponseWriter, r *dns.Msg) {
m := new(dns.Msg)
defer func() {
w.WriteMsg(m)
}()
if len(r.Question) < 1 {
log.Print("[unexpected] nameserver received a request with no questions\n")
m = r.SetRcodeFormatError(r)
return
}
// TODO (irbekrm): maybe set message compression
switch r.Question[0].Qtype {
case dns.TypeA:
q := r.Question[0].Name
fqdn, err := dnsname.ToFQDN(q)
if err != nil {
m = r.SetRcodeFormatError(r)
return
}
// The only supported use of this nameserver is as a
// single source of truth for MagicDNS names by
// non-tailnet Kubernetes workloads.
m.Authoritative = true
m.RecursionAvailable = false
ips := n.lookupIP4(fqdn)
if ips == nil || len(ips) == 0 {
// As we are the authoritative nameserver for MagicDNS
// names, if we do not have a record for this MagicDNS
// name, it does not exist.
m = m.SetRcode(r, dns.RcodeNameError)
return
}
// TODO (irbekrm): what TTL?
for _, ip := range ips {
rr := &dns.A{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, A: ip}
m.SetRcode(r, dns.RcodeSuccess)
m.Answer = append(m.Answer, rr)
}
case dns.TypeAAAA:
// TODO (irbekrm): implement IPv6 support
fallthrough
default:
log.Printf("[unexpected] nameserver received a query for an unsupported record type: %s\n", r.Question[0].String())
m.SetRcode(r, dns.RcodeNotImplemented)
}
}
return h
}
// runRecordsReconciler ensures that nameserver's in-memory records are
// reset when the provided configuration changes.
func (n *nameserver) runRecordsReconciler(ctx context.Context) {
log.Print("updating nameserver's records from the provided configuration...\n")
if err := n.resetRecords(); err != nil { // ensure records are up to date before the nameserver starts
log.Fatalf("error setting nameserver's records: %v\n", err)
}
log.Print("nameserver's records were updated\n")
go func() {
for {
select {
case <-ctx.Done():
log.Printf("context cancelled, exiting records reconciler\n")
return
case <-n.configWatcher:
log.Print("configuration update detected, resetting records\n")
if err := n.resetRecords(); err != nil {
// TODO (irbekrm): this runs in a
// container that will be thrown away,
// so this should be ok. But maybe still
// need to ensure that the DNS server
// terminates connections more
// gracefully.
log.Fatalf("error resetting records: %v\n", err)
}
log.Print("nameserver records were reset\n")
}
}
}()
}
// resetRecords sets the in-memory DNS records of this nameserver from the
// provided configuration. It does not check for the diff, so the caller is
// expected to ensure that this is only called when reset is needed.
func (n *nameserver) resetRecords() error {
dnsCfgBytes, err := n.configReader()
if err != nil {
log.Printf("error reading nameserver's configuration: %v\n", err)
return err
}
if dnsCfgBytes == nil || len(dnsCfgBytes) < 1 {
log.Print("nameserver's configuration is empty, any in-memory records will be unset\n")
n.mu.Lock()
n.ip4 = make(map[dnsname.FQDN][]net.IP)
n.mu.Unlock()
return nil
}
dnsCfg := &operatorutils.Records{}
err = json.Unmarshal(dnsCfgBytes, dnsCfg)
if err != nil {
return fmt.Errorf("error unmarshalling nameserver configuration: %v\n", err)
}
if dnsCfg.Version != operatorutils.Alpha1Version {
return fmt.Errorf("unsupported configuration version %s, supported versions are %s\n", dnsCfg.Version, operatorutils.Alpha1Version)
}
ip4 := make(map[dnsname.FQDN][]net.IP)
defer func() {
n.mu.Lock()
defer n.mu.Unlock()
n.ip4 = ip4
}()
if dnsCfg.IP4 == nil || len(dnsCfg.IP4) == 0 {
log.Print("nameserver's configuration contains no records, any in-memory records will be unset\n")
return nil
}
for fqdn, ips := range dnsCfg.IP4 {
fqdn, err := dnsname.ToFQDN(fqdn)
if err != nil {
log.Printf("invalid nameserver's configuration: %s is not a valid FQDN: %v; skipping this record\n", fqdn, err)
continue // one invalid hostname should not break the whole nameserver
}
for _, ipS := range ips {
ip := net.ParseIP(ipS).To4()
if ip == nil { // To4 returns nil if IP is not a IPv4 address
log.Printf("invalid nameserver's configuration: %v does not appear to be an IPv4 address; skipping this record\n", ipS)
continue // one invalid IP address should not break the whole nameserver
}
ip4[fqdn] = []net.IP{ip}
}
}
return nil
}
// listenAndServe starts a DNS server for the provided network and address.
func listenAndServe(net, addr string, shutdown chan os.Signal) {
s := &dns.Server{Addr: addr, Net: net}
go func() {
<-shutdown
log.Printf("shutting down server for %s\n", net)
s.Shutdown()
}()
log.Printf("listening for %s queries on %s\n", net, addr)
if err := s.ListenAndServe(); err != nil {
log.Fatalf("error running %s server: %v\n", net, err)
}
}
// ensureWatcherForKubeConfigMap sets up a new file watcher for the ConfigMap
// that's expected to be mounted at /config. Returns a channel that receives an
// event every time the contents get updated.
func ensureWatcherForKubeConfigMap(ctx context.Context) chan string {
c := make(chan string)
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatalf("error creating a new watcher for the mounted ConfigMap: %v\n", err)
}
// kubelet mounts configmap to a Pod using a series of symlinks, one of
// which is <mount-dir>/..data that Kubernetes recommends consumers to
// use if they need to monitor changes
// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
toWatch := filepath.Join(defaultDNSConfigDir, kubeletMountedConfigLn)
go func() {
defer watcher.Close()
log.Printf("starting file watch for %s\n", defaultDNSConfigDir)
for {
select {
case <-ctx.Done():
log.Print("context cancelled, exiting ConfigMap watcher\n")
return
case event, ok := <-watcher.Events:
if !ok {
log.Fatal("watcher finished; exiting")
}
if event.Name == toWatch {
msg := fmt.Sprintf("ConfigMap update received: %s\n", event)
log.Print(msg)
c <- msg
}
case err, ok := <-watcher.Errors:
if !ok {
// TODO (irbekrm): this runs in a
// container that will be thrown away,
// so this should be ok. But maybe still
// need to ensure that the DNS server
// terminates connections more
// gracefully.
log.Fatalf("[unexpected] configuration watcher error: errors watcher finished: %v\n", err)
}
if err != nil {
// TODO (irbekrm): this runs in a
// container that will be thrown away,
// so this should be ok. But maybe still
// need to ensure that the DNS server
// terminates connections more
// gracefully.
log.Fatalf("[unexpected] error watching configuration: %v\n", err)
}
}
}
}()
if err = watcher.Add(defaultDNSConfigDir); err != nil {
log.Fatalf("failed setting up a watcher for the mounted ConfigMap: %v\n", err)
}
return c
}
// configReaderFunc is a function that returns the desired nameserver configuration.
type configReaderFunc func() ([]byte, error)
// configMapConfigReader reads the desired nameserver configuration from a
// dns.json file in a ConfigMap mounted at /config.
var configMapConfigReader configReaderFunc = func() ([]byte, error) {
if contents, err := os.ReadFile(filepath.Join(defaultDNSConfigDir, defaultDNSFile)); err == nil {
return contents, nil
} else if os.IsNotExist(err) {
return nil, nil
} else {
return nil, err
}
}
// lookupIP4 returns any IPv4 addresses for the given FQDN from nameserver's
// in-memory records.
func (n *nameserver) lookupIP4(fqdn dnsname.FQDN) []net.IP {
if n.ip4 == nil {
return nil
}
n.mu.Lock()
defer n.mu.Unlock()
f := n.ip4[fqdn]
return f
}

View File

@@ -1,227 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"net"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/miekg/dns"
"tailscale.com/util/dnsname"
)
func TestNameserver(t *testing.T) {
tests := []struct {
name string
ip4 map[dnsname.FQDN][]net.IP
query *dns.Msg
wantResp *dns.Msg
}{
{
name: "A record query, record exists",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{Id: 1, RecursionDesired: true},
},
wantResp: &dns.Msg{
Answer: []dns.RR{&dns.A{Hdr: dns.RR_Header{
Name: "foo.bar.com", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0},
A: net.IP{1, 2, 3, 4}}},
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeSuccess,
RecursionAvailable: false,
RecursionDesired: true,
Response: true,
Opcode: dns.OpcodeQuery,
Authoritative: true,
}},
},
{
name: "A record query, record does not exist",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{Id: 1},
},
wantResp: &dns.Msg{
Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeNameError,
RecursionAvailable: false,
Response: true,
Opcode: dns.OpcodeQuery,
Authoritative: true,
}},
},
{
name: "A record query, but the name is not a valid FQDN",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{Id: 1},
},
wantResp: &dns.Msg{
Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeFormatError,
Response: true,
Opcode: dns.OpcodeQuery,
}},
},
{
name: "AAAA record query",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
MsgHdr: dns.MsgHdr{Id: 1},
},
wantResp: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeNotImplemented,
Response: true,
Opcode: dns.OpcodeQuery,
}},
},
{
name: "AAAA record query",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
MsgHdr: dns.MsgHdr{Id: 1},
},
wantResp: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeNotImplemented,
Response: true,
Opcode: dns.OpcodeQuery,
}},
},
{
name: "CNAME record query",
ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
query: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
MsgHdr: dns.MsgHdr{Id: 1},
},
wantResp: &dns.Msg{
Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}},
MsgHdr: dns.MsgHdr{
Id: 1,
Rcode: dns.RcodeNotImplemented,
Response: true,
Opcode: dns.OpcodeQuery,
}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ns := &nameserver{
ip4: tt.ip4,
}
handler := ns.handleFunc()
fakeRespW := &fakeResponseWriter{}
handler(fakeRespW, tt.query)
if diff := cmp.Diff(*fakeRespW.msg, *tt.wantResp); diff != "" {
t.Fatalf("unexpected response (-got +want): \n%s", diff)
}
})
}
}
func TestResetRecords(t *testing.T) {
tests := []struct {
name string
config []byte
hasIp4 map[dnsname.FQDN][]net.IP
wantsIp4 map[dnsname.FQDN][]net.IP
wantsErr bool
}{
{
name: "previously empty nameserver.ip4 gets set",
config: []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
},
{
name: "nameserver.ip4 gets reset",
hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
config: []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}},
},
{
name: "configuration with incompatible version",
hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
config: []byte(`{"version": "v1beta1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`),
wantsIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
wantsErr: true,
},
{
name: "nameserver.ip4 gets reset to empty config when no configuration is provided",
hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
wantsIp4: make(map[dnsname.FQDN][]net.IP),
},
{
name: "nameserver.ip4 gets reset to empty config when the provided configuration is empty",
hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}},
config: []byte(`{"version": "v1alpha1", "ip4": {}}`),
wantsIp4: make(map[dnsname.FQDN][]net.IP),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ns := &nameserver{
ip4: tt.hasIp4,
configReader: func() ([]byte, error) { return tt.config, nil },
}
if err := ns.resetRecords(); err == nil == tt.wantsErr {
t.Errorf("resetRecords() returned err: %v, wantsErr: %v", err, tt.wantsErr)
}
if diff := cmp.Diff(ns.ip4, tt.wantsIp4); diff != "" {
t.Fatalf("unexpected nameserver.ip4 contents (-got +want): \n%s", diff)
}
})
}
}
// fakeResponseWriter is a faked out dns.ResponseWriter that can be used in
// tests that need to read the response message that was written.
type fakeResponseWriter struct {
msg *dns.Msg
}
var _ dns.ResponseWriter = &fakeResponseWriter{}
func (fr *fakeResponseWriter) WriteMsg(msg *dns.Msg) error {
fr.msg = msg
return nil
}
func (fr *fakeResponseWriter) LocalAddr() net.Addr {
return nil
}
func (fr *fakeResponseWriter) RemoteAddr() net.Addr {
return nil
}
func (fr *fakeResponseWriter) Write([]byte) (int, error) {
return 0, nil
}
func (fr *fakeResponseWriter) Close() error {
return nil
}
func (fr *fakeResponseWriter) TsigStatus() error {
return nil
}
func (fr *fakeResponseWriter) TsigTimersOnly(bool) {}
func (fr *fakeResponseWriter) Hijack() {}