WIP

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/examples/proxygroup.yaml

@@ -1,7 +1,20 @@
 apiVersion: tailscale.com/v1alpha1
 kind: ProxyGroup
 metadata:
-  name: egress-proxies
+  name: ingress-proxies
 spec:
-  type: egress
-  replicas: 3
+  type: ingress 
+  replicas: 2
+  proxyClass: prod
+---
+apiVersion: tailscale.com/v1alpha1
+kind: ProxyClass
+metadata:
+  name: prod
+spec:
+  statefulSet:
+    pod:
+      tailscaleContainer:
+         env:
+         - name: TS_DEBUG_FIREWALL_MODE
+           value: "iptables"

cmd/k8s-operator/ingress-for-pg.go

@@ -54,7 +54,8 @@ const (
 	// well as the default HTTPS endpoint).
 	annotationHTTPEndpoint = "tailscale.com/http-endpoint"
 
-	labelDomain = "tailscale.com/domain"
+	labelDomain              = "tailscale.com/domain"
+	managedVIPServiceComment = "This VIPService is managed by the Tailscale Kubernetes Operator, do not modify"
 )
 
 var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
@@ -314,7 +315,6 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		vipPorts = append(vipPorts, "80")
 	}
 
-	const managedVIPServiceComment = "This VIPService is managed by the Tailscale Kubernetes Operator, do not modify"
 	vipSvc := &tailscale.VIPService{
 		Name:        serviceName,
 		Tags:        tags,

cmd/k8s-operator/operator.go

@@ -372,6 +372,8 @@ func runReconcilers(opts reconcilerOpts) {
 		ControllerManagedBy(mgr).
 		For(&corev1.Service{}).
 		Named("service-pg-reconciler").
+		// TODO: this watch does not seem to work- does not if ProxyGroup created later
+		// maybe need to watch the ProxyGroup
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))).
 		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
 		Complete(&HAServiceReconciler{

cmd/k8s-operator/proxygroup_specs.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/ingressservices"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
@@ -175,6 +176,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				Name:  "TS_INTERNAL_APP",
 				Value: kubetypes.AppProxyGroupIngress,
 			},
+				corev1.EnvVar{
+					Name:  "TS_INGRESS_PROXIES_CONFIG_PATH",
+					Value: fmt.Sprintf("/etc/proxies/%s", ingressservices.IngressConfigKey),
+				},
 				corev1.EnvVar{
 					Name:  "TS_SERVE_CONFIG",
 					Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),

cmd/k8s-operator/svc-for-pg.go

@@ -325,17 +325,23 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
 		}
 
 		if ip.Is4() {
-			mak.Set(&cfg.IPv4Mapping, vipv4, ip)
+			cfg.IPv4Mapping = &ingressservices.Mapping{
+				ClusterIP:    ip,
+				VIPServiceIP: vipv4,
+			}
 		} else if ip.Is6() {
-			mak.Set(&cfg.IPv6Mapping, vipv6, ip)
+			cfg.IPv6Mapping = &ingressservices.Mapping{
+				ClusterIP:    ip,
+				VIPServiceIP: vipv6,
+			}
 		}
 	}
 
 	existingCfg := cfgs[serviceName.String()]
 	if !reflect.DeepEqual(existingCfg, cfg) {
-		logger.Infof("Updating ingress config")
+		logger.Infof("Updating ingress config adding %+#v", cfg)
 		mak.Set(&cfgs, serviceName.String(), cfg)
-		cfgBytes, err := json.Marshal(cfg)
+		cfgBytes, err := json.Marshal(cfgs)
 		if err != nil {
 			return false, fmt.Errorf("error marshaling ingress config: %w", err)
 		}
@@ -347,9 +353,9 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
 
 	// 5. Update tailscaled's AdvertiseServices config, which should add the VIPService
 	// IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved.
-	// if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, mode, logger); err != nil {
-	// 	return false, fmt.Errorf("failed to update tailscaled config: %w", err)
-	// }
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, mode, logger); err != nil {
+		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
+	}
 
 	// 6. Update Ingress status if ProxyGroup Pods are ready.
 	// count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName)
@@ -628,6 +634,7 @@ func (r *HAServiceReconciler) cleanupVIPService(ctx context.Context, name tailcf
 
 func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) {
 	// Get all config Secrets for this ProxyGroup.
+	// Get all Pods
 	secrets := &corev1.SecretList{}
 	if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
 		return fmt.Errorf("failed to list config Secrets: %w", err)