diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 26ee0b7c6..9750824f2 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -184,7 +184,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge Connector: &connector{ isExitNode: cn.Spec.ExitNode, }, - ProxyClass: proxyClass, + ProxyClassName: proxyClass, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 07f0e2d97..34353471c 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -74,7 +74,7 @@ func TestConnector(t *testing.T) { isExitNode: true, subnetRoutes: "10.40.0.0/14", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) // Add another route to be advertised. @@ -152,7 +152,7 @@ func TestConnector(t *testing.T) { subnetRoutes: "10.40.0.0/14", hostname: "test-connector", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) // Add an exit node. @@ -237,7 +237,7 @@ func TestConnectorWithProxyClass(t *testing.T) { isExitNode: true, subnetRoutes: "10.40.0.0/14", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 9d1d587a1..111a1a1b6 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -1011,6 +1011,13 @@ spec: value: description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string + tailscale: + description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies. + type: object + properties: + acceptRoutes: + description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false. + type: boolean status: description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 78553542f..828db3685 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -1263,6 +1263,13 @@ spec: type: array type: object type: object + tailscale: + description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies. + properties: + acceptRoutes: + description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false. + type: boolean + type: object type: object status: description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 8335506bc..b093c52be 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -264,7 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ServeConfig: sc, Tags: tags, ChildResourceLabels: crl, - ProxyClass: proxyClass, + ProxyClassName: proxyClass, } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 8fc68a8c7..afd6f3853 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -100,7 +100,7 @@ func TestTailscaleIngress(t *testing.T) { } opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) @@ -231,7 +231,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { } opts.serveConfig = serveConfig - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil) expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation) diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index b9bd9f783..ed579da9b 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -75,7 +75,7 @@ func TestLoadBalancerClass(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) @@ -216,7 +216,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { hostname: "default-test", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) want := &corev1.Service{ @@ -240,7 +240,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { }, } expectEqual(t, fc, want, nil) - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) @@ -326,7 +326,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { hostname: "default-test", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) want := &corev1.Service{ @@ -350,7 +350,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { }, } expectEqual(t, fc, want, nil) - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) @@ -433,7 +433,7 @@ func TestAnnotations(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) want := &corev1.Service{ @@ -541,7 +541,7 @@ func TestAnnotationIntoLB(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) @@ -672,7 +672,7 @@ func TestLBIntoAnnotation(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) @@ -813,7 +813,7 @@ func TestCustomHostname(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, o), nil) + expectEqual(t, fc, expectedSecret(t, fc, o), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation) want := &corev1.Service{ @@ -935,10 +935,14 @@ func TestProxyClassForService(t *testing.T) { // Setup pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, - Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ - Labels: map[string]string{"foo": "bar"}, - Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Spec: tsapi.ProxyClassSpec{ + TailscaleConfig: &tsapi.TailscaleConfig{ + AcceptRoutes: true, + }, + StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -989,7 +993,7 @@ func TestProxyClassForService(t *testing.T) { hostname: "default-test", clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) @@ -1001,6 +1005,7 @@ func TestProxyClassForService(t *testing.T) { }) expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) // 3. ProxyClass is set to Ready, the Service gets reconciled by the // services-reconciler and the customization from the ProxyClass is @@ -1016,6 +1021,7 @@ func TestProxyClassForService(t *testing.T) { opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t)) // 4. tailscale.com/proxy-class label is removed from the Service, the // configuration from the ProxyClass is removed from the cluster @@ -1477,7 +1483,7 @@ func Test_externalNameService(t *testing.T) { clusterTargetDNS: "foo.com", } - expectEqual(t, fc, expectedSecret(t, opts), nil) + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil) expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 4d9417ac0..dbfe456e3 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -125,7 +125,9 @@ type tailscaleSTSConfig struct { // what this StatefulSet should be created for. Connector *connector - ProxyClass string + ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy + + ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one) } type connector struct { @@ -171,6 +173,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } + proxyClass := new(tsapi.ProxyClass) + if sts.ProxyClassName != "" { + if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil { + return nil, fmt.Errorf("failed to get ProxyClass: %w", err) + } + if !tsoperator.ProxyClassIsReady(proxyClass) { + logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") + return nil, nil + } + } + sts.ProxyClass = proxyClass + secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) @@ -465,16 +479,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } pod := &ss.Spec.Template container := &pod.Spec.Containers[0] - proxyClass := new(tsapi.ProxyClass) - if sts.ProxyClass != "" { - if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil { - return nil, fmt.Errorf("failed to get ProxyClass: %w", err) - } - if !tsoperator.ProxyClassIsReady(proxyClass) { - logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") - return nil, nil - } - } container.Image = a.proxyImage ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, @@ -589,9 +593,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName()) - if sts.ProxyClass != "" { - logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass) - ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger) + if sts.ProxyClassName != "" { + logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName) + ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { s.Spec = ss.Spec @@ -765,6 +769,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co } conf.AdvertiseRoutes = routes } + if shouldAcceptRoutes(stsC.ProxyClass) { + conf.AcceptRoutes = "true" + } + if newAuthkey != "" { conf.AuthKey = &newAuthkey } else if oldSecret != nil { @@ -803,6 +811,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co return capVerConfigs, nil } +func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool { + return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes +} + // ptrObject is a type constraint for pointer types that implement // client.Object. type ptrObject[T any] interface { diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 09e292e4f..e825837d5 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -204,7 +204,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Hostname: hostname, Tags: tags, ChildResourceLabels: crl, - ProxyClass: proxyClass, + ProxyClassName: proxyClass, } a.mu.Lock() diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 684b75433..f56366f5e 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -328,7 +328,7 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service { } } -func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { +func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret { t.Helper() s := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -355,6 +355,16 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { AuthKey: ptr.To("secret-authkey"), AcceptRoutes: "false", } + if opts.proxyClass != "" { + t.Logf("applying configuration from ProxyClass %s", opts.proxyClass) + proxyClass := new(tsapi.ProxyClass) + if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { + t.Fatalf("error getting ProxyClass: %v", err) + } + if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes { + conf.AcceptRoutes = "true" + } + } var routes []netip.Prefix if opts.subnetRoutes != "" || opts.isExitNode { r := opts.subnetRoutes @@ -474,6 +484,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want got.SetResourceVersion("") want.SetResourceVersion("") if modifier != nil { + modifier(want) modifier(got) } if diff := cmp.Diff(got, want); diff != "" { @@ -608,3 +619,33 @@ func (c *fakeTSClient) Deleted() []string { func removeHashAnnotation(sts *appsv1.StatefulSet) { delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash) } + +func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) { + return func(secret *corev1.Secret) { + t.Helper() + if len(secret.StringData["tailscaled"]) != 0 { + conf := &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil { + t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err) + } + conf.AuthKey = nil + b, err := json.Marshal(conf) + if err != nil { + t.Fatalf("error marshalling updated 'tailscaled' config: %v", err) + } + mak.Set(&secret.StringData, "tailscaled", string(b)) + } + if len(secret.StringData["cap-95.hujson"]) != 0 { + conf := &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil { + t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err) + } + conf.AuthKey = nil + b, err := json.Marshal(conf) + if err != nil { + t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err) + } + mak.Set(&secret.StringData, "cap-95.hujson", string(b)) + } + } +} diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 4b1d59178..5ae31aa8a 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -613,6 +613,13 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
false + + tailscale + object + + TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
+ + false @@ -3302,6 +3309,33 @@ The pod this Toleration is attached to tolerates any taint that matches the trip +### ProxyClass.spec.tailscale +[↩ Parent](#proxyclassspec) + + + +TailscaleConfig contains options to configure the tailscale-specific parameters of proxies. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
acceptRoutesboolean + AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.
+
false
+ + ### ProxyClass.status [↩ Parent](#proxyclass) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 82db0f69e..96554cac2 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -62,6 +62,20 @@ type ProxyClassSpec struct { // recommend that you use those for debugging purposes. // +optional Metrics *Metrics `json:"metrics,omitempty"` + // TailscaleConfig contains options to configure the tailscale-specific + // parameters of proxies. + // +optional + TailscaleConfig *TailscaleConfig `json:"tailscale,omitempty"` +} + +type TailscaleConfig struct { + // AcceptRoutes can be set to true to make the proxy instance accept + // routes advertized by other nodes on the tailnet, such as subnet + // routes. + // This is equivalent of passing --accept-routes flag to a tailscale Linux client. + // https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines + // Defaults to false. + AcceptRoutes bool `json:"acceptRoutes,omitempty"` } type StatefulSet struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 3d5840ad2..3ba6ee098 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -489,6 +489,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(Metrics) **out = **in } + if in.TailscaleConfig != nil { + in, out := &in.TailscaleConfig, &out.TailscaleConfig + *out = new(TailscaleConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec. @@ -614,3 +619,18 @@ func (in Tags) DeepCopy() Tags { in.DeepCopyInto(out) return *out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaleConfig. +func (in *TailscaleConfig) DeepCopy() *TailscaleConfig { + if in == nil { + return nil + } + out := new(TailscaleConfig) + in.DeepCopyInto(out) + return out +}