diff --git a/cmd/start/start.go b/cmd/start/start.go
index 4f134b0443..58571abc10 100644
--- a/cmd/start/start.go
+++ b/cmd/start/start.go
@@ -39,6 +39,7 @@ import (
http_util "github.com/zitadel/zitadel/internal/api/http"
"github.com/zitadel/zitadel/internal/api/http/middleware"
"github.com/zitadel/zitadel/internal/api/oidc"
+ "github.com/zitadel/zitadel/internal/api/robots_txt"
"github.com/zitadel/zitadel/internal/api/saml"
"github.com/zitadel/zitadel/internal/api/ui/console"
"github.com/zitadel/zitadel/internal/api/ui/login"
@@ -305,6 +306,13 @@ func startAPIs(
return err
}
+ // robots.txt handler
+ robotsTxtHandler, err := robots_txt.Start()
+ if err != nil {
+ return fmt.Errorf("unable to start robots txt handler: %w", err)
+ }
+ apis.RegisterHandlerOnPrefix(robots_txt.HandlerPrefix, robotsTxtHandler)
+
// TODO: Record openapi access logs?
openAPIHandler, err := openapi.Start()
if err != nil {
diff --git a/console/src/index.html b/console/src/index.html
index 14dc2035eb..fe500d8d28 100644
--- a/console/src/index.html
+++ b/console/src/index.html
@@ -21,6 +21,7 @@
+
diff --git a/internal/api/api.go b/internal/api/api.go
index d634e50109..4efdb322a0 100644
--- a/internal/api/api.go
+++ b/internal/api/api.go
@@ -16,6 +16,7 @@ import (
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/grpc/server"
http_util "github.com/zitadel/zitadel/internal/api/http"
+ http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
"github.com/zitadel/zitadel/internal/api/ui/login"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/logstore"
@@ -167,6 +168,7 @@ func (a *API) routeGRPCWeb() {
return true
}),
)
+ a.router.Use(http_mw.RobotsTagHandler)
a.router.NewRoute().
Methods(http.MethodPost, http.MethodOptions).
MatcherFunc(
diff --git a/internal/api/grpc/server/gateway.go b/internal/api/grpc/server/gateway.go
index a4cbbbe614..faba435940 100644
--- a/internal/api/grpc/server/gateway.go
+++ b/internal/api/grpc/server/gateway.go
@@ -149,6 +149,7 @@ func addInterceptors(handler http.Handler, http1HostName string) http.Handler {
handler = http_mw.CallDurationHandler(handler)
handler = http1Host(handler, http1HostName)
handler = http_mw.CORSInterceptor(handler)
+ handler = http_mw.RobotsTagHandler(handler)
handler = http_mw.DefaultTelemetryHandler(handler)
return http_mw.DefaultMetricsHandler(handler)
}
diff --git a/internal/api/http/header.go b/internal/api/http/header.go
index 8402c271d6..f2d3e19c13 100644
--- a/internal/api/http/header.go
+++ b/internal/api/http/header.go
@@ -23,6 +23,7 @@ const (
XUserAgent = "x-user-agent"
XGrpcWeb = "x-grpc-web"
XRequestedWith = "x-requested-with"
+ XRobotsTag = "x-robots-tag"
IfNoneMatch = "If-None-Match"
LastModified = "Last-Modified"
Etag = "Etag"
diff --git a/internal/api/http/middleware/robots_tag_interceptor.go b/internal/api/http/middleware/robots_tag_interceptor.go
new file mode 100644
index 0000000000..076a91cfa2
--- /dev/null
+++ b/internal/api/http/middleware/robots_tag_interceptor.go
@@ -0,0 +1,14 @@
+package middleware
+
+import (
+ "net/http"
+
+ http_utils "github.com/zitadel/zitadel/internal/api/http"
+)
+
+func RobotsTagHandler(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set(http_utils.XRobotsTag, "none")
+ next.ServeHTTP(w, r)
+ })
+}
diff --git a/internal/api/http/middleware/robots_tag_interceptor_test.go b/internal/api/http/middleware/robots_tag_interceptor_test.go
new file mode 100644
index 0000000000..7e30d4a9b9
--- /dev/null
+++ b/internal/api/http/middleware/robots_tag_interceptor_test.go
@@ -0,0 +1,24 @@
+package middleware
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_RobotsTagInterceptor(t *testing.T) {
+ testHandler := func(w http.ResponseWriter, r *http.Request) {}
+ req := httptest.NewRequest(http.MethodGet, "/", nil)
+ recorder := httptest.NewRecorder()
+
+ handler := RobotsTagHandler(http.HandlerFunc(testHandler))
+ handler.ServeHTTP(recorder, req)
+
+ res := recorder.Result()
+ exp := res.Header.Get("X-Robots-Tag")
+ assert.Equal(t, "none", exp)
+
+ defer res.Body.Close()
+}
diff --git a/internal/api/robots_txt/robots_txt.go b/internal/api/robots_txt/robots_txt.go
new file mode 100644
index 0000000000..be487740dc
--- /dev/null
+++ b/internal/api/robots_txt/robots_txt.go
@@ -0,0 +1,19 @@
+package robots_txt
+
+import (
+ "fmt"
+ "net/http"
+)
+
+const (
+ HandlerPrefix = "/robots.txt"
+)
+
+func Start() (http.Handler, error) {
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ w.Header().Set("Content-Type", "application/text")
+ fmt.Fprintf(w, "User-agent: *\nDisallow: /\n")
+ })
+ return handler, nil
+}
diff --git a/internal/api/robots_txt/robots_txt_test.go b/internal/api/robots_txt/robots_txt_test.go
new file mode 100644
index 0000000000..ca70b6bc3c
--- /dev/null
+++ b/internal/api/robots_txt/robots_txt_test.go
@@ -0,0 +1,28 @@
+package robots_txt
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_RobotsTxt(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
+ recorder := httptest.NewRecorder()
+
+ handler, err := Start()
+ handler.ServeHTTP(recorder, req)
+ assert.Equal(t, nil, err)
+
+ res := recorder.Result()
+ body, err := io.ReadAll(res.Body)
+ assert.Equal(t, nil, err)
+
+ assert.Equal(t, 200, res.StatusCode)
+ assert.Equal(t, "User-agent: *\nDisallow: /\n", string(body))
+
+ defer res.Body.Close()
+}
diff --git a/internal/api/ui/login/static/templates/main.html b/internal/api/ui/login/static/templates/main.html
index 08aac7c1bb..04ec281ebe 100644
--- a/internal/api/ui/login/static/templates/main.html
+++ b/internal/api/ui/login/static/templates/main.html
@@ -23,6 +23,7 @@
{{ .Title }}
+