fix: introduce measures to avoid bots crawling and indexing activities (#5728)

* fix: 404 for robots.txt and meta robots tags

* fix: add unit tests for robots txt and tag

* fix: add meta tag robots none for login pages

* fix: weird format issue in header.go

* fix: add x-robots-tag=none to grpcwebserver

* fix linting

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>
Co-authored-by: Livio Spring <livio.a@gmail.com>

internal/api/robots_txt/robots_txt.go

@@ -0,0 +1,19 @@
+package robots_txt
+
+import (
+	"fmt"
+	"net/http"
+)
+
+const (
+	HandlerPrefix = "/robots.txt"
+)
+
+func Start() (http.Handler, error) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/text")
+		fmt.Fprintf(w, "User-agent: *\nDisallow: /\n")
+	})
+	return handler, nil
+}