add sitemap, robots, and noindex admin routes (#85)

Serve a runtime sitemap.xml with the indexable marketing pages only; endgame play pages are left out to avoid indexing tens of thousands of near-identical pages. Generate robots.txt at runtime so it points at the sitemap on the deployment's host, and send X-Robots-Tag: noindex from every /admin response.
This commit is contained in:
2026-11-06 23:04:10 +01:00
parent 924a3e08dc
commit eaf72a7686
9 changed files with 174 additions and 7 deletions
@@ -0,0 +1,23 @@
defmodule ChesstrainerWeb.RobotsController do
@moduledoc """
Serves `/robots.txt` with a sitemap URL that follows the deployment's host.
Built at runtime from the endpoint configuration rather than checked in as a
static file, so the sitemap stays correct across environments.
"""
use ChesstrainerWeb, :controller
def index(conn, _params) do
body = """
User-agent: *
Allow: /
Sitemap: #{url(conn, ~p"/sitemap.xml")}
"""
conn
|> put_resp_content_type("text/plain")
|> send_resp(200, body)
end
end
@@ -0,0 +1,45 @@
defmodule ChesstrainerWeb.SitemapController do
@moduledoc """
Generates `/sitemap.xml` for the indexable public pages.
Endgame play pages are deliberately left out: there are tens of thousands of
them and they render the same interactive surface with only the position
differing, so listing them all would be index bloat. Revisit that if public
browse pages (by Key or theme) exist and are worth ranking.
"""
use ChesstrainerWeb, :controller
def index(conn, _params) do
urls = [
url(conn, ~p"/"),
url(conn, ~p"/privacy"),
url(conn, ~p"/terms-conditions")
]
conn
|> put_resp_content_type("application/xml")
|> send_resp(200, render_sitemap(urls))
end
defp render_sitemap(urls) do
entries = Enum.map_join(urls, &" <url><loc>#{escape(&1)}</loc></url>\n")
"""
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
#{entries}</urlset>
"""
end
# Templates escape automatically; this response is built as a plain string,
# so XML-significant characters must be escaped here.
defp escape(value) do
value
|> String.replace("&", "&amp;")
|> String.replace("<", "&lt;")
|> String.replace(">", "&gt;")
|> String.replace("\"", "&quot;")
|> String.replace("'", "&apos;")
end
end
+17
View File
@@ -0,0 +1,17 @@
defmodule ChesstrainerWeb.Plugs.NoIndex do
@moduledoc """
Tells crawlers not to index the response.
Uses the `X-Robots-Tag` header rather than a `<meta name="robots">` tag: it
survives redirects and authentication failures, and covers LiveView pages
without every template having to assign `:noindex`.
"""
import Plug.Conn
def init(opts), do: opts
def call(conn, _opts) do
put_resp_header(conn, "x-robots-tag", "noindex, nofollow")
end
end
+19 -1
View File
@@ -17,6 +17,15 @@ defmodule ChesstrainerWeb.Router do
plug :accepts, ["json"]
end
# Crawler files are plain text and XML, so they must not negotiate HTML.
pipeline :seo do
end
# Never let search engines index private or authenticated areas.
pipeline :noindex do
plug ChesstrainerWeb.Plugs.NoIndex
end
scope "/", ChesstrainerWeb do
pipe_through :browser
@@ -28,7 +37,7 @@ defmodule ChesstrainerWeb.Router do
end
scope "/admin", ChesstrainerWeb do
pipe_through [:browser, :require_admin_user]
pipe_through [:browser, :noindex, :require_admin_user]
live_session :endgames,
on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do
@@ -101,6 +110,15 @@ defmodule ChesstrainerWeb.Router do
delete "/users/log-out", UserSessionController, :delete
end
# Crawler files live outside the catch-all so they are real responses, not
# the branded 404 page.
scope "/", ChesstrainerWeb do
pipe_through :seo
get "/robots.txt", RobotsController, :index
get "/sitemap.xml", SitemapController, :index
end
# Keep this last: routes match in definition order. It gives unmatched paths a
# branded 404 page with the site chrome instead of Phoenix's bare status page.
scope "/", ChesstrainerWeb do