add sitemap, robots, and noindex admin routes (#85)

Serve a runtime sitemap.xml with the indexable marketing pages only; endgame play pages are left out to avoid indexing tens of thousands of near-identical pages. Generate robots.txt at runtime so it points at the sitemap on the deployment's host, and send X-Robots-Tag: noindex from every /admin response.
This commit is contained in:
2026-11-06 23:04:10 +01:00
parent 924a3e08dc
commit eaf72a7686
9 changed files with 174 additions and 7 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ defmodule ChesstrainerWeb do
those modules here.
"""
def static_paths, do: ~w(assets fonts images favicon.ico robots.txt)
def static_paths, do: ~w(assets fonts images favicon.ico)
def router do
quote do
@@ -0,0 +1,23 @@
defmodule ChesstrainerWeb.RobotsController do
@moduledoc """
Serves `/robots.txt` with a sitemap URL that follows the deployment's host.
Built at runtime from the endpoint configuration rather than checked in as a
static file, so the sitemap stays correct across environments.
"""
use ChesstrainerWeb, :controller
def index(conn, _params) do
body = """
User-agent: *
Allow: /
Sitemap: #{url(conn, ~p"/sitemap.xml")}
"""
conn
|> put_resp_content_type("text/plain")
|> send_resp(200, body)
end
end
@@ -0,0 +1,45 @@
defmodule ChesstrainerWeb.SitemapController do
@moduledoc """
Generates `/sitemap.xml` for the indexable public pages.
Endgame play pages are deliberately left out: there are tens of thousands of
them and they render the same interactive surface with only the position
differing, so listing them all would be index bloat. Revisit that if public
browse pages (by Key or theme) exist and are worth ranking.
"""
use ChesstrainerWeb, :controller
def index(conn, _params) do
urls = [
url(conn, ~p"/"),
url(conn, ~p"/privacy"),
url(conn, ~p"/terms-conditions")
]
conn
|> put_resp_content_type("application/xml")
|> send_resp(200, render_sitemap(urls))
end
defp render_sitemap(urls) do
entries = Enum.map_join(urls, &" <url><loc>#{escape(&1)}</loc></url>\n")
"""
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
#{entries}</urlset>
"""
end
# Templates escape automatically; this response is built as a plain string,
# so XML-significant characters must be escaped here.
defp escape(value) do
value
|> String.replace("&", "&amp;")
|> String.replace("<", "&lt;")
|> String.replace(">", "&gt;")
|> String.replace("\"", "&quot;")
|> String.replace("'", "&apos;")
end
end
+17
View File
@@ -0,0 +1,17 @@
defmodule ChesstrainerWeb.Plugs.NoIndex do
@moduledoc """
Tells crawlers not to index the response.
Uses the `X-Robots-Tag` header rather than a `<meta name="robots">` tag: it
survives redirects and authentication failures, and covers LiveView pages
without every template having to assign `:noindex`.
"""
import Plug.Conn
def init(opts), do: opts
def call(conn, _opts) do
put_resp_header(conn, "x-robots-tag", "noindex, nofollow")
end
end
+19 -1
View File
@@ -17,6 +17,15 @@ defmodule ChesstrainerWeb.Router do
plug :accepts, ["json"]
end
# Crawler files are plain text and XML, so they must not negotiate HTML.
pipeline :seo do
end
# Never let search engines index private or authenticated areas.
pipeline :noindex do
plug ChesstrainerWeb.Plugs.NoIndex
end
scope "/", ChesstrainerWeb do
pipe_through :browser
@@ -28,7 +37,7 @@ defmodule ChesstrainerWeb.Router do
end
scope "/admin", ChesstrainerWeb do
pipe_through [:browser, :require_admin_user]
pipe_through [:browser, :noindex, :require_admin_user]
live_session :endgames,
on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do
@@ -101,6 +110,15 @@ defmodule ChesstrainerWeb.Router do
delete "/users/log-out", UserSessionController, :delete
end
# Crawler files live outside the catch-all so they are real responses, not
# the branded 404 page.
scope "/", ChesstrainerWeb do
pipe_through :seo
get "/robots.txt", RobotsController, :index
get "/sitemap.xml", SitemapController, :index
end
# Keep this last: routes match in definition order. It gives unmatched paths a
# branded 404 page with the site chrome instead of Phoenix's bare status page.
scope "/", ChesstrainerWeb do
-5
View File
@@ -1,5 +0,0 @@
# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /
@@ -59,6 +59,27 @@ defmodule ChesstrainerWeb.AdminGatingTest do
assert html_response(conn, 200)
end
test "admin routes are marked noindex for crawlers", %{conn: conn} do
conn = get(conn, ~p"/admin/endgames")
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
{:ok, admin} = Accounts.update_user_role(user_fixture(), :admin)
conn =
build_conn()
|> log_in_user(admin)
|> get(~p"/admin/endgames")
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
end
test "public pages are not marked noindex", %{conn: conn} do
conn = get(conn, ~p"/")
assert get_resp_header(conn, "x-robots-tag") == []
end
test "the navbar hides the Admin link from members", %{conn: conn} do
conn =
conn
@@ -0,0 +1,22 @@
defmodule ChesstrainerWeb.RobotsControllerTest do
use ChesstrainerWeb.ConnCase, async: true
alias ChesstrainerWeb.Endpoint
describe "GET /robots.txt" do
test "allows crawlers and points at the sitemap", %{conn: conn} do
body = conn |> get(~p"/robots.txt") |> response(200)
assert body =~ "User-agent: *"
assert body =~ "Allow: /"
assert body =~ "Sitemap: #{Endpoint.url()}/sitemap.xml"
end
test "is served as plain text", %{conn: conn} do
conn = get(conn, ~p"/robots.txt")
assert [content_type] = get_resp_header(conn, "content-type")
assert content_type =~ "text/plain"
end
end
end
@@ -0,0 +1,26 @@
defmodule ChesstrainerWeb.SitemapControllerTest do
use ChesstrainerWeb.ConnCase, async: true
alias ChesstrainerWeb.Endpoint
describe "GET /sitemap.xml" do
test "lists only the public marketing pages", %{conn: conn} do
body = conn |> get(~p"/sitemap.xml") |> response(200)
base_url = Endpoint.url()
assert body =~ ~s(<?xml version="1.0" encoding="UTF-8"?>)
assert body =~ ~s(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">)
assert body =~ "<loc>#{base_url}/</loc>"
assert body =~ "<loc>#{base_url}/privacy</loc>"
assert body =~ "<loc>#{base_url}/terms-conditions</loc>"
assert length(:binary.matches(body, "<url>")) == 3
end
test "is served as XML", %{conn: conn} do
conn = get(conn, ~p"/sitemap.xml")
assert [content_type] = get_resp_header(conn, "content-type")
assert content_type =~ "application/xml"
end
end
end