From eaf72a7686bc9956d6e962bcfa709871979bb27a Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 6 Nov 2026 23:04:10 +0100 Subject: [PATCH] add sitemap, robots, and noindex admin routes (#85) Serve a runtime sitemap.xml with the indexable marketing pages only; endgame play pages are left out to avoid indexing tens of thousands of near-identical pages. Generate robots.txt at runtime so it points at the sitemap on the deployment's host, and send X-Robots-Tag: noindex from every /admin response. --- lib/chesstrainer_web.ex | 2 +- .../controllers/robots_controller.ex | 23 ++++++++++ .../controllers/sitemap_controller.ex | 45 +++++++++++++++++++ lib/chesstrainer_web/plugs/no_index.ex | 17 +++++++ lib/chesstrainer_web/router.ex | 20 ++++++++- priv/static/robots.txt | 5 --- test/chesstrainer_web/admin_gating_test.exs | 21 +++++++++ .../controllers/robots_controller_test.exs | 22 +++++++++ .../controllers/sitemap_controller_test.exs | 26 +++++++++++ 9 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 lib/chesstrainer_web/controllers/robots_controller.ex create mode 100644 lib/chesstrainer_web/controllers/sitemap_controller.ex create mode 100644 lib/chesstrainer_web/plugs/no_index.ex delete mode 100644 priv/static/robots.txt create mode 100644 test/chesstrainer_web/controllers/robots_controller_test.exs create mode 100644 test/chesstrainer_web/controllers/sitemap_controller_test.exs diff --git a/lib/chesstrainer_web.ex b/lib/chesstrainer_web.ex index 26ef443..542ae0d 100644 --- a/lib/chesstrainer_web.ex +++ b/lib/chesstrainer_web.ex @@ -17,7 +17,7 @@ defmodule ChesstrainerWeb do those modules here. """ - def static_paths, do: ~w(assets fonts images favicon.ico robots.txt) + def static_paths, do: ~w(assets fonts images favicon.ico) def router do quote do diff --git a/lib/chesstrainer_web/controllers/robots_controller.ex b/lib/chesstrainer_web/controllers/robots_controller.ex new file mode 100644 index 0000000..62e0062 --- /dev/null +++ b/lib/chesstrainer_web/controllers/robots_controller.ex @@ -0,0 +1,23 @@ +defmodule ChesstrainerWeb.RobotsController do + @moduledoc """ + Serves `/robots.txt` with a sitemap URL that follows the deployment's host. + + Built at runtime from the endpoint configuration rather than checked in as a + static file, so the sitemap stays correct across environments. + """ + + use ChesstrainerWeb, :controller + + def index(conn, _params) do + body = """ + User-agent: * + Allow: / + + Sitemap: #{url(conn, ~p"/sitemap.xml")} + """ + + conn + |> put_resp_content_type("text/plain") + |> send_resp(200, body) + end +end diff --git a/lib/chesstrainer_web/controllers/sitemap_controller.ex b/lib/chesstrainer_web/controllers/sitemap_controller.ex new file mode 100644 index 0000000..e9b0b14 --- /dev/null +++ b/lib/chesstrainer_web/controllers/sitemap_controller.ex @@ -0,0 +1,45 @@ +defmodule ChesstrainerWeb.SitemapController do + @moduledoc """ + Generates `/sitemap.xml` for the indexable public pages. + + Endgame play pages are deliberately left out: there are tens of thousands of + them and they render the same interactive surface with only the position + differing, so listing them all would be index bloat. Revisit that if public + browse pages (by Key or theme) exist and are worth ranking. + """ + + use ChesstrainerWeb, :controller + + def index(conn, _params) do + urls = [ + url(conn, ~p"/"), + url(conn, ~p"/privacy"), + url(conn, ~p"/terms-conditions") + ] + + conn + |> put_resp_content_type("application/xml") + |> send_resp(200, render_sitemap(urls)) + end + + defp render_sitemap(urls) do + entries = Enum.map_join(urls, &" #{escape(&1)}\n") + + """ + + + #{entries} + """ + end + + # Templates escape automatically; this response is built as a plain string, + # so XML-significant characters must be escaped here. + defp escape(value) do + value + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + |> String.replace("\"", """) + |> String.replace("'", "'") + end +end diff --git a/lib/chesstrainer_web/plugs/no_index.ex b/lib/chesstrainer_web/plugs/no_index.ex new file mode 100644 index 0000000..13b0c09 --- /dev/null +++ b/lib/chesstrainer_web/plugs/no_index.ex @@ -0,0 +1,17 @@ +defmodule ChesstrainerWeb.Plugs.NoIndex do + @moduledoc """ + Tells crawlers not to index the response. + + Uses the `X-Robots-Tag` header rather than a `` tag: it + survives redirects and authentication failures, and covers LiveView pages + without every template having to assign `:noindex`. + """ + + import Plug.Conn + + def init(opts), do: opts + + def call(conn, _opts) do + put_resp_header(conn, "x-robots-tag", "noindex, nofollow") + end +end diff --git a/lib/chesstrainer_web/router.ex b/lib/chesstrainer_web/router.ex index 6e9c1c0..abe16de 100644 --- a/lib/chesstrainer_web/router.ex +++ b/lib/chesstrainer_web/router.ex @@ -17,6 +17,15 @@ defmodule ChesstrainerWeb.Router do plug :accepts, ["json"] end + # Crawler files are plain text and XML, so they must not negotiate HTML. + pipeline :seo do + end + + # Never let search engines index private or authenticated areas. + pipeline :noindex do + plug ChesstrainerWeb.Plugs.NoIndex + end + scope "/", ChesstrainerWeb do pipe_through :browser @@ -28,7 +37,7 @@ defmodule ChesstrainerWeb.Router do end scope "/admin", ChesstrainerWeb do - pipe_through [:browser, :require_admin_user] + pipe_through [:browser, :noindex, :require_admin_user] live_session :endgames, on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do @@ -101,6 +110,15 @@ defmodule ChesstrainerWeb.Router do delete "/users/log-out", UserSessionController, :delete end + # Crawler files live outside the catch-all so they are real responses, not + # the branded 404 page. + scope "/", ChesstrainerWeb do + pipe_through :seo + + get "/robots.txt", RobotsController, :index + get "/sitemap.xml", SitemapController, :index + end + # Keep this last: routes match in definition order. It gives unmatched paths a # branded 404 page with the site chrome instead of Phoenix's bare status page. scope "/", ChesstrainerWeb do diff --git a/priv/static/robots.txt b/priv/static/robots.txt deleted file mode 100644 index 26e06b5..0000000 --- a/priv/static/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / diff --git a/test/chesstrainer_web/admin_gating_test.exs b/test/chesstrainer_web/admin_gating_test.exs index b4e608a..8745a38 100644 --- a/test/chesstrainer_web/admin_gating_test.exs +++ b/test/chesstrainer_web/admin_gating_test.exs @@ -59,6 +59,27 @@ defmodule ChesstrainerWeb.AdminGatingTest do assert html_response(conn, 200) end + test "admin routes are marked noindex for crawlers", %{conn: conn} do + conn = get(conn, ~p"/admin/endgames") + + assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"] + + {:ok, admin} = Accounts.update_user_role(user_fixture(), :admin) + + conn = + build_conn() + |> log_in_user(admin) + |> get(~p"/admin/endgames") + + assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"] + end + + test "public pages are not marked noindex", %{conn: conn} do + conn = get(conn, ~p"/") + + assert get_resp_header(conn, "x-robots-tag") == [] + end + test "the navbar hides the Admin link from members", %{conn: conn} do conn = conn diff --git a/test/chesstrainer_web/controllers/robots_controller_test.exs b/test/chesstrainer_web/controllers/robots_controller_test.exs new file mode 100644 index 0000000..0f0adfe --- /dev/null +++ b/test/chesstrainer_web/controllers/robots_controller_test.exs @@ -0,0 +1,22 @@ +defmodule ChesstrainerWeb.RobotsControllerTest do + use ChesstrainerWeb.ConnCase, async: true + + alias ChesstrainerWeb.Endpoint + + describe "GET /robots.txt" do + test "allows crawlers and points at the sitemap", %{conn: conn} do + body = conn |> get(~p"/robots.txt") |> response(200) + + assert body =~ "User-agent: *" + assert body =~ "Allow: /" + assert body =~ "Sitemap: #{Endpoint.url()}/sitemap.xml" + end + + test "is served as plain text", %{conn: conn} do + conn = get(conn, ~p"/robots.txt") + + assert [content_type] = get_resp_header(conn, "content-type") + assert content_type =~ "text/plain" + end + end +end diff --git a/test/chesstrainer_web/controllers/sitemap_controller_test.exs b/test/chesstrainer_web/controllers/sitemap_controller_test.exs new file mode 100644 index 0000000..6ca75c2 --- /dev/null +++ b/test/chesstrainer_web/controllers/sitemap_controller_test.exs @@ -0,0 +1,26 @@ +defmodule ChesstrainerWeb.SitemapControllerTest do + use ChesstrainerWeb.ConnCase, async: true + + alias ChesstrainerWeb.Endpoint + + describe "GET /sitemap.xml" do + test "lists only the public marketing pages", %{conn: conn} do + body = conn |> get(~p"/sitemap.xml") |> response(200) + base_url = Endpoint.url() + + assert body =~ ~s() + assert body =~ ~s() + assert body =~ "#{base_url}/" + assert body =~ "#{base_url}/privacy" + assert body =~ "#{base_url}/terms-conditions" + assert length(:binary.matches(body, "")) == 3 + end + + test "is served as XML", %{conn: conn} do + conn = get(conn, ~p"/sitemap.xml") + + assert [content_type] = get_resp_header(conn, "content-type") + assert content_type =~ "application/xml" + end + end +end