diff --git a/lib/chesstrainer_web.ex b/lib/chesstrainer_web.ex
index 26ef443..542ae0d 100644
--- a/lib/chesstrainer_web.ex
+++ b/lib/chesstrainer_web.ex
@@ -17,7 +17,7 @@ defmodule ChesstrainerWeb do
those modules here.
"""
- def static_paths, do: ~w(assets fonts images favicon.ico robots.txt)
+ def static_paths, do: ~w(assets fonts images favicon.ico)
def router do
quote do
diff --git a/lib/chesstrainer_web/controllers/robots_controller.ex b/lib/chesstrainer_web/controllers/robots_controller.ex
new file mode 100644
index 0000000..62e0062
--- /dev/null
+++ b/lib/chesstrainer_web/controllers/robots_controller.ex
@@ -0,0 +1,23 @@
+defmodule ChesstrainerWeb.RobotsController do
+ @moduledoc """
+ Serves `/robots.txt` with a sitemap URL that follows the deployment's host.
+
+ Built at runtime from the endpoint configuration rather than checked in as a
+ static file, so the sitemap stays correct across environments.
+ """
+
+ use ChesstrainerWeb, :controller
+
+ def index(conn, _params) do
+ body = """
+ User-agent: *
+ Allow: /
+
+ Sitemap: #{url(conn, ~p"/sitemap.xml")}
+ """
+
+ conn
+ |> put_resp_content_type("text/plain")
+ |> send_resp(200, body)
+ end
+end
diff --git a/lib/chesstrainer_web/controllers/sitemap_controller.ex b/lib/chesstrainer_web/controllers/sitemap_controller.ex
new file mode 100644
index 0000000..e9b0b14
--- /dev/null
+++ b/lib/chesstrainer_web/controllers/sitemap_controller.ex
@@ -0,0 +1,45 @@
+defmodule ChesstrainerWeb.SitemapController do
+ @moduledoc """
+ Generates `/sitemap.xml` for the indexable public pages.
+
+ Endgame play pages are deliberately left out: there are tens of thousands of
+ them and they render the same interactive surface with only the position
+ differing, so listing them all would be index bloat. Revisit that if public
+ browse pages (by Key or theme) exist and are worth ranking.
+ """
+
+ use ChesstrainerWeb, :controller
+
+ def index(conn, _params) do
+ urls = [
+ url(conn, ~p"/"),
+ url(conn, ~p"/privacy"),
+ url(conn, ~p"/terms-conditions")
+ ]
+
+ conn
+ |> put_resp_content_type("application/xml")
+ |> send_resp(200, render_sitemap(urls))
+ end
+
+ defp render_sitemap(urls) do
+ entries = Enum.map_join(urls, &" #{escape(&1)}\n")
+
+ """
+
+
+ #{entries}
+ """
+ end
+
+ # Templates escape automatically; this response is built as a plain string,
+ # so XML-significant characters must be escaped here.
+ defp escape(value) do
+ value
+ |> String.replace("&", "&")
+ |> String.replace("<", "<")
+ |> String.replace(">", ">")
+ |> String.replace("\"", """)
+ |> String.replace("'", "'")
+ end
+end
diff --git a/lib/chesstrainer_web/plugs/no_index.ex b/lib/chesstrainer_web/plugs/no_index.ex
new file mode 100644
index 0000000..13b0c09
--- /dev/null
+++ b/lib/chesstrainer_web/plugs/no_index.ex
@@ -0,0 +1,17 @@
+defmodule ChesstrainerWeb.Plugs.NoIndex do
+ @moduledoc """
+ Tells crawlers not to index the response.
+
+ Uses the `X-Robots-Tag` header rather than a `` tag: it
+ survives redirects and authentication failures, and covers LiveView pages
+ without every template having to assign `:noindex`.
+ """
+
+ import Plug.Conn
+
+ def init(opts), do: opts
+
+ def call(conn, _opts) do
+ put_resp_header(conn, "x-robots-tag", "noindex, nofollow")
+ end
+end
diff --git a/lib/chesstrainer_web/router.ex b/lib/chesstrainer_web/router.ex
index 6e9c1c0..abe16de 100644
--- a/lib/chesstrainer_web/router.ex
+++ b/lib/chesstrainer_web/router.ex
@@ -17,6 +17,15 @@ defmodule ChesstrainerWeb.Router do
plug :accepts, ["json"]
end
+ # Crawler files are plain text and XML, so they must not negotiate HTML.
+ pipeline :seo do
+ end
+
+ # Never let search engines index private or authenticated areas.
+ pipeline :noindex do
+ plug ChesstrainerWeb.Plugs.NoIndex
+ end
+
scope "/", ChesstrainerWeb do
pipe_through :browser
@@ -28,7 +37,7 @@ defmodule ChesstrainerWeb.Router do
end
scope "/admin", ChesstrainerWeb do
- pipe_through [:browser, :require_admin_user]
+ pipe_through [:browser, :noindex, :require_admin_user]
live_session :endgames,
on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do
@@ -101,6 +110,15 @@ defmodule ChesstrainerWeb.Router do
delete "/users/log-out", UserSessionController, :delete
end
+ # Crawler files live outside the catch-all so they are real responses, not
+ # the branded 404 page.
+ scope "/", ChesstrainerWeb do
+ pipe_through :seo
+
+ get "/robots.txt", RobotsController, :index
+ get "/sitemap.xml", SitemapController, :index
+ end
+
# Keep this last: routes match in definition order. It gives unmatched paths a
# branded 404 page with the site chrome instead of Phoenix's bare status page.
scope "/", ChesstrainerWeb do
diff --git a/priv/static/robots.txt b/priv/static/robots.txt
deleted file mode 100644
index 26e06b5..0000000
--- a/priv/static/robots.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
diff --git a/test/chesstrainer_web/admin_gating_test.exs b/test/chesstrainer_web/admin_gating_test.exs
index b4e608a..8745a38 100644
--- a/test/chesstrainer_web/admin_gating_test.exs
+++ b/test/chesstrainer_web/admin_gating_test.exs
@@ -59,6 +59,27 @@ defmodule ChesstrainerWeb.AdminGatingTest do
assert html_response(conn, 200)
end
+ test "admin routes are marked noindex for crawlers", %{conn: conn} do
+ conn = get(conn, ~p"/admin/endgames")
+
+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
+
+ {:ok, admin} = Accounts.update_user_role(user_fixture(), :admin)
+
+ conn =
+ build_conn()
+ |> log_in_user(admin)
+ |> get(~p"/admin/endgames")
+
+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
+ end
+
+ test "public pages are not marked noindex", %{conn: conn} do
+ conn = get(conn, ~p"/")
+
+ assert get_resp_header(conn, "x-robots-tag") == []
+ end
+
test "the navbar hides the Admin link from members", %{conn: conn} do
conn =
conn
diff --git a/test/chesstrainer_web/controllers/robots_controller_test.exs b/test/chesstrainer_web/controllers/robots_controller_test.exs
new file mode 100644
index 0000000..0f0adfe
--- /dev/null
+++ b/test/chesstrainer_web/controllers/robots_controller_test.exs
@@ -0,0 +1,22 @@
+defmodule ChesstrainerWeb.RobotsControllerTest do
+ use ChesstrainerWeb.ConnCase, async: true
+
+ alias ChesstrainerWeb.Endpoint
+
+ describe "GET /robots.txt" do
+ test "allows crawlers and points at the sitemap", %{conn: conn} do
+ body = conn |> get(~p"/robots.txt") |> response(200)
+
+ assert body =~ "User-agent: *"
+ assert body =~ "Allow: /"
+ assert body =~ "Sitemap: #{Endpoint.url()}/sitemap.xml"
+ end
+
+ test "is served as plain text", %{conn: conn} do
+ conn = get(conn, ~p"/robots.txt")
+
+ assert [content_type] = get_resp_header(conn, "content-type")
+ assert content_type =~ "text/plain"
+ end
+ end
+end
diff --git a/test/chesstrainer_web/controllers/sitemap_controller_test.exs b/test/chesstrainer_web/controllers/sitemap_controller_test.exs
new file mode 100644
index 0000000..6ca75c2
--- /dev/null
+++ b/test/chesstrainer_web/controllers/sitemap_controller_test.exs
@@ -0,0 +1,26 @@
+defmodule ChesstrainerWeb.SitemapControllerTest do
+ use ChesstrainerWeb.ConnCase, async: true
+
+ alias ChesstrainerWeb.Endpoint
+
+ describe "GET /sitemap.xml" do
+ test "lists only the public marketing pages", %{conn: conn} do
+ body = conn |> get(~p"/sitemap.xml") |> response(200)
+ base_url = Endpoint.url()
+
+ assert body =~ ~s()
+ assert body =~ ~s()
+ assert body =~ "#{base_url}/"
+ assert body =~ "#{base_url}/privacy"
+ assert body =~ "#{base_url}/terms-conditions"
+ assert length(:binary.matches(body, "")) == 3
+ end
+
+ test "is served as XML", %{conn: conn} do
+ conn = get(conn, ~p"/sitemap.xml")
+
+ assert [content_type] = get_resp_header(conn, "content-type")
+ assert content_type =~ "application/xml"
+ end
+ end
+end