mirror of
https://github.com/TheRealOwenRees/chess-endgame-trainer.git
synced 2026-11-18 11:42:07 +00:00
add sitemap, robots, and noindex admin routes (#85)
Serve a runtime sitemap.xml with the indexable marketing pages only; endgame play pages are left out to avoid indexing tens of thousands of near-identical pages. Generate robots.txt at runtime so it points at the sitemap on the deployment's host, and send X-Robots-Tag: noindex from every /admin response.
This commit is contained in:
@@ -17,7 +17,7 @@ defmodule ChesstrainerWeb do
|
|||||||
those modules here.
|
those modules here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def static_paths, do: ~w(assets fonts images favicon.ico robots.txt)
|
def static_paths, do: ~w(assets fonts images favicon.ico)
|
||||||
|
|
||||||
def router do
|
def router do
|
||||||
quote do
|
quote do
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
defmodule ChesstrainerWeb.RobotsController do
|
||||||
|
@moduledoc """
|
||||||
|
Serves `/robots.txt` with a sitemap URL that follows the deployment's host.
|
||||||
|
|
||||||
|
Built at runtime from the endpoint configuration rather than checked in as a
|
||||||
|
static file, so the sitemap stays correct across environments.
|
||||||
|
"""
|
||||||
|
|
||||||
|
use ChesstrainerWeb, :controller
|
||||||
|
|
||||||
|
def index(conn, _params) do
|
||||||
|
body = """
|
||||||
|
User-agent: *
|
||||||
|
Allow: /
|
||||||
|
|
||||||
|
Sitemap: #{url(conn, ~p"/sitemap.xml")}
|
||||||
|
"""
|
||||||
|
|
||||||
|
conn
|
||||||
|
|> put_resp_content_type("text/plain")
|
||||||
|
|> send_resp(200, body)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
defmodule ChesstrainerWeb.SitemapController do
|
||||||
|
@moduledoc """
|
||||||
|
Generates `/sitemap.xml` for the indexable public pages.
|
||||||
|
|
||||||
|
Endgame play pages are deliberately left out: there are tens of thousands of
|
||||||
|
them and they render the same interactive surface with only the position
|
||||||
|
differing, so listing them all would be index bloat. Revisit that if public
|
||||||
|
browse pages (by Key or theme) exist and are worth ranking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
use ChesstrainerWeb, :controller
|
||||||
|
|
||||||
|
def index(conn, _params) do
|
||||||
|
urls = [
|
||||||
|
url(conn, ~p"/"),
|
||||||
|
url(conn, ~p"/privacy"),
|
||||||
|
url(conn, ~p"/terms-conditions")
|
||||||
|
]
|
||||||
|
|
||||||
|
conn
|
||||||
|
|> put_resp_content_type("application/xml")
|
||||||
|
|> send_resp(200, render_sitemap(urls))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp render_sitemap(urls) do
|
||||||
|
entries = Enum.map_join(urls, &" <url><loc>#{escape(&1)}</loc></url>\n")
|
||||||
|
|
||||||
|
"""
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||||
|
#{entries}</urlset>
|
||||||
|
"""
|
||||||
|
end
|
||||||
|
|
||||||
|
# Templates escape automatically; this response is built as a plain string,
|
||||||
|
# so XML-significant characters must be escaped here.
|
||||||
|
defp escape(value) do
|
||||||
|
value
|
||||||
|
|> String.replace("&", "&")
|
||||||
|
|> String.replace("<", "<")
|
||||||
|
|> String.replace(">", ">")
|
||||||
|
|> String.replace("\"", """)
|
||||||
|
|> String.replace("'", "'")
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
defmodule ChesstrainerWeb.Plugs.NoIndex do
|
||||||
|
@moduledoc """
|
||||||
|
Tells crawlers not to index the response.
|
||||||
|
|
||||||
|
Uses the `X-Robots-Tag` header rather than a `<meta name="robots">` tag: it
|
||||||
|
survives redirects and authentication failures, and covers LiveView pages
|
||||||
|
without every template having to assign `:noindex`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import Plug.Conn
|
||||||
|
|
||||||
|
def init(opts), do: opts
|
||||||
|
|
||||||
|
def call(conn, _opts) do
|
||||||
|
put_resp_header(conn, "x-robots-tag", "noindex, nofollow")
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -17,6 +17,15 @@ defmodule ChesstrainerWeb.Router do
|
|||||||
plug :accepts, ["json"]
|
plug :accepts, ["json"]
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Crawler files are plain text and XML, so they must not negotiate HTML.
|
||||||
|
pipeline :seo do
|
||||||
|
end
|
||||||
|
|
||||||
|
# Never let search engines index private or authenticated areas.
|
||||||
|
pipeline :noindex do
|
||||||
|
plug ChesstrainerWeb.Plugs.NoIndex
|
||||||
|
end
|
||||||
|
|
||||||
scope "/", ChesstrainerWeb do
|
scope "/", ChesstrainerWeb do
|
||||||
pipe_through :browser
|
pipe_through :browser
|
||||||
|
|
||||||
@@ -28,7 +37,7 @@ defmodule ChesstrainerWeb.Router do
|
|||||||
end
|
end
|
||||||
|
|
||||||
scope "/admin", ChesstrainerWeb do
|
scope "/admin", ChesstrainerWeb do
|
||||||
pipe_through [:browser, :require_admin_user]
|
pipe_through [:browser, :noindex, :require_admin_user]
|
||||||
|
|
||||||
live_session :endgames,
|
live_session :endgames,
|
||||||
on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do
|
on_mount: [{ChesstrainerWeb.UserAuth, :require_admin}] do
|
||||||
@@ -101,6 +110,15 @@ defmodule ChesstrainerWeb.Router do
|
|||||||
delete "/users/log-out", UserSessionController, :delete
|
delete "/users/log-out", UserSessionController, :delete
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Crawler files live outside the catch-all so they are real responses, not
|
||||||
|
# the branded 404 page.
|
||||||
|
scope "/", ChesstrainerWeb do
|
||||||
|
pipe_through :seo
|
||||||
|
|
||||||
|
get "/robots.txt", RobotsController, :index
|
||||||
|
get "/sitemap.xml", SitemapController, :index
|
||||||
|
end
|
||||||
|
|
||||||
# Keep this last: routes match in definition order. It gives unmatched paths a
|
# Keep this last: routes match in definition order. It gives unmatched paths a
|
||||||
# branded 404 page with the site chrome instead of Phoenix's bare status page.
|
# branded 404 page with the site chrome instead of Phoenix's bare status page.
|
||||||
scope "/", ChesstrainerWeb do
|
scope "/", ChesstrainerWeb do
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
|
|
||||||
#
|
|
||||||
# To ban all spiders from the entire site uncomment the next two lines:
|
|
||||||
# User-agent: *
|
|
||||||
# Disallow: /
|
|
||||||
@@ -59,6 +59,27 @@ defmodule ChesstrainerWeb.AdminGatingTest do
|
|||||||
assert html_response(conn, 200)
|
assert html_response(conn, 200)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "admin routes are marked noindex for crawlers", %{conn: conn} do
|
||||||
|
conn = get(conn, ~p"/admin/endgames")
|
||||||
|
|
||||||
|
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
|
||||||
|
|
||||||
|
{:ok, admin} = Accounts.update_user_role(user_fixture(), :admin)
|
||||||
|
|
||||||
|
conn =
|
||||||
|
build_conn()
|
||||||
|
|> log_in_user(admin)
|
||||||
|
|> get(~p"/admin/endgames")
|
||||||
|
|
||||||
|
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "public pages are not marked noindex", %{conn: conn} do
|
||||||
|
conn = get(conn, ~p"/")
|
||||||
|
|
||||||
|
assert get_resp_header(conn, "x-robots-tag") == []
|
||||||
|
end
|
||||||
|
|
||||||
test "the navbar hides the Admin link from members", %{conn: conn} do
|
test "the navbar hides the Admin link from members", %{conn: conn} do
|
||||||
conn =
|
conn =
|
||||||
conn
|
conn
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
defmodule ChesstrainerWeb.RobotsControllerTest do
|
||||||
|
use ChesstrainerWeb.ConnCase, async: true
|
||||||
|
|
||||||
|
alias ChesstrainerWeb.Endpoint
|
||||||
|
|
||||||
|
describe "GET /robots.txt" do
|
||||||
|
test "allows crawlers and points at the sitemap", %{conn: conn} do
|
||||||
|
body = conn |> get(~p"/robots.txt") |> response(200)
|
||||||
|
|
||||||
|
assert body =~ "User-agent: *"
|
||||||
|
assert body =~ "Allow: /"
|
||||||
|
assert body =~ "Sitemap: #{Endpoint.url()}/sitemap.xml"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "is served as plain text", %{conn: conn} do
|
||||||
|
conn = get(conn, ~p"/robots.txt")
|
||||||
|
|
||||||
|
assert [content_type] = get_resp_header(conn, "content-type")
|
||||||
|
assert content_type =~ "text/plain"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
defmodule ChesstrainerWeb.SitemapControllerTest do
|
||||||
|
use ChesstrainerWeb.ConnCase, async: true
|
||||||
|
|
||||||
|
alias ChesstrainerWeb.Endpoint
|
||||||
|
|
||||||
|
describe "GET /sitemap.xml" do
|
||||||
|
test "lists only the public marketing pages", %{conn: conn} do
|
||||||
|
body = conn |> get(~p"/sitemap.xml") |> response(200)
|
||||||
|
base_url = Endpoint.url()
|
||||||
|
|
||||||
|
assert body =~ ~s(<?xml version="1.0" encoding="UTF-8"?>)
|
||||||
|
assert body =~ ~s(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">)
|
||||||
|
assert body =~ "<loc>#{base_url}/</loc>"
|
||||||
|
assert body =~ "<loc>#{base_url}/privacy</loc>"
|
||||||
|
assert body =~ "<loc>#{base_url}/terms-conditions</loc>"
|
||||||
|
assert length(:binary.matches(body, "<url>")) == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "is served as XML", %{conn: conn} do
|
||||||
|
conn = get(conn, ~p"/sitemap.xml")
|
||||||
|
|
||||||
|
assert [content_type] = get_resp_header(conn, "content-type")
|
||||||
|
assert content_type =~ "application/xml"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
Reference in New Issue
Block a user