Add structured production logging (#105, #107) (#141)

* Update README.md

* Add structured JSON production logging (#105)

* Log requests and background failures (#107)

* Document production logging and the VPS runbook
This commit is contained in:
2026-11-10 22:00:05 +01:00
committed by GitHub
parent f8897fa5ae
commit 202b9b1f6a
20 changed files with 1380 additions and 15 deletions
+63
View File
@@ -0,0 +1,63 @@
defmodule Chesstrainer.Background do
@moduledoc """
Runs background work and logs any failure.
Background failures otherwise disappear: a task exits, the supervisor keeps
the rest of the system running, and nobody sees why the work did not happen.
This wrapper turns crashes, throws, exits and `{:error, reason}` returns into
one structured error line with the reason and stacktrace.
The label is a short description of the work; metadata is added to the log
line unchanged (minus the formatter's own redaction).
"""
require Logger
@type result :: term()
@doc """
Runs `fun` and logs a failure. Returns the function's value, or `:error` when
it crashed or returned an error tuple.
"""
@spec run(String.t(), (-> result)) :: result() | :error
def run(label, fun), do: run(label, [], fun)
@doc """
Like `run/2`, with extra metadata for the error line.
"""
@spec run(String.t(), keyword(), (-> result)) :: result() | :error
def run(label, metadata, fun)
when is_binary(label) and is_list(metadata) and is_function(fun, 0) do
case fun.() do
{:error, reason} ->
Logger.error(
"#{label} failed: #{format_term(reason)}",
Keyword.merge(metadata, reason: reason)
)
:error
result ->
result
end
rescue
exception ->
Logger.error(
"#{label} crashed: #{Exception.message(exception)}",
Keyword.merge(metadata, exception: exception, stacktrace: __STACKTRACE__)
)
:error
catch
kind, reason ->
Logger.error(
"#{label} crashed: #{format_term(reason)}",
Keyword.merge(metadata, kind: kind, reason: reason, stacktrace: __STACKTRACE__)
)
:error
end
defp format_term(term) when is_binary(term), do: term
defp format_term(term), do: inspect(term, limit: 10, printable_limit: 500)
end
+16 -2
View File
@@ -11,7 +11,8 @@ defmodule Chesstrainer.Cache do
that were used heavily once but have not been used since eventually age out.
A daily backup runs off the cache process at a configurable UTC time, so reads
and writes stay responsive while the snapshot is written.
and writes stay responsive while the snapshot is written. Failures are logged
rather than lost.
## Examples
@@ -19,7 +20,9 @@ defmodule Chesstrainer.Cache do
"""
use GenServer
require Logger
alias Chesstrainer.Background
alias Chesstrainer.Cache.BackupSchedule
alias Chesstrainer.Cache.Persist
@@ -121,7 +124,18 @@ defmodule Chesstrainer.Cache do
table = state.table
backup_fun = state.backup_fun
Task.Supervisor.start_child(Chesstrainer.Cache.BackupSupervisor, fn -> backup_fun.(table) end)
task = fn ->
Background.run("cache backup", [table: table], fn -> backup_fun.(table) end)
end
case Task.Supervisor.start_child(Chesstrainer.Cache.BackupSupervisor, task) do
{:ok, _pid} ->
:ok
{:error, reason} ->
Logger.error("cache backup could not start", table: table, reason: reason)
:ok
end
end
defp schedule_backup(state) do
+7 -1
View File
@@ -5,8 +5,13 @@ defmodule Chesstrainer.Cache.Persist do
A backup is written to a temporary DETS file in the same directory and then
atomically renamed into place. A crash during the write therefore leaves the
previous backup untouched instead of a truncated file that fails to restore.
A backup that cannot be opened at boot is logged and skipped, so a damaged
file does not stop the application from starting with an empty cache.
"""
require Logger
@backup_path "priv/static/backups"
@format_version "v2"
@@ -54,7 +59,8 @@ defmodule Chesstrainer.Cache.Persist do
:dets.close(table)
{:error, _} ->
{:error, reason} ->
Logger.warning("cache backup restore failed", table: table, reason: reason)
:ok
end
end
@@ -11,6 +11,8 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
import Chesstrainer.Ratelimiters
alias Chesstrainer.Endgames.Tablebase
require Logger
@lichess_tablebase_url "https://tablebase.lichess.ovh/standard?fen="
@spec from_fen(String.t()) ::
@@ -74,6 +76,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
# Server errors
{:ok, %Req.Response{status: status, body: body}} when status in 500..599 ->
Logger.warning("tablebase fetch failed with HTTP #{status}", status: status)
{:error, {:server_error, status, body}}
# Catch‑all for other status codes
@@ -82,6 +85,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
# Network / client‑side failure
{:error, reason} ->
Logger.warning("tablebase fetch failed: #{inspect(reason)}", reason: reason)
{:error, reason}
end
end
+307
View File
@@ -0,0 +1,307 @@
defmodule Chesstrainer.JSONLogFormatter do
@moduledoc """
An OTP logger formatter that emits one JSON object per log line.
Production uses this formatter (see `config/prod.exs`), while development and
test keep `Logger.Formatter`'s readable output. Every line carries a
timestamp, level and message, plus a request id when one is in the logger
metadata. Access lines add `method`, `path`, `status` and `duration_ms`;
exceptions and OTP crash reports add an `error` object with `kind`, `reason`
and `stacktrace`.
The formatter is defensive. Metadata is sanitised into JSON-safe values, and
an event that cannot be encoded still produces a single JSON line instead of
taking the logger down. Email addresses and credentials found in metadata,
params or the rendered message are replaced with `[REDACTED]`.
"""
@max_message_bytes 8_096
@max_stacktrace_entries 50
@redacted "[REDACTED]"
@email_regex ~r/[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~-]+@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+/
@secret_assignment_regex ~r/\b(token|password|secret|api[_-]?key|authorization)=([^&\s"']+)/i
@sensitive_key_fragments ~w(password secret token authorization cookie api_key private_key credential)
@sensitive_email_keys ~w(email email_address user_email contact_email recipient)
# These are unpacked into the line itself rather than copied as metadata.
@internal_keys [
:time,
:gl,
:report_cb,
:error_logger,
:exception,
:stacktrace,
:crash_reason,
:reason,
:kind,
:conn,
:socket
]
# Fixed fields always win over a metadata key with the same name.
@reserved_keys [
:timestamp,
:level,
:message,
:error,
:request_id,
:method,
:path,
:status,
:duration_ms
]
@doc """
Formats a `:logger` event as a newline-terminated JSON object.
"""
@spec format(:logger.log_event(), :logger.formatter_config()) :: iodata()
def format(event, _config) do
event
|> build()
|> Jason.encode!()
|> then(&[&1, ?\n])
rescue
exception -> fallback(exception)
catch
kind, reason -> fallback({kind, reason})
end
defp build(event) do
meta = event.meta || %{}
%{
"timestamp" => timestamp(meta),
"level" => to_string(event.level),
"message" => message(event)
}
|> put_access(meta)
|> put_metadata(meta)
|> put_error(meta)
end
defp timestamp(%{time: time}) when is_integer(time) do
time
|> DateTime.from_unix!(:microsecond)
|> DateTime.to_iso8601()
end
defp timestamp(_meta), do: DateTime.utc_now() |> DateTime.to_iso8601()
defp message(event) do
event
|> Logger.Formatter.format_event(@max_message_bytes)
|> Logger.Formatter.prune()
|> IO.iodata_to_binary()
|> redact_text()
rescue
_exception ->
event.msg
|> inspect(limit: 20, printable_limit: @max_message_bytes)
|> redact_text()
catch
_kind, reason ->
{event.msg, reason}
|> inspect(limit: 20, printable_limit: @max_message_bytes)
|> redact_text()
end
defp put_access(entry, meta) do
entry
|> put_if_present("request_id", meta[:request_id])
|> put_if_present("method", meta[:method])
|> put_if_present("path", meta[:path])
|> put_if_present("status", meta[:status])
|> put_if_present("duration_ms", meta[:duration_ms])
end
defp put_if_present(entry, _key, nil), do: entry
defp put_if_present(entry, key, value) do
Map.put(entry, key, sanitize(value, key))
end
defp put_metadata(entry, meta) do
Enum.reduce(meta, entry, fn
{key, _value}, acc when key in @internal_keys ->
acc
{key, _value}, acc when key in @reserved_keys ->
acc
{key, value}, acc ->
Map.put(acc, key |> key_string() |> redact_text(), metadata_value(key, value))
end)
end
defp metadata_value(:mfa, {module, function, arity})
when is_atom(module) and is_atom(function) and is_integer(arity) do
Exception.format_mfa(module, function, arity)
end
defp metadata_value(key, value), do: sanitize(value, key)
# Only the first exception/reason source is rendered. OTP crash reports carry
# `crash_reason`, while Logger.error(..., exception: ..., stacktrace: ...)
# carries the two keys directly.
defp put_error(entry, meta) do
case error(meta) do
nil -> entry
error when map_size(error) == 0 -> entry
error -> Map.put(entry, "error", error)
end
end
defp error(%{exception: %{__exception__: true} = exception} = meta) do
%{
"kind" => inspect(exception.__struct__),
"reason" => Exception.message(exception),
"stacktrace" => stacktrace(meta[:stacktrace])
}
|> drop_nil()
end
defp error(%{crash_reason: crash_reason} = meta) do
{reason, trace} = split_crash_reason(crash_reason, meta[:stacktrace])
%{
"kind" => error_kind(reason),
"reason" => error_reason(reason),
"stacktrace" => stacktrace(trace)
}
|> drop_nil()
end
defp error(meta) do
if meta[:reason] || meta[:stacktrace] do
%{
"kind" => meta[:kind] && to_string(meta[:kind]),
"reason" => error_reason(meta[:reason]),
"stacktrace" => stacktrace(meta[:stacktrace])
}
|> drop_nil()
end
end
defp split_crash_reason({reason, trace}, _fallback) when is_list(trace), do: {reason, trace}
defp split_crash_reason(reason, fallback), do: {reason, fallback}
defp error_kind(%{__exception__: true} = exception), do: inspect(exception.__struct__)
defp error_kind(reason) when is_atom(reason), do: Atom.to_string(reason)
defp error_kind(reason) when is_tuple(reason), do: reason |> elem(0) |> error_kind()
defp error_kind(_reason), do: nil
defp error_reason(nil), do: nil
defp error_reason(%{__exception__: true} = exception), do: Exception.message(exception)
defp error_reason(reason) when is_binary(reason), do: redact_text(reason)
defp error_reason(reason) do
reason
|> inspect(limit: 30, printable_limit: 2_000)
|> redact_text()
end
defp stacktrace(stacktrace) when is_list(stacktrace) and stacktrace != [] do
stacktrace
|> Enum.take(@max_stacktrace_entries)
|> Enum.map(&format_stacktrace_entry/1)
|> Enum.map(&redact_text/1)
end
defp stacktrace(_stacktrace), do: nil
defp format_stacktrace_entry(entry) do
Exception.format_stacktrace_entry(entry)
rescue
_exception -> inspect(entry, limit: 10, printable_limit: 1_000)
end
defp drop_nil(map), do: Map.reject(map, fn {_key, value} -> is_nil(value) end)
defp key_string(key) when is_atom(key), do: Atom.to_string(key)
defp key_string(key) when is_binary(key), do: key
defp key_string(key), do: inspect(key)
defp sanitize(nil, _key), do: nil
defp sanitize(value, _key) when is_boolean(value), do: value
defp sanitize(value, _key) when is_integer(value) or is_float(value), do: value
defp sanitize(value, key) when is_binary(value) do
cond do
sensitive_key?(key) -> @redacted
String.valid?(value) -> redact_text(value)
true -> inspect(value)
end
end
defp sanitize(value, _key) when is_atom(value), do: Atom.to_string(value)
defp sanitize(value, _key) when is_struct(value) do
value
|> inspect(limit: 20, printable_limit: 2_000)
|> redact_text()
end
defp sanitize(value, key) when is_list(value) do
cond do
sensitive_key?(key) -> @redacted
value != [] and List.ascii_printable?(value) -> value |> List.to_string() |> redact_text()
true -> Enum.map(value, &sanitize(&1, nil))
end
end
defp sanitize(value, _key) when is_tuple(value) do
value
|> Tuple.to_list()
|> Enum.map(&sanitize(&1, nil))
end
defp sanitize(value, _key) when is_map(value) do
Enum.into(value, %{}, fn {key, nested} ->
{key |> key_string() |> redact_text(), sanitize(nested, key)}
end)
end
defp sanitize(value, _key) do
value
|> inspect(limit: 20, printable_limit: 2_000)
|> redact_text()
end
defp sensitive_key?(nil), do: false
defp sensitive_key?(key) do
key = key |> key_string() |> String.downcase()
key in @sensitive_email_keys or String.ends_with?(key, "_email") or
Enum.any?(@sensitive_key_fragments, &String.contains?(key, &1))
end
defp redact_text(text) when is_binary(text) do
text = if String.valid?(text), do: text, else: inspect(text)
text
|> then(&Regex.replace(@email_regex, &1, @redacted))
|> then(&Regex.replace(@secret_assignment_regex, &1, "\\1=#{@redacted}"))
end
defp fallback(reason) do
entry = %{
"timestamp" => DateTime.utc_now() |> DateTime.to_iso8601(),
"level" => "error",
"message" => "could not format log event",
"error" => %{
"reason" => reason |> inspect(limit: 10, printable_limit: 1_000) |> redact_text()
}
}
json =
case Jason.encode(entry) do
{:ok, json} -> json
{:error, _reason} -> ~s({"level":"error","message":"could not format log event"})
end
[json, ?\n]
end
end
+4 -1
View File
@@ -41,7 +41,10 @@ defmodule ChesstrainerWeb.Endpoint do
cookie_key: "request_logger"
plug Plug.RequestId
plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]
# Access lines come from ChesstrainerWeb.RequestLogger (one structured line
# per request); disable Phoenix.Logger's endpoint start/stop pair.
plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint], log: false
plug Plug.Parsers,
parsers: [:urlencoded, :multipart, :json],
@@ -0,0 +1,25 @@
defmodule ChesstrainerWeb.Plugs.PutRequestId do
@moduledoc """
Copies `Plug.RequestId`'s request id into the session.
LiveView processes handle later events after the original HTTP request has
finished. Keeping the id in the session lets `UserAuth.mount_current_scope/2`
put it into `Logger.metadata/1`, so LiveView logs remain correlatable with the
request that mounted them.
"""
@behaviour Plug
import Plug.Conn
@impl true
def init(opts), do: opts
@impl true
def call(conn, _opts) do
case conn.assigns[:request_id] do
nil -> conn
request_id -> put_session(conn, :request_id, request_id)
end
end
end
+66
View File
@@ -0,0 +1,66 @@
defmodule ChesstrainerWeb.RequestLogger do
@moduledoc """
Emits one structured access line per HTTP request.
The line carries the method, path, status, duration in milliseconds and the
request id when one is available. Query parameters and bodies are never
logged. Path segments that look like tokens or identifiers (20 or more
URL-safe characters) are replaced with `[REDACTED]`.
`Phoenix.Logger`'s own endpoint start/stop lines are disabled in
`ChesstrainerWeb.Endpoint` so each request produces exactly one line.
"""
require Logger
@event [:phoenix, :endpoint, :stop]
@handler_id {__MODULE__, :access_log}
@token_segment ~r/\A[A-Za-z0-9_.-]{20,}\z/
@doc false
def install do
case :telemetry.attach(@handler_id, @event, &__MODULE__.handle_event/4, :ok) do
:ok -> :ok
{:error, :already_exists} -> :ok
end
end
@doc false
def handle_event(_event, measurements, metadata, _config) do
conn = metadata.conn
duration_ms = duration_ms(measurements.duration)
path = redact_path(conn.request_path)
status = conn.status || 0
metadata =
[
method: conn.method,
path: path,
status: status,
duration_ms: duration_ms
]
|> put_request_id(conn.assigns[:request_id])
Logger.info("#{conn.method} #{path} #{status} in #{duration_ms}ms", metadata)
end
defp duration_ms(native) do
native
|> System.convert_time_unit(:native, :microsecond)
|> Kernel./(1000)
|> Float.round(1)
end
defp put_request_id(metadata, nil), do: metadata
defp put_request_id(metadata, request_id), do: Keyword.put(metadata, :request_id, request_id)
defp redact_path(path) do
path
|> String.split("/")
|> Enum.map_join("/", &redact_segment/1)
end
defp redact_segment(segment) do
if Regex.match?(@token_segment, segment), do: "[REDACTED]", else: segment
end
end
+1
View File
@@ -6,6 +6,7 @@ defmodule ChesstrainerWeb.Router do
pipeline :browser do
plug :accepts, ["html"]
plug :fetch_session
plug ChesstrainerWeb.Plugs.PutRequestId
plug :fetch_live_flash
plug :put_root_layout, html: {ChesstrainerWeb.Layouts, :root}
plug :protect_from_forgery
+2
View File
@@ -8,6 +8,8 @@ defmodule ChesstrainerWeb.Telemetry do
@impl true
def init(_arg) do
ChesstrainerWeb.RequestLogger.install()
children = [
# Telemetry poller will execute the given period measurements
# every 10_000ms. Learn more here: https://telemetry-metrics.hexdocs.pm
+4
View File
@@ -290,6 +290,10 @@ defmodule ChesstrainerWeb.UserAuth do
end
defp mount_current_scope(socket, session) do
if request_id = session["request_id"] do
Logger.metadata(request_id: request_id)
end
Phoenix.Component.assign_new(socket, :current_scope, fn ->
{user, _} =
if user_token = session["user_token"] do