From a5bb1cb670aabba2354a3f9dccf391b577349eac Mon Sep 17 00:00:00 2001 From: Owen Rees Date: Mon, 2 Nov 2026 23:02:01 +0100 Subject: [PATCH] Scale the tablebase cache to ~100k entries (#24) * Slim the cached tablebase value to the fields the trainer reads (#20) * Sample for LFU eviction instead of scanning the cache (#21) * Make cache size configurable and back up daily off-peak (#22) * Document the cache design in the glossary and an ADR (#23) * Address cache-scaling code review findings --- GLOSSARY.md | 4 +- config/config.exs | 6 + config/runtime.exs | 54 ++++++ .../0004-tablebase-cache-shape-and-scaling.md | 27 +++ lib/chesstrainer/application.ex | 5 +- lib/chesstrainer/cache.ex | 64 ++++--- lib/chesstrainer/cache/backup_schedule.ex | 70 ++++++++ lib/chesstrainer/cache/evict.ex | 92 ++++++---- lib/chesstrainer/cache/persist.ex | 61 ++++++- lib/chesstrainer/endgames/tablebase.ex | 11 +- lib/chesstrainer/endgames/tablebase/move.ex | 27 +-- lib/chesstrainer/endgames/tablebase/parser.ex | 14 +- .../cache/backup_schedule_test.exs | 57 +++++++ test/chesstrainer/cache/evict_test.exs | 78 +++++++++ test/chesstrainer/cache_test.exs | 159 +++++++++++++++++- test/chesstrainer/endgames/tablebase_test.exs | 86 ++++++++-- 16 files changed, 695 insertions(+), 120 deletions(-) create mode 100644 docs/adr/0004-tablebase-cache-shape-and-scaling.md create mode 100644 lib/chesstrainer/cache/backup_schedule.ex create mode 100644 test/chesstrainer/cache/backup_schedule_test.exs create mode 100644 test/chesstrainer/cache/evict_test.exs diff --git a/GLOSSARY.md b/GLOSSARY.md index ef78538..d1d85dc 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -63,7 +63,9 @@ promotion (still winning) for conversion puzzles, mate otherwise. **Tablebase**: A database of precomputed optimal outcomes for every position with at most -seven pieces, queried via the lichess API and cached. +seven pieces, queried via the lichess API and cached. The trainer caches only +the outcome data it grades on — enough to tell a winning move from a losing +one — not every metric the tablebase reports, and refetches anything else. **Cooldown**: Rate-limit state entered after a 429 from the lichess tablebase API, during diff --git a/config/config.exs b/config/config.exs index 54b44fd..c348dbe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -58,6 +58,12 @@ config :tailwind, env: %{"NODE_PATH" => [Path.expand("../deps", __DIR__), Mix.Project.build_path()]} ] +# Configure the tablebase cache: maximum entries and the UTC time of the daily +# backup. Both can be overridden with environment variables at runtime. +config :chesstrainer, Chesstrainer.Cache, + max_size: 100_000, + backup_time: {3, 0} + # Configure Elixir's Logger config :logger, :default_formatter, format: "$time $metadata[$level] $message\n", diff --git a/config/runtime.exs b/config/runtime.exs index 221c71d..4f22e79 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -20,6 +20,60 @@ if System.get_env("PHX_SERVER") do config :chesstrainer, ChesstrainerWeb.Endpoint, server: true end +# Tablebase cache: size and the UTC time of the daily backup. Environment +# variables win over the defaults in config/config.exs so deployments can tune +# the cache without a rebuild. Invalid values are reported and ignored rather +# than crashing boot with an opaque parse error. +cache_config = Application.get_env(:chesstrainer, Chesstrainer.Cache, []) +default_max_size = Keyword.fetch!(cache_config, :max_size) +default_backup_time = Keyword.fetch!(cache_config, :backup_time) + +cache_max_size = + case System.get_env("TABLEBASE_CACHE_MAX_SIZE") do + nil -> + default_max_size + + value -> + case Integer.parse(value) do + {size, ""} when size > 0 -> + size + + _ -> + IO.warn( + "ignoring TABLEBASE_CACHE_MAX_SIZE=#{inspect(value)}: " <> + "expected a positive integer, using #{default_max_size}", + [] + ) + + default_max_size + end + end + +cache_backup_time = + case System.get_env("TABLEBASE_BACKUP_TIME") do + nil -> + default_backup_time + + value -> + case Chesstrainer.Cache.BackupSchedule.parse_time(value) do + {:ok, time} -> + time + + :error -> + IO.warn( + "ignoring TABLEBASE_BACKUP_TIME=#{inspect(value)}: " <> + "expected HH:MM in UTC, using #{inspect(default_backup_time)}", + [] + ) + + default_backup_time + end + end + +config :chesstrainer, Chesstrainer.Cache, + max_size: cache_max_size, + backup_time: cache_backup_time + config :chesstrainer, ChesstrainerWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))] diff --git a/docs/adr/0004-tablebase-cache-shape-and-scaling.md b/docs/adr/0004-tablebase-cache-shape-and-scaling.md new file mode 100644 index 0000000..a52e4d7 --- /dev/null +++ b/docs/adr/0004-tablebase-cache-shape-and-scaling.md @@ -0,0 +1,27 @@ +# Cache only the graded tablebase outcome, at ~100k entries with sampled eviction and daily best-effort backups + +The cached value keeps only what the trainer reads: the position's category, +checkmate flag and moves (each with category, `dtz`, `san` and `uci`), plus +`dtm`/`dtc` as a small, deliberate hedge for a future "mate in N" or +distance-to-conversion feature. The rest of the API response — `dtw`, precise +DTZ, per-move terminal and variant flags, material flags — is dropped: it is +never read, and slimming each entry from ~10 KB to ~4 KB keeps the working set +in memory. Cursed-win and blessed-loss are normalised to a draw at parse time, +so the variant and `dtw` fields mean nothing here. The cost is that a richer +trainer would need a cache shape (and DETS format) change and a refill; +`dtm`/`dtc` keep the most likely such feature cheap. + +The cache holds 100k entries by default and is configurable. At ~4 KB per entry +that is roughly 400 MB of BEAM memory per node — accepted because the cache is a +warm, per-node optimization rather than the source of truth, and the cap is +configurable for smaller deployments. That is large enough that the recurring +working set rarely evicts, and at capacity eviction samples ~64 entries and +drops the least-frequently-used of those rather than scanning the whole table +for the exact victim: a cache that re-fetches on a miss only needs *a* cold +entry, and the O(n) scan per put was what stopped the cache scaling past ~100k. +The weekly frequency-halving sweep stays, because it is time-based aging rather +than victim selection. The cache is warm state — the +API is the source of truth — so a daily backup at 03:00 UTC, written off the +cache process to a temporary file and atomically renamed, is the right +durability level: a crash can lose up to a day of cache, costing rate-limited +re-fetches rather than data, and a torn write can never replace a good backup. diff --git a/lib/chesstrainer/application.ex b/lib/chesstrainer/application.ex index 65f5b77..10519b6 100644 --- a/lib/chesstrainer/application.ex +++ b/lib/chesstrainer/application.ex @@ -7,6 +7,8 @@ defmodule Chesstrainer.Application do @impl true def start(_type, _args) do + cache_opts = Application.get_env(:chesstrainer, Chesstrainer.Cache, []) + children = [ ChesstrainerWeb.Telemetry, Chesstrainer.Repo, @@ -17,7 +19,8 @@ defmodule Chesstrainer.Application do # Start to serve requests, typically the last entry ChesstrainerWeb.Endpoint, {Registry, keys: :unique, name: Chesstrainer.CacheRegistry}, - {Chesstrainer.Cache, :tablebase} + {Task.Supervisor, name: Chesstrainer.Cache.BackupSupervisor}, + {Chesstrainer.Cache, {:tablebase, cache_opts}} ] # See https://elixir.hexdocs.pm/Supervisor.html diff --git a/lib/chesstrainer/cache.ex b/lib/chesstrainer/cache.ex index c101aa4..ab4f4ec 100644 --- a/lib/chesstrainer/cache.ex +++ b/lib/chesstrainer/cache.ex @@ -4,9 +4,14 @@ defmodule Chesstrainer.Cache do Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump the frequency; when the cache is over its maximum size the least frequently - used entry is evicted (ties broken by least recent use). A weekly sweep halves - every frequency so entries that were used heavily once but have not been used - since eventually age out. + used entry *of a rotating bounded sample* is evicted (ties broken by least + recent use). Eviction is therefore an approximation of the global LFU victim, + which keeps each put cheap at the cost of occasionally dropping an entry that + was not quite the coldest. A weekly sweep halves every frequency so entries + that were used heavily once but have not been used since eventually age out. + + A daily backup runs off the cache process at a configurable UTC time, so reads + and writes stay responsive while the snapshot is written. ## Examples @@ -15,11 +20,15 @@ defmodule Chesstrainer.Cache do use GenServer - @default_max_size 1000 - @default_sweep_interval :timer.hours(24 * 7) - @backup_interval_minutes 60 + alias Chesstrainer.Cache.BackupSchedule + alias Chesstrainer.Cache.Persist - def start_link(table), do: start_link(table, []) + @default_max_size 100_000 + @default_sweep_interval :timer.hours(24 * 7) + + def start_link(table) when is_atom(table), do: start_link(table, []) + + def start_link({table, opts}), do: start_link(table, opts) def start_link(table, opts) when is_list(opts) do GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table)) @@ -28,7 +37,7 @@ defmodule Chesstrainer.Cache do def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value}) def get(table, key), do: GenServer.call(via_tuple(table), {:get, key}) def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key}) - def backup(table), do: GenServer.call(via_tuple(table), :backup) + def backup(table), do: Persist.backup(table) def sweep(table), do: GenServer.call(via_tuple(table), :sweep) def show(table) do @@ -43,18 +52,21 @@ defmodule Chesstrainer.Cache do state = %{ table: table, max_size: Keyword.get(opts, :max_size, @default_max_size), - sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval) + sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval), + backup_time: BackupSchedule.time!(Keyword.get(opts, :backup_time)), + backup_fun: Keyword.get(opts, :backup_fun, &Persist.backup/1), + backup_timer: nil, + evict_cursor: :start } # Create and restore synchronously so the cache is never observed empty # between `start_link/1` returning and the restore finishing. create_table(table) - __MODULE__.Persist.restore(table) - __MODULE__.Evict.evict(table, state.max_size) - :timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup) + Persist.restore(table) + evict_cursor = __MODULE__.Evict.evict(table, state.max_size, state.evict_cursor) :timer.send_interval(state.sweep_interval, :sweep) - {:ok, state} + {:ok, schedule_backup(%{state | evict_cursor: evict_cursor})} end @impl true @@ -62,8 +74,8 @@ defmodule Chesstrainer.Cache do ensure_table(state.table) access = System.system_time(:microsecond) :ets.insert(state.table, {key, value, 1, access}) - __MODULE__.Evict.evict(state.table, state.max_size) - {:reply, :ok, state} + evict_cursor = __MODULE__.Evict.evict(state.table, state.max_size, state.evict_cursor) + {:reply, :ok, %{state | evict_cursor: evict_cursor}} end @impl true @@ -87,12 +99,6 @@ defmodule Chesstrainer.Cache do {:reply, :ok, state} end - @impl true - def handle_call(:backup, _from, state) do - __MODULE__.Persist.backup(state.table) - {:reply, :ok, state} - end - @impl true def handle_call(:sweep, _from, state) do __MODULE__.Sweep.halve_frequencies(state.table) @@ -101,8 +107,8 @@ defmodule Chesstrainer.Cache do @impl true def handle_info(:backup, state) do - __MODULE__.Persist.backup(state.table) - {:noreply, state} + run_backup(state) + {:noreply, schedule_backup(state)} end @impl true @@ -111,6 +117,18 @@ defmodule Chesstrainer.Cache do {:noreply, state} end + defp run_backup(state) do + table = state.table + backup_fun = state.backup_fun + + Task.Supervisor.start_child(Chesstrainer.Cache.BackupSupervisor, fn -> backup_fun.(table) end) + end + + defp schedule_backup(state) do + timer = Process.send_after(self(), :backup, BackupSchedule.next_delay(state.backup_time)) + %{state | backup_timer: timer} + end + defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}} defp ensure_table(table) do diff --git a/lib/chesstrainer/cache/backup_schedule.ex b/lib/chesstrainer/cache/backup_schedule.ex new file mode 100644 index 0000000..33c7fa8 --- /dev/null +++ b/lib/chesstrainer/cache/backup_schedule.ex @@ -0,0 +1,70 @@ +defmodule Chesstrainer.Cache.BackupSchedule do + @moduledoc """ + Daily off-peak schedule for cache backups. + + The cache backs up once a day at a configurable UTC time (03:00 by default). + The delay is computed from the wall clock, so restarts and clock drift do not + shift the run towards peak hours the way a fixed interval from boot would. + + This module only computes the schedule; writing the backup is + `Chesstrainer.Cache.Persist.backup/1`. + """ + + @default_time {3, 0} + + @doc """ + Parses a backup time given as `{hour, minute}` or an `"HH:MM"` string. + """ + def parse_time({hour, minute}) when hour in 0..23 and minute in 0..59, + do: {:ok, {hour, minute}} + + def parse_time(value) when is_binary(value) do + with [hour, minute] <- String.split(value, ":"), + {hour, ""} <- Integer.parse(hour), + {minute, ""} <- Integer.parse(minute), + true <- hour in 0..23 and minute in 0..59 do + {:ok, {hour, minute}} + else + _ -> :error + end + end + + def parse_time(_value), do: :error + + @doc """ + Normalises a backup time, falling back to #{inspect(@default_time)} UTC when + no time is configured. + """ + def time!(nil), do: @default_time + + def time!(value) do + case parse_time(value) do + {:ok, time} -> + time + + :error -> + raise ArgumentError, "invalid cache backup time: #{inspect(value)}" + end + end + + @doc """ + Milliseconds from `now` until the next occurrence of `time`, in UTC. + + When `now` is exactly the scheduled time the next run is a full day away, so + a run always re-arms the following day rather than firing twice. + """ + def next_delay(time, now \\ DateTime.utc_now()) do + {hour, minute} = time!(time) + + today = DateTime.new!(DateTime.to_date(now), Time.new!(hour, minute, 0)) + + next_run = + if DateTime.compare(today, now) == :gt do + today + else + DateTime.add(today, 1, :day) + end + + DateTime.diff(next_run, now, :millisecond) + end +end diff --git a/lib/chesstrainer/cache/evict.ex b/lib/chesstrainer/cache/evict.ex index c4eff39..7ed43ee 100644 --- a/lib/chesstrainer/cache/evict.ex +++ b/lib/chesstrainer/cache/evict.ex @@ -1,44 +1,78 @@ defmodule Chesstrainer.Cache.Evict do @moduledoc """ Eviction for the LFU cache. + + Eviction is deliberately approximate. Folding over the whole table to find the + exact least-frequently-used entry costs O(n) on every put once the cache is at + capacity, which stops the cache scaling past ~100k entries. Instead each pass + samples a bounded number of entries and evicts the coldest of those: a cache + that re-fetches on a miss only needs "a cold entry", not the coldest one. + + The sample walks the table through an `:ets.select/3` continuation, so + successive passes look at different slices instead of rescanning the same + first few entries. Over many puts every entry gets a turn to be considered, + while each pass still costs the same bounded amount. """ + @sample_size 64 + + @match_spec [{{:"$1", :_, :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}] + + @typedoc """ + Position in the table's select traversal. + + `:start` begins a new pass; any other value resumes the pass where the + previous eviction left off. + """ + @type cursor :: :start | term() + @doc """ - Evicts least-frequently-used entries until `table` holds at most `max_size` - rows. Ties in frequency are broken by least recent use (`last_access`). - """ - def evict(table, max_size) do - if :ets.info(table, :size) > max_size do - case find_least_used_entry(table) do - {key, _frequency, _last_access} -> - :ets.delete(table, key) - evict(table, max_size) + Evicts entries until `table` holds at most `max_size` rows, returning the + cursor to pass to the next call. - nil -> - :ok + Each pass samples at most #{@sample_size} entries and deletes the least + frequently used one of them, then resumes from the advanced cursor. Ties in + frequency are broken by least recent use (`last_access`). + """ + @spec evict(:ets.table(), non_neg_integer(), cursor()) :: cursor() + def evict(table, max_size, cursor \\ :start) do + if :ets.info(table, :size) > max_size do + case sample(table, cursor) do + {[], :start} -> + # End of the table: wrap and keep evicting. If we were already at the + # start there is nothing left to sample (an empty table), so stop. + if cursor == :start, do: :start, else: evict(table, max_size, :start) + + {entries, next} -> + {key, _frequency, _last_access} = + Enum.min_by(entries, fn {_key, frequency, last_access} -> + {frequency, last_access} + end) + + :ets.delete(table, key) + evict(table, max_size, next) end else - :ok + cursor end end - defp find_least_used_entry(table) do - :ets.foldl( - fn {k, _v, frequency, last_access}, acc -> - case acc do - nil -> - {k, frequency, last_access} + defp sample(table, :start) do + case :ets.select(table, @match_spec, @sample_size) do + {entries, continuation} -> {entries, continuation} + :"$end_of_table" -> {[], :start} + end + end - {_k, best_frequency, best_access} - when {frequency, last_access} < {best_frequency, best_access} -> - {k, frequency, last_access} - - _ -> - acc - end - end, - nil, - table - ) + defp sample(table, cursor) do + case :ets.select(cursor) do + {entries, next} -> {entries, next} + :"$end_of_table" -> {[], :start} + end + rescue + # A continuation is only valid for the table instance that produced it. + # A deleted and recreated table, or a restore, invalidates it, so restart + # the rotation instead of crashing the cache. + ArgumentError -> sample(table, :start) end end diff --git a/lib/chesstrainer/cache/persist.ex b/lib/chesstrainer/cache/persist.ex index 7840f65..8ffa6f9 100644 --- a/lib/chesstrainer/cache/persist.ex +++ b/lib/chesstrainer/cache/persist.ex @@ -1,25 +1,38 @@ defmodule Chesstrainer.Cache.Persist do @moduledoc """ Persistence for the cache. Backup and restore functions. + + A backup is written to a temporary DETS file in the same directory and then + atomically renamed into place. A crash during the write therefore leaves the + previous backup untouched instead of a truncated file that fails to restore. """ @backup_path "priv/static/backups" @format_version "v2" + @doc """ + Writes the `table` to its backup file. Returns `:ok` or `{:error, reason}`. + """ def backup(table) do File.mkdir_p!(@backup_path) - name = backup_filename(table) |> String.to_charlist() - {:ok, dets_table} = :dets.open_file(name, type: :set) - :ets.to_dets(table, dets_table) - :dets.sync(dets_table) - :dets.close(dets_table) + path = backup_path(table) + + :global.trans( + {__MODULE__, table}, + fn -> write(table, path <> ".tmp", path) end, + [node()], + :infinity + ) end + @doc """ + Restores the backup file for `table`, if a readable one exists. + """ def restore(table) do File.mkdir_p!(@backup_path) drop_legacy_backup(table) - path = backup_filename(table) + path = backup_path(table) if File.exists?(path) do name = String.to_charlist(path) @@ -49,7 +62,41 @@ defmodule Chesstrainer.Cache.Persist do :ok end - defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det") + @doc """ + The path of the backup file for `table`. + """ + def backup_path(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det") + + defp write(table, temp, path) do + # Remove a stale temporary file left by an earlier crash; if the path is + # unusable the open below fails and the existing backup stays in place. + _ = File.rm(temp) + + result = + with {:ok, dets_table} <- :dets.open_file(String.to_charlist(temp), type: :set) do + try do + _ = :ets.to_dets(table, dets_table) + :dets.sync(dets_table) + after + :dets.close(dets_table) + end + end + + case result do + :ok -> + case File.rename(temp, path) do + :ok -> + :ok + + {:error, reason} -> + _ = File.rm(temp) + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end defp drop_legacy_backup(table) do legacy = Path.join(@backup_path, "#{table}.det") diff --git a/lib/chesstrainer/endgames/tablebase.ex b/lib/chesstrainer/endgames/tablebase.ex index 3fcb674..2ad3a85 100644 --- a/lib/chesstrainer/endgames/tablebase.ex +++ b/lib/chesstrainer/endgames/tablebase.ex @@ -1,6 +1,11 @@ defmodule Chesstrainer.Endgames.Tablebase do @moduledoc """ All endgame tablebase related functions. + + The cached value carries only what the trainer reads: `category`, + `checkmate`, and `moves` drive play, while `dtm`/`dtc` are small distance + hedges. Cursed-win/blessed-loss are normalised to `:draw` at parse time, so + the raw `dtw`/`dtz`/`insufficient_material` fields are not kept here. """ alias Chesstrainer.Endgames.Tablebase @@ -12,9 +17,6 @@ defmodule Chesstrainer.Endgames.Tablebase do checkmate: boolean(), dtc: integer() | nil, dtm: integer() | nil, - dtw: integer() | nil, - dtz: integer() | nil, - insufficient_material: boolean(), moves: [Tablebase.Move.t()] } @@ -22,9 +24,6 @@ defmodule Chesstrainer.Endgames.Tablebase do checkmate: false, dtc: nil, dtm: nil, - dtw: nil, - dtz: nil, - insufficient_material: false, moves: nil @doc """ diff --git a/lib/chesstrainer/endgames/tablebase/move.ex b/lib/chesstrainer/endgames/tablebase/move.ex index 1650eea..f0aabc2 100644 --- a/lib/chesstrainer/endgames/tablebase/move.ex +++ b/lib/chesstrainer/endgames/tablebase/move.ex @@ -1,41 +1,28 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do @moduledoc """ Moves struct for the tablebase + + Carries only the fields the trainer reads: `category` and `dtz` for grading, + `san`/`uci` for display and matching, and `dtm`/`dtc` as small distance + hedges. Terminal detection is structural (the next position's `checkmate` or + its empty `moves`), so no per-move terminal flags are kept. """ @type result :: :win | :draw | :loss | :unknown @type t :: %__MODULE__{ category: result, - checkmate: boolean(), - conversion: boolean(), dtc: integer() | nil, dtm: integer() | nil, - dtw: integer() | nil, dtz: integer() | nil, - precise_dtz: integer() | nil, san: String.t() | nil, - stalemate: boolean(), - uci: String.t() | nil, - variant_loss: boolean(), - variant_win: boolean(), - zeroing: boolean(), - insufficient_material: boolean() + uci: String.t() | nil } defstruct category: nil, - checkmate: false, - conversion: false, dtc: nil, dtm: nil, - dtw: nil, dtz: nil, - precise_dtz: nil, san: nil, - stalemate: false, - uci: nil, - variant_loss: false, - variant_win: false, - zeroing: false, - insufficient_material: false + uci: nil end diff --git a/lib/chesstrainer/endgames/tablebase/parser.ex b/lib/chesstrainer/endgames/tablebase/parser.ex index 2e05ed5..33211c8 100644 --- a/lib/chesstrainer/endgames/tablebase/parser.ex +++ b/lib/chesstrainer/endgames/tablebase/parser.ex @@ -45,9 +45,6 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do checkmate: map["checkmate"], dtc: map["dtc"], dtm: map["dtm"], - dtw: map["dtw"], - dtz: map["dtz"], - insufficient_material: map["insufficient_material"], moves: Enum.map(map["moves"], &moves_from_map/1) }} end @@ -55,20 +52,11 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do defp moves_from_map(map) do %Tablebase.Move{ category: Tablebase.normalize_category(map["category"]), - checkmate: map["checkmate"], - conversion: map["conversion"], dtc: map["dtc"], dtm: map["dtm"], - dtw: map["dtw"], dtz: map["dtz"], - insufficient_material: map["insufficient_material"], - precise_dtz: map["precise_dtz"], san: map["san"], - stalemate: map["stalemate"], - uci: map["uci"], - variant_loss: map["variant_loss"], - variant_win: map["variant_win"], - zeroing: map["zeroing"] + uci: map["uci"] } end diff --git a/test/chesstrainer/cache/backup_schedule_test.exs b/test/chesstrainer/cache/backup_schedule_test.exs new file mode 100644 index 0000000..4fde5eb --- /dev/null +++ b/test/chesstrainer/cache/backup_schedule_test.exs @@ -0,0 +1,57 @@ +defmodule Chesstrainer.Cache.BackupScheduleTest do + use ExUnit.Case, async: true + + alias Chesstrainer.Cache.BackupSchedule + + describe "parse_time/1" do + test "accepts HH:MM strings and {hour, minute} tuples" do + assert {:ok, {3, 0}} = BackupSchedule.parse_time("03:00") + assert {:ok, {23, 59}} = BackupSchedule.parse_time("23:59") + assert {:ok, {0, 5}} = BackupSchedule.parse_time("0:05") + assert {:ok, {4, 30}} = BackupSchedule.parse_time({4, 30}) + end + + test "rejects values that are not a valid UTC time" do + assert :error = BackupSchedule.parse_time("24:00") + assert :error = BackupSchedule.parse_time("03:60") + assert :error = BackupSchedule.parse_time("03:00:00") + assert :error = BackupSchedule.parse_time("three") + assert :error = BackupSchedule.parse_time({24, 0}) + assert :error = BackupSchedule.parse_time(nil) + end + end + + describe "time!/1" do + test "defaults to 03:00 UTC" do + assert {3, 0} = BackupSchedule.time!(nil) + assert {3, 0} = BackupSchedule.time!("03:00") + end + end + + describe "next_delay/2" do + test "returns the delay until today's run when it is still ahead" do + now = ~U[2026-11-02 01:30:00Z] + + assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(1) + :timer.minutes(30) + end + + test "rolls over to tomorrow once today's run has passed" do + now = ~U[2026-11-02 04:00:00Z] + + assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(23) + end + + test "waits a full day when called exactly at the scheduled time" do + now = ~U[2026-11-02 03:00:00Z] + + assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(24) + end + + test "computes the delay from the wall clock by default" do + delay = BackupSchedule.next_delay({0, 0}) + + assert delay > 0 + assert delay <= :timer.hours(24) + end + end +end diff --git a/test/chesstrainer/cache/evict_test.exs b/test/chesstrainer/cache/evict_test.exs new file mode 100644 index 0000000..33367d3 --- /dev/null +++ b/test/chesstrainer/cache/evict_test.exs @@ -0,0 +1,78 @@ +defmodule Chesstrainer.Cache.EvictTest do + use ExUnit.Case, async: true + + alias Chesstrainer.Cache.Evict + + @sample_size 64 + + setup do + table = :"evict_test_#{System.unique_integer([:positive])}" + :ets.new(table, [:named_table, :public, :set]) + + {:ok, table: table} + end + + describe "evict/3" do + test "shrinks a table that is far over capacity", %{table: table} do + for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i}) + + Evict.evict(table, 900) + + assert :ets.info(table, :size) == 900 + end + + test "keeps a frequently used entry while cold entries are evicted", %{table: table} do + :ets.insert(table, {:hot, :hot, 100, 0}) + for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i}) + + Evict.evict(table, 100) + + assert :ets.info(table, :size) == 100 + assert [{:hot, :hot, 100, 0}] = :ets.lookup(table, :hot) + end + + test "the returned cursor resumes the sample instead of restarting it", %{table: table} do + for i <- 1..5, do: :ets.insert(table, {i, i, 1, i}) + + assert Evict.evict(table, 10) == :start + + cursor = Evict.evict(table, 4) + + assert :ets.info(table, :size) == 4 + refute cursor == :start + end + + test "rotates across the table, eventually evicting entries outside the first sample", + %{table: table} do + for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i}) + + # Find the slice a fresh pass would sample and make it the hottest part of + # the table. A pass that always restarted from `:start` would keep + # choosing from this slice (every one of those entries is hotter than + # everything else), so cold entries elsewhere could never be evicted. + hot_keys = first_sample_keys(table) + for key <- hot_keys, do: :ets.insert(table, {key, key, 1_000, key}) + + Enum.reduce(1..@sample_size, :start, fn _, cursor -> + Evict.evict(table, :ets.info(table, :size) - 1, cursor) + end) + + frequencies = Map.new(:ets.tab2list(table), fn {key, _v, f, _t} -> {key, f} end) + cold_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1 end) + hot_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1_000 end) + + assert :ets.info(table, :size) == 1_000 - @sample_size + assert cold_left < 1_000 - @sample_size + assert hot_left > 0 + end + end + + defp first_sample_keys(table) do + match_spec = [{{:"$1", :_, :"$2", :"$3"}, [], [:"$1"]}] + + case :ets.select(table, match_spec, @sample_size) do + {keys, _continuation} -> keys + :"$end_of_table" -> [] + end + end +end diff --git a/test/chesstrainer/cache_test.exs b/test/chesstrainer/cache_test.exs index 0c6d264..9d48538 100644 --- a/test/chesstrainer/cache_test.exs +++ b/test/chesstrainer/cache_test.exs @@ -5,8 +5,9 @@ defmodule Chesstrainer.CacheTest do setup context do table = :"test_#{System.unique_integer([:positive])}" - {:ok, pid} = Cache.start_link(table, Map.get(context, :cache_opts, [])) - on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) + opts = Map.get(context, :cache_opts, []) + pid = start_supervised!(Supervisor.child_spec({Cache, {table, opts}}, id: table)) + {:ok, %{pid: pid, table: table}} end @@ -37,6 +38,32 @@ defmodule Chesstrainer.CacheTest do end end + describe "configuration" do + test "defaults to a max size of 100_000", %{pid: pid} do + assert :sys.get_state(pid).max_size == 100_000 + end + + test "starts from a {table, opts} child spec argument" do + table = :"test_child_spec_#{System.unique_integer([:positive])}" + + pid = + start_supervised!(Supervisor.child_spec({Cache, {table, max_size: 3}}, id: table)) + + assert :sys.get_state(pid).max_size == 3 + end + + test "the application starts the tablebase cache with the configured options" do + config = Application.get_env(:chesstrainer, Chesstrainer.Cache, []) + pid = GenServer.whereis({:via, Registry, {Chesstrainer.CacheRegistry, :tablebase}}) + state = :sys.get_state(pid) + + assert state.max_size == Keyword.get(config, :max_size, 100_000) + + assert state.backup_time == + Chesstrainer.Cache.BackupSchedule.time!(Keyword.get(config, :backup_time)) + end + end + describe "frequency tracking" do test "get/2 increments the entry frequency", %{table: table} do Cache.put(table, "key", "value") @@ -77,6 +104,26 @@ defmodule Chesstrainer.CacheTest do end end + describe "eviction on a cache larger than the sample" do + @describetag cache_opts: [max_size: 100] + + test "stays at the maximum size under churn", %{table: table} do + for i <- 1..500, do: Cache.put(table, i, i) + + assert Cache.size(table) == 100 + end + + test "a frequently used entry survives churn from cold entries", %{table: table} do + Cache.put(table, :hot, "value") + for _ <- 1..50, do: Cache.get(table, :hot) + + for i <- 1..500, do: Cache.put(table, i, i) + + assert Cache.size(table) == 100 + assert {:ok, "value"} = Cache.get(table, :hot) + end + end + describe "weekly sweep" do test "halves every entry frequency", %{table: table} do Cache.put(table, "busy", "value") @@ -92,23 +139,123 @@ defmodule Chesstrainer.CacheTest do end end + describe "backup" do + test "runs off the cache process so reads and writes stay responsive" do + table = :"test_async_backup_#{System.unique_integer([:positive])}" + parent = self() + ref = make_ref() + + backup_fun = fn _table -> + send(parent, {:backup_started, ref, self()}) + + receive do + {:finish_backup, ^ref} -> :ok + end + end + + pid = + start_supervised!( + Supervisor.child_spec({Cache, {table, [backup_fun: backup_fun]}}, id: table) + ) + + assert :ok = Cache.put(table, "key", "value") + + send(pid, :backup) + + assert_receive {:backup_started, ^ref, backup_pid}, 1_000 + backup_monitor = Process.monitor(backup_pid) + + assert {:ok, "value"} = Cache.get(table, "key") + assert :ok = Cache.put(table, "other", "value") + + send(backup_pid, {:finish_backup, ref}) + assert_receive {:DOWN, ^backup_monitor, :process, ^backup_pid, :normal}, 1_000 + end + + test "re-arms the daily backup after each run" do + table = :"test_rearm_#{System.unique_integer([:positive])}" + + pid = + start_supervised!( + Supervisor.child_spec( + {Cache, {table, [backup_fun: fn _table -> :ok end, backup_time: "04:30"]}}, + id: table + ) + ) + + send(pid, :backup) + + state = :sys.get_state(pid) + + assert state.backup_time == {4, 30} + assert is_reference(state.backup_timer) + assert :erlang.read_timer(state.backup_timer) > 0 + end + end + describe "persistence" do + test "writes the backup atomically, leaving no temporary file" do + table = :"test_atomic_#{System.unique_integer([:positive])}" + backup = Path.join("priv/static/backups", "#{table}.v2.det") + + on_exit(fn -> + File.rm_rf(backup <> ".tmp") + File.rm(backup) + end) + + start_supervised!(Supervisor.child_spec({Cache, table}, id: table)) + + Cache.put(table, "key", "value") + + assert :ok = Cache.backup(table) + assert File.exists?(backup) + refute File.exists?(backup <> ".tmp") + end + + test "a failed backup leaves the previous backup intact" do + table = :"test_failed_backup_#{System.unique_integer([:positive])}" + backup = Path.join("priv/static/backups", "#{table}.v2.det") + temp = backup <> ".tmp" + + on_exit(fn -> + File.rm_rf(temp) + File.rm(backup) + end) + + start_supervised!(Supervisor.child_spec({Cache, table}, id: table)) + + Cache.put(table, "key", "value") + assert :ok = Cache.backup(table) + + # A directory at the temporary path makes the write fail before the rename. + File.mkdir_p!(temp) + Cache.put(table, "another", "value") + + assert {:error, _reason} = Cache.Persist.backup(table) + assert File.exists?(backup) + assert :ok = stop_supervised(table) + + start_supervised!(Supervisor.child_spec({Cache, table}, id: table)) + + assert {:ok, "value"} = Cache.get(table, "key") + assert :error = Cache.get(table, "another") + end + test "backup and restore preserve frequencies" do table = :"test_persist_#{System.unique_integer([:positive])}" backup = Path.join("priv/static/backups", "#{table}.v2.det") on_exit(fn -> File.rm(backup) end) - {:ok, pid} = Cache.start_link(table) + start_supervised!(Supervisor.child_spec({Cache, table}, id: table)) Cache.put(table, "key", "value") Cache.get(table, "key") Cache.get(table, "key") assert :ok = Cache.backup(table) - assert :ok = GenServer.stop(pid) + assert :ok = stop_supervised(table) - {:ok, restored_pid} = Cache.start_link(table) - on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end) + start_supervised!(Supervisor.child_spec({Cache, table}, id: table)) assert [{"key", "value", frequency, _last_access}] = Cache.show(table) assert frequency == 3 diff --git a/test/chesstrainer/endgames/tablebase_test.exs b/test/chesstrainer/endgames/tablebase_test.exs index 9e8bf5c..bd63269 100644 --- a/test/chesstrainer/endgames/tablebase_test.exs +++ b/test/chesstrainer/endgames/tablebase_test.exs @@ -32,6 +32,18 @@ defmodule Chesstrainer.Endgames.TablebaseTest do end end + describe "struct shape" do + test "Tablebase carries only the fields the trainer reads" do + assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() == + [:category, :checkmate, :dtc, :dtm, :moves] + end + + test "Tablebase.Move carries only the fields the trainer reads" do + assert %Tablebase.Move{} |> Map.from_struct() |> Map.keys() |> Enum.sort() == + [:category, :dtc, :dtm, :dtz, :san, :uci] + end + end + describe "Parser.from_map/1" do test "normalises the position and move categories" do assert {:ok, tablebase} = @@ -40,32 +52,78 @@ defmodule Chesstrainer.Endgames.TablebaseTest do "checkmate" => false, "dtc" => nil, "dtm" => 20, - "dtw" => nil, "dtz" => 3, - "insufficient_material" => false, "moves" => [ %{ "category" => "blessed-loss", - "checkmate" => false, - "conversion" => false, "dtc" => nil, "dtm" => -19, - "dtw" => nil, "dtz" => -2, - "insufficient_material" => false, - "precise_dtz" => -2, "san" => "Kd7", - "stalemate" => false, - "uci" => "e8d7", - "variant_loss" => false, - "variant_win" => false, - "zeroing" => false + "uci" => "e8d7" } ] }) - assert tablebase.category == :draw - assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves + assert %Tablebase{category: :draw, checkmate: false, dtc: nil, dtm: 20} = tablebase + + assert [%Tablebase.Move{category: :draw, dtc: nil, dtm: -19, dtz: -2}] = tablebase.moves + assert [%Tablebase.Move{san: "Kd7", uci: "e8d7"}] = tablebase.moves + end + + test "drops the fields the trainer never reads" do + assert {:ok, tablebase} = + Parser.from_map(%{ + "category" => "win", + "checkmate" => true, + "dtc" => 1, + "dtm" => 2, + "dtw" => 3, + "dtz" => 4, + "insufficient_material" => false, + "moves" => [ + %{ + "category" => "loss", + "checkmate" => true, + "conversion" => true, + "dtc" => 5, + "dtm" => 6, + "dtw" => 7, + "dtz" => 8, + "insufficient_material" => false, + "precise_dtz" => 9, + "san" => "Qd7#", + "stalemate" => false, + "uci" => "d1d7", + "variant_loss" => false, + "variant_win" => false, + "zeroing" => true + } + ] + }) + + assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() == + [:category, :checkmate, :dtc, :dtm, :moves] + + assert [move] = tablebase.moves + + assert move |> Map.from_struct() |> Map.keys() |> Enum.sort() == + [:category, :dtc, :dtm, :dtz, :san, :uci] + end + + test "keeps checkmate and empty moves, which drive terminal detection" do + assert {:ok, tablebase} = + Parser.from_map(%{ + "category" => "win", + "checkmate" => true, + "dtc" => 1, + "dtm" => 2, + "dtz" => 3, + "moves" => [] + }) + + assert tablebase.checkmate + assert tablebase.moves == [] end end