Scale the tablebase cache to ~100k entries (#24)

* Slim the cached tablebase value to the fields the trainer reads (#20)

* Sample for LFU eviction instead of scanning the cache (#21)

* Make cache size configurable and back up daily off-peak (#22)

* Document the cache design in the glossary and an ADR (#23)

* Address cache-scaling code review findings
This commit is contained in:
2026-11-02 23:02:01 +01:00
committed by GitHub
parent fa71783e55
commit a5bb1cb670
16 changed files with 695 additions and 120 deletions
+3 -1
View File
@@ -63,7 +63,9 @@ promotion (still winning) for conversion puzzles, mate otherwise.
**Tablebase**:
A database of precomputed optimal outcomes for every position with at most
seven pieces, queried via the lichess API and cached.
seven pieces, queried via the lichess API and cached. The trainer caches only
the outcome data it grades on — enough to tell a winning move from a losing
one — not every metric the tablebase reports, and refetches anything else.
**Cooldown**:
Rate-limit state entered after a 429 from the lichess tablebase API, during
+6
View File
@@ -58,6 +58,12 @@ config :tailwind,
env: %{"NODE_PATH" => [Path.expand("../deps", __DIR__), Mix.Project.build_path()]}
]
# Configure the tablebase cache: maximum entries and the UTC time of the daily
# backup. Both can be overridden with environment variables at runtime.
config :chesstrainer, Chesstrainer.Cache,
max_size: 100_000,
backup_time: {3, 0}
# Configure Elixir's Logger
config :logger, :default_formatter,
format: "$time $metadata[$level] $message\n",
+54
View File
@@ -20,6 +20,60 @@ if System.get_env("PHX_SERVER") do
config :chesstrainer, ChesstrainerWeb.Endpoint, server: true
end
# Tablebase cache: size and the UTC time of the daily backup. Environment
# variables win over the defaults in config/config.exs so deployments can tune
# the cache without a rebuild. Invalid values are reported and ignored rather
# than crashing boot with an opaque parse error.
cache_config = Application.get_env(:chesstrainer, Chesstrainer.Cache, [])
default_max_size = Keyword.fetch!(cache_config, :max_size)
default_backup_time = Keyword.fetch!(cache_config, :backup_time)
cache_max_size =
case System.get_env("TABLEBASE_CACHE_MAX_SIZE") do
nil ->
default_max_size
value ->
case Integer.parse(value) do
{size, ""} when size > 0 ->
size
_ ->
IO.warn(
"ignoring TABLEBASE_CACHE_MAX_SIZE=#{inspect(value)}: " <>
"expected a positive integer, using #{default_max_size}",
[]
)
default_max_size
end
end
cache_backup_time =
case System.get_env("TABLEBASE_BACKUP_TIME") do
nil ->
default_backup_time
value ->
case Chesstrainer.Cache.BackupSchedule.parse_time(value) do
{:ok, time} ->
time
:error ->
IO.warn(
"ignoring TABLEBASE_BACKUP_TIME=#{inspect(value)}: " <>
"expected HH:MM in UTC, using #{inspect(default_backup_time)}",
[]
)
default_backup_time
end
end
config :chesstrainer, Chesstrainer.Cache,
max_size: cache_max_size,
backup_time: cache_backup_time
config :chesstrainer, ChesstrainerWeb.Endpoint,
http: [port: String.to_integer(System.get_env("PORT", "4000"))]
@@ -0,0 +1,27 @@
# Cache only the graded tablebase outcome, at ~100k entries with sampled eviction and daily best-effort backups
The cached value keeps only what the trainer reads: the position's category,
checkmate flag and moves (each with category, `dtz`, `san` and `uci`), plus
`dtm`/`dtc` as a small, deliberate hedge for a future "mate in N" or
distance-to-conversion feature. The rest of the API response — `dtw`, precise
DTZ, per-move terminal and variant flags, material flags — is dropped: it is
never read, and slimming each entry from ~10 KB to ~4 KB keeps the working set
in memory. Cursed-win and blessed-loss are normalised to a draw at parse time,
so the variant and `dtw` fields mean nothing here. The cost is that a richer
trainer would need a cache shape (and DETS format) change and a refill;
`dtm`/`dtc` keep the most likely such feature cheap.
The cache holds 100k entries by default and is configurable. At ~4 KB per entry
that is roughly 400 MB of BEAM memory per node — accepted because the cache is a
warm, per-node optimization rather than the source of truth, and the cap is
configurable for smaller deployments. That is large enough that the recurring
working set rarely evicts, and at capacity eviction samples ~64 entries and
drops the least-frequently-used of those rather than scanning the whole table
for the exact victim: a cache that re-fetches on a miss only needs *a* cold
entry, and the O(n) scan per put was what stopped the cache scaling past ~100k.
The weekly frequency-halving sweep stays, because it is time-based aging rather
than victim selection. The cache is warm state — the
API is the source of truth — so a daily backup at 03:00 UTC, written off the
cache process to a temporary file and atomically renamed, is the right
durability level: a crash can lose up to a day of cache, costing rate-limited
re-fetches rather than data, and a torn write can never replace a good backup.
+4 -1
View File
@@ -7,6 +7,8 @@ defmodule Chesstrainer.Application do
@impl true
def start(_type, _args) do
cache_opts = Application.get_env(:chesstrainer, Chesstrainer.Cache, [])
children = [
ChesstrainerWeb.Telemetry,
Chesstrainer.Repo,
@@ -17,7 +19,8 @@ defmodule Chesstrainer.Application do
# Start to serve requests, typically the last entry
ChesstrainerWeb.Endpoint,
{Registry, keys: :unique, name: Chesstrainer.CacheRegistry},
{Chesstrainer.Cache, :tablebase}
{Task.Supervisor, name: Chesstrainer.Cache.BackupSupervisor},
{Chesstrainer.Cache, {:tablebase, cache_opts}}
]
# See https://elixir.hexdocs.pm/Supervisor.html
+41 -23
View File
@@ -4,9 +4,14 @@ defmodule Chesstrainer.Cache do
Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump
the frequency; when the cache is over its maximum size the least frequently
used entry is evicted (ties broken by least recent use). A weekly sweep halves
every frequency so entries that were used heavily once but have not been used
since eventually age out.
used entry *of a rotating bounded sample* is evicted (ties broken by least
recent use). Eviction is therefore an approximation of the global LFU victim,
which keeps each put cheap at the cost of occasionally dropping an entry that
was not quite the coldest. A weekly sweep halves every frequency so entries
that were used heavily once but have not been used since eventually age out.
A daily backup runs off the cache process at a configurable UTC time, so reads
and writes stay responsive while the snapshot is written.
## Examples
@@ -15,11 +20,15 @@ defmodule Chesstrainer.Cache do
use GenServer
@default_max_size 1000
@default_sweep_interval :timer.hours(24 * 7)
@backup_interval_minutes 60
alias Chesstrainer.Cache.BackupSchedule
alias Chesstrainer.Cache.Persist
def start_link(table), do: start_link(table, [])
@default_max_size 100_000
@default_sweep_interval :timer.hours(24 * 7)
def start_link(table) when is_atom(table), do: start_link(table, [])
def start_link({table, opts}), do: start_link(table, opts)
def start_link(table, opts) when is_list(opts) do
GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table))
@@ -28,7 +37,7 @@ defmodule Chesstrainer.Cache do
def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value})
def get(table, key), do: GenServer.call(via_tuple(table), {:get, key})
def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key})
def backup(table), do: GenServer.call(via_tuple(table), :backup)
def backup(table), do: Persist.backup(table)
def sweep(table), do: GenServer.call(via_tuple(table), :sweep)
def show(table) do
@@ -43,18 +52,21 @@ defmodule Chesstrainer.Cache do
state = %{
table: table,
max_size: Keyword.get(opts, :max_size, @default_max_size),
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval)
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval),
backup_time: BackupSchedule.time!(Keyword.get(opts, :backup_time)),
backup_fun: Keyword.get(opts, :backup_fun, &Persist.backup/1),
backup_timer: nil,
evict_cursor: :start
}
# Create and restore synchronously so the cache is never observed empty
# between `start_link/1` returning and the restore finishing.
create_table(table)
__MODULE__.Persist.restore(table)
__MODULE__.Evict.evict(table, state.max_size)
:timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup)
Persist.restore(table)
evict_cursor = __MODULE__.Evict.evict(table, state.max_size, state.evict_cursor)
:timer.send_interval(state.sweep_interval, :sweep)
{:ok, state}
{:ok, schedule_backup(%{state | evict_cursor: evict_cursor})}
end
@impl true
@@ -62,8 +74,8 @@ defmodule Chesstrainer.Cache do
ensure_table(state.table)
access = System.system_time(:microsecond)
:ets.insert(state.table, {key, value, 1, access})
__MODULE__.Evict.evict(state.table, state.max_size)
{:reply, :ok, state}
evict_cursor = __MODULE__.Evict.evict(state.table, state.max_size, state.evict_cursor)
{:reply, :ok, %{state | evict_cursor: evict_cursor}}
end
@impl true
@@ -87,12 +99,6 @@ defmodule Chesstrainer.Cache do
{:reply, :ok, state}
end
@impl true
def handle_call(:backup, _from, state) do
__MODULE__.Persist.backup(state.table)
{:reply, :ok, state}
end
@impl true
def handle_call(:sweep, _from, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
@@ -101,8 +107,8 @@ defmodule Chesstrainer.Cache do
@impl true
def handle_info(:backup, state) do
__MODULE__.Persist.backup(state.table)
{:noreply, state}
run_backup(state)
{:noreply, schedule_backup(state)}
end
@impl true
@@ -111,6 +117,18 @@ defmodule Chesstrainer.Cache do
{:noreply, state}
end
defp run_backup(state) do
table = state.table
backup_fun = state.backup_fun
Task.Supervisor.start_child(Chesstrainer.Cache.BackupSupervisor, fn -> backup_fun.(table) end)
end
defp schedule_backup(state) do
timer = Process.send_after(self(), :backup, BackupSchedule.next_delay(state.backup_time))
%{state | backup_timer: timer}
end
defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}}
defp ensure_table(table) do
+70
View File
@@ -0,0 +1,70 @@
defmodule Chesstrainer.Cache.BackupSchedule do
@moduledoc """
Daily off-peak schedule for cache backups.
The cache backs up once a day at a configurable UTC time (03:00 by default).
The delay is computed from the wall clock, so restarts and clock drift do not
shift the run towards peak hours the way a fixed interval from boot would.
This module only computes the schedule; writing the backup is
`Chesstrainer.Cache.Persist.backup/1`.
"""
@default_time {3, 0}
@doc """
Parses a backup time given as `{hour, minute}` or an `"HH:MM"` string.
"""
def parse_time({hour, minute}) when hour in 0..23 and minute in 0..59,
do: {:ok, {hour, minute}}
def parse_time(value) when is_binary(value) do
with [hour, minute] <- String.split(value, ":"),
{hour, ""} <- Integer.parse(hour),
{minute, ""} <- Integer.parse(minute),
true <- hour in 0..23 and minute in 0..59 do
{:ok, {hour, minute}}
else
_ -> :error
end
end
def parse_time(_value), do: :error
@doc """
Normalises a backup time, falling back to #{inspect(@default_time)} UTC when
no time is configured.
"""
def time!(nil), do: @default_time
def time!(value) do
case parse_time(value) do
{:ok, time} ->
time
:error ->
raise ArgumentError, "invalid cache backup time: #{inspect(value)}"
end
end
@doc """
Milliseconds from `now` until the next occurrence of `time`, in UTC.
When `now` is exactly the scheduled time the next run is a full day away, so
a run always re-arms the following day rather than firing twice.
"""
def next_delay(time, now \\ DateTime.utc_now()) do
{hour, minute} = time!(time)
today = DateTime.new!(DateTime.to_date(now), Time.new!(hour, minute, 0))
next_run =
if DateTime.compare(today, now) == :gt do
today
else
DateTime.add(today, 1, :day)
end
DateTime.diff(next_run, now, :millisecond)
end
end
+63 -29
View File
@@ -1,44 +1,78 @@
defmodule Chesstrainer.Cache.Evict do
@moduledoc """
Eviction for the LFU cache.
Eviction is deliberately approximate. Folding over the whole table to find the
exact least-frequently-used entry costs O(n) on every put once the cache is at
capacity, which stops the cache scaling past ~100k entries. Instead each pass
samples a bounded number of entries and evicts the coldest of those: a cache
that re-fetches on a miss only needs "a cold entry", not the coldest one.
The sample walks the table through an `:ets.select/3` continuation, so
successive passes look at different slices instead of rescanning the same
first few entries. Over many puts every entry gets a turn to be considered,
while each pass still costs the same bounded amount.
"""
@sample_size 64
@match_spec [{{:"$1", :_, :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]
@typedoc """
Position in the table's select traversal.
`:start` begins a new pass; any other value resumes the pass where the
previous eviction left off.
"""
@type cursor :: :start | term()
@doc """
Evicts least-frequently-used entries until `table` holds at most `max_size`
rows. Ties in frequency are broken by least recent use (`last_access`).
"""
def evict(table, max_size) do
if :ets.info(table, :size) > max_size do
case find_least_used_entry(table) do
{key, _frequency, _last_access} ->
:ets.delete(table, key)
evict(table, max_size)
Evicts entries until `table` holds at most `max_size` rows, returning the
cursor to pass to the next call.
nil ->
:ok
Each pass samples at most #{@sample_size} entries and deletes the least
frequently used one of them, then resumes from the advanced cursor. Ties in
frequency are broken by least recent use (`last_access`).
"""
@spec evict(:ets.table(), non_neg_integer(), cursor()) :: cursor()
def evict(table, max_size, cursor \\ :start) do
if :ets.info(table, :size) > max_size do
case sample(table, cursor) do
{[], :start} ->
# End of the table: wrap and keep evicting. If we were already at the
# start there is nothing left to sample (an empty table), so stop.
if cursor == :start, do: :start, else: evict(table, max_size, :start)
{entries, next} ->
{key, _frequency, _last_access} =
Enum.min_by(entries, fn {_key, frequency, last_access} ->
{frequency, last_access}
end)
:ets.delete(table, key)
evict(table, max_size, next)
end
else
:ok
cursor
end
end
defp find_least_used_entry(table) do
:ets.foldl(
fn {k, _v, frequency, last_access}, acc ->
case acc do
nil ->
{k, frequency, last_access}
{_k, best_frequency, best_access}
when {frequency, last_access} < {best_frequency, best_access} ->
{k, frequency, last_access}
_ ->
acc
defp sample(table, :start) do
case :ets.select(table, @match_spec, @sample_size) do
{entries, continuation} -> {entries, continuation}
:"$end_of_table" -> {[], :start}
end
end,
nil,
table
)
end
defp sample(table, cursor) do
case :ets.select(cursor) do
{entries, next} -> {entries, next}
:"$end_of_table" -> {[], :start}
end
rescue
# A continuation is only valid for the table instance that produced it.
# A deleted and recreated table, or a restore, invalidates it, so restart
# the rotation instead of crashing the cache.
ArgumentError -> sample(table, :start)
end
end
+54 -7
View File
@@ -1,25 +1,38 @@
defmodule Chesstrainer.Cache.Persist do
@moduledoc """
Persistence for the cache. Backup and restore functions.
A backup is written to a temporary DETS file in the same directory and then
atomically renamed into place. A crash during the write therefore leaves the
previous backup untouched instead of a truncated file that fails to restore.
"""
@backup_path "priv/static/backups"
@format_version "v2"
@doc """
Writes the `table` to its backup file. Returns `:ok` or `{:error, reason}`.
"""
def backup(table) do
File.mkdir_p!(@backup_path)
name = backup_filename(table) |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(name, type: :set)
:ets.to_dets(table, dets_table)
:dets.sync(dets_table)
:dets.close(dets_table)
path = backup_path(table)
:global.trans(
{__MODULE__, table},
fn -> write(table, path <> ".tmp", path) end,
[node()],
:infinity
)
end
@doc """
Restores the backup file for `table`, if a readable one exists.
"""
def restore(table) do
File.mkdir_p!(@backup_path)
drop_legacy_backup(table)
path = backup_filename(table)
path = backup_path(table)
if File.exists?(path) do
name = String.to_charlist(path)
@@ -49,7 +62,41 @@ defmodule Chesstrainer.Cache.Persist do
:ok
end
defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
@doc """
The path of the backup file for `table`.
"""
def backup_path(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
defp write(table, temp, path) do
# Remove a stale temporary file left by an earlier crash; if the path is
# unusable the open below fails and the existing backup stays in place.
_ = File.rm(temp)
result =
with {:ok, dets_table} <- :dets.open_file(String.to_charlist(temp), type: :set) do
try do
_ = :ets.to_dets(table, dets_table)
:dets.sync(dets_table)
after
:dets.close(dets_table)
end
end
case result do
:ok ->
case File.rename(temp, path) do
:ok ->
:ok
{:error, reason} ->
_ = File.rm(temp)
{:error, reason}
end
{:error, reason} ->
{:error, reason}
end
end
defp drop_legacy_backup(table) do
legacy = Path.join(@backup_path, "#{table}.det")
+5 -6
View File
@@ -1,6 +1,11 @@
defmodule Chesstrainer.Endgames.Tablebase do
@moduledoc """
All endgame tablebase related functions.
The cached value carries only what the trainer reads: `category`,
`checkmate`, and `moves` drive play, while `dtm`/`dtc` are small distance
hedges. Cursed-win/blessed-loss are normalised to `:draw` at parse time, so
the raw `dtw`/`dtz`/`insufficient_material` fields are not kept here.
"""
alias Chesstrainer.Endgames.Tablebase
@@ -12,9 +17,6 @@ defmodule Chesstrainer.Endgames.Tablebase do
checkmate: boolean(),
dtc: integer() | nil,
dtm: integer() | nil,
dtw: integer() | nil,
dtz: integer() | nil,
insufficient_material: boolean(),
moves: [Tablebase.Move.t()]
}
@@ -22,9 +24,6 @@ defmodule Chesstrainer.Endgames.Tablebase do
checkmate: false,
dtc: nil,
dtm: nil,
dtw: nil,
dtz: nil,
insufficient_material: false,
moves: nil
@doc """
+7 -20
View File
@@ -1,41 +1,28 @@
defmodule Chesstrainer.Endgames.Tablebase.Move do
@moduledoc """
Moves struct for the tablebase
Carries only the fields the trainer reads: `category` and `dtz` for grading,
`san`/`uci` for display and matching, and `dtm`/`dtc` as small distance
hedges. Terminal detection is structural (the next position's `checkmate` or
its empty `moves`), so no per-move terminal flags are kept.
"""
@type result :: :win | :draw | :loss | :unknown
@type t :: %__MODULE__{
category: result,
checkmate: boolean(),
conversion: boolean(),
dtc: integer() | nil,
dtm: integer() | nil,
dtw: integer() | nil,
dtz: integer() | nil,
precise_dtz: integer() | nil,
san: String.t() | nil,
stalemate: boolean(),
uci: String.t() | nil,
variant_loss: boolean(),
variant_win: boolean(),
zeroing: boolean(),
insufficient_material: boolean()
uci: String.t() | nil
}
defstruct category: nil,
checkmate: false,
conversion: false,
dtc: nil,
dtm: nil,
dtw: nil,
dtz: nil,
precise_dtz: nil,
san: nil,
stalemate: false,
uci: nil,
variant_loss: false,
variant_win: false,
zeroing: false,
insufficient_material: false
uci: nil
end
+1 -13
View File
@@ -45,9 +45,6 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
checkmate: map["checkmate"],
dtc: map["dtc"],
dtm: map["dtm"],
dtw: map["dtw"],
dtz: map["dtz"],
insufficient_material: map["insufficient_material"],
moves: Enum.map(map["moves"], &moves_from_map/1)
}}
end
@@ -55,20 +52,11 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
defp moves_from_map(map) do
%Tablebase.Move{
category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"],
conversion: map["conversion"],
dtc: map["dtc"],
dtm: map["dtm"],
dtw: map["dtw"],
dtz: map["dtz"],
insufficient_material: map["insufficient_material"],
precise_dtz: map["precise_dtz"],
san: map["san"],
stalemate: map["stalemate"],
uci: map["uci"],
variant_loss: map["variant_loss"],
variant_win: map["variant_win"],
zeroing: map["zeroing"]
uci: map["uci"]
}
end
+57
View File
@@ -0,0 +1,57 @@
defmodule Chesstrainer.Cache.BackupScheduleTest do
use ExUnit.Case, async: true
alias Chesstrainer.Cache.BackupSchedule
describe "parse_time/1" do
test "accepts HH:MM strings and {hour, minute} tuples" do
assert {:ok, {3, 0}} = BackupSchedule.parse_time("03:00")
assert {:ok, {23, 59}} = BackupSchedule.parse_time("23:59")
assert {:ok, {0, 5}} = BackupSchedule.parse_time("0:05")
assert {:ok, {4, 30}} = BackupSchedule.parse_time({4, 30})
end
test "rejects values that are not a valid UTC time" do
assert :error = BackupSchedule.parse_time("24:00")
assert :error = BackupSchedule.parse_time("03:60")
assert :error = BackupSchedule.parse_time("03:00:00")
assert :error = BackupSchedule.parse_time("three")
assert :error = BackupSchedule.parse_time({24, 0})
assert :error = BackupSchedule.parse_time(nil)
end
end
describe "time!/1" do
test "defaults to 03:00 UTC" do
assert {3, 0} = BackupSchedule.time!(nil)
assert {3, 0} = BackupSchedule.time!("03:00")
end
end
describe "next_delay/2" do
test "returns the delay until today's run when it is still ahead" do
now = ~U[2026-11-02 01:30:00Z]
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(1) + :timer.minutes(30)
end
test "rolls over to tomorrow once today's run has passed" do
now = ~U[2026-11-02 04:00:00Z]
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(23)
end
test "waits a full day when called exactly at the scheduled time" do
now = ~U[2026-11-02 03:00:00Z]
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(24)
end
test "computes the delay from the wall clock by default" do
delay = BackupSchedule.next_delay({0, 0})
assert delay > 0
assert delay <= :timer.hours(24)
end
end
end
+78
View File
@@ -0,0 +1,78 @@
defmodule Chesstrainer.Cache.EvictTest do
use ExUnit.Case, async: true
alias Chesstrainer.Cache.Evict
@sample_size 64
setup do
table = :"evict_test_#{System.unique_integer([:positive])}"
:ets.new(table, [:named_table, :public, :set])
{:ok, table: table}
end
describe "evict/3" do
test "shrinks a table that is far over capacity", %{table: table} do
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
Evict.evict(table, 900)
assert :ets.info(table, :size) == 900
end
test "keeps a frequently used entry while cold entries are evicted", %{table: table} do
:ets.insert(table, {:hot, :hot, 100, 0})
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
Evict.evict(table, 100)
assert :ets.info(table, :size) == 100
assert [{:hot, :hot, 100, 0}] = :ets.lookup(table, :hot)
end
test "the returned cursor resumes the sample instead of restarting it", %{table: table} do
for i <- 1..5, do: :ets.insert(table, {i, i, 1, i})
assert Evict.evict(table, 10) == :start
cursor = Evict.evict(table, 4)
assert :ets.info(table, :size) == 4
refute cursor == :start
end
test "rotates across the table, eventually evicting entries outside the first sample",
%{table: table} do
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
# Find the slice a fresh pass would sample and make it the hottest part of
# the table. A pass that always restarted from `:start` would keep
# choosing from this slice (every one of those entries is hotter than
# everything else), so cold entries elsewhere could never be evicted.
hot_keys = first_sample_keys(table)
for key <- hot_keys, do: :ets.insert(table, {key, key, 1_000, key})
Enum.reduce(1..@sample_size, :start, fn _, cursor ->
Evict.evict(table, :ets.info(table, :size) - 1, cursor)
end)
frequencies = Map.new(:ets.tab2list(table), fn {key, _v, f, _t} -> {key, f} end)
cold_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1 end)
hot_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1_000 end)
assert :ets.info(table, :size) == 1_000 - @sample_size
assert cold_left < 1_000 - @sample_size
assert hot_left > 0
end
end
defp first_sample_keys(table) do
match_spec = [{{:"$1", :_, :"$2", :"$3"}, [], [:"$1"]}]
case :ets.select(table, match_spec, @sample_size) do
{keys, _continuation} -> keys
:"$end_of_table" -> []
end
end
end
+153 -6
View File
@@ -5,8 +5,9 @@ defmodule Chesstrainer.CacheTest do
setup context do
table = :"test_#{System.unique_integer([:positive])}"
{:ok, pid} = Cache.start_link(table, Map.get(context, :cache_opts, []))
on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end)
opts = Map.get(context, :cache_opts, [])
pid = start_supervised!(Supervisor.child_spec({Cache, {table, opts}}, id: table))
{:ok, %{pid: pid, table: table}}
end
@@ -37,6 +38,32 @@ defmodule Chesstrainer.CacheTest do
end
end
describe "configuration" do
test "defaults to a max size of 100_000", %{pid: pid} do
assert :sys.get_state(pid).max_size == 100_000
end
test "starts from a {table, opts} child spec argument" do
table = :"test_child_spec_#{System.unique_integer([:positive])}"
pid =
start_supervised!(Supervisor.child_spec({Cache, {table, max_size: 3}}, id: table))
assert :sys.get_state(pid).max_size == 3
end
test "the application starts the tablebase cache with the configured options" do
config = Application.get_env(:chesstrainer, Chesstrainer.Cache, [])
pid = GenServer.whereis({:via, Registry, {Chesstrainer.CacheRegistry, :tablebase}})
state = :sys.get_state(pid)
assert state.max_size == Keyword.get(config, :max_size, 100_000)
assert state.backup_time ==
Chesstrainer.Cache.BackupSchedule.time!(Keyword.get(config, :backup_time))
end
end
describe "frequency tracking" do
test "get/2 increments the entry frequency", %{table: table} do
Cache.put(table, "key", "value")
@@ -77,6 +104,26 @@ defmodule Chesstrainer.CacheTest do
end
end
describe "eviction on a cache larger than the sample" do
@describetag cache_opts: [max_size: 100]
test "stays at the maximum size under churn", %{table: table} do
for i <- 1..500, do: Cache.put(table, i, i)
assert Cache.size(table) == 100
end
test "a frequently used entry survives churn from cold entries", %{table: table} do
Cache.put(table, :hot, "value")
for _ <- 1..50, do: Cache.get(table, :hot)
for i <- 1..500, do: Cache.put(table, i, i)
assert Cache.size(table) == 100
assert {:ok, "value"} = Cache.get(table, :hot)
end
end
describe "weekly sweep" do
test "halves every entry frequency", %{table: table} do
Cache.put(table, "busy", "value")
@@ -92,23 +139,123 @@ defmodule Chesstrainer.CacheTest do
end
end
describe "backup" do
test "runs off the cache process so reads and writes stay responsive" do
table = :"test_async_backup_#{System.unique_integer([:positive])}"
parent = self()
ref = make_ref()
backup_fun = fn _table ->
send(parent, {:backup_started, ref, self()})
receive do
{:finish_backup, ^ref} -> :ok
end
end
pid =
start_supervised!(
Supervisor.child_spec({Cache, {table, [backup_fun: backup_fun]}}, id: table)
)
assert :ok = Cache.put(table, "key", "value")
send(pid, :backup)
assert_receive {:backup_started, ^ref, backup_pid}, 1_000
backup_monitor = Process.monitor(backup_pid)
assert {:ok, "value"} = Cache.get(table, "key")
assert :ok = Cache.put(table, "other", "value")
send(backup_pid, {:finish_backup, ref})
assert_receive {:DOWN, ^backup_monitor, :process, ^backup_pid, :normal}, 1_000
end
test "re-arms the daily backup after each run" do
table = :"test_rearm_#{System.unique_integer([:positive])}"
pid =
start_supervised!(
Supervisor.child_spec(
{Cache, {table, [backup_fun: fn _table -> :ok end, backup_time: "04:30"]}},
id: table
)
)
send(pid, :backup)
state = :sys.get_state(pid)
assert state.backup_time == {4, 30}
assert is_reference(state.backup_timer)
assert :erlang.read_timer(state.backup_timer) > 0
end
end
describe "persistence" do
test "writes the backup atomically, leaving no temporary file" do
table = :"test_atomic_#{System.unique_integer([:positive])}"
backup = Path.join("priv/static/backups", "#{table}.v2.det")
on_exit(fn ->
File.rm_rf(backup <> ".tmp")
File.rm(backup)
end)
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
Cache.put(table, "key", "value")
assert :ok = Cache.backup(table)
assert File.exists?(backup)
refute File.exists?(backup <> ".tmp")
end
test "a failed backup leaves the previous backup intact" do
table = :"test_failed_backup_#{System.unique_integer([:positive])}"
backup = Path.join("priv/static/backups", "#{table}.v2.det")
temp = backup <> ".tmp"
on_exit(fn ->
File.rm_rf(temp)
File.rm(backup)
end)
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
Cache.put(table, "key", "value")
assert :ok = Cache.backup(table)
# A directory at the temporary path makes the write fail before the rename.
File.mkdir_p!(temp)
Cache.put(table, "another", "value")
assert {:error, _reason} = Cache.Persist.backup(table)
assert File.exists?(backup)
assert :ok = stop_supervised(table)
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
assert {:ok, "value"} = Cache.get(table, "key")
assert :error = Cache.get(table, "another")
end
test "backup and restore preserve frequencies" do
table = :"test_persist_#{System.unique_integer([:positive])}"
backup = Path.join("priv/static/backups", "#{table}.v2.det")
on_exit(fn -> File.rm(backup) end)
{:ok, pid} = Cache.start_link(table)
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
Cache.put(table, "key", "value")
Cache.get(table, "key")
Cache.get(table, "key")
assert :ok = Cache.backup(table)
assert :ok = GenServer.stop(pid)
assert :ok = stop_supervised(table)
{:ok, restored_pid} = Cache.start_link(table)
on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end)
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
assert frequency == 3
+72 -14
View File
@@ -32,6 +32,18 @@ defmodule Chesstrainer.Endgames.TablebaseTest do
end
end
describe "struct shape" do
test "Tablebase carries only the fields the trainer reads" do
assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
[:category, :checkmate, :dtc, :dtm, :moves]
end
test "Tablebase.Move carries only the fields the trainer reads" do
assert %Tablebase.Move{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
[:category, :dtc, :dtm, :dtz, :san, :uci]
end
end
describe "Parser.from_map/1" do
test "normalises the position and move categories" do
assert {:ok, tablebase} =
@@ -40,32 +52,78 @@ defmodule Chesstrainer.Endgames.TablebaseTest do
"checkmate" => false,
"dtc" => nil,
"dtm" => 20,
"dtw" => nil,
"dtz" => 3,
"insufficient_material" => false,
"moves" => [
%{
"category" => "blessed-loss",
"checkmate" => false,
"conversion" => false,
"dtc" => nil,
"dtm" => -19,
"dtw" => nil,
"dtz" => -2,
"insufficient_material" => false,
"precise_dtz" => -2,
"san" => "Kd7",
"stalemate" => false,
"uci" => "e8d7",
"variant_loss" => false,
"variant_win" => false,
"zeroing" => false
"uci" => "e8d7"
}
]
})
assert tablebase.category == :draw
assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves
assert %Tablebase{category: :draw, checkmate: false, dtc: nil, dtm: 20} = tablebase
assert [%Tablebase.Move{category: :draw, dtc: nil, dtm: -19, dtz: -2}] = tablebase.moves
assert [%Tablebase.Move{san: "Kd7", uci: "e8d7"}] = tablebase.moves
end
test "drops the fields the trainer never reads" do
assert {:ok, tablebase} =
Parser.from_map(%{
"category" => "win",
"checkmate" => true,
"dtc" => 1,
"dtm" => 2,
"dtw" => 3,
"dtz" => 4,
"insufficient_material" => false,
"moves" => [
%{
"category" => "loss",
"checkmate" => true,
"conversion" => true,
"dtc" => 5,
"dtm" => 6,
"dtw" => 7,
"dtz" => 8,
"insufficient_material" => false,
"precise_dtz" => 9,
"san" => "Qd7#",
"stalemate" => false,
"uci" => "d1d7",
"variant_loss" => false,
"variant_win" => false,
"zeroing" => true
}
]
})
assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
[:category, :checkmate, :dtc, :dtm, :moves]
assert [move] = tablebase.moves
assert move |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
[:category, :dtc, :dtm, :dtz, :san, :uci]
end
test "keeps checkmate and empty moves, which drive terminal detection" do
assert {:ok, tablebase} =
Parser.from_map(%{
"category" => "win",
"checkmate" => true,
"dtc" => 1,
"dtm" => 2,
"dtz" => 3,
"moves" => []
})
assert tablebase.checkmate
assert tablebase.moves == []
end
end