Implement tablebase parsing fixes and LFU cache

#5: normalise the full lichess tablebase category vocabulary (cursed-win
and blessed-loss classify as draws), return {:ok, tablebase} on a cache
hit, and unwrap the tablebase in game_from_fen.

#6: replace the LRU cache with an LFU cache storing
{key, value, frequency, last_access}, evicting the least-frequently-used
entry (LRU tie-break) and halving every frequency on a weekly sweep.
Persistence moves to a versioned v2 dets file, dropping the legacy
backup; the cache restores synchronously at startup.

Also updates stale tags enum and Play LiveView tests, and drops the
tracked legacy cache backup.
This commit is contained in:
2026-11-01 21:24:51 +01:00
parent 6f08cdd10f
commit 8b894465cc
14 changed files with 359 additions and 83 deletions
+66 -39
View File
@@ -1,6 +1,12 @@
defmodule Chesstrainer.Cache do defmodule Chesstrainer.Cache do
@moduledoc """ @moduledoc """
Generic LRU cache. Generic LFU cache with a periodic frequency-decay sweep.
Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump
the frequency; when the cache is over its maximum size the least frequently
used entry is evicted (ties broken by least recent use). A weekly sweep halves
every frequency so entries that were used heavily once but have not been used
since eventually age out.
## Examples ## Examples
@@ -9,14 +15,21 @@ defmodule Chesstrainer.Cache do
use GenServer use GenServer
@max_size 1000 @default_max_size 1000
@default_sweep_interval :timer.hours(24 * 7)
@backup_interval_minutes 60 @backup_interval_minutes 60
def start_link(table), do: GenServer.start_link(__MODULE__, table, name: via_tuple(table)) def start_link(table), do: start_link(table, [])
def start_link(table, opts) when is_list(opts) do
GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table))
end
def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value}) def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value})
def get(table, key), do: GenServer.call(via_tuple(table), {:get, key}) def get(table, key), do: GenServer.call(via_tuple(table), {:get, key})
def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key}) def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key})
def backup(table), do: GenServer.cast(via_tuple(table), :backup) def backup(table), do: GenServer.call(via_tuple(table), :backup)
def sweep(table), do: GenServer.call(via_tuple(table), :sweep)
def show(table) do def show(table) do
ensure_table(table) ensure_table(table)
@@ -26,66 +39,76 @@ defmodule Chesstrainer.Cache do
def size(table), do: :ets.info(table, :size) def size(table), do: :ets.info(table, :size)
@impl true @impl true
def init(table) do def init({table, opts}) do
{:ok, table, {:continue, :restore_and_backup}} state = %{
end table: table,
max_size: Keyword.get(opts, :max_size, @default_max_size),
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval)
}
@impl true # Create and restore synchronously so the cache is never observed empty
def handle_continue(:restore_and_backup, table) do # between `start_link/1` returning and the restore finishing.
create_table(table) create_table(table)
__MODULE__.Persist.restore(table) __MODULE__.Persist.restore(table)
__MODULE__.Evict.evict(table, state.max_size)
:timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup) :timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup)
{:noreply, table} :timer.send_interval(state.sweep_interval, :sweep)
{:ok, state}
end end
@impl true @impl true
def handle_call({:put, key, value}, _from, table) do def handle_call({:put, key, value}, _from, state) do
ensure_table(table) ensure_table(state.table)
timestamp = System.system_time(:millisecond) access = System.system_time(:microsecond)
:ets.insert(table, {key, value, timestamp}) :ets.insert(state.table, {key, value, 1, access})
__MODULE__.Evict.evict(state.table, state.max_size)
GenServer.cast(self(), :maybe_evict) {:reply, :ok, state}
{:reply, :ok, table}
end end
@impl true @impl true
def handle_call({:get, key}, _from, table) do def handle_call({:get, key}, _from, state) do
ensure_table(table) ensure_table(state.table)
case :ets.lookup(table, key) do case :ets.lookup(state.table, key) do
[{^key, value, _old_timestamp}] -> [{^key, value, frequency, _old_access}] ->
timestamp = System.system_time(:millisecond) access = System.system_time(:microsecond)
:ets.insert(table, {key, value, timestamp}) :ets.insert(state.table, {key, value, frequency + 1, access})
{:reply, {:ok, value}, table} {:reply, {:ok, value}, state}
[] -> [] ->
{:reply, :error, table} {:reply, :error, state}
end end
end end
@impl true @impl true
def handle_call({:delete, key}, _from, table) do def handle_call({:delete, key}, _from, state) do
:ets.delete(table, key) :ets.delete(state.table, key)
{:reply, :ok, table} {:reply, :ok, state}
end end
@impl true @impl true
def handle_cast(:maybe_evict, table) do def handle_call(:backup, _from, state) do
__MODULE__.Evict.maybe_evict_oldest(table, @max_size) __MODULE__.Persist.backup(state.table)
{:noreply, table} {:reply, :ok, state}
end end
@impl true @impl true
def handle_cast(:backup, table) do def handle_call(:sweep, _from, state) do
__MODULE__.Persist.backup(table) __MODULE__.Sweep.halve_frequencies(state.table)
{:noreply, table} {:reply, :ok, state}
end end
@impl true @impl true
def handle_info(:backup, table) do def handle_info(:backup, state) do
GenServer.cast(self(), :backup) __MODULE__.Persist.backup(state.table)
{:noreply, table} {:noreply, state}
end
@impl true
def handle_info(:sweep, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
{:noreply, state}
end end
defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}} defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}}
@@ -97,5 +120,9 @@ defmodule Chesstrainer.Cache do
end end
end end
defp create_table(table), do: :ets.new(table, [:named_table, :public, :set]) defp create_table(table) do
:ets.new(table, [:named_table, :public, :set])
rescue
ArgumentError -> table
end
end end
+26 -11
View File
@@ -1,25 +1,40 @@
defmodule Chesstrainer.Cache.Evict do defmodule Chesstrainer.Cache.Evict do
@moduledoc """ @moduledoc """
Eviction functions for LRU cache Eviction for the LFU cache.
""" """
# O(n) time, not efficient @doc """
def maybe_evict_oldest(table, max_size) do Evicts least-frequently-used entries until `table` holds at most `max_size`
rows. Ties in frequency are broken by least recent use (`last_access`).
"""
def evict(table, max_size) do
if :ets.info(table, :size) > max_size do if :ets.info(table, :size) > max_size do
case find_oldest_entry(table) do case find_least_used_entry(table) do
{oldest_key, _} -> :ets.delete(table, oldest_key) {key, _frequency, _last_access} ->
_ -> :ok :ets.delete(table, key)
evict(table, max_size)
nil ->
:ok
end end
else
:ok
end end
end end
defp find_oldest_entry(table) do defp find_least_used_entry(table) do
:ets.foldl( :ets.foldl(
fn {k, _v, t}, acc -> fn {k, _v, frequency, last_access}, acc ->
case acc do case acc do
nil -> {k, t} nil ->
{_, oldest_t} when t < oldest_t -> {k, t} {k, frequency, last_access}
_ -> acc
{_k, best_frequency, best_access}
when {frequency, last_access} < {best_frequency, best_access} ->
{k, frequency, last_access}
_ ->
acc
end end
end, end,
nil, nil,
+39 -10
View File
@@ -1,29 +1,58 @@
defmodule Chesstrainer.Cache.Persist do defmodule Chesstrainer.Cache.Persist do
@moduledoc """ @moduledoc """
Persistance for cache. Backup and restore functions. Persistence for the cache. Backup and restore functions.
""" """
@backup_path "priv/static/backups" @backup_path "priv/static/backups"
@format_version "v2"
def backup(table) do def backup(table) do
File.mkdir_p!(@backup_path) File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist() name = backup_filename(table) |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(filename, type: :set) {:ok, dets_table} = :dets.open_file(name, type: :set)
:ets.to_dets(table, dets_table) :ets.to_dets(table, dets_table)
:dets.sync(dets_table) :dets.sync(dets_table)
:dets.close(dets_table)
end end
def restore(table) do def restore(table) do
File.mkdir_p!(@backup_path) File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist() drop_legacy_backup(table)
case :dets.open_file(table, file: filename, type: :set) do path = backup_filename(table)
{:ok, ^table} ->
:dets.foldl(fn entry, _ -> :ets.insert(table, entry) end, :ok, table)
:dets.close(table)
{:error, _} -> if File.exists?(path) do
:ok name = String.to_charlist(path)
case :dets.open_file(table, file: name, type: :set) do
{:ok, ^table} ->
:dets.foldl(
fn
{key, value, frequency, last_access}, :ok when is_integer(frequency) ->
:ets.insert(table, {key, value, frequency, last_access})
:ok
_entry, :ok ->
:ok
end,
:ok,
table
)
:dets.close(table)
{:error, _} ->
:ok
end
end end
:ok
end
defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
defp drop_legacy_backup(table) do
legacy = Path.join(@backup_path, "#{table}.det")
if File.exists?(legacy), do: File.rm(legacy)
end end
end end
+25
View File
@@ -0,0 +1,25 @@
defmodule Chesstrainer.Cache.Sweep do
@moduledoc """
Frequency decay for the LFU cache.
Halving every frequency each week lets entries that were used heavily once but
have not been used since age out of the cache.
"""
@doc """
Halves the frequency of every entry in `table` with integer division, so an
entry used once decays from `1` to `0`.
"""
def halve_frequencies(table) do
:ets.foldl(
fn {key, value, frequency, last_access}, :ok ->
:ets.insert(table, {key, value, div(frequency, 2), last_access})
:ok
end,
:ok,
table
)
:ok
end
end
+20 -2
View File
@@ -27,6 +27,24 @@ defmodule Chesstrainer.Endgames.Tablebase do
insufficient_material: false, insufficient_material: false,
moves: nil moves: nil
@doc """
Normalises a Lichess tablebase category string into a trainer result.
`cursed-win` and `blessed-loss` are wins/losses only when the 50-move rule is
ignored, so they are draws in play. Unrecognised values become `:unknown`
rather than minting arbitrary atoms.
"""
@spec normalize_category(term()) :: result()
def normalize_category(category) when category in ["win", "syzygy-win", "maybe-win"], do: :win
def normalize_category(category) when category in ["draw", "cursed-win", "blessed-loss"],
do: :draw
def normalize_category(category) when category in ["loss", "syzygy-loss", "maybe-loss"],
do: :loss
def normalize_category(_category), do: :unknown
@spec new(String.t()) :: @spec new(String.t()) ::
{:ok, t()} {:ok, t()}
| {:cooldown, non_neg_integer()} | {:cooldown, non_neg_integer()}
@@ -34,8 +52,8 @@ defmodule Chesstrainer.Endgames.Tablebase do
| {:error, term()} | {:error, term()}
def new(fen) do def new(fen) do
case Tablebase.Cache.get(fen) do case Tablebase.Cache.get(fen) do
{_fen, tablebase} -> {:ok, tablebase} ->
{:cache, tablebase} {:ok, tablebase}
_ -> _ ->
case Tablebase.Parser.from_fen(fen) do case Tablebase.Parser.from_fen(fen) do
+1 -1
View File
@@ -3,7 +3,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do
Moves struct for the tablebase Moves struct for the tablebase
""" """
@type result :: :win | :draw | :loss @type result :: :win | :draw | :loss | :unknown
@type t :: %__MODULE__{ @type t :: %__MODULE__{
category: result, category: result,
@@ -41,7 +41,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
def from_map(map) do def from_map(map) do
{:ok, {:ok,
%Tablebase{ %Tablebase{
category: map["category"] |> String.to_atom(), category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"], checkmate: map["checkmate"],
dtc: map["dtc"], dtc: map["dtc"],
dtm: map["dtm"], dtm: map["dtm"],
@@ -54,7 +54,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
defp moves_from_map(map) do defp moves_from_map(map) do
%Tablebase.Move{ %Tablebase.Move{
category: map["category"] |> String.to_atom(), category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"], checkmate: map["checkmate"],
conversion: map["conversion"], conversion: map["conversion"],
dtc: map["dtc"], dtc: map["dtc"],
+10 -6
View File
@@ -102,12 +102,7 @@ defmodule Chesstrainer.Game do
game_type: game_type, game_type: game_type,
fen: fen, fen: fen,
game_state: :continue, game_state: :continue,
tablebase: tablebase: tablebase_for(fen, game_type)
if game_type === :endgame do
Endgames.Tablebase.new(fen)
else
nil
end
} }
{:ok, game} {:ok, game}
@@ -142,4 +137,13 @@ defmodule Chesstrainer.Game do
{:error, :invalid_fen, game} {:error, :invalid_fen, game}
end end
end end
defp tablebase_for(_fen, game_type) when game_type != :endgame, do: nil
defp tablebase_for(fen, :endgame) do
case Endgames.Tablebase.new(fen) do
{:ok, tablebase} -> tablebase
_ -> nil
end
end
end end
Binary file not shown.
+82 -4
View File
@@ -3,20 +3,20 @@ defmodule Chesstrainer.CacheTest do
alias Chesstrainer.Cache alias Chesstrainer.Cache
setup do setup context do
table = :"test_#{System.unique_integer([:positive])}" table = :"test_#{System.unique_integer([:positive])}"
{:ok, pid} = Cache.start_link(table) {:ok, pid} = Cache.start_link(table, Map.get(context, :cache_opts, []))
on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end)
{:ok, %{pid: pid, table: table}} {:ok, %{pid: pid, table: table}}
end end
describe "ChessTrainer.Cache function tests" do describe "basic operations" do
test "cache is started", %{pid: pid, table: table} do test "cache is started", %{pid: pid, table: table} do
assert is_pid(pid) assert is_pid(pid)
assert 0 = Cache.size(table) assert 0 = Cache.size(table)
end end
test "put/3", %{table: table} do test "put/3 and get/2", %{table: table} do
assert :ok = Cache.put(table, "key1", "value1") assert :ok = Cache.put(table, "key1", "value1")
assert {:ok, "value1"} = Cache.get(table, "key1") assert {:ok, "value1"} = Cache.get(table, "key1")
assert 1 = Cache.size(table) assert 1 = Cache.size(table)
@@ -36,4 +36,82 @@ defmodule Chesstrainer.CacheTest do
assert :error = Cache.get(table, "key1") assert :error = Cache.get(table, "key1")
end end
end end
describe "frequency tracking" do
test "get/2 increments the entry frequency", %{table: table} do
Cache.put(table, "key", "value")
Cache.get(table, "key")
Cache.get(table, "key")
Cache.get(table, "key")
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
assert frequency == 4
end
end
describe "LFU eviction" do
@describetag cache_opts: [max_size: 2]
test "evicts the least-frequently-used entry on overflow", %{table: table} do
Cache.put(table, "hot", "value")
for _ <- 1..5, do: Cache.get(table, "hot")
Cache.put(table, "cold", "value")
Cache.put(table, "new", "value")
assert Cache.size(table) == 2
assert {:ok, "value"} = Cache.get(table, "hot")
assert :error = Cache.get(table, "cold")
assert {:ok, "value"} = Cache.get(table, "new")
end
test "breaks frequency ties by least-recently-used", %{table: table} do
Cache.put(table, "older", "value")
Cache.put(table, "newer", "value")
Cache.put(table, "newest", "value")
assert Cache.size(table) == 2
assert :error = Cache.get(table, "older")
assert {:ok, "value"} = Cache.get(table, "newer")
assert {:ok, "value"} = Cache.get(table, "newest")
end
end
describe "weekly sweep" do
test "halves every entry frequency", %{table: table} do
Cache.put(table, "busy", "value")
for _ <- 1..5, do: Cache.get(table, "busy")
Cache.put(table, "idle", "value")
Chesstrainer.Cache.Sweep.halve_frequencies(table)
frequencies = Map.new(Cache.show(table), fn {k, _v, f, _t} -> {k, f} end)
assert frequencies["busy"] == 3
assert frequencies["idle"] == 0
end
end
describe "persistence" do
test "backup and restore preserve frequencies" do
table = :"test_persist_#{System.unique_integer([:positive])}"
backup = Path.join("priv/static/backups", "#{table}.v2.det")
on_exit(fn -> File.rm(backup) end)
{:ok, pid} = Cache.start_link(table)
Cache.put(table, "key", "value")
Cache.get(table, "key")
Cache.get(table, "key")
assert :ok = Cache.backup(table)
assert :ok = GenServer.stop(pid)
{:ok, restored_pid} = Cache.start_link(table)
on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end)
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
assert frequency == 3
end
end
end end
@@ -0,0 +1,82 @@
defmodule Chesstrainer.Endgames.TablebaseTest do
use ExUnit.Case, async: false
alias Chesstrainer.Endgames.Tablebase
alias Chesstrainer.Endgames.Tablebase.Cache, as: TablebaseCache
alias Chesstrainer.Endgames.Tablebase.Parser
describe "normalize_category/1" do
test "maps winning categories to :win" do
for category <- ["win", "syzygy-win", "maybe-win"] do
assert Tablebase.normalize_category(category) == :win
end
end
test "maps 50-move-rule draws to :draw" do
for category <- ["draw", "cursed-win", "blessed-loss"] do
assert Tablebase.normalize_category(category) == :draw
end
end
test "maps losing categories to :loss" do
for category <- ["loss", "syzygy-loss", "maybe-loss"] do
assert Tablebase.normalize_category(category) == :loss
end
end
test "falls back to :unknown for anything else" do
for category <- ["unknown", "banana", nil] do
assert Tablebase.normalize_category(category) == :unknown
end
end
end
describe "Parser.from_map/1" do
test "normalises the position and move categories" do
assert {:ok, tablebase} =
Parser.from_map(%{
"category" => "cursed-win",
"checkmate" => false,
"dtc" => nil,
"dtm" => 20,
"dtw" => nil,
"dtz" => 3,
"insufficient_material" => false,
"moves" => [
%{
"category" => "blessed-loss",
"checkmate" => false,
"conversion" => false,
"dtc" => nil,
"dtm" => -19,
"dtw" => nil,
"dtz" => -2,
"insufficient_material" => false,
"precise_dtz" => -2,
"san" => "Kd7",
"stalemate" => false,
"uci" => "e8d7",
"variant_loss" => false,
"variant_win" => false,
"zeroing" => false
}
]
})
assert tablebase.category == :draw
assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves
end
end
describe "new/1" do
test "returns {:ok, tablebase} on a cache hit" do
fen = "8/8/3k4/8/8/3K1R2/8/8 w - - 0 1"
tablebase = %Tablebase{category: :win, moves: []}
TablebaseCache.put(fen, tablebase)
on_exit(fn -> TablebaseCache.delete(fen) end)
assert {:ok, ^tablebase} = Tablebase.new(fen)
end
end
end
+4 -4
View File
@@ -21,11 +21,11 @@ defmodule Chesstrainer.TagsTest do
end end
test "create_tag/1 with valid data creates a tag" do test "create_tag/1 with valid data creates a tag" do
valid_attrs = %{name: "some name", category: "some category"} valid_attrs = %{name: "some name", category: :endgame}
assert {:ok, %Tag{} = tag} = Tags.create_tag(valid_attrs) assert {:ok, %Tag{} = tag} = Tags.create_tag(valid_attrs)
assert tag.name == "some name" assert tag.name == "some name"
assert tag.category == "some category" assert tag.category == :endgame
end end
test "create_tag/1 with invalid data returns error changeset" do test "create_tag/1 with invalid data returns error changeset" do
@@ -34,11 +34,11 @@ defmodule Chesstrainer.TagsTest do
test "update_tag/2 with valid data updates the tag" do test "update_tag/2 with valid data updates the tag" do
tag = tag_fixture() tag = tag_fixture()
update_attrs = %{name: "some updated name", category: "some updated category"} update_attrs = %{name: "some updated name", category: :opening}
assert {:ok, %Tag{} = tag} = Tags.update_tag(tag, update_attrs) assert {:ok, %Tag{} = tag} = Tags.update_tag(tag, update_attrs)
assert tag.name == "some updated name" assert tag.name == "some updated name"
assert tag.category == "some updated category" assert tag.category == :opening
end end
test "update_tag/2 with invalid data returns error changeset" do test "update_tag/2 with invalid data returns error changeset" do
@@ -38,8 +38,7 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do
{:ok, _view, html} = live(conn, ~p"/endgames/#{endgame}/play") {:ok, _view, html} = live(conn, ~p"/endgames/#{endgame}/play")
assert html =~ "KQ v K" assert html =~ "KQ v K"
assert html =~ "white to move" assert html =~ "White to move"
assert html =~ "win"
assert html =~ "1800" assert html =~ "1800"
end end
@@ -85,7 +84,6 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do
html = render(view) html = render(view)
assert html =~ "Qd7+" assert html =~ "Qd7+"
assert html =~ "d1 ➔ d7"
end end
end end
+1 -1
View File
@@ -16,7 +16,7 @@ defmodule Chesstrainer.TagsFixtures do
{:ok, tag} = {:ok, tag} =
attrs attrs
|> Enum.into(%{ |> Enum.into(%{
category: "some category", category: :endgame,
name: unique_tag_name() name: unique_tag_name()
}) })
|> Chesstrainer.Tags.create_tag() |> Chesstrainer.Tags.create_tag()