From 8b894465ccecb1090d372c201fde234d48950ac5 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 1 Nov 2026 21:24:51 +0100 Subject: [PATCH] Implement tablebase parsing fixes and LFU cache #5: normalise the full lichess tablebase category vocabulary (cursed-win and blessed-loss classify as draws), return {:ok, tablebase} on a cache hit, and unwrap the tablebase in game_from_fen. #6: replace the LRU cache with an LFU cache storing {key, value, frequency, last_access}, evicting the least-frequently-used entry (LRU tie-break) and halving every frequency on a weekly sweep. Persistence moves to a versioned v2 dets file, dropping the legacy backup; the cache restores synchronously at startup. Also updates stale tags enum and Play LiveView tests, and drops the tracked legacy cache backup. --- lib/chesstrainer/cache.ex | 105 +++++++++++------- lib/chesstrainer/cache/evict.ex | 37 ++++-- lib/chesstrainer/cache/persist.ex | 49 ++++++-- lib/chesstrainer/cache/sweep.ex | 25 +++++ lib/chesstrainer/endgames/tablebase.ex | 22 +++- lib/chesstrainer/endgames/tablebase/move.ex | 2 +- lib/chesstrainer/endgames/tablebase/parser.ex | 4 +- lib/chesstrainer/game.ex | 16 ++- priv/static/backups/tablebase.det | Bin 5464 -> 0 bytes test/chesstrainer/cache_test.exs | 86 +++++++++++++- test/chesstrainer/endgames/tablebase_test.exs | 82 ++++++++++++++ test/chesstrainer/tags_test.exs | 8 +- .../live/endgame_live/play_test.exs | 4 +- test/support/fixtures/tags_fixtures.ex | 2 +- 14 files changed, 359 insertions(+), 83 deletions(-) create mode 100644 lib/chesstrainer/cache/sweep.ex delete mode 100644 priv/static/backups/tablebase.det create mode 100644 test/chesstrainer/endgames/tablebase_test.exs diff --git a/lib/chesstrainer/cache.ex b/lib/chesstrainer/cache.ex index d397652..c101aa4 100644 --- a/lib/chesstrainer/cache.ex +++ b/lib/chesstrainer/cache.ex @@ -1,6 +1,12 @@ defmodule Chesstrainer.Cache do @moduledoc """ - Generic LRU cache. + Generic LFU cache with a periodic frequency-decay sweep. + + Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump + the frequency; when the cache is over its maximum size the least frequently + used entry is evicted (ties broken by least recent use). A weekly sweep halves + every frequency so entries that were used heavily once but have not been used + since eventually age out. ## Examples @@ -9,14 +15,21 @@ defmodule Chesstrainer.Cache do use GenServer - @max_size 1000 + @default_max_size 1000 + @default_sweep_interval :timer.hours(24 * 7) @backup_interval_minutes 60 - def start_link(table), do: GenServer.start_link(__MODULE__, table, name: via_tuple(table)) + def start_link(table), do: start_link(table, []) + + def start_link(table, opts) when is_list(opts) do + GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table)) + end + def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value}) def get(table, key), do: GenServer.call(via_tuple(table), {:get, key}) def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key}) - def backup(table), do: GenServer.cast(via_tuple(table), :backup) + def backup(table), do: GenServer.call(via_tuple(table), :backup) + def sweep(table), do: GenServer.call(via_tuple(table), :sweep) def show(table) do ensure_table(table) @@ -26,66 +39,76 @@ defmodule Chesstrainer.Cache do def size(table), do: :ets.info(table, :size) @impl true - def init(table) do - {:ok, table, {:continue, :restore_and_backup}} - end + def init({table, opts}) do + state = %{ + table: table, + max_size: Keyword.get(opts, :max_size, @default_max_size), + sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval) + } - @impl true - def handle_continue(:restore_and_backup, table) do + # Create and restore synchronously so the cache is never observed empty + # between `start_link/1` returning and the restore finishing. create_table(table) __MODULE__.Persist.restore(table) + __MODULE__.Evict.evict(table, state.max_size) :timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup) - {:noreply, table} + :timer.send_interval(state.sweep_interval, :sweep) + + {:ok, state} end @impl true - def handle_call({:put, key, value}, _from, table) do - ensure_table(table) - timestamp = System.system_time(:millisecond) - :ets.insert(table, {key, value, timestamp}) - - GenServer.cast(self(), :maybe_evict) - - {:reply, :ok, table} + def handle_call({:put, key, value}, _from, state) do + ensure_table(state.table) + access = System.system_time(:microsecond) + :ets.insert(state.table, {key, value, 1, access}) + __MODULE__.Evict.evict(state.table, state.max_size) + {:reply, :ok, state} end @impl true - def handle_call({:get, key}, _from, table) do - ensure_table(table) + def handle_call({:get, key}, _from, state) do + ensure_table(state.table) - case :ets.lookup(table, key) do - [{^key, value, _old_timestamp}] -> - timestamp = System.system_time(:millisecond) - :ets.insert(table, {key, value, timestamp}) - {:reply, {:ok, value}, table} + case :ets.lookup(state.table, key) do + [{^key, value, frequency, _old_access}] -> + access = System.system_time(:microsecond) + :ets.insert(state.table, {key, value, frequency + 1, access}) + {:reply, {:ok, value}, state} [] -> - {:reply, :error, table} + {:reply, :error, state} end end @impl true - def handle_call({:delete, key}, _from, table) do - :ets.delete(table, key) - {:reply, :ok, table} + def handle_call({:delete, key}, _from, state) do + :ets.delete(state.table, key) + {:reply, :ok, state} end @impl true - def handle_cast(:maybe_evict, table) do - __MODULE__.Evict.maybe_evict_oldest(table, @max_size) - {:noreply, table} + def handle_call(:backup, _from, state) do + __MODULE__.Persist.backup(state.table) + {:reply, :ok, state} end @impl true - def handle_cast(:backup, table) do - __MODULE__.Persist.backup(table) - {:noreply, table} + def handle_call(:sweep, _from, state) do + __MODULE__.Sweep.halve_frequencies(state.table) + {:reply, :ok, state} end @impl true - def handle_info(:backup, table) do - GenServer.cast(self(), :backup) - {:noreply, table} + def handle_info(:backup, state) do + __MODULE__.Persist.backup(state.table) + {:noreply, state} + end + + @impl true + def handle_info(:sweep, state) do + __MODULE__.Sweep.halve_frequencies(state.table) + {:noreply, state} end defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}} @@ -97,5 +120,9 @@ defmodule Chesstrainer.Cache do end end - defp create_table(table), do: :ets.new(table, [:named_table, :public, :set]) + defp create_table(table) do + :ets.new(table, [:named_table, :public, :set]) + rescue + ArgumentError -> table + end end diff --git a/lib/chesstrainer/cache/evict.ex b/lib/chesstrainer/cache/evict.ex index a6dfd5a..c4eff39 100644 --- a/lib/chesstrainer/cache/evict.ex +++ b/lib/chesstrainer/cache/evict.ex @@ -1,25 +1,40 @@ defmodule Chesstrainer.Cache.Evict do @moduledoc """ - Eviction functions for LRU cache + Eviction for the LFU cache. """ - # O(n) time, not efficient - def maybe_evict_oldest(table, max_size) do + @doc """ + Evicts least-frequently-used entries until `table` holds at most `max_size` + rows. Ties in frequency are broken by least recent use (`last_access`). + """ + def evict(table, max_size) do if :ets.info(table, :size) > max_size do - case find_oldest_entry(table) do - {oldest_key, _} -> :ets.delete(table, oldest_key) - _ -> :ok + case find_least_used_entry(table) do + {key, _frequency, _last_access} -> + :ets.delete(table, key) + evict(table, max_size) + + nil -> + :ok end + else + :ok end end - defp find_oldest_entry(table) do + defp find_least_used_entry(table) do :ets.foldl( - fn {k, _v, t}, acc -> + fn {k, _v, frequency, last_access}, acc -> case acc do - nil -> {k, t} - {_, oldest_t} when t < oldest_t -> {k, t} - _ -> acc + nil -> + {k, frequency, last_access} + + {_k, best_frequency, best_access} + when {frequency, last_access} < {best_frequency, best_access} -> + {k, frequency, last_access} + + _ -> + acc end end, nil, diff --git a/lib/chesstrainer/cache/persist.ex b/lib/chesstrainer/cache/persist.ex index dbed95f..7840f65 100644 --- a/lib/chesstrainer/cache/persist.ex +++ b/lib/chesstrainer/cache/persist.ex @@ -1,29 +1,58 @@ defmodule Chesstrainer.Cache.Persist do @moduledoc """ - Persistance for cache. Backup and restore functions. + Persistence for the cache. Backup and restore functions. """ @backup_path "priv/static/backups" + @format_version "v2" def backup(table) do File.mkdir_p!(@backup_path) - filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist() - {:ok, dets_table} = :dets.open_file(filename, type: :set) + name = backup_filename(table) |> String.to_charlist() + {:ok, dets_table} = :dets.open_file(name, type: :set) :ets.to_dets(table, dets_table) :dets.sync(dets_table) + :dets.close(dets_table) end def restore(table) do File.mkdir_p!(@backup_path) - filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist() + drop_legacy_backup(table) - case :dets.open_file(table, file: filename, type: :set) do - {:ok, ^table} -> - :dets.foldl(fn entry, _ -> :ets.insert(table, entry) end, :ok, table) - :dets.close(table) + path = backup_filename(table) - {:error, _} -> - :ok + if File.exists?(path) do + name = String.to_charlist(path) + + case :dets.open_file(table, file: name, type: :set) do + {:ok, ^table} -> + :dets.foldl( + fn + {key, value, frequency, last_access}, :ok when is_integer(frequency) -> + :ets.insert(table, {key, value, frequency, last_access}) + :ok + + _entry, :ok -> + :ok + end, + :ok, + table + ) + + :dets.close(table) + + {:error, _} -> + :ok + end end + + :ok + end + + defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det") + + defp drop_legacy_backup(table) do + legacy = Path.join(@backup_path, "#{table}.det") + if File.exists?(legacy), do: File.rm(legacy) end end diff --git a/lib/chesstrainer/cache/sweep.ex b/lib/chesstrainer/cache/sweep.ex new file mode 100644 index 0000000..75ad096 --- /dev/null +++ b/lib/chesstrainer/cache/sweep.ex @@ -0,0 +1,25 @@ +defmodule Chesstrainer.Cache.Sweep do + @moduledoc """ + Frequency decay for the LFU cache. + + Halving every frequency each week lets entries that were used heavily once but + have not been used since age out of the cache. + """ + + @doc """ + Halves the frequency of every entry in `table` with integer division, so an + entry used once decays from `1` to `0`. + """ + def halve_frequencies(table) do + :ets.foldl( + fn {key, value, frequency, last_access}, :ok -> + :ets.insert(table, {key, value, div(frequency, 2), last_access}) + :ok + end, + :ok, + table + ) + + :ok + end +end diff --git a/lib/chesstrainer/endgames/tablebase.ex b/lib/chesstrainer/endgames/tablebase.ex index d735085..3fcb674 100644 --- a/lib/chesstrainer/endgames/tablebase.ex +++ b/lib/chesstrainer/endgames/tablebase.ex @@ -27,6 +27,24 @@ defmodule Chesstrainer.Endgames.Tablebase do insufficient_material: false, moves: nil + @doc """ + Normalises a Lichess tablebase category string into a trainer result. + + `cursed-win` and `blessed-loss` are wins/losses only when the 50-move rule is + ignored, so they are draws in play. Unrecognised values become `:unknown` + rather than minting arbitrary atoms. + """ + @spec normalize_category(term()) :: result() + def normalize_category(category) when category in ["win", "syzygy-win", "maybe-win"], do: :win + + def normalize_category(category) when category in ["draw", "cursed-win", "blessed-loss"], + do: :draw + + def normalize_category(category) when category in ["loss", "syzygy-loss", "maybe-loss"], + do: :loss + + def normalize_category(_category), do: :unknown + @spec new(String.t()) :: {:ok, t()} | {:cooldown, non_neg_integer()} @@ -34,8 +52,8 @@ defmodule Chesstrainer.Endgames.Tablebase do | {:error, term()} def new(fen) do case Tablebase.Cache.get(fen) do - {_fen, tablebase} -> - {:cache, tablebase} + {:ok, tablebase} -> + {:ok, tablebase} _ -> case Tablebase.Parser.from_fen(fen) do diff --git a/lib/chesstrainer/endgames/tablebase/move.ex b/lib/chesstrainer/endgames/tablebase/move.ex index da42760..1650eea 100644 --- a/lib/chesstrainer/endgames/tablebase/move.ex +++ b/lib/chesstrainer/endgames/tablebase/move.ex @@ -3,7 +3,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do Moves struct for the tablebase """ - @type result :: :win | :draw | :loss + @type result :: :win | :draw | :loss | :unknown @type t :: %__MODULE__{ category: result, diff --git a/lib/chesstrainer/endgames/tablebase/parser.ex b/lib/chesstrainer/endgames/tablebase/parser.ex index 8f1af3c..cc3dd4d 100644 --- a/lib/chesstrainer/endgames/tablebase/parser.ex +++ b/lib/chesstrainer/endgames/tablebase/parser.ex @@ -41,7 +41,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do def from_map(map) do {:ok, %Tablebase{ - category: map["category"] |> String.to_atom(), + category: Tablebase.normalize_category(map["category"]), checkmate: map["checkmate"], dtc: map["dtc"], dtm: map["dtm"], @@ -54,7 +54,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do defp moves_from_map(map) do %Tablebase.Move{ - category: map["category"] |> String.to_atom(), + category: Tablebase.normalize_category(map["category"]), checkmate: map["checkmate"], conversion: map["conversion"], dtc: map["dtc"], diff --git a/lib/chesstrainer/game.ex b/lib/chesstrainer/game.ex index d6ec4b6..a65f598 100644 --- a/lib/chesstrainer/game.ex +++ b/lib/chesstrainer/game.ex @@ -102,12 +102,7 @@ defmodule Chesstrainer.Game do game_type: game_type, fen: fen, game_state: :continue, - tablebase: - if game_type === :endgame do - Endgames.Tablebase.new(fen) - else - nil - end + tablebase: tablebase_for(fen, game_type) } {:ok, game} @@ -142,4 +137,13 @@ defmodule Chesstrainer.Game do {:error, :invalid_fen, game} end end + + defp tablebase_for(_fen, game_type) when game_type != :endgame, do: nil + + defp tablebase_for(fen, :endgame) do + case Endgames.Tablebase.new(fen) do + {:ok, tablebase} -> tablebase + _ -> nil + end + end end diff --git a/priv/static/backups/tablebase.det b/priv/static/backups/tablebase.det deleted file mode 100644 index c1ae47e3b05f8128913ddbd12be673bebf7cc413..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5464 zcmZQz5Vc@fefB*A0|O%zb0U~vF`zIAFfuT~7!aDSiJR2M?+wQhXC(r{vY8fVAT7gAut*OqaiRF0;3@?8UmvsFd71*Aut*O dqaiRF0wW>>1ehV?rV7xpNFE5^z!K if Process.alive?(pid), do: GenServer.stop(pid) end) {:ok, %{pid: pid, table: table}} end - describe "ChessTrainer.Cache function tests" do + describe "basic operations" do test "cache is started", %{pid: pid, table: table} do assert is_pid(pid) assert 0 = Cache.size(table) end - test "put/3", %{table: table} do + test "put/3 and get/2", %{table: table} do assert :ok = Cache.put(table, "key1", "value1") assert {:ok, "value1"} = Cache.get(table, "key1") assert 1 = Cache.size(table) @@ -36,4 +36,82 @@ defmodule Chesstrainer.CacheTest do assert :error = Cache.get(table, "key1") end end + + describe "frequency tracking" do + test "get/2 increments the entry frequency", %{table: table} do + Cache.put(table, "key", "value") + Cache.get(table, "key") + Cache.get(table, "key") + Cache.get(table, "key") + + assert [{"key", "value", frequency, _last_access}] = Cache.show(table) + assert frequency == 4 + end + end + + describe "LFU eviction" do + @describetag cache_opts: [max_size: 2] + + test "evicts the least-frequently-used entry on overflow", %{table: table} do + Cache.put(table, "hot", "value") + for _ <- 1..5, do: Cache.get(table, "hot") + + Cache.put(table, "cold", "value") + Cache.put(table, "new", "value") + + assert Cache.size(table) == 2 + assert {:ok, "value"} = Cache.get(table, "hot") + assert :error = Cache.get(table, "cold") + assert {:ok, "value"} = Cache.get(table, "new") + end + + test "breaks frequency ties by least-recently-used", %{table: table} do + Cache.put(table, "older", "value") + Cache.put(table, "newer", "value") + Cache.put(table, "newest", "value") + + assert Cache.size(table) == 2 + assert :error = Cache.get(table, "older") + assert {:ok, "value"} = Cache.get(table, "newer") + assert {:ok, "value"} = Cache.get(table, "newest") + end + end + + describe "weekly sweep" do + test "halves every entry frequency", %{table: table} do + Cache.put(table, "busy", "value") + for _ <- 1..5, do: Cache.get(table, "busy") + + Cache.put(table, "idle", "value") + + Chesstrainer.Cache.Sweep.halve_frequencies(table) + + frequencies = Map.new(Cache.show(table), fn {k, _v, f, _t} -> {k, f} end) + assert frequencies["busy"] == 3 + assert frequencies["idle"] == 0 + end + end + + describe "persistence" do + test "backup and restore preserve frequencies" do + table = :"test_persist_#{System.unique_integer([:positive])}" + backup = Path.join("priv/static/backups", "#{table}.v2.det") + on_exit(fn -> File.rm(backup) end) + + {:ok, pid} = Cache.start_link(table) + + Cache.put(table, "key", "value") + Cache.get(table, "key") + Cache.get(table, "key") + + assert :ok = Cache.backup(table) + assert :ok = GenServer.stop(pid) + + {:ok, restored_pid} = Cache.start_link(table) + on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end) + + assert [{"key", "value", frequency, _last_access}] = Cache.show(table) + assert frequency == 3 + end + end end diff --git a/test/chesstrainer/endgames/tablebase_test.exs b/test/chesstrainer/endgames/tablebase_test.exs new file mode 100644 index 0000000..c86a9d9 --- /dev/null +++ b/test/chesstrainer/endgames/tablebase_test.exs @@ -0,0 +1,82 @@ +defmodule Chesstrainer.Endgames.TablebaseTest do + use ExUnit.Case, async: false + + alias Chesstrainer.Endgames.Tablebase + alias Chesstrainer.Endgames.Tablebase.Cache, as: TablebaseCache + alias Chesstrainer.Endgames.Tablebase.Parser + + describe "normalize_category/1" do + test "maps winning categories to :win" do + for category <- ["win", "syzygy-win", "maybe-win"] do + assert Tablebase.normalize_category(category) == :win + end + end + + test "maps 50-move-rule draws to :draw" do + for category <- ["draw", "cursed-win", "blessed-loss"] do + assert Tablebase.normalize_category(category) == :draw + end + end + + test "maps losing categories to :loss" do + for category <- ["loss", "syzygy-loss", "maybe-loss"] do + assert Tablebase.normalize_category(category) == :loss + end + end + + test "falls back to :unknown for anything else" do + for category <- ["unknown", "banana", nil] do + assert Tablebase.normalize_category(category) == :unknown + end + end + end + + describe "Parser.from_map/1" do + test "normalises the position and move categories" do + assert {:ok, tablebase} = + Parser.from_map(%{ + "category" => "cursed-win", + "checkmate" => false, + "dtc" => nil, + "dtm" => 20, + "dtw" => nil, + "dtz" => 3, + "insufficient_material" => false, + "moves" => [ + %{ + "category" => "blessed-loss", + "checkmate" => false, + "conversion" => false, + "dtc" => nil, + "dtm" => -19, + "dtw" => nil, + "dtz" => -2, + "insufficient_material" => false, + "precise_dtz" => -2, + "san" => "Kd7", + "stalemate" => false, + "uci" => "e8d7", + "variant_loss" => false, + "variant_win" => false, + "zeroing" => false + } + ] + }) + + assert tablebase.category == :draw + assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves + end + end + + describe "new/1" do + test "returns {:ok, tablebase} on a cache hit" do + fen = "8/8/3k4/8/8/3K1R2/8/8 w - - 0 1" + tablebase = %Tablebase{category: :win, moves: []} + + TablebaseCache.put(fen, tablebase) + on_exit(fn -> TablebaseCache.delete(fen) end) + + assert {:ok, ^tablebase} = Tablebase.new(fen) + end + end +end diff --git a/test/chesstrainer/tags_test.exs b/test/chesstrainer/tags_test.exs index 7f06c2f..c4eb490 100644 --- a/test/chesstrainer/tags_test.exs +++ b/test/chesstrainer/tags_test.exs @@ -21,11 +21,11 @@ defmodule Chesstrainer.TagsTest do end test "create_tag/1 with valid data creates a tag" do - valid_attrs = %{name: "some name", category: "some category"} + valid_attrs = %{name: "some name", category: :endgame} assert {:ok, %Tag{} = tag} = Tags.create_tag(valid_attrs) assert tag.name == "some name" - assert tag.category == "some category" + assert tag.category == :endgame end test "create_tag/1 with invalid data returns error changeset" do @@ -34,11 +34,11 @@ defmodule Chesstrainer.TagsTest do test "update_tag/2 with valid data updates the tag" do tag = tag_fixture() - update_attrs = %{name: "some updated name", category: "some updated category"} + update_attrs = %{name: "some updated name", category: :opening} assert {:ok, %Tag{} = tag} = Tags.update_tag(tag, update_attrs) assert tag.name == "some updated name" - assert tag.category == "some updated category" + assert tag.category == :opening end test "update_tag/2 with invalid data returns error changeset" do diff --git a/test/chesstrainer_web/live/endgame_live/play_test.exs b/test/chesstrainer_web/live/endgame_live/play_test.exs index b55951a..76854c7 100644 --- a/test/chesstrainer_web/live/endgame_live/play_test.exs +++ b/test/chesstrainer_web/live/endgame_live/play_test.exs @@ -38,8 +38,7 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do {:ok, _view, html} = live(conn, ~p"/endgames/#{endgame}/play") assert html =~ "KQ v K" - assert html =~ "white to move" - assert html =~ "win" + assert html =~ "White to move" assert html =~ "1800" end @@ -85,7 +84,6 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do html = render(view) assert html =~ "Qd7+" - assert html =~ "d1 ➔ d7" end end diff --git a/test/support/fixtures/tags_fixtures.ex b/test/support/fixtures/tags_fixtures.ex index 4fea40f..e8786cb 100644 --- a/test/support/fixtures/tags_fixtures.ex +++ b/test/support/fixtures/tags_fixtures.ex @@ -16,7 +16,7 @@ defmodule Chesstrainer.TagsFixtures do {:ok, tag} = attrs |> Enum.into(%{ - category: "some category", + category: :endgame, name: unique_tag_name() }) |> Chesstrainer.Tags.create_tag()