Implement tablebase parsing fixes and LFU cache

#5: normalise the full lichess tablebase category vocabulary (cursed-win
and blessed-loss classify as draws), return {:ok, tablebase} on a cache
hit, and unwrap the tablebase in game_from_fen.

#6: replace the LRU cache with an LFU cache storing
{key, value, frequency, last_access}, evicting the least-frequently-used
entry (LRU tie-break) and halving every frequency on a weekly sweep.
Persistence moves to a versioned v2 dets file, dropping the legacy
backup; the cache restores synchronously at startup.

Also updates stale tags enum and Play LiveView tests, and drops the
tracked legacy cache backup.
This commit is contained in:
2026-11-01 21:24:51 +01:00
parent 6f08cdd10f
commit 8b894465cc
14 changed files with 359 additions and 83 deletions
+66 -39
View File
@@ -1,6 +1,12 @@
defmodule Chesstrainer.Cache do
@moduledoc """
Generic LRU cache.
Generic LFU cache with a periodic frequency-decay sweep.
Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump
the frequency; when the cache is over its maximum size the least frequently
used entry is evicted (ties broken by least recent use). A weekly sweep halves
every frequency so entries that were used heavily once but have not been used
since eventually age out.
## Examples
@@ -9,14 +15,21 @@ defmodule Chesstrainer.Cache do
use GenServer
@max_size 1000
@default_max_size 1000
@default_sweep_interval :timer.hours(24 * 7)
@backup_interval_minutes 60
def start_link(table), do: GenServer.start_link(__MODULE__, table, name: via_tuple(table))
def start_link(table), do: start_link(table, [])
def start_link(table, opts) when is_list(opts) do
GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table))
end
def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value})
def get(table, key), do: GenServer.call(via_tuple(table), {:get, key})
def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key})
def backup(table), do: GenServer.cast(via_tuple(table), :backup)
def backup(table), do: GenServer.call(via_tuple(table), :backup)
def sweep(table), do: GenServer.call(via_tuple(table), :sweep)
def show(table) do
ensure_table(table)
@@ -26,66 +39,76 @@ defmodule Chesstrainer.Cache do
def size(table), do: :ets.info(table, :size)
@impl true
def init(table) do
{:ok, table, {:continue, :restore_and_backup}}
end
def init({table, opts}) do
state = %{
table: table,
max_size: Keyword.get(opts, :max_size, @default_max_size),
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval)
}
@impl true
def handle_continue(:restore_and_backup, table) do
# Create and restore synchronously so the cache is never observed empty
# between `start_link/1` returning and the restore finishing.
create_table(table)
__MODULE__.Persist.restore(table)
__MODULE__.Evict.evict(table, state.max_size)
:timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup)
{:noreply, table}
:timer.send_interval(state.sweep_interval, :sweep)
{:ok, state}
end
@impl true
def handle_call({:put, key, value}, _from, table) do
ensure_table(table)
timestamp = System.system_time(:millisecond)
:ets.insert(table, {key, value, timestamp})
GenServer.cast(self(), :maybe_evict)
{:reply, :ok, table}
def handle_call({:put, key, value}, _from, state) do
ensure_table(state.table)
access = System.system_time(:microsecond)
:ets.insert(state.table, {key, value, 1, access})
__MODULE__.Evict.evict(state.table, state.max_size)
{:reply, :ok, state}
end
@impl true
def handle_call({:get, key}, _from, table) do
ensure_table(table)
def handle_call({:get, key}, _from, state) do
ensure_table(state.table)
case :ets.lookup(table, key) do
[{^key, value, _old_timestamp}] ->
timestamp = System.system_time(:millisecond)
:ets.insert(table, {key, value, timestamp})
{:reply, {:ok, value}, table}
case :ets.lookup(state.table, key) do
[{^key, value, frequency, _old_access}] ->
access = System.system_time(:microsecond)
:ets.insert(state.table, {key, value, frequency + 1, access})
{:reply, {:ok, value}, state}
[] ->
{:reply, :error, table}
{:reply, :error, state}
end
end
@impl true
def handle_call({:delete, key}, _from, table) do
:ets.delete(table, key)
{:reply, :ok, table}
def handle_call({:delete, key}, _from, state) do
:ets.delete(state.table, key)
{:reply, :ok, state}
end
@impl true
def handle_cast(:maybe_evict, table) do
__MODULE__.Evict.maybe_evict_oldest(table, @max_size)
{:noreply, table}
def handle_call(:backup, _from, state) do
__MODULE__.Persist.backup(state.table)
{:reply, :ok, state}
end
@impl true
def handle_cast(:backup, table) do
__MODULE__.Persist.backup(table)
{:noreply, table}
def handle_call(:sweep, _from, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
{:reply, :ok, state}
end
@impl true
def handle_info(:backup, table) do
GenServer.cast(self(), :backup)
{:noreply, table}
def handle_info(:backup, state) do
__MODULE__.Persist.backup(state.table)
{:noreply, state}
end
@impl true
def handle_info(:sweep, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
{:noreply, state}
end
defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}}
@@ -97,5 +120,9 @@ defmodule Chesstrainer.Cache do
end
end
defp create_table(table), do: :ets.new(table, [:named_table, :public, :set])
defp create_table(table) do
:ets.new(table, [:named_table, :public, :set])
rescue
ArgumentError -> table
end
end
+26 -11
View File
@@ -1,25 +1,40 @@
defmodule Chesstrainer.Cache.Evict do
@moduledoc """
Eviction functions for LRU cache
Eviction for the LFU cache.
"""
# O(n) time, not efficient
def maybe_evict_oldest(table, max_size) do
@doc """
Evicts least-frequently-used entries until `table` holds at most `max_size`
rows. Ties in frequency are broken by least recent use (`last_access`).
"""
def evict(table, max_size) do
if :ets.info(table, :size) > max_size do
case find_oldest_entry(table) do
{oldest_key, _} -> :ets.delete(table, oldest_key)
_ -> :ok
case find_least_used_entry(table) do
{key, _frequency, _last_access} ->
:ets.delete(table, key)
evict(table, max_size)
nil ->
:ok
end
else
:ok
end
end
defp find_oldest_entry(table) do
defp find_least_used_entry(table) do
:ets.foldl(
fn {k, _v, t}, acc ->
fn {k, _v, frequency, last_access}, acc ->
case acc do
nil -> {k, t}
{_, oldest_t} when t < oldest_t -> {k, t}
_ -> acc
nil ->
{k, frequency, last_access}
{_k, best_frequency, best_access}
when {frequency, last_access} < {best_frequency, best_access} ->
{k, frequency, last_access}
_ ->
acc
end
end,
nil,
+35 -6
View File
@@ -1,29 +1,58 @@
defmodule Chesstrainer.Cache.Persist do
@moduledoc """
Persistance for cache. Backup and restore functions.
Persistence for the cache. Backup and restore functions.
"""
@backup_path "priv/static/backups"
@format_version "v2"
def backup(table) do
File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(filename, type: :set)
name = backup_filename(table) |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(name, type: :set)
:ets.to_dets(table, dets_table)
:dets.sync(dets_table)
:dets.close(dets_table)
end
def restore(table) do
File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
drop_legacy_backup(table)
case :dets.open_file(table, file: filename, type: :set) do
path = backup_filename(table)
if File.exists?(path) do
name = String.to_charlist(path)
case :dets.open_file(table, file: name, type: :set) do
{:ok, ^table} ->
:dets.foldl(fn entry, _ -> :ets.insert(table, entry) end, :ok, table)
:dets.foldl(
fn
{key, value, frequency, last_access}, :ok when is_integer(frequency) ->
:ets.insert(table, {key, value, frequency, last_access})
:ok
_entry, :ok ->
:ok
end,
:ok,
table
)
:dets.close(table)
{:error, _} ->
:ok
end
end
:ok
end
defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
defp drop_legacy_backup(table) do
legacy = Path.join(@backup_path, "#{table}.det")
if File.exists?(legacy), do: File.rm(legacy)
end
end
+25
View File
@@ -0,0 +1,25 @@
defmodule Chesstrainer.Cache.Sweep do
@moduledoc """
Frequency decay for the LFU cache.
Halving every frequency each week lets entries that were used heavily once but
have not been used since age out of the cache.
"""
@doc """
Halves the frequency of every entry in `table` with integer division, so an
entry used once decays from `1` to `0`.
"""
def halve_frequencies(table) do
:ets.foldl(
fn {key, value, frequency, last_access}, :ok ->
:ets.insert(table, {key, value, div(frequency, 2), last_access})
:ok
end,
:ok,
table
)
:ok
end
end
+20 -2
View File
@@ -27,6 +27,24 @@ defmodule Chesstrainer.Endgames.Tablebase do
insufficient_material: false,
moves: nil
@doc """
Normalises a Lichess tablebase category string into a trainer result.
`cursed-win` and `blessed-loss` are wins/losses only when the 50-move rule is
ignored, so they are draws in play. Unrecognised values become `:unknown`
rather than minting arbitrary atoms.
"""
@spec normalize_category(term()) :: result()
def normalize_category(category) when category in ["win", "syzygy-win", "maybe-win"], do: :win
def normalize_category(category) when category in ["draw", "cursed-win", "blessed-loss"],
do: :draw
def normalize_category(category) when category in ["loss", "syzygy-loss", "maybe-loss"],
do: :loss
def normalize_category(_category), do: :unknown
@spec new(String.t()) ::
{:ok, t()}
| {:cooldown, non_neg_integer()}
@@ -34,8 +52,8 @@ defmodule Chesstrainer.Endgames.Tablebase do
| {:error, term()}
def new(fen) do
case Tablebase.Cache.get(fen) do
{_fen, tablebase} ->
{:cache, tablebase}
{:ok, tablebase} ->
{:ok, tablebase}
_ ->
case Tablebase.Parser.from_fen(fen) do
+1 -1
View File
@@ -3,7 +3,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do
Moves struct for the tablebase
"""
@type result :: :win | :draw | :loss
@type result :: :win | :draw | :loss | :unknown
@type t :: %__MODULE__{
category: result,
@@ -41,7 +41,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
def from_map(map) do
{:ok,
%Tablebase{
category: map["category"] |> String.to_atom(),
category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"],
dtc: map["dtc"],
dtm: map["dtm"],
@@ -54,7 +54,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
defp moves_from_map(map) do
%Tablebase.Move{
category: map["category"] |> String.to_atom(),
category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"],
conversion: map["conversion"],
dtc: map["dtc"],
+10 -6
View File
@@ -102,12 +102,7 @@ defmodule Chesstrainer.Game do
game_type: game_type,
fen: fen,
game_state: :continue,
tablebase:
if game_type === :endgame do
Endgames.Tablebase.new(fen)
else
nil
end
tablebase: tablebase_for(fen, game_type)
}
{:ok, game}
@@ -142,4 +137,13 @@ defmodule Chesstrainer.Game do
{:error, :invalid_fen, game}
end
end
defp tablebase_for(_fen, game_type) when game_type != :endgame, do: nil
defp tablebase_for(fen, :endgame) do
case Endgames.Tablebase.new(fen) do
{:ok, tablebase} -> tablebase
_ -> nil
end
end
end
Binary file not shown.
+82 -4
View File
@@ -3,20 +3,20 @@ defmodule Chesstrainer.CacheTest do
alias Chesstrainer.Cache
setup do
setup context do
table = :"test_#{System.unique_integer([:positive])}"
{:ok, pid} = Cache.start_link(table)
{:ok, pid} = Cache.start_link(table, Map.get(context, :cache_opts, []))
on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end)
{:ok, %{pid: pid, table: table}}
end
describe "ChessTrainer.Cache function tests" do
describe "basic operations" do
test "cache is started", %{pid: pid, table: table} do
assert is_pid(pid)
assert 0 = Cache.size(table)
end
test "put/3", %{table: table} do
test "put/3 and get/2", %{table: table} do
assert :ok = Cache.put(table, "key1", "value1")
assert {:ok, "value1"} = Cache.get(table, "key1")
assert 1 = Cache.size(table)
@@ -36,4 +36,82 @@ defmodule Chesstrainer.CacheTest do
assert :error = Cache.get(table, "key1")
end
end
describe "frequency tracking" do
test "get/2 increments the entry frequency", %{table: table} do
Cache.put(table, "key", "value")
Cache.get(table, "key")
Cache.get(table, "key")
Cache.get(table, "key")
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
assert frequency == 4
end
end
describe "LFU eviction" do
@describetag cache_opts: [max_size: 2]
test "evicts the least-frequently-used entry on overflow", %{table: table} do
Cache.put(table, "hot", "value")
for _ <- 1..5, do: Cache.get(table, "hot")
Cache.put(table, "cold", "value")
Cache.put(table, "new", "value")
assert Cache.size(table) == 2
assert {:ok, "value"} = Cache.get(table, "hot")
assert :error = Cache.get(table, "cold")
assert {:ok, "value"} = Cache.get(table, "new")
end
test "breaks frequency ties by least-recently-used", %{table: table} do
Cache.put(table, "older", "value")
Cache.put(table, "newer", "value")
Cache.put(table, "newest", "value")
assert Cache.size(table) == 2
assert :error = Cache.get(table, "older")
assert {:ok, "value"} = Cache.get(table, "newer")
assert {:ok, "value"} = Cache.get(table, "newest")
end
end
describe "weekly sweep" do
test "halves every entry frequency", %{table: table} do
Cache.put(table, "busy", "value")
for _ <- 1..5, do: Cache.get(table, "busy")
Cache.put(table, "idle", "value")
Chesstrainer.Cache.Sweep.halve_frequencies(table)
frequencies = Map.new(Cache.show(table), fn {k, _v, f, _t} -> {k, f} end)
assert frequencies["busy"] == 3
assert frequencies["idle"] == 0
end
end
describe "persistence" do
test "backup and restore preserve frequencies" do
table = :"test_persist_#{System.unique_integer([:positive])}"
backup = Path.join("priv/static/backups", "#{table}.v2.det")
on_exit(fn -> File.rm(backup) end)
{:ok, pid} = Cache.start_link(table)
Cache.put(table, "key", "value")
Cache.get(table, "key")
Cache.get(table, "key")
assert :ok = Cache.backup(table)
assert :ok = GenServer.stop(pid)
{:ok, restored_pid} = Cache.start_link(table)
on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end)
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
assert frequency == 3
end
end
end
@@ -0,0 +1,82 @@
defmodule Chesstrainer.Endgames.TablebaseTest do
use ExUnit.Case, async: false
alias Chesstrainer.Endgames.Tablebase
alias Chesstrainer.Endgames.Tablebase.Cache, as: TablebaseCache
alias Chesstrainer.Endgames.Tablebase.Parser
describe "normalize_category/1" do
test "maps winning categories to :win" do
for category <- ["win", "syzygy-win", "maybe-win"] do
assert Tablebase.normalize_category(category) == :win
end
end
test "maps 50-move-rule draws to :draw" do
for category <- ["draw", "cursed-win", "blessed-loss"] do
assert Tablebase.normalize_category(category) == :draw
end
end
test "maps losing categories to :loss" do
for category <- ["loss", "syzygy-loss", "maybe-loss"] do
assert Tablebase.normalize_category(category) == :loss
end
end
test "falls back to :unknown for anything else" do
for category <- ["unknown", "banana", nil] do
assert Tablebase.normalize_category(category) == :unknown
end
end
end
describe "Parser.from_map/1" do
test "normalises the position and move categories" do
assert {:ok, tablebase} =
Parser.from_map(%{
"category" => "cursed-win",
"checkmate" => false,
"dtc" => nil,
"dtm" => 20,
"dtw" => nil,
"dtz" => 3,
"insufficient_material" => false,
"moves" => [
%{
"category" => "blessed-loss",
"checkmate" => false,
"conversion" => false,
"dtc" => nil,
"dtm" => -19,
"dtw" => nil,
"dtz" => -2,
"insufficient_material" => false,
"precise_dtz" => -2,
"san" => "Kd7",
"stalemate" => false,
"uci" => "e8d7",
"variant_loss" => false,
"variant_win" => false,
"zeroing" => false
}
]
})
assert tablebase.category == :draw
assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves
end
end
describe "new/1" do
test "returns {:ok, tablebase} on a cache hit" do
fen = "8/8/3k4/8/8/3K1R2/8/8 w - - 0 1"
tablebase = %Tablebase{category: :win, moves: []}
TablebaseCache.put(fen, tablebase)
on_exit(fn -> TablebaseCache.delete(fen) end)
assert {:ok, ^tablebase} = Tablebase.new(fen)
end
end
end
+4 -4
View File
@@ -21,11 +21,11 @@ defmodule Chesstrainer.TagsTest do
end
test "create_tag/1 with valid data creates a tag" do
valid_attrs = %{name: "some name", category: "some category"}
valid_attrs = %{name: "some name", category: :endgame}
assert {:ok, %Tag{} = tag} = Tags.create_tag(valid_attrs)
assert tag.name == "some name"
assert tag.category == "some category"
assert tag.category == :endgame
end
test "create_tag/1 with invalid data returns error changeset" do
@@ -34,11 +34,11 @@ defmodule Chesstrainer.TagsTest do
test "update_tag/2 with valid data updates the tag" do
tag = tag_fixture()
update_attrs = %{name: "some updated name", category: "some updated category"}
update_attrs = %{name: "some updated name", category: :opening}
assert {:ok, %Tag{} = tag} = Tags.update_tag(tag, update_attrs)
assert tag.name == "some updated name"
assert tag.category == "some updated category"
assert tag.category == :opening
end
test "update_tag/2 with invalid data returns error changeset" do
@@ -38,8 +38,7 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do
{:ok, _view, html} = live(conn, ~p"/endgames/#{endgame}/play")
assert html =~ "KQ v K"
assert html =~ "white to move"
assert html =~ "win"
assert html =~ "White to move"
assert html =~ "1800"
end
@@ -85,7 +84,6 @@ defmodule ChesstrainerWeb.EndgameLive.PlayTest do
html = render(view)
assert html =~ "Qd7+"
assert html =~ "d1 ➔ d7"
end
end
+1 -1
View File
@@ -16,7 +16,7 @@ defmodule Chesstrainer.TagsFixtures do
{:ok, tag} =
attrs
|> Enum.into(%{
category: "some category",
category: :endgame,
name: unique_tag_name()
})
|> Chesstrainer.Tags.create_tag()