Implement tablebase parsing fixes and LFU cache

#5: normalise the full lichess tablebase category vocabulary (cursed-win
and blessed-loss classify as draws), return {:ok, tablebase} on a cache
hit, and unwrap the tablebase in game_from_fen.

#6: replace the LRU cache with an LFU cache storing
{key, value, frequency, last_access}, evicting the least-frequently-used
entry (LRU tie-break) and halving every frequency on a weekly sweep.
Persistence moves to a versioned v2 dets file, dropping the legacy
backup; the cache restores synchronously at startup.

Also updates stale tags enum and Play LiveView tests, and drops the
tracked legacy cache backup.
This commit is contained in:
2026-11-01 21:24:51 +01:00
parent 6f08cdd10f
commit 8b894465cc
14 changed files with 359 additions and 83 deletions
+66 -39
View File
@@ -1,6 +1,12 @@
defmodule Chesstrainer.Cache do
@moduledoc """
Generic LRU cache.
Generic LFU cache with a periodic frequency-decay sweep.
Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump
the frequency; when the cache is over its maximum size the least frequently
used entry is evicted (ties broken by least recent use). A weekly sweep halves
every frequency so entries that were used heavily once but have not been used
since eventually age out.
## Examples
@@ -9,14 +15,21 @@ defmodule Chesstrainer.Cache do
use GenServer
@max_size 1000
@default_max_size 1000
@default_sweep_interval :timer.hours(24 * 7)
@backup_interval_minutes 60
def start_link(table), do: GenServer.start_link(__MODULE__, table, name: via_tuple(table))
def start_link(table), do: start_link(table, [])
def start_link(table, opts) when is_list(opts) do
GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table))
end
def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value})
def get(table, key), do: GenServer.call(via_tuple(table), {:get, key})
def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key})
def backup(table), do: GenServer.cast(via_tuple(table), :backup)
def backup(table), do: GenServer.call(via_tuple(table), :backup)
def sweep(table), do: GenServer.call(via_tuple(table), :sweep)
def show(table) do
ensure_table(table)
@@ -26,66 +39,76 @@ defmodule Chesstrainer.Cache do
def size(table), do: :ets.info(table, :size)
@impl true
def init(table) do
{:ok, table, {:continue, :restore_and_backup}}
end
def init({table, opts}) do
state = %{
table: table,
max_size: Keyword.get(opts, :max_size, @default_max_size),
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval)
}
@impl true
def handle_continue(:restore_and_backup, table) do
# Create and restore synchronously so the cache is never observed empty
# between `start_link/1` returning and the restore finishing.
create_table(table)
__MODULE__.Persist.restore(table)
__MODULE__.Evict.evict(table, state.max_size)
:timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup)
{:noreply, table}
:timer.send_interval(state.sweep_interval, :sweep)
{:ok, state}
end
@impl true
def handle_call({:put, key, value}, _from, table) do
ensure_table(table)
timestamp = System.system_time(:millisecond)
:ets.insert(table, {key, value, timestamp})
GenServer.cast(self(), :maybe_evict)
{:reply, :ok, table}
def handle_call({:put, key, value}, _from, state) do
ensure_table(state.table)
access = System.system_time(:microsecond)
:ets.insert(state.table, {key, value, 1, access})
__MODULE__.Evict.evict(state.table, state.max_size)
{:reply, :ok, state}
end
@impl true
def handle_call({:get, key}, _from, table) do
ensure_table(table)
def handle_call({:get, key}, _from, state) do
ensure_table(state.table)
case :ets.lookup(table, key) do
[{^key, value, _old_timestamp}] ->
timestamp = System.system_time(:millisecond)
:ets.insert(table, {key, value, timestamp})
{:reply, {:ok, value}, table}
case :ets.lookup(state.table, key) do
[{^key, value, frequency, _old_access}] ->
access = System.system_time(:microsecond)
:ets.insert(state.table, {key, value, frequency + 1, access})
{:reply, {:ok, value}, state}
[] ->
{:reply, :error, table}
{:reply, :error, state}
end
end
@impl true
def handle_call({:delete, key}, _from, table) do
:ets.delete(table, key)
{:reply, :ok, table}
def handle_call({:delete, key}, _from, state) do
:ets.delete(state.table, key)
{:reply, :ok, state}
end
@impl true
def handle_cast(:maybe_evict, table) do
__MODULE__.Evict.maybe_evict_oldest(table, @max_size)
{:noreply, table}
def handle_call(:backup, _from, state) do
__MODULE__.Persist.backup(state.table)
{:reply, :ok, state}
end
@impl true
def handle_cast(:backup, table) do
__MODULE__.Persist.backup(table)
{:noreply, table}
def handle_call(:sweep, _from, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
{:reply, :ok, state}
end
@impl true
def handle_info(:backup, table) do
GenServer.cast(self(), :backup)
{:noreply, table}
def handle_info(:backup, state) do
__MODULE__.Persist.backup(state.table)
{:noreply, state}
end
@impl true
def handle_info(:sweep, state) do
__MODULE__.Sweep.halve_frequencies(state.table)
{:noreply, state}
end
defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}}
@@ -97,5 +120,9 @@ defmodule Chesstrainer.Cache do
end
end
defp create_table(table), do: :ets.new(table, [:named_table, :public, :set])
defp create_table(table) do
:ets.new(table, [:named_table, :public, :set])
rescue
ArgumentError -> table
end
end
+26 -11
View File
@@ -1,25 +1,40 @@
defmodule Chesstrainer.Cache.Evict do
@moduledoc """
Eviction functions for LRU cache
Eviction for the LFU cache.
"""
# O(n) time, not efficient
def maybe_evict_oldest(table, max_size) do
@doc """
Evicts least-frequently-used entries until `table` holds at most `max_size`
rows. Ties in frequency are broken by least recent use (`last_access`).
"""
def evict(table, max_size) do
if :ets.info(table, :size) > max_size do
case find_oldest_entry(table) do
{oldest_key, _} -> :ets.delete(table, oldest_key)
_ -> :ok
case find_least_used_entry(table) do
{key, _frequency, _last_access} ->
:ets.delete(table, key)
evict(table, max_size)
nil ->
:ok
end
else
:ok
end
end
defp find_oldest_entry(table) do
defp find_least_used_entry(table) do
:ets.foldl(
fn {k, _v, t}, acc ->
fn {k, _v, frequency, last_access}, acc ->
case acc do
nil -> {k, t}
{_, oldest_t} when t < oldest_t -> {k, t}
_ -> acc
nil ->
{k, frequency, last_access}
{_k, best_frequency, best_access}
when {frequency, last_access} < {best_frequency, best_access} ->
{k, frequency, last_access}
_ ->
acc
end
end,
nil,
+39 -10
View File
@@ -1,29 +1,58 @@
defmodule Chesstrainer.Cache.Persist do
@moduledoc """
Persistance for cache. Backup and restore functions.
Persistence for the cache. Backup and restore functions.
"""
@backup_path "priv/static/backups"
@format_version "v2"
def backup(table) do
File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(filename, type: :set)
name = backup_filename(table) |> String.to_charlist()
{:ok, dets_table} = :dets.open_file(name, type: :set)
:ets.to_dets(table, dets_table)
:dets.sync(dets_table)
:dets.close(dets_table)
end
def restore(table) do
File.mkdir_p!(@backup_path)
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
drop_legacy_backup(table)
case :dets.open_file(table, file: filename, type: :set) do
{:ok, ^table} ->
:dets.foldl(fn entry, _ -> :ets.insert(table, entry) end, :ok, table)
:dets.close(table)
path = backup_filename(table)
{:error, _} ->
:ok
if File.exists?(path) do
name = String.to_charlist(path)
case :dets.open_file(table, file: name, type: :set) do
{:ok, ^table} ->
:dets.foldl(
fn
{key, value, frequency, last_access}, :ok when is_integer(frequency) ->
:ets.insert(table, {key, value, frequency, last_access})
:ok
_entry, :ok ->
:ok
end,
:ok,
table
)
:dets.close(table)
{:error, _} ->
:ok
end
end
:ok
end
defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
defp drop_legacy_backup(table) do
legacy = Path.join(@backup_path, "#{table}.det")
if File.exists?(legacy), do: File.rm(legacy)
end
end
+25
View File
@@ -0,0 +1,25 @@
defmodule Chesstrainer.Cache.Sweep do
@moduledoc """
Frequency decay for the LFU cache.
Halving every frequency each week lets entries that were used heavily once but
have not been used since age out of the cache.
"""
@doc """
Halves the frequency of every entry in `table` with integer division, so an
entry used once decays from `1` to `0`.
"""
def halve_frequencies(table) do
:ets.foldl(
fn {key, value, frequency, last_access}, :ok ->
:ets.insert(table, {key, value, div(frequency, 2), last_access})
:ok
end,
:ok,
table
)
:ok
end
end
+20 -2
View File
@@ -27,6 +27,24 @@ defmodule Chesstrainer.Endgames.Tablebase do
insufficient_material: false,
moves: nil
@doc """
Normalises a Lichess tablebase category string into a trainer result.
`cursed-win` and `blessed-loss` are wins/losses only when the 50-move rule is
ignored, so they are draws in play. Unrecognised values become `:unknown`
rather than minting arbitrary atoms.
"""
@spec normalize_category(term()) :: result()
def normalize_category(category) when category in ["win", "syzygy-win", "maybe-win"], do: :win
def normalize_category(category) when category in ["draw", "cursed-win", "blessed-loss"],
do: :draw
def normalize_category(category) when category in ["loss", "syzygy-loss", "maybe-loss"],
do: :loss
def normalize_category(_category), do: :unknown
@spec new(String.t()) ::
{:ok, t()}
| {:cooldown, non_neg_integer()}
@@ -34,8 +52,8 @@ defmodule Chesstrainer.Endgames.Tablebase do
| {:error, term()}
def new(fen) do
case Tablebase.Cache.get(fen) do
{_fen, tablebase} ->
{:cache, tablebase}
{:ok, tablebase} ->
{:ok, tablebase}
_ ->
case Tablebase.Parser.from_fen(fen) do
+1 -1
View File
@@ -3,7 +3,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do
Moves struct for the tablebase
"""
@type result :: :win | :draw | :loss
@type result :: :win | :draw | :loss | :unknown
@type t :: %__MODULE__{
category: result,
@@ -41,7 +41,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
def from_map(map) do
{:ok,
%Tablebase{
category: map["category"] |> String.to_atom(),
category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"],
dtc: map["dtc"],
dtm: map["dtm"],
@@ -54,7 +54,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
defp moves_from_map(map) do
%Tablebase.Move{
category: map["category"] |> String.to_atom(),
category: Tablebase.normalize_category(map["category"]),
checkmate: map["checkmate"],
conversion: map["conversion"],
dtc: map["dtc"],
+10 -6
View File
@@ -102,12 +102,7 @@ defmodule Chesstrainer.Game do
game_type: game_type,
fen: fen,
game_state: :continue,
tablebase:
if game_type === :endgame do
Endgames.Tablebase.new(fen)
else
nil
end
tablebase: tablebase_for(fen, game_type)
}
{:ok, game}
@@ -142,4 +137,13 @@ defmodule Chesstrainer.Game do
{:error, :invalid_fen, game}
end
end
defp tablebase_for(_fen, game_type) when game_type != :endgame, do: nil
defp tablebase_for(fen, :endgame) do
case Endgames.Tablebase.new(fen) do
{:ok, tablebase} -> tablebase
_ -> nil
end
end
end