mirror of
https://github.com/TheRealOwenRees/chess-endgame-trainer.git
synced 2026-11-18 03:32:06 +00:00
Implement tablebase parsing fixes and LFU cache
#5: normalise the full lichess tablebase category vocabulary (cursed-win and blessed-loss classify as draws), return {:ok, tablebase} on a cache hit, and unwrap the tablebase in game_from_fen. #6: replace the LRU cache with an LFU cache storing {key, value, frequency, last_access}, evicting the least-frequently-used entry (LRU tie-break) and halving every frequency on a weekly sweep. Persistence moves to a versioned v2 dets file, dropping the legacy backup; the cache restores synchronously at startup. Also updates stale tags enum and Play LiveView tests, and drops the tracked legacy cache backup.
This commit is contained in:
+66
-39
@@ -1,6 +1,12 @@
|
||||
defmodule Chesstrainer.Cache do
|
||||
@moduledoc """
|
||||
Generic LRU cache.
|
||||
Generic LFU cache with a periodic frequency-decay sweep.
|
||||
|
||||
Entries are stored in ETS as `{key, value, frequency, last_access}`. Reads bump
|
||||
the frequency; when the cache is over its maximum size the least frequently
|
||||
used entry is evicted (ties broken by least recent use). A weekly sweep halves
|
||||
every frequency so entries that were used heavily once but have not been used
|
||||
since eventually age out.
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -9,14 +15,21 @@ defmodule Chesstrainer.Cache do
|
||||
|
||||
use GenServer
|
||||
|
||||
@max_size 1000
|
||||
@default_max_size 1000
|
||||
@default_sweep_interval :timer.hours(24 * 7)
|
||||
@backup_interval_minutes 60
|
||||
|
||||
def start_link(table), do: GenServer.start_link(__MODULE__, table, name: via_tuple(table))
|
||||
def start_link(table), do: start_link(table, [])
|
||||
|
||||
def start_link(table, opts) when is_list(opts) do
|
||||
GenServer.start_link(__MODULE__, {table, opts}, name: via_tuple(table))
|
||||
end
|
||||
|
||||
def put(table, key, value), do: GenServer.call(via_tuple(table), {:put, key, value})
|
||||
def get(table, key), do: GenServer.call(via_tuple(table), {:get, key})
|
||||
def delete(table, key), do: GenServer.call(via_tuple(table), {:delete, key})
|
||||
def backup(table), do: GenServer.cast(via_tuple(table), :backup)
|
||||
def backup(table), do: GenServer.call(via_tuple(table), :backup)
|
||||
def sweep(table), do: GenServer.call(via_tuple(table), :sweep)
|
||||
|
||||
def show(table) do
|
||||
ensure_table(table)
|
||||
@@ -26,66 +39,76 @@ defmodule Chesstrainer.Cache do
|
||||
def size(table), do: :ets.info(table, :size)
|
||||
|
||||
@impl true
|
||||
def init(table) do
|
||||
{:ok, table, {:continue, :restore_and_backup}}
|
||||
end
|
||||
def init({table, opts}) do
|
||||
state = %{
|
||||
table: table,
|
||||
max_size: Keyword.get(opts, :max_size, @default_max_size),
|
||||
sweep_interval: Keyword.get(opts, :sweep_interval, @default_sweep_interval)
|
||||
}
|
||||
|
||||
@impl true
|
||||
def handle_continue(:restore_and_backup, table) do
|
||||
# Create and restore synchronously so the cache is never observed empty
|
||||
# between `start_link/1` returning and the restore finishing.
|
||||
create_table(table)
|
||||
__MODULE__.Persist.restore(table)
|
||||
__MODULE__.Evict.evict(table, state.max_size)
|
||||
:timer.send_interval(:timer.minutes(@backup_interval_minutes), :backup)
|
||||
{:noreply, table}
|
||||
:timer.send_interval(state.sweep_interval, :sweep)
|
||||
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:put, key, value}, _from, table) do
|
||||
ensure_table(table)
|
||||
timestamp = System.system_time(:millisecond)
|
||||
:ets.insert(table, {key, value, timestamp})
|
||||
|
||||
GenServer.cast(self(), :maybe_evict)
|
||||
|
||||
{:reply, :ok, table}
|
||||
def handle_call({:put, key, value}, _from, state) do
|
||||
ensure_table(state.table)
|
||||
access = System.system_time(:microsecond)
|
||||
:ets.insert(state.table, {key, value, 1, access})
|
||||
__MODULE__.Evict.evict(state.table, state.max_size)
|
||||
{:reply, :ok, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:get, key}, _from, table) do
|
||||
ensure_table(table)
|
||||
def handle_call({:get, key}, _from, state) do
|
||||
ensure_table(state.table)
|
||||
|
||||
case :ets.lookup(table, key) do
|
||||
[{^key, value, _old_timestamp}] ->
|
||||
timestamp = System.system_time(:millisecond)
|
||||
:ets.insert(table, {key, value, timestamp})
|
||||
{:reply, {:ok, value}, table}
|
||||
case :ets.lookup(state.table, key) do
|
||||
[{^key, value, frequency, _old_access}] ->
|
||||
access = System.system_time(:microsecond)
|
||||
:ets.insert(state.table, {key, value, frequency + 1, access})
|
||||
{:reply, {:ok, value}, state}
|
||||
|
||||
[] ->
|
||||
{:reply, :error, table}
|
||||
{:reply, :error, state}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:delete, key}, _from, table) do
|
||||
:ets.delete(table, key)
|
||||
{:reply, :ok, table}
|
||||
def handle_call({:delete, key}, _from, state) do
|
||||
:ets.delete(state.table, key)
|
||||
{:reply, :ok, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(:maybe_evict, table) do
|
||||
__MODULE__.Evict.maybe_evict_oldest(table, @max_size)
|
||||
{:noreply, table}
|
||||
def handle_call(:backup, _from, state) do
|
||||
__MODULE__.Persist.backup(state.table)
|
||||
{:reply, :ok, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(:backup, table) do
|
||||
__MODULE__.Persist.backup(table)
|
||||
{:noreply, table}
|
||||
def handle_call(:sweep, _from, state) do
|
||||
__MODULE__.Sweep.halve_frequencies(state.table)
|
||||
{:reply, :ok, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:backup, table) do
|
||||
GenServer.cast(self(), :backup)
|
||||
{:noreply, table}
|
||||
def handle_info(:backup, state) do
|
||||
__MODULE__.Persist.backup(state.table)
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:sweep, state) do
|
||||
__MODULE__.Sweep.halve_frequencies(state.table)
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
defp via_tuple(table), do: {:via, Registry, {Chesstrainer.CacheRegistry, table}}
|
||||
@@ -97,5 +120,9 @@ defmodule Chesstrainer.Cache do
|
||||
end
|
||||
end
|
||||
|
||||
defp create_table(table), do: :ets.new(table, [:named_table, :public, :set])
|
||||
defp create_table(table) do
|
||||
:ets.new(table, [:named_table, :public, :set])
|
||||
rescue
|
||||
ArgumentError -> table
|
||||
end
|
||||
end
|
||||
|
||||
Vendored
+26
-11
@@ -1,25 +1,40 @@
|
||||
defmodule Chesstrainer.Cache.Evict do
|
||||
@moduledoc """
|
||||
Eviction functions for LRU cache
|
||||
Eviction for the LFU cache.
|
||||
"""
|
||||
|
||||
# O(n) time, not efficient
|
||||
def maybe_evict_oldest(table, max_size) do
|
||||
@doc """
|
||||
Evicts least-frequently-used entries until `table` holds at most `max_size`
|
||||
rows. Ties in frequency are broken by least recent use (`last_access`).
|
||||
"""
|
||||
def evict(table, max_size) do
|
||||
if :ets.info(table, :size) > max_size do
|
||||
case find_oldest_entry(table) do
|
||||
{oldest_key, _} -> :ets.delete(table, oldest_key)
|
||||
_ -> :ok
|
||||
case find_least_used_entry(table) do
|
||||
{key, _frequency, _last_access} ->
|
||||
:ets.delete(table, key)
|
||||
evict(table, max_size)
|
||||
|
||||
nil ->
|
||||
:ok
|
||||
end
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp find_oldest_entry(table) do
|
||||
defp find_least_used_entry(table) do
|
||||
:ets.foldl(
|
||||
fn {k, _v, t}, acc ->
|
||||
fn {k, _v, frequency, last_access}, acc ->
|
||||
case acc do
|
||||
nil -> {k, t}
|
||||
{_, oldest_t} when t < oldest_t -> {k, t}
|
||||
_ -> acc
|
||||
nil ->
|
||||
{k, frequency, last_access}
|
||||
|
||||
{_k, best_frequency, best_access}
|
||||
when {frequency, last_access} < {best_frequency, best_access} ->
|
||||
{k, frequency, last_access}
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end,
|
||||
nil,
|
||||
|
||||
Vendored
+39
-10
@@ -1,29 +1,58 @@
|
||||
defmodule Chesstrainer.Cache.Persist do
|
||||
@moduledoc """
|
||||
Persistance for cache. Backup and restore functions.
|
||||
Persistence for the cache. Backup and restore functions.
|
||||
"""
|
||||
|
||||
@backup_path "priv/static/backups"
|
||||
@format_version "v2"
|
||||
|
||||
def backup(table) do
|
||||
File.mkdir_p!(@backup_path)
|
||||
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
|
||||
{:ok, dets_table} = :dets.open_file(filename, type: :set)
|
||||
name = backup_filename(table) |> String.to_charlist()
|
||||
{:ok, dets_table} = :dets.open_file(name, type: :set)
|
||||
:ets.to_dets(table, dets_table)
|
||||
:dets.sync(dets_table)
|
||||
:dets.close(dets_table)
|
||||
end
|
||||
|
||||
def restore(table) do
|
||||
File.mkdir_p!(@backup_path)
|
||||
filename = Path.join(@backup_path, "#{table}.det") |> String.to_charlist()
|
||||
drop_legacy_backup(table)
|
||||
|
||||
case :dets.open_file(table, file: filename, type: :set) do
|
||||
{:ok, ^table} ->
|
||||
:dets.foldl(fn entry, _ -> :ets.insert(table, entry) end, :ok, table)
|
||||
:dets.close(table)
|
||||
path = backup_filename(table)
|
||||
|
||||
{:error, _} ->
|
||||
:ok
|
||||
if File.exists?(path) do
|
||||
name = String.to_charlist(path)
|
||||
|
||||
case :dets.open_file(table, file: name, type: :set) do
|
||||
{:ok, ^table} ->
|
||||
:dets.foldl(
|
||||
fn
|
||||
{key, value, frequency, last_access}, :ok when is_integer(frequency) ->
|
||||
:ets.insert(table, {key, value, frequency, last_access})
|
||||
:ok
|
||||
|
||||
_entry, :ok ->
|
||||
:ok
|
||||
end,
|
||||
:ok,
|
||||
table
|
||||
)
|
||||
|
||||
:dets.close(table)
|
||||
|
||||
{:error, _} ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp backup_filename(table), do: Path.join(@backup_path, "#{table}.#{@format_version}.det")
|
||||
|
||||
defp drop_legacy_backup(table) do
|
||||
legacy = Path.join(@backup_path, "#{table}.det")
|
||||
if File.exists?(legacy), do: File.rm(legacy)
|
||||
end
|
||||
end
|
||||
|
||||
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
defmodule Chesstrainer.Cache.Sweep do
|
||||
@moduledoc """
|
||||
Frequency decay for the LFU cache.
|
||||
|
||||
Halving every frequency each week lets entries that were used heavily once but
|
||||
have not been used since age out of the cache.
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Halves the frequency of every entry in `table` with integer division, so an
|
||||
entry used once decays from `1` to `0`.
|
||||
"""
|
||||
def halve_frequencies(table) do
|
||||
:ets.foldl(
|
||||
fn {key, value, frequency, last_access}, :ok ->
|
||||
:ets.insert(table, {key, value, div(frequency, 2), last_access})
|
||||
:ok
|
||||
end,
|
||||
:ok,
|
||||
table
|
||||
)
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
@@ -27,6 +27,24 @@ defmodule Chesstrainer.Endgames.Tablebase do
|
||||
insufficient_material: false,
|
||||
moves: nil
|
||||
|
||||
@doc """
|
||||
Normalises a Lichess tablebase category string into a trainer result.
|
||||
|
||||
`cursed-win` and `blessed-loss` are wins/losses only when the 50-move rule is
|
||||
ignored, so they are draws in play. Unrecognised values become `:unknown`
|
||||
rather than minting arbitrary atoms.
|
||||
"""
|
||||
@spec normalize_category(term()) :: result()
|
||||
def normalize_category(category) when category in ["win", "syzygy-win", "maybe-win"], do: :win
|
||||
|
||||
def normalize_category(category) when category in ["draw", "cursed-win", "blessed-loss"],
|
||||
do: :draw
|
||||
|
||||
def normalize_category(category) when category in ["loss", "syzygy-loss", "maybe-loss"],
|
||||
do: :loss
|
||||
|
||||
def normalize_category(_category), do: :unknown
|
||||
|
||||
@spec new(String.t()) ::
|
||||
{:ok, t()}
|
||||
| {:cooldown, non_neg_integer()}
|
||||
@@ -34,8 +52,8 @@ defmodule Chesstrainer.Endgames.Tablebase do
|
||||
| {:error, term()}
|
||||
def new(fen) do
|
||||
case Tablebase.Cache.get(fen) do
|
||||
{_fen, tablebase} ->
|
||||
{:cache, tablebase}
|
||||
{:ok, tablebase} ->
|
||||
{:ok, tablebase}
|
||||
|
||||
_ ->
|
||||
case Tablebase.Parser.from_fen(fen) do
|
||||
|
||||
@@ -3,7 +3,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Move do
|
||||
Moves struct for the tablebase
|
||||
"""
|
||||
|
||||
@type result :: :win | :draw | :loss
|
||||
@type result :: :win | :draw | :loss | :unknown
|
||||
|
||||
@type t :: %__MODULE__{
|
||||
category: result,
|
||||
|
||||
@@ -41,7 +41,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
|
||||
def from_map(map) do
|
||||
{:ok,
|
||||
%Tablebase{
|
||||
category: map["category"] |> String.to_atom(),
|
||||
category: Tablebase.normalize_category(map["category"]),
|
||||
checkmate: map["checkmate"],
|
||||
dtc: map["dtc"],
|
||||
dtm: map["dtm"],
|
||||
@@ -54,7 +54,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Parser do
|
||||
|
||||
defp moves_from_map(map) do
|
||||
%Tablebase.Move{
|
||||
category: map["category"] |> String.to_atom(),
|
||||
category: Tablebase.normalize_category(map["category"]),
|
||||
checkmate: map["checkmate"],
|
||||
conversion: map["conversion"],
|
||||
dtc: map["dtc"],
|
||||
|
||||
@@ -102,12 +102,7 @@ defmodule Chesstrainer.Game do
|
||||
game_type: game_type,
|
||||
fen: fen,
|
||||
game_state: :continue,
|
||||
tablebase:
|
||||
if game_type === :endgame do
|
||||
Endgames.Tablebase.new(fen)
|
||||
else
|
||||
nil
|
||||
end
|
||||
tablebase: tablebase_for(fen, game_type)
|
||||
}
|
||||
|
||||
{:ok, game}
|
||||
@@ -142,4 +137,13 @@ defmodule Chesstrainer.Game do
|
||||
{:error, :invalid_fen, game}
|
||||
end
|
||||
end
|
||||
|
||||
defp tablebase_for(_fen, game_type) when game_type != :endgame, do: nil
|
||||
|
||||
defp tablebase_for(fen, :endgame) do
|
||||
case Endgames.Tablebase.new(fen) do
|
||||
{:ok, tablebase} -> tablebase
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user