mirror of
https://github.com/TheRealOwenRees/chess-endgame-trainer.git
synced 2026-11-18 03:32:06 +00:00
Scale the tablebase cache to ~100k entries (#24)
* Slim the cached tablebase value to the fields the trainer reads (#20) * Sample for LFU eviction instead of scanning the cache (#21) * Make cache size configurable and back up daily off-peak (#22) * Document the cache design in the glossary and an ADR (#23) * Address cache-scaling code review findings
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
defmodule Chesstrainer.Cache.BackupScheduleTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Chesstrainer.Cache.BackupSchedule
|
||||
|
||||
describe "parse_time/1" do
|
||||
test "accepts HH:MM strings and {hour, minute} tuples" do
|
||||
assert {:ok, {3, 0}} = BackupSchedule.parse_time("03:00")
|
||||
assert {:ok, {23, 59}} = BackupSchedule.parse_time("23:59")
|
||||
assert {:ok, {0, 5}} = BackupSchedule.parse_time("0:05")
|
||||
assert {:ok, {4, 30}} = BackupSchedule.parse_time({4, 30})
|
||||
end
|
||||
|
||||
test "rejects values that are not a valid UTC time" do
|
||||
assert :error = BackupSchedule.parse_time("24:00")
|
||||
assert :error = BackupSchedule.parse_time("03:60")
|
||||
assert :error = BackupSchedule.parse_time("03:00:00")
|
||||
assert :error = BackupSchedule.parse_time("three")
|
||||
assert :error = BackupSchedule.parse_time({24, 0})
|
||||
assert :error = BackupSchedule.parse_time(nil)
|
||||
end
|
||||
end
|
||||
|
||||
describe "time!/1" do
|
||||
test "defaults to 03:00 UTC" do
|
||||
assert {3, 0} = BackupSchedule.time!(nil)
|
||||
assert {3, 0} = BackupSchedule.time!("03:00")
|
||||
end
|
||||
end
|
||||
|
||||
describe "next_delay/2" do
|
||||
test "returns the delay until today's run when it is still ahead" do
|
||||
now = ~U[2026-11-02 01:30:00Z]
|
||||
|
||||
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(1) + :timer.minutes(30)
|
||||
end
|
||||
|
||||
test "rolls over to tomorrow once today's run has passed" do
|
||||
now = ~U[2026-11-02 04:00:00Z]
|
||||
|
||||
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(23)
|
||||
end
|
||||
|
||||
test "waits a full day when called exactly at the scheduled time" do
|
||||
now = ~U[2026-11-02 03:00:00Z]
|
||||
|
||||
assert BackupSchedule.next_delay({3, 0}, now) == :timer.hours(24)
|
||||
end
|
||||
|
||||
test "computes the delay from the wall clock by default" do
|
||||
delay = BackupSchedule.next_delay({0, 0})
|
||||
|
||||
assert delay > 0
|
||||
assert delay <= :timer.hours(24)
|
||||
end
|
||||
end
|
||||
end
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
defmodule Chesstrainer.Cache.EvictTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Chesstrainer.Cache.Evict
|
||||
|
||||
@sample_size 64
|
||||
|
||||
setup do
|
||||
table = :"evict_test_#{System.unique_integer([:positive])}"
|
||||
:ets.new(table, [:named_table, :public, :set])
|
||||
|
||||
{:ok, table: table}
|
||||
end
|
||||
|
||||
describe "evict/3" do
|
||||
test "shrinks a table that is far over capacity", %{table: table} do
|
||||
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
|
||||
|
||||
Evict.evict(table, 900)
|
||||
|
||||
assert :ets.info(table, :size) == 900
|
||||
end
|
||||
|
||||
test "keeps a frequently used entry while cold entries are evicted", %{table: table} do
|
||||
:ets.insert(table, {:hot, :hot, 100, 0})
|
||||
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
|
||||
|
||||
Evict.evict(table, 100)
|
||||
|
||||
assert :ets.info(table, :size) == 100
|
||||
assert [{:hot, :hot, 100, 0}] = :ets.lookup(table, :hot)
|
||||
end
|
||||
|
||||
test "the returned cursor resumes the sample instead of restarting it", %{table: table} do
|
||||
for i <- 1..5, do: :ets.insert(table, {i, i, 1, i})
|
||||
|
||||
assert Evict.evict(table, 10) == :start
|
||||
|
||||
cursor = Evict.evict(table, 4)
|
||||
|
||||
assert :ets.info(table, :size) == 4
|
||||
refute cursor == :start
|
||||
end
|
||||
|
||||
test "rotates across the table, eventually evicting entries outside the first sample",
|
||||
%{table: table} do
|
||||
for i <- 1..1_000, do: :ets.insert(table, {i, i, 1, i})
|
||||
|
||||
# Find the slice a fresh pass would sample and make it the hottest part of
|
||||
# the table. A pass that always restarted from `:start` would keep
|
||||
# choosing from this slice (every one of those entries is hotter than
|
||||
# everything else), so cold entries elsewhere could never be evicted.
|
||||
hot_keys = first_sample_keys(table)
|
||||
for key <- hot_keys, do: :ets.insert(table, {key, key, 1_000, key})
|
||||
|
||||
Enum.reduce(1..@sample_size, :start, fn _, cursor ->
|
||||
Evict.evict(table, :ets.info(table, :size) - 1, cursor)
|
||||
end)
|
||||
|
||||
frequencies = Map.new(:ets.tab2list(table), fn {key, _v, f, _t} -> {key, f} end)
|
||||
cold_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1 end)
|
||||
hot_left = Enum.count(frequencies, fn {_key, frequency} -> frequency == 1_000 end)
|
||||
|
||||
assert :ets.info(table, :size) == 1_000 - @sample_size
|
||||
assert cold_left < 1_000 - @sample_size
|
||||
assert hot_left > 0
|
||||
end
|
||||
end
|
||||
|
||||
defp first_sample_keys(table) do
|
||||
match_spec = [{{:"$1", :_, :"$2", :"$3"}, [], [:"$1"]}]
|
||||
|
||||
case :ets.select(table, match_spec, @sample_size) do
|
||||
{keys, _continuation} -> keys
|
||||
:"$end_of_table" -> []
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -5,8 +5,9 @@ defmodule Chesstrainer.CacheTest do
|
||||
|
||||
setup context do
|
||||
table = :"test_#{System.unique_integer([:positive])}"
|
||||
{:ok, pid} = Cache.start_link(table, Map.get(context, :cache_opts, []))
|
||||
on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end)
|
||||
opts = Map.get(context, :cache_opts, [])
|
||||
pid = start_supervised!(Supervisor.child_spec({Cache, {table, opts}}, id: table))
|
||||
|
||||
{:ok, %{pid: pid, table: table}}
|
||||
end
|
||||
|
||||
@@ -37,6 +38,32 @@ defmodule Chesstrainer.CacheTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "configuration" do
|
||||
test "defaults to a max size of 100_000", %{pid: pid} do
|
||||
assert :sys.get_state(pid).max_size == 100_000
|
||||
end
|
||||
|
||||
test "starts from a {table, opts} child spec argument" do
|
||||
table = :"test_child_spec_#{System.unique_integer([:positive])}"
|
||||
|
||||
pid =
|
||||
start_supervised!(Supervisor.child_spec({Cache, {table, max_size: 3}}, id: table))
|
||||
|
||||
assert :sys.get_state(pid).max_size == 3
|
||||
end
|
||||
|
||||
test "the application starts the tablebase cache with the configured options" do
|
||||
config = Application.get_env(:chesstrainer, Chesstrainer.Cache, [])
|
||||
pid = GenServer.whereis({:via, Registry, {Chesstrainer.CacheRegistry, :tablebase}})
|
||||
state = :sys.get_state(pid)
|
||||
|
||||
assert state.max_size == Keyword.get(config, :max_size, 100_000)
|
||||
|
||||
assert state.backup_time ==
|
||||
Chesstrainer.Cache.BackupSchedule.time!(Keyword.get(config, :backup_time))
|
||||
end
|
||||
end
|
||||
|
||||
describe "frequency tracking" do
|
||||
test "get/2 increments the entry frequency", %{table: table} do
|
||||
Cache.put(table, "key", "value")
|
||||
@@ -77,6 +104,26 @@ defmodule Chesstrainer.CacheTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "eviction on a cache larger than the sample" do
|
||||
@describetag cache_opts: [max_size: 100]
|
||||
|
||||
test "stays at the maximum size under churn", %{table: table} do
|
||||
for i <- 1..500, do: Cache.put(table, i, i)
|
||||
|
||||
assert Cache.size(table) == 100
|
||||
end
|
||||
|
||||
test "a frequently used entry survives churn from cold entries", %{table: table} do
|
||||
Cache.put(table, :hot, "value")
|
||||
for _ <- 1..50, do: Cache.get(table, :hot)
|
||||
|
||||
for i <- 1..500, do: Cache.put(table, i, i)
|
||||
|
||||
assert Cache.size(table) == 100
|
||||
assert {:ok, "value"} = Cache.get(table, :hot)
|
||||
end
|
||||
end
|
||||
|
||||
describe "weekly sweep" do
|
||||
test "halves every entry frequency", %{table: table} do
|
||||
Cache.put(table, "busy", "value")
|
||||
@@ -92,23 +139,123 @@ defmodule Chesstrainer.CacheTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "backup" do
|
||||
test "runs off the cache process so reads and writes stay responsive" do
|
||||
table = :"test_async_backup_#{System.unique_integer([:positive])}"
|
||||
parent = self()
|
||||
ref = make_ref()
|
||||
|
||||
backup_fun = fn _table ->
|
||||
send(parent, {:backup_started, ref, self()})
|
||||
|
||||
receive do
|
||||
{:finish_backup, ^ref} -> :ok
|
||||
end
|
||||
end
|
||||
|
||||
pid =
|
||||
start_supervised!(
|
||||
Supervisor.child_spec({Cache, {table, [backup_fun: backup_fun]}}, id: table)
|
||||
)
|
||||
|
||||
assert :ok = Cache.put(table, "key", "value")
|
||||
|
||||
send(pid, :backup)
|
||||
|
||||
assert_receive {:backup_started, ^ref, backup_pid}, 1_000
|
||||
backup_monitor = Process.monitor(backup_pid)
|
||||
|
||||
assert {:ok, "value"} = Cache.get(table, "key")
|
||||
assert :ok = Cache.put(table, "other", "value")
|
||||
|
||||
send(backup_pid, {:finish_backup, ref})
|
||||
assert_receive {:DOWN, ^backup_monitor, :process, ^backup_pid, :normal}, 1_000
|
||||
end
|
||||
|
||||
test "re-arms the daily backup after each run" do
|
||||
table = :"test_rearm_#{System.unique_integer([:positive])}"
|
||||
|
||||
pid =
|
||||
start_supervised!(
|
||||
Supervisor.child_spec(
|
||||
{Cache, {table, [backup_fun: fn _table -> :ok end, backup_time: "04:30"]}},
|
||||
id: table
|
||||
)
|
||||
)
|
||||
|
||||
send(pid, :backup)
|
||||
|
||||
state = :sys.get_state(pid)
|
||||
|
||||
assert state.backup_time == {4, 30}
|
||||
assert is_reference(state.backup_timer)
|
||||
assert :erlang.read_timer(state.backup_timer) > 0
|
||||
end
|
||||
end
|
||||
|
||||
describe "persistence" do
|
||||
test "writes the backup atomically, leaving no temporary file" do
|
||||
table = :"test_atomic_#{System.unique_integer([:positive])}"
|
||||
backup = Path.join("priv/static/backups", "#{table}.v2.det")
|
||||
|
||||
on_exit(fn ->
|
||||
File.rm_rf(backup <> ".tmp")
|
||||
File.rm(backup)
|
||||
end)
|
||||
|
||||
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
|
||||
|
||||
Cache.put(table, "key", "value")
|
||||
|
||||
assert :ok = Cache.backup(table)
|
||||
assert File.exists?(backup)
|
||||
refute File.exists?(backup <> ".tmp")
|
||||
end
|
||||
|
||||
test "a failed backup leaves the previous backup intact" do
|
||||
table = :"test_failed_backup_#{System.unique_integer([:positive])}"
|
||||
backup = Path.join("priv/static/backups", "#{table}.v2.det")
|
||||
temp = backup <> ".tmp"
|
||||
|
||||
on_exit(fn ->
|
||||
File.rm_rf(temp)
|
||||
File.rm(backup)
|
||||
end)
|
||||
|
||||
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
|
||||
|
||||
Cache.put(table, "key", "value")
|
||||
assert :ok = Cache.backup(table)
|
||||
|
||||
# A directory at the temporary path makes the write fail before the rename.
|
||||
File.mkdir_p!(temp)
|
||||
Cache.put(table, "another", "value")
|
||||
|
||||
assert {:error, _reason} = Cache.Persist.backup(table)
|
||||
assert File.exists?(backup)
|
||||
assert :ok = stop_supervised(table)
|
||||
|
||||
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
|
||||
|
||||
assert {:ok, "value"} = Cache.get(table, "key")
|
||||
assert :error = Cache.get(table, "another")
|
||||
end
|
||||
|
||||
test "backup and restore preserve frequencies" do
|
||||
table = :"test_persist_#{System.unique_integer([:positive])}"
|
||||
backup = Path.join("priv/static/backups", "#{table}.v2.det")
|
||||
on_exit(fn -> File.rm(backup) end)
|
||||
|
||||
{:ok, pid} = Cache.start_link(table)
|
||||
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
|
||||
|
||||
Cache.put(table, "key", "value")
|
||||
Cache.get(table, "key")
|
||||
Cache.get(table, "key")
|
||||
|
||||
assert :ok = Cache.backup(table)
|
||||
assert :ok = GenServer.stop(pid)
|
||||
assert :ok = stop_supervised(table)
|
||||
|
||||
{:ok, restored_pid} = Cache.start_link(table)
|
||||
on_exit(fn -> if Process.alive?(restored_pid), do: GenServer.stop(restored_pid) end)
|
||||
start_supervised!(Supervisor.child_spec({Cache, table}, id: table))
|
||||
|
||||
assert [{"key", "value", frequency, _last_access}] = Cache.show(table)
|
||||
assert frequency == 3
|
||||
|
||||
@@ -32,6 +32,18 @@ defmodule Chesstrainer.Endgames.TablebaseTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "struct shape" do
|
||||
test "Tablebase carries only the fields the trainer reads" do
|
||||
assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
|
||||
[:category, :checkmate, :dtc, :dtm, :moves]
|
||||
end
|
||||
|
||||
test "Tablebase.Move carries only the fields the trainer reads" do
|
||||
assert %Tablebase.Move{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
|
||||
[:category, :dtc, :dtm, :dtz, :san, :uci]
|
||||
end
|
||||
end
|
||||
|
||||
describe "Parser.from_map/1" do
|
||||
test "normalises the position and move categories" do
|
||||
assert {:ok, tablebase} =
|
||||
@@ -40,32 +52,78 @@ defmodule Chesstrainer.Endgames.TablebaseTest do
|
||||
"checkmate" => false,
|
||||
"dtc" => nil,
|
||||
"dtm" => 20,
|
||||
"dtw" => nil,
|
||||
"dtz" => 3,
|
||||
"insufficient_material" => false,
|
||||
"moves" => [
|
||||
%{
|
||||
"category" => "blessed-loss",
|
||||
"checkmate" => false,
|
||||
"conversion" => false,
|
||||
"dtc" => nil,
|
||||
"dtm" => -19,
|
||||
"dtw" => nil,
|
||||
"dtz" => -2,
|
||||
"insufficient_material" => false,
|
||||
"precise_dtz" => -2,
|
||||
"san" => "Kd7",
|
||||
"stalemate" => false,
|
||||
"uci" => "e8d7",
|
||||
"variant_loss" => false,
|
||||
"variant_win" => false,
|
||||
"zeroing" => false
|
||||
"uci" => "e8d7"
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
assert tablebase.category == :draw
|
||||
assert [%Tablebase.Move{category: :draw, uci: "e8d7"}] = tablebase.moves
|
||||
assert %Tablebase{category: :draw, checkmate: false, dtc: nil, dtm: 20} = tablebase
|
||||
|
||||
assert [%Tablebase.Move{category: :draw, dtc: nil, dtm: -19, dtz: -2}] = tablebase.moves
|
||||
assert [%Tablebase.Move{san: "Kd7", uci: "e8d7"}] = tablebase.moves
|
||||
end
|
||||
|
||||
test "drops the fields the trainer never reads" do
|
||||
assert {:ok, tablebase} =
|
||||
Parser.from_map(%{
|
||||
"category" => "win",
|
||||
"checkmate" => true,
|
||||
"dtc" => 1,
|
||||
"dtm" => 2,
|
||||
"dtw" => 3,
|
||||
"dtz" => 4,
|
||||
"insufficient_material" => false,
|
||||
"moves" => [
|
||||
%{
|
||||
"category" => "loss",
|
||||
"checkmate" => true,
|
||||
"conversion" => true,
|
||||
"dtc" => 5,
|
||||
"dtm" => 6,
|
||||
"dtw" => 7,
|
||||
"dtz" => 8,
|
||||
"insufficient_material" => false,
|
||||
"precise_dtz" => 9,
|
||||
"san" => "Qd7#",
|
||||
"stalemate" => false,
|
||||
"uci" => "d1d7",
|
||||
"variant_loss" => false,
|
||||
"variant_win" => false,
|
||||
"zeroing" => true
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
assert %Tablebase{} |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
|
||||
[:category, :checkmate, :dtc, :dtm, :moves]
|
||||
|
||||
assert [move] = tablebase.moves
|
||||
|
||||
assert move |> Map.from_struct() |> Map.keys() |> Enum.sort() ==
|
||||
[:category, :dtc, :dtm, :dtz, :san, :uci]
|
||||
end
|
||||
|
||||
test "keeps checkmate and empty moves, which drive terminal detection" do
|
||||
assert {:ok, tablebase} =
|
||||
Parser.from_map(%{
|
||||
"category" => "win",
|
||||
"checkmate" => true,
|
||||
"dtc" => 1,
|
||||
"dtm" => 2,
|
||||
"dtz" => 3,
|
||||
"moves" => []
|
||||
})
|
||||
|
||||
assert tablebase.checkmate
|
||||
assert tablebase.moves == []
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user