Add admin bulk FEN importer (#156)

* change position of theme picker

* Add admin bulk FEN importer

Paste FENs, resolve results from the local Syzygy tablebase only, preview each line's status, and bulk insert with themes and a batch rating.
This commit is contained in:
2026-11-12 22:44:13 +01:00
committed by GitHub
parent 20b53dae6f
commit fddb94e2bd
13 changed files with 1010 additions and 107 deletions
+218
View File
@@ -0,0 +1,218 @@
defmodule Chesstrainer.Endgames.BulkImport do
@moduledoc """
Classifies pasted FENs for the admin bulk importer and inserts the positions
the local Syzygy tablebase can answer.
Results always come from `Chesstrainer.Endgames.Tablebase.probe_local_result/1`,
so a bulk import never consults the remote (Lichess) backend.
"""
import Ecto.Query, warn: false
alias Chesstrainer.Endgames.Endgame
alias Chesstrainer.Endgames.Tablebase
alias Chesstrainer.{FEN, Key, Repo}
@max_fens 1000
@max_pieces 5
@default_rating 1500.0
@type status ::
:ready
| :duplicate
| :invalid_fen
| :too_many_pieces
| :illegal
| :terminal
| :unsupported
@type entry :: %{
line: pos_integer(),
fen: String.t(),
key: String.t() | nil,
color: :white | :black | nil,
result: Tablebase.result() | nil,
status: status(),
reason: String.t() | nil
}
@doc """
The most FENs a single import will consider.
"""
@spec max_fens() :: pos_integer()
def max_fens, do: @max_fens
@doc """
Parses one FEN per line, skipping blank lines and lines beginning with `#`.
Returns entries with a status explaining whether the line can be imported,
and `too_many?` when the input exceeded `max_fens/0` (only the first
`max_fens/0` lines are considered).
FENs already in the database and duplicates within the batch are reported as
duplicates without probing the tablebase.
"""
@spec preview(String.t()) :: %{entries: [entry()], too_many?: boolean()}
def preview(text) when is_binary(text) do
lines = parse_lines(text)
{lines, rest} = Enum.split(lines, @max_fens)
fens = Enum.map(lines, fn {_line, fen} -> FEN.normalize(fen) end)
existing = existing_fens(fens)
{entries, _seen} =
Enum.map_reduce(lines, %{}, fn {line, raw_fen}, seen ->
classify(line, FEN.normalize(raw_fen), seen, existing)
end)
%{entries: entries, too_many?: rest != []}
end
@doc """
Inserts every `:ready` entry, attaching `themes` and `rating` to each.
Rows that collide with an existing FEN are skipped. Returns the number
imported and the number of entries that were not ready or were skipped.
"""
@spec commit([entry()], [Chesstrainer.Themes.Theme.t()], number()) ::
%{imported: non_neg_integer(), skipped: non_neg_integer()}
def commit(entries, themes \\ [], rating \\ @default_rating) do
ready = Enum.filter(entries, &(&1.status == :ready))
now = DateTime.utc_now() |> DateTime.truncate(:second)
rows =
Enum.map(ready, fn entry ->
%{
id: Ecto.UUID.generate(),
public_code: Endgame.generate_public_code(),
fen: entry.fen,
color: entry.color,
key: entry.key,
result: entry.result,
rating: (rating || @default_rating) * 1.0,
rating_deviation: 350.0,
times_attempted: 0,
times_solved: 0,
inserted_at: now,
updated_at: now
}
end)
{imported, inserted_rows} =
Repo.insert_all(Endgame, rows, on_conflict: :nothing, returning: [:id])
join_rows =
for %Endgame{id: id} <- inserted_rows, theme <- themes do
%{
endgame_id: Ecto.UUID.dump!(id),
theme_id: theme.id
}
end
if join_rows != [] do
Repo.insert_all("endgames_themes", join_rows, on_conflict: :nothing)
end
%{imported: imported, skipped: length(entries) - imported}
end
defp parse_lines(text) do
text
|> String.split(~r/\r?\n/)
|> Enum.with_index(1)
|> Enum.reduce([], fn {raw, index}, acc ->
trimmed = String.trim(raw)
if trimmed == "" or String.starts_with?(trimmed, "#") do
acc
else
[{index, trimmed} | acc]
end
end)
|> Enum.reverse()
end
defp existing_fens(fens) do
fens
|> Enum.uniq()
|> then(fn fens -> Repo.all(from e in Endgame, where: e.fen in ^fens, select: e.fen) end)
|> Map.new(&{&1, true})
end
defp classify(line, fen, seen, existing) do
cond do
Map.has_key?(seen, fen) ->
{entry(line, fen, :duplicate, "Duplicate FEN in this batch"), seen}
Map.has_key?(existing, fen) ->
{entry(line, fen, :duplicate, "FEN already exists"), seen}
true ->
{classify_new(line, fen), Map.put(seen, fen, true)}
end
end
defp classify_new(line, fen) do
issues = Endgame.fen_issues(fen)
pieces = FEN.piece_count(fen)
cond do
issues != [] ->
entry(line, fen, :invalid_fen, Enum.join(issues, "; "))
is_integer(pieces) and pieces > @max_pieces ->
entry(
line,
fen,
:too_many_pieces,
"Only positions with #{@max_pieces} or fewer pieces are supported"
)
true ->
classify_probe(line, fen)
end
end
defp classify_probe(line, fen) do
case Tablebase.probe_local_result(fen) do
{:ok, result} ->
entry(line, fen, :ready, nil) |> Map.put(:result, result)
{:error, :terminal} ->
entry(line, fen, :terminal, "Checkmate, stalemate, or no legal moves")
{:error, :illegal} ->
entry(line, fen, :illegal, "The tablebase rejected this position")
{:error, :not_configured} ->
entry(
line,
fen,
:unsupported,
"Local tablebase is not configured (set TABLEBASE_PATH)"
)
{:error, :unsupported} ->
entry(line, fen, :unsupported, "The local tablebase cannot answer this position")
end
end
defp entry(line, fen, status, reason) do
%{
line: line,
fen: fen,
key: key_for(fen),
color: FEN.color_from_fen(fen),
result: nil,
status: status,
reason: reason
}
end
defp key_for(fen) do
case Key.from_fen(fen) do
{:ok, key} -> key
_ -> nil
end
end
end
+9 -4
View File
@@ -90,10 +90,15 @@ defmodule Chesstrainer.Endgames.Endgame do
defp normalize_attrs(attrs), do: attrs
# Strict structural validation. Mirrors chess.js rules so the FEN survives
# both the server-side parser (Chex, permissive) and the client-side parser
# (chess.js, strict).
defp fen_issues(fen) do
@doc """
Returns the structural problems with a FEN, or `[]` when it is well formed.
Strict structural validation. Mirrors chess.js rules so the FEN survives
both the server-side parser (Chex, permissive) and the client-side parser
(chess.js, strict). Used by the admin bulk importer to explain why a line
was rejected.
"""
def fen_issues(fen) do
case String.split(fen, " ") do
[board, active, _castling, _en_passant, halfmove, fullmove] ->
board_valid? = valid_board?(board)
+14
View File
@@ -12,6 +12,7 @@ defmodule Chesstrainer.Endgames.Tablebase do
"""
alias Chesstrainer.Endgames.Tablebase
alias Chesstrainer.Endgames.Tablebase.Backend.Fathom, as: FathomBackend
alias Chesstrainer.FEN
@type result :: :win | :draw | :loss | :unknown
@@ -67,6 +68,19 @@ defmodule Chesstrainer.Endgames.Tablebase do
def flip(:draw), do: :draw
def flip(_outcome), do: :unknown
@doc """
Probes a FEN using only the local Syzygy tablebase and returns the result
from the side-to-move's perspective.
Unlike `new/1` this never falls back to the remote backend and never reads or
writes the cache. It is the primitive behind the admin bulk importer, so an
imported result always comes from our own tablebases.
"""
@spec probe_local_result(String.t()) ::
{:ok, result()}
| {:error, :not_configured | :illegal | :terminal | :unsupported}
def probe_local_result(fen), do: FathomBackend.probe_result_local(fen)
@spec new(String.t()) ::
{:ok, t()}
| {:cooldown, non_neg_integer()}
@@ -18,6 +18,7 @@ defmodule Chesstrainer.Endgames.Tablebase.Backend.Fathom do
alias Chesstrainer.Endgames.Tablebase.Local
alias Chesstrainer.Endgames.Tablebase.Move
alias Chesstrainer.Endgames.Tablebase.San
alias Chesstrainer.FEN
@impl true
def probe(fen) do
@@ -32,6 +33,53 @@ defmodule Chesstrainer.Endgames.Tablebase.Backend.Fathom do
end
end
@doc """
Probes the local tablebase without the remote fallback and returns only the
position's result.
Used by the admin bulk importer, which must never import a result from
Lichess. The result is from the side-to-move's perspective, matching the
`category` on `Chesstrainer.Endgames.Tablebase`.
Returns `{:error, :terminal}` for checkmate, stalemate, and positions with no
legal moves, because they cannot be played as puzzles.
"""
@spec probe_result_local(String.t()) ::
{:ok, Tablebase.result()}
| {:error, :not_configured | :illegal | :terminal | :unsupported}
def probe_result_local(fen) do
with {:ok, largest} <- Local.check(),
pieces when is_integer(pieces) <- FEN.piece_count(fen),
true <- pieces <= largest || {:error, :unsupported},
{:ok, {root, moves}} <- Nif.probe(fen) do
case root do
:failed ->
{:error, :illegal}
:checkmate ->
{:error, :terminal}
:stalemate ->
{:error, :terminal}
{wdl, _from, _to, _promotes, _ep, _dtz} ->
if moves == [] do
{:error, :terminal}
else
case Tablebase.normalize_wdl(wdl) do
:unknown -> {:error, :unsupported}
category -> {:ok, category}
end
end
end
else
{:error, :not_configured} = error -> error
{:error, :unsupported} = error -> error
{:error, _reason} -> {:error, :unsupported}
nil -> {:error, :unsupported}
end
end
defp fallback(fen) do
(Application.get_env(:chesstrainer, :tablebase_fallback) || Lichess).probe(fen)
end