Improve ergonomics of batch operations on entire database

Use a stream for better memory management, and return IDs for records
which haven't been processed correctly.
This commit is contained in:
Claudio Ortolina
2024-11-14 22:15:31 +00:00
parent e002c57aa0
commit ac6b398afc
+26 -8
View File
@@ -3,12 +3,11 @@ defmodule MusicLibrary.Records.Batch do
alias MusicLibrary.Records.Record
alias MusicLibrary.Repo
import Ecto.Query
def refresh_musicbrainz_data do
Record
|> Repo.all()
|> Enum.each(fn r ->
import_musicbrainz_data(r)
run_on_all_records(fn record ->
import_musicbrainz_data(record)
Process.sleep(1000)
end)
end
@@ -22,15 +21,34 @@ defmodule MusicLibrary.Records.Batch do
end
def update_release_ids do
Record
|> Repo.all()
|> Enum.each(&update_release_ids/1)
run_on_all_records(&update_release_ids/1)
end
def update_release_ids(record) do
record
|> Record.update_release_ids()
|> Repo.update!()
|> Repo.update()
end
defp run_on_all_records(fun) do
q = from(r in Record)
stream = Repo.stream(q)
Repo.transaction(fn ->
Enum.reduce(stream, [], fn record, acc ->
case fun.(record) do
{:error, reason} ->
Logger.error("Failed to run function on record #{record.id} with #{inspect(reason)}")
[record.id | acc]
:ok ->
acc
{:ok, _record} ->
acc
end
end)
end)
end
defp musicbrainz do