Batch scrobble rule application by type to reduce N queries to 2 (#61)
* Implement optimized batch rule application - Add apply_all_album_rules/1 to batch apply all album rules in single query - Add apply_all_artist_rules/1 to batch apply all artist rules in single query - Update apply_all_rules/0 to use new batch functions - Add comprehensive tests for batch application - Use CASE statement in SQL to apply multiple rules efficiently Co-authored-by: cloud8421 <537608+cloud8421@users.noreply.github.com> * Add batch application support for track-filtered rules - Extend apply_all_album_rules/2 to support filtering by tracks - Extend apply_all_artist_rules/2 to support filtering by tracks - Update apply_all_rules/1 to use batch functions for track-filtered application - Remove duplicate function definitions - Ensures both main use cases (all tracks and specific tracks) are optimized Co-authored-by: cloud8421 <537608+cloud8421@users.noreply.github.com> * Add comprehensive documentation for optimization - Document the problem and solution approach - Explain SQL generation and performance impact - Detail trade-offs and backward compatibility - Include future considerations and scalability notes Co-authored-by: cloud8421 <537608+cloud8421@users.noreply.github.com> * Apply credo suggestions and format --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: cloud8421 <537608+cloud8421@users.noreply.github.com> Co-authored-by: Claudio Ortolina <cloud8421@gmail.com>
This commit is contained in:
Copilot
@@ -326,9 +326,229 @@ defmodule MusicLibrary.ScrobbleRules do
|
||||
apply_artist_rule(rule, tracks)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Applies all album rules in a single query.
|
||||
|
||||
Uses a CASE statement to update the musicbrainz_id for all matching albums
|
||||
in one database operation, which is more efficient than applying rules individually.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_album_rules([rule1, rule2])
|
||||
{:ok, 15}
|
||||
|
||||
"""
|
||||
def apply_all_album_rules([]), do: {:ok, 0}
|
||||
|
||||
def apply_all_album_rules(rules) when is_list(rules) do
|
||||
# Build CASE WHEN clauses dynamically
|
||||
{case_clauses, case_params} =
|
||||
rules
|
||||
|> Enum.reduce({"", []}, fn rule, {sql_acc, params_acc} ->
|
||||
clause =
|
||||
"WHEN json_extract(album, '$.title') = ? THEN json_set(album, '$.musicbrainz_id', ?) "
|
||||
|
||||
{sql_acc <> clause, params_acc ++ [rule.match_value, rule.target_musicbrainz_id]}
|
||||
end)
|
||||
|
||||
# Build complete UPDATE statement
|
||||
case_sql = "CASE #{case_clauses}ELSE album END"
|
||||
|
||||
# Build WHERE IN clause
|
||||
match_values = Enum.map(rules, & &1.match_value)
|
||||
in_placeholders = Enum.map_join(match_values, ", ", fn _ -> "?" end)
|
||||
where_sql = "json_extract(album, '$.title') IN (#{in_placeholders})"
|
||||
|
||||
# Complete SQL
|
||||
sql = """
|
||||
UPDATE scrobbled_tracks
|
||||
SET album = #{case_sql}
|
||||
WHERE #{where_sql}
|
||||
"""
|
||||
|
||||
# All parameters: case params + where params
|
||||
all_params = case_params ++ match_values
|
||||
|
||||
# Execute the query
|
||||
case Repo.query(sql, all_params) do
|
||||
{:ok, %{num_rows: count}} -> {:ok, count}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Applies all album rules to a specific set of tracks in a single query.
|
||||
|
||||
Uses a CASE statement to update the musicbrainz_id for all matching albums
|
||||
in one database operation, filtering by the provided tracks.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_album_rules([rule1, rule2], tracks)
|
||||
{:ok, 3}
|
||||
|
||||
"""
|
||||
def apply_all_album_rules([], _tracks), do: {:ok, 0}
|
||||
def apply_all_album_rules(_rules, []), do: {:ok, 0}
|
||||
|
||||
def apply_all_album_rules(rules, tracks) when is_list(rules) and is_list(tracks) do
|
||||
# Build CASE WHEN clauses dynamically
|
||||
{case_clauses, case_params} =
|
||||
rules
|
||||
|> Enum.reduce({"", []}, fn rule, {sql_acc, params_acc} ->
|
||||
clause =
|
||||
"WHEN json_extract(album, '$.title') = ? THEN json_set(album, '$.musicbrainz_id', ?) "
|
||||
|
||||
{sql_acc <> clause, params_acc ++ [rule.match_value, rule.target_musicbrainz_id]}
|
||||
end)
|
||||
|
||||
# Build complete UPDATE statement
|
||||
case_sql = "CASE #{case_clauses}ELSE album END"
|
||||
|
||||
# Build WHERE IN clause for album titles
|
||||
match_values = Enum.map(rules, & &1.match_value)
|
||||
album_placeholders = Enum.map_join(match_values, ", ", fn _ -> "?" end)
|
||||
|
||||
# Build WHERE IN clause for track timestamps
|
||||
track_scrobbled_at_uts = Enum.map(tracks, & &1.scrobbled_at_uts)
|
||||
track_placeholders = Enum.map_join(track_scrobbled_at_uts, ", ", fn _ -> "?" end)
|
||||
|
||||
where_sql =
|
||||
"json_extract(album, '$.title') IN (#{album_placeholders}) AND " <>
|
||||
"scrobbled_at_uts IN (#{track_placeholders})"
|
||||
|
||||
# Complete SQL
|
||||
sql = """
|
||||
UPDATE scrobbled_tracks
|
||||
SET album = #{case_sql}
|
||||
WHERE #{where_sql}
|
||||
"""
|
||||
|
||||
# All parameters: case params + album match values + track timestamps
|
||||
all_params = case_params ++ match_values ++ track_scrobbled_at_uts
|
||||
|
||||
# Execute the query
|
||||
case Repo.query(sql, all_params) do
|
||||
{:ok, %{num_rows: count}} -> {:ok, count}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Applies all artist rules in a single query.
|
||||
|
||||
Uses a CASE statement to update the musicbrainz_id for all matching artists
|
||||
in one database operation, which is more efficient than applying rules individually.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_artist_rules([rule1, rule2])
|
||||
{:ok, 25}
|
||||
|
||||
"""
|
||||
def apply_all_artist_rules([]), do: {:ok, 0}
|
||||
|
||||
def apply_all_artist_rules(rules) when is_list(rules) do
|
||||
# Build CASE WHEN clauses dynamically
|
||||
{case_clauses, case_params} =
|
||||
rules
|
||||
|> Enum.reduce({"", []}, fn rule, {sql_acc, params_acc} ->
|
||||
clause =
|
||||
"WHEN json_extract(artist, '$.name') = ? THEN json_set(artist, '$.musicbrainz_id', ?) "
|
||||
|
||||
{sql_acc <> clause, params_acc ++ [rule.match_value, rule.target_musicbrainz_id]}
|
||||
end)
|
||||
|
||||
# Build complete UPDATE statement
|
||||
case_sql = "CASE #{case_clauses}ELSE artist END"
|
||||
|
||||
# Build WHERE IN clause
|
||||
match_values = Enum.map(rules, & &1.match_value)
|
||||
in_placeholders = Enum.map_join(match_values, ", ", fn _ -> "?" end)
|
||||
where_sql = "json_extract(artist, '$.name') IN (#{in_placeholders})"
|
||||
|
||||
# Complete SQL
|
||||
sql = """
|
||||
UPDATE scrobbled_tracks
|
||||
SET artist = #{case_sql}
|
||||
WHERE #{where_sql}
|
||||
"""
|
||||
|
||||
# All parameters: case params + where params
|
||||
all_params = case_params ++ match_values
|
||||
|
||||
# Execute the query
|
||||
case Repo.query(sql, all_params) do
|
||||
{:ok, %{num_rows: count}} -> {:ok, count}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Applies all artist rules to a specific set of tracks in a single query.
|
||||
|
||||
Uses a CASE statement to update the musicbrainz_id for all matching artists
|
||||
in one database operation, filtering by the provided tracks.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_artist_rules([rule1, rule2], tracks)
|
||||
{:ok, 7}
|
||||
|
||||
"""
|
||||
def apply_all_artist_rules([], _tracks), do: {:ok, 0}
|
||||
def apply_all_artist_rules(_rules, []), do: {:ok, 0}
|
||||
|
||||
def apply_all_artist_rules(rules, tracks) when is_list(rules) and is_list(tracks) do
|
||||
# Build CASE WHEN clauses dynamically
|
||||
{case_clauses, case_params} =
|
||||
rules
|
||||
|> Enum.reduce({"", []}, fn rule, {sql_acc, params_acc} ->
|
||||
clause =
|
||||
"WHEN json_extract(artist, '$.name') = ? THEN json_set(artist, '$.musicbrainz_id', ?) "
|
||||
|
||||
{sql_acc <> clause, params_acc ++ [rule.match_value, rule.target_musicbrainz_id]}
|
||||
end)
|
||||
|
||||
# Build complete UPDATE statement
|
||||
case_sql = "CASE #{case_clauses}ELSE artist END"
|
||||
|
||||
# Build WHERE IN clause for artist names
|
||||
match_values = Enum.map(rules, & &1.match_value)
|
||||
artist_placeholders = Enum.map_join(match_values, ", ", fn _ -> "?" end)
|
||||
|
||||
# Build WHERE IN clause for track timestamps
|
||||
track_scrobbled_at_uts = Enum.map(tracks, & &1.scrobbled_at_uts)
|
||||
track_placeholders = Enum.map_join(track_scrobbled_at_uts, ", ", fn _ -> "?" end)
|
||||
|
||||
where_sql =
|
||||
"json_extract(artist, '$.name') IN (#{artist_placeholders}) AND " <>
|
||||
"scrobbled_at_uts IN (#{track_placeholders})"
|
||||
|
||||
# Complete SQL
|
||||
sql = """
|
||||
UPDATE scrobbled_tracks
|
||||
SET artist = #{case_sql}
|
||||
WHERE #{where_sql}
|
||||
"""
|
||||
|
||||
# All parameters: case params + artist match values + track timestamps
|
||||
all_params = case_params ++ match_values ++ track_scrobbled_at_uts
|
||||
|
||||
# Execute the query
|
||||
case Repo.query(sql, all_params) do
|
||||
{:ok, %{num_rows: count}} -> {:ok, count}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Applies all enabled rules.
|
||||
|
||||
This optimized version groups rules by type and applies all rules of each type
|
||||
in a single database query, which is much more efficient than applying each rule
|
||||
individually.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_rules()
|
||||
@@ -337,14 +557,45 @@ defmodule MusicLibrary.ScrobbleRules do
|
||||
"""
|
||||
def apply_all_rules do
|
||||
:telemetry.span([:music_library, :scrobble_rules, :apply_all_rules], %{}, fn ->
|
||||
result =
|
||||
list_enabled_rules()
|
||||
|> Enum.map(fn rule ->
|
||||
case apply_rule(rule) do
|
||||
{:ok, count} -> {:ok, {rule.type, rule.match_value, count}}
|
||||
{:error, reason} -> {:error, {rule.type, rule.match_value, reason}}
|
||||
end
|
||||
end)
|
||||
enabled_rules = list_enabled_rules()
|
||||
|
||||
# Group rules by type
|
||||
{album_rules, artist_rules} =
|
||||
Enum.split_with(enabled_rules, fn rule -> rule.type == :album end)
|
||||
|
||||
# Apply all album rules in one query
|
||||
album_result =
|
||||
case apply_all_album_rules(album_rules) do
|
||||
{:ok, count} ->
|
||||
# Return the count for each album rule (total updated)
|
||||
# Note: this returns the same count for each rule since they're applied together
|
||||
Enum.map(album_rules, fn rule ->
|
||||
{:ok, {rule.type, rule.match_value, count}}
|
||||
end)
|
||||
|
||||
{:error, reason} ->
|
||||
Enum.map(album_rules, fn rule ->
|
||||
{:error, {rule.type, rule.match_value, reason}}
|
||||
end)
|
||||
end
|
||||
|
||||
# Apply all artist rules in one query
|
||||
artist_result =
|
||||
case apply_all_artist_rules(artist_rules) do
|
||||
{:ok, count} ->
|
||||
# Return the count for each artist rule (total updated)
|
||||
# Note: this returns the same count for each rule since they're applied together
|
||||
Enum.map(artist_rules, fn rule ->
|
||||
{:ok, {rule.type, rule.match_value, count}}
|
||||
end)
|
||||
|
||||
{:error, reason} ->
|
||||
Enum.map(artist_rules, fn rule ->
|
||||
{:error, {rule.type, rule.match_value, reason}}
|
||||
end)
|
||||
end
|
||||
|
||||
result = album_result ++ artist_result
|
||||
|
||||
{result, %{scrobble_track_count: :all}}
|
||||
end)
|
||||
@@ -353,6 +604,9 @@ defmodule MusicLibrary.ScrobbleRules do
|
||||
@doc """
|
||||
Applies all enabled rules to a specific set of tracks.
|
||||
|
||||
This optimized version groups rules by type and applies all rules of each type
|
||||
in a single database query, filtering by the provided tracks.
|
||||
|
||||
## Examples
|
||||
|
||||
iex> apply_all_rules(tracks)
|
||||
@@ -368,14 +622,41 @@ defmodule MusicLibrary.ScrobbleRules do
|
||||
|
||||
def apply_all_rules(tracks) do
|
||||
:telemetry.span([:music_library, :scrobble_rules, :apply_all_rules], %{}, fn ->
|
||||
result =
|
||||
list_enabled_rules()
|
||||
|> Enum.map(fn rule ->
|
||||
case apply_rule(rule, tracks) do
|
||||
{:ok, count} -> {:ok, {rule.type, rule.match_value, count}}
|
||||
{:error, reason} -> {:error, {rule.type, rule.match_value, reason}}
|
||||
end
|
||||
end)
|
||||
enabled_rules = list_enabled_rules()
|
||||
|
||||
# Group rules by type
|
||||
{album_rules, artist_rules} =
|
||||
Enum.split_with(enabled_rules, fn rule -> rule.type == :album end)
|
||||
|
||||
# Apply all album rules in one query
|
||||
album_result =
|
||||
case apply_all_album_rules(album_rules, tracks) do
|
||||
{:ok, count} ->
|
||||
Enum.map(album_rules, fn rule ->
|
||||
{:ok, {rule.type, rule.match_value, count}}
|
||||
end)
|
||||
|
||||
{:error, reason} ->
|
||||
Enum.map(album_rules, fn rule ->
|
||||
{:error, {rule.type, rule.match_value, reason}}
|
||||
end)
|
||||
end
|
||||
|
||||
# Apply all artist rules in one query
|
||||
artist_result =
|
||||
case apply_all_artist_rules(artist_rules, tracks) do
|
||||
{:ok, count} ->
|
||||
Enum.map(artist_rules, fn rule ->
|
||||
{:ok, {rule.type, rule.match_value, count}}
|
||||
end)
|
||||
|
||||
{:error, reason} ->
|
||||
Enum.map(artist_rules, fn rule ->
|
||||
{:error, {rule.type, rule.match_value, reason}}
|
||||
end)
|
||||
end
|
||||
|
||||
result = album_result ++ artist_result
|
||||
|
||||
{result, %{scrobble_track_count: Enum.count(tracks)}}
|
||||
end)
|
||||
|
||||
Reference in New Issue
Block a user