Optimize ListeningStats query performance
Three independent refactors of lib/music_library/listening_stats.ex, measured against the dev DB (104k tracks) via bench/listening_stats.exs: | Query | Baseline | After | Speedup | |---------------------------------|-----------|-----------|---------| | recent_activity(tz, 100) | 38.98 ms | 2.87 ms | 13.6x | | list_tracks(page 1, 200) | 47.30 ms | 4.81 ms | 9.8x | | get_top_artists_by_days(7) | 66.51 ms | 1.02 ms | 65x | | get_top_artists_by_days(30) | 67.41 ms | 2.84 ms | 23.7x | | get_top_artists_by_days(365) | 76.03 ms | 25.57 ms | 3.0x | | get_top_albums_by_days(7) | 92.63 ms | 1.61 ms | 57.5x | | get_top_albums_by_days(30) | 94.52 ms | 3.69 ms | 25.6x | | get_top_albums_by_days(365) | 105.23 ms | 32.81 ms | 3.2x | | get_top_artists(limit: 10) | 123.96 ms | 101.68 ms | 1.22x | | get_top_albums(limit: 10) | 209.64 ms | 108.23 ms | 1.94x | tracks_with_record_info_query/0 now uses correlated scalar subqueries against record_releases and artist_records instead of materializing helper subqueries on every call. The cost scales with the outer LIMIT, not with the size of record_releases. top_albums_base_query/0 and top_artists_base_query/0 are replaced by aggregate_query + attach_metadata pairs. The pattern is aggregate first, attach metadata second: GROUP BY runs against the raw track scan, then a tiny outer SELECT attaches record_releases / artist_infos lookups for the <= 10 result rows via correlated subqueries. tracks_since_query/1 wraps the date-filtered inner scan with limit: -1. SQLite cannot flatten a subquery that has a LIMIT, so it materializes the date-bounded subset and the optimizer uses the timestamp index for the range scan instead of the album/artist composite index. This trick replaces SQLite's WITH ... AS MATERIALIZED (which ecto_sqlite3 doesn't expose). All json_extract(?, '\$.path') fragments use the canonical form rather than the equivalent ? ->> '\$.path' shorthand. SQLite's index matcher requires the GROUP BY expression to match the index expression textually to use the composite index for natural ordering. collected_releases_query/0 and wishlisted_releases_query/0 are removed from Collection and Wishlist — they had a single internal caller that no longer exists after the refactor. New regression tests lock the semantics that the optimized queries must preserve: - count(DISTINCT scrobbled_at_uts) — 579 duplicate timestamps exist in the dev DB from rapid Last.fm scrobbles, so replacing with count(*) would silently change results - :artist_id key in list_tracks result maps — ScrobbledTracksLive.Index destructures it even though the template body never references it Closes #148
This commit is contained in:
@@ -411,6 +411,73 @@ defmodule MusicLibrary.ListeningStatsTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "play_count uses count(DISTINCT scrobbled_at_uts)" do
|
||||
test "two tracks at the same scrobbled_at_uts count as one play in get_top_albums" do
|
||||
shared_uts = System.system_time(:second) - 100
|
||||
|
||||
# Two distinct rows (different titles, so the (uts, title) UNIQUE
|
||||
# constraint is satisfied) sharing the same scrobble timestamp.
|
||||
# Last.fm sometimes scrobbles multiple tracks at the same UTS — the
|
||||
# play_count must reflect unique listening events, not row count.
|
||||
track_fixture(%{
|
||||
title: "Same UTS Track A",
|
||||
album_title: "Same UTS Album",
|
||||
artist_name: "Same UTS Artist",
|
||||
scrobbled_at_uts: shared_uts
|
||||
})
|
||||
|
||||
track_fixture(%{
|
||||
title: "Same UTS Track B",
|
||||
album_title: "Same UTS Album",
|
||||
artist_name: "Same UTS Artist",
|
||||
scrobbled_at_uts: shared_uts
|
||||
})
|
||||
|
||||
results = ListeningStats.get_top_albums(limit: 10)
|
||||
[entry] = Enum.filter(results, fn r -> r.album_title == "Same UTS Album" end)
|
||||
|
||||
assert entry.play_count == 1,
|
||||
"expected count(DISTINCT scrobbled_at_uts) semantics — got #{entry.play_count}"
|
||||
end
|
||||
|
||||
test "two tracks at the same scrobbled_at_uts count as one play in get_top_artists" do
|
||||
shared_uts = System.system_time(:second) - 100
|
||||
|
||||
track_fixture(%{
|
||||
title: "Same UTS Track 1",
|
||||
artist_name: "Same UTS Solo Artist",
|
||||
scrobbled_at_uts: shared_uts
|
||||
})
|
||||
|
||||
track_fixture(%{
|
||||
title: "Same UTS Track 2",
|
||||
artist_name: "Same UTS Solo Artist",
|
||||
scrobbled_at_uts: shared_uts
|
||||
})
|
||||
|
||||
results = ListeningStats.get_top_artists(limit: 10)
|
||||
[entry] = Enum.filter(results, fn r -> r.name == "Same UTS Solo Artist" end)
|
||||
|
||||
assert entry.play_count == 1,
|
||||
"expected count(DISTINCT scrobbled_at_uts) semantics — got #{entry.play_count}"
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_tracks result-map shape" do
|
||||
test "always includes :artist_id" do
|
||||
# `lib/music_library_web/live/scrobbled_tracks_live/index.ex` (around lines
|
||||
# 104–113) destructures `artist_id` from each stream item even though the
|
||||
# template body never references it. Removing the key would crash the
|
||||
# LiveView with `MatchError`. Lock the public contract here.
|
||||
track_fixture(%{title: "Shape Test"})
|
||||
|
||||
[result | _] = ListeningStats.list_tracks(%{page: 1, page_size: 5})
|
||||
|
||||
assert Map.has_key?(result, :artist_id),
|
||||
"list_tracks result map must include :artist_id (consumer: ScrobbledTracksLive.Index)"
|
||||
end
|
||||
end
|
||||
|
||||
describe "get_top_artists/1" do
|
||||
test "counts tracks with missing artist_infos records" do
|
||||
artist_info =
|
||||
|
||||
Reference in New Issue
Block a user