Resolve all data in one query in Scrobble Activity

Aim is to avoid having to do this in multiple rounds with in-memory
operations which are harder to follow

foo.exs

@@ -0,0 +1,2 @@
+MusicLibrary.ScrobbleActivity.recent_tracks("Europe/London", 1)
+|> IO.inspect()

lib/music_library/scrobble_activity.ex

@@ -3,7 +3,7 @@ defmodule MusicLibrary.ScrobbleActivity do
 
   alias LastFm.{Scrobble, Track}
   alias MusicBrainz.Release
-  alias MusicLibrary.{Artists, Collection, Repo, Secrets, Wishlist}
+  alias MusicLibrary.{Artists, Collection, Records.ArtistRecord, Repo, Secrets, Wishlist}
 
   def can_scrobble? do
     Secrets.get("last_fm_session_key") !== nil
@@ -111,78 +111,102 @@ defmodule MusicLibrary.ScrobbleActivity do
     Repo.aggregate(Track, :count, :scrobbled_at_uts)
   end
 
-  def from_recent_tracks(recent_tracks, timezone) do
+  def recent_activity(timezone, limit \\ 100) do
-    all_artist_pairs = Artists.get_all_artist_pairs()
+    # When we get recent tracks, we need to:
-    recent_release_ids = recent_release_ids(recent_tracks)
+    #
-    collected_releases = Collection.collected_releases(recent_release_ids)
+    # - Map each track to a record in the collection (if it exists)
-    wishlisted_releases = Wishlist.wishlisted_releases(recent_release_ids)
+    # - Map each track to a record in the wishlist (if it exists)
-
+    # - Map each track to an artist, knowing that sometimes track artists do
-    localized_recent_tracks =
+    #   not have the necessary information. In that case we can go from
-      Enum.map(recent_tracks, fn t ->
+    #   track -> album -> record -> artist
-        %{
+    collected_releases_query =
-          t
+      from r in fragment("records, json_each(records.release_ids)"),
-          | scrobbled_at_label: localize_scrobbled_at(t.scrobbled_at_uts, timezone),
+        where: fragment("records.purchased_at IS NOT NULL"),
-            artist: polyfill_artist(t, collected_releases, wishlisted_releases, all_artist_pairs)
+        select: %{
+          record_id: fragment("records.id"),
+          cover_hash: fragment("records.cover_hash"),
+          release_id: r.value
         }
-      end)
 
-    all_artist_ids = Artists.get_all_artist_ids()
+    wishlisted_releases_query =
-    recent_artist_ids = recent_artist_ids(localized_recent_tracks)
+      from r in fragment("records, json_each(records.release_ids)"),
-    artist_ids = MapSet.intersection(all_artist_ids, recent_artist_ids)
+        where: fragment("records.purchased_at IS NULL"),
+        select: %{
+          record_id: fragment("records.id"),
+          cover_hash: fragment("records.cover_hash"),
+          release_id: r.value
+        }
+
+    all_artists_query =
+      from ar in ArtistRecord,
+        distinct: true
+
+    tracks_query =
+      from t in Track,
+        left_join: cr in subquery(collected_releases_query),
+        on: cr.release_id == fragment("? ->> '$.musicbrainz_id'", t.album),
+        left_join: wr in subquery(wishlisted_releases_query),
+        on: wr.release_id == fragment("? ->> '$.musicbrainz_id'", t.album),
+        left_join: ar in subquery(all_artists_query),
+        on: wr.record_id == ar.record_id or cr.record_id == ar.record_id,
+        order_by: [desc: t.scrobbled_at_uts],
+        limit: ^limit,
+        select: %{
+          track: t,
+          collected_record_id: cr.record_id,
+          wishlisted_record_id: wr.record_id,
+          artist_id: ar.musicbrainz_id,
+          cover_hash: coalesce(cr.cover_hash, wr.cover_hash)
+        }
+
+    recent_tracks = Repo.all(tracks_query)
+
+    recent_tracks =
+      recent_tracks
+      |> Enum.map(fn %{track: track, artist_id: artist_id} = rt ->
+        %{rt | track: polifyll_track(track, timezone, artist_id)}
+      end)
 
     recent_albums =
-      localized_recent_tracks
+      recent_tracks
-      |> Enum.dedup_by(fn t -> t.album end)
+      |> Enum.dedup_by(fn %{track: track} -> track.album end)
-      |> Enum.map(fn t ->
+      |> Enum.map(fn %{track: track} = tr ->
-        %{
+        tr
-          scrobbled_at_uts: t.scrobbled_at_uts,
+        |> Map.delete(:track)
-          scrobbled_at_label: t.scrobbled_at_label,
+        |> Map.put(
-          metadata: t.album,
+          :album,
-          artist: t.artist,
+          %{
-          cover_url: t.cover_url
+            scrobbled_at_uts: track.scrobbled_at_uts,
-        }
+            scrobbled_at_label: track.scrobbled_at_label,
+            metadata: track.album,
+            artist: track.artist,
+            cover_url: track.cover_url
+          }
+        )
       end)
 
     %{
-      localized_recent_tracks: localized_recent_tracks,
+      recent_tracks: recent_tracks,
-      localized_recent_albums: recent_albums,
+      recent_albums: recent_albums
-      collected_releases: collected_releases,
-      wishlisted_releases: wishlisted_releases,
-      artist_ids: artist_ids
     }
   end
 
-  defp polyfill_artist(track, collected_releases, wishlisted_releases, all_artist_pairs) do
+  defp polifyll_track(track, timezone, artist_id) do
     %{
-      track.artist
+      track
-      | musicbrainz_id:
+      | scrobbled_at_label: localize_scrobbled_at(track.scrobbled_at_uts, timezone),
-          find_artist_id(track, collected_releases, wishlisted_releases, all_artist_pairs)
+        artist: polyfill_artist(track.artist, artist_id)
     }
   end
 
-  defguardp has_no_artist_id(track)
+  defp polyfill_artist(artist, musicbrainz_id) do
-            when is_nil(track.artist.musicbrainz_id) or track.artist.musicbrainz_id == ""
+    if is_nil(artist.musicbrainz_id) or artist.musicbrainz_id == "" do
-
+      %{artist | musicbrainz_id: musicbrainz_id}
-  defp find_artist_id(track, collected_releases, wishlisted_releases, all_artist_pairs)
+    else
-       when has_no_artist_id(track) do
+      artist
-    matched_release =
-      Enum.find(collected_releases ++ wishlisted_releases, fn r ->
-        r.release_id == track.album.musicbrainz_id
-      end)
-
-    record_id = if matched_release, do: matched_release.record_id
-
-    if record_id do
-      Enum.find_value(all_artist_pairs, fn pair ->
-        if pair.record_id == record_id, do: pair.artist_id
-      end)
     end
   end
 
-  defp find_artist_id(track, _collected_releases, _wishlisted_releases, _all_artist_pairs) do
-    track.artist.musicbrainz_id
-  end
-
   def localize_scrobbled_at(uts, timezone) do
     ldt =
       uts
@@ -192,21 +216,6 @@ defmodule MusicLibrary.ScrobbleActivity do
     Calendar.strftime(ldt, "%d/%m/%Y %X")
   end
 
-  defp recent_release_ids(recent_tracks) do
-    recent_tracks
-    |> Enum.map(fn t -> t.album.musicbrainz_id end)
-    |> Enum.uniq()
-    |> Enum.reject(fn musicbrainz_id -> musicbrainz_id == "" end)
-  end
-
-  defp recent_artist_ids(recent_tracks) do
-    recent_tracks
-    |> Enum.map(fn t -> t.artist.musicbrainz_id end)
-    |> Enum.uniq()
-    |> Enum.reject(fn musicbrainz_id -> musicbrainz_id == "" end)
-    |> MapSet.new()
-  end
-
   @doc """
   Gets the top albums by scrobble count for the given number of days.
   Returns a list of maps with album information and play counts.

lib/music_library_web/live/stats_live/index.ex

@@ -6,13 +6,13 @@ defmodule MusicLibraryWeb.StatsLive.Index do
   import MusicLibraryWeb.RecordComponents, only: [format_label: 1, type_label: 1]
   import MusicLibraryWeb.StatsComponents
 
+  alias MusicLibrary.Assets.Transform
   alias MusicLibrary.{Collection, Records, ScrobbleActivity, Wishlist}
   alias MusicLibraryWeb.StatsLive.{TopAlbums, TopArtists}
 
   def mount(_params, _session, socket) do
     current_date = Date.utc_today()
     latest_record = Collection.get_latest_record!()
-    recent_tracks = LastFm.get_scrobbled_tracks(50)
     records_by_artists = Collection.count_records_by_artist(limit: 20)
     records_by_genre = Collection.count_records_by_genre(limit: 20)
     records_on_this_day = Collection.get_records_on_this_day(current_date)
@@ -24,14 +24,14 @@ defmodule MusicLibraryWeb.StatsLive.Index do
     {:ok,
      socket
      |> stream_configure(:recent_tracks,
-       dom_id: fn track -> "track-#{track.scrobbled_at_uts}" end
+       dom_id: fn %{track: track} -> "track-#{track.scrobbled_at_uts}" end
      )
      |> stream_configure(:recent_albums,
-       dom_id: fn album -> "album-#{album.scrobbled_at_uts}" end
+       dom_id: fn %{album: album} -> "album-#{album.scrobbled_at_uts}" end
      )
      |> stream(:records_on_this_day, records_on_this_day, reset: true)
      |> assign_counts()
-     |> assign_scrobble_activity(recent_tracks)
+     |> assign_scrobble_activity()
      |> assign(
        current_date: current_date,
        scrobble_activity_mode: "albums",
@@ -101,11 +101,9 @@ defmodule MusicLibraryWeb.StatsLive.Index do
   end
 
   def handle_info(%{track_count: _count}, socket) do
-    recent_tracks = LastFm.get_scrobbled_tracks()
-
     {:noreply,
      socket
-     |> assign_scrobble_activity(recent_tracks)}
+     |> assign_scrobble_activity()}
   end
 
   defp assign_counts(socket) do
@@ -126,32 +124,24 @@ defmodule MusicLibraryWeb.StatsLive.Index do
     )
   end
 
-  defp assign_scrobble_activity(socket, recent_tracks) do
+  defp assign_scrobble_activity(socket) do
     %{
-      localized_recent_tracks: localized_recent_tracks,
+      recent_tracks: recent_tracks,
-      localized_recent_albums: recent_albums,
+      recent_albums: recent_albums
-      collected_releases: collected_releases,
+    } = ScrobbleActivity.recent_activity(socket.assigns.timezone)
-      wishlisted_releases: wishlisted_releases,
-      artist_ids: artist_ids
-    } = ScrobbleActivity.from_recent_tracks(recent_tracks, socket.assigns.timezone)
 
     scrobble_count = ScrobbleActivity.scrobble_count()
 
     last_updated_uts =
-      if track = List.first(localized_recent_tracks) do
+      if rt = List.first(recent_tracks) do
-        track.scrobbled_at_uts
+        rt.track.scrobbled_at_uts
       end
 
     socket
     |> assign(:last_updated_uts, last_updated_uts)
     |> assign(:scrobble_count, scrobble_count)
-    |> stream(:recent_tracks, localized_recent_tracks, reset: true)
+    |> stream(:recent_tracks, recent_tracks, reset: true)
     |> stream(:recent_albums, recent_albums, reset: true)
-    |> assign(
-      collected_releases: collected_releases,
-      wishlisted_releases: wishlisted_releases,
-      artist_ids: artist_ids
-    )
   end
 
   defp format_scrobbled_at_uts(uts) do
@@ -159,4 +149,18 @@ defmodule MusicLibraryWeb.StatsLive.Index do
     |> DateTime.from_unix!()
     |> DateTime.to_iso8601()
   end
+
+  @last_fm_fallback_cover_url "https://lastfm.freetls.fastly.net/i/u/64s/2a96cbd8b46e442fc41c2b86b821562f.png"
+
+  defp track_or_album_cover_url(track_or_album, cover_hash) do
+    if track_or_album.cover_url == @last_fm_fallback_cover_url do
+      payload =
+        Transform.new(hash: cover_hash, width: 96)
+        |> Transform.encode!()
+
+      ~p"/assets/#{payload}"
+    else
+      track_or_album.cover_url
+    end
+  end
 end

lib/music_library_web/live/stats_live/index.html.heex

@@ -85,7 +85,20 @@
           class="mt-5 p-6 bg-white dark:bg-zinc-800 rounded-md shadow-sm"
           phx-update="stream"
         >
-          <li :for={{id, album} <- @streams.recent_albums} id={id} class="group">
+          <li
+            :for={
+              {id,
+               %{
+                 album: album,
+                 artist_id: artist_id,
+                 collected_record_id: collected_record_id,
+                 wishlisted_record_id: wishlisted_record_id,
+                 cover_hash: cover_hash
+               }} <- @streams.recent_albums
+            }
+            id={id}
+            class="group"
+          >
             <div class="relative pb-4 group-last:pb-0">
               <span
                 class="group-last:hidden absolute left-6 top-6 -ml-px h-full w-0.5 bg-zinc-200"
@@ -96,18 +109,18 @@
                 <div class="flex min-w-0 justify-between space-x-4 items-center">
                   <img
                     class="h-12 w-12 rounded-md shadow-sm"
-                    src={album.cover_url}
+                    src={track_or_album_cover_url(album, cover_hash)}
                     alt={album.metadata.title}
                   />
                   <div>
                     <p
-                      :if={album.artist.musicbrainz_id not in @artist_ids}
+                      :if={!artist_id}
                       class="font-semibold text-sm block text-zinc-500 dark:text-zinc-400"
                     >
                       {album.artist.name}
                     </p>
                     <.link
-                      :if={album.artist.musicbrainz_id in @artist_ids}
+                      :if={artist_id}
                       class="font-semibold text-sm block text-zinc-700 hover:text-zinc-500 dark:text-zinc-400 dark:hover:text-zinc-300"
                       navigate={~p"/artists/#{album.artist.musicbrainz_id}"}
                     >
@@ -196,22 +209,16 @@
                     {gettext("No MB ID")}
                   </.badge>
                   <.link
-                    :if={
+                    :if={collected_record_id}
-                      record_id =
+                    navigate={~p"/collection/#{collected_record_id}"}
-                        tracked_record?(@collected_releases, album.metadata.musicbrainz_id)
-                    }
-                    navigate={~p"/collection/#{record_id}"}
                   >
                     <.badge color="success">
                       {gettext("Collected")}
                     </.badge>
                   </.link>
                   <.link
-                    :if={
+                    :if={wishlisted_record_id}
-                      record_id =
+                    navigate={~p"/wishlist/#{wishlisted_record_id}"}
-                        tracked_record?(@wishlisted_releases, album.metadata.musicbrainz_id)
-                    }
-                    navigate={~p"/wishlist/#{record_id}"}
                   >
                     <.badge color="warning">
                       {gettext("Wishlisted")}
@@ -221,11 +228,8 @@
 
                 <.dropdown
                   :if={
-                    album.metadata.musicbrainz_id !== "" and
+                    album.metadata.musicbrainz_id !== "" and !collected_record_id and
-                      !tracked_record?(
+                      !wishlisted_record_id
-                        @collected_releases ++ @wishlisted_releases,
-                        album.metadata.musicbrainz_id
-                      )
                   }
                   id={"actions-#{album.scrobbled_at_uts}-albums"}
                   placement="bottom-end"
@@ -266,7 +270,20 @@
           class="mt-5 p-6 bg-white dark:bg-zinc-800 rounded-md shadow-sm"
           phx-update="stream"
         >
-          <li :for={{id, track} <- @streams.recent_tracks} id={id} class="group">
+          <li
+            :for={
+              {id,
+               %{
+                 track: track,
+                 artist_id: artist_id,
+                 collected_record_id: collected_record_id,
+                 wishlisted_record_id: wishlisted_record_id,
+                 cover_hash: cover_hash
+               }} <- @streams.recent_tracks
+            }
+            id={id}
+            class="group"
+          >
             <div class="relative pb-4 group-last:pb-0">
               <span
                 class="group-last:hidden absolute left-6 top-6 -ml-px h-full w-0.5 bg-zinc-200"
@@ -277,20 +294,20 @@
                 <div class="flex min-w-0 justify-between space-x-4 items-center">
                   <img
                     class="h-12 w-12 rounded-md shadow-sm"
-                    src={track.cover_url}
+                    src={track_or_album_cover_url(track, cover_hash)}
                     alt={track.title}
                   />
                   <div>
                     <p
-                      :if={track.artist.musicbrainz_id not in @artist_ids}
+                      :if={!artist_id}
                       class="font-semibold text-sm block text-zinc-500 dark:text-zinc-400"
                     >
                       {track.artist.name}
                     </p>
                     <.link
-                      :if={track.artist.musicbrainz_id in @artist_ids}
+                      :if={artist_id}
                       class="font-semibold text-sm block text-zinc-700 hover:text-zinc-500 dark:text-zinc-400 dark:hover:text-zinc-300"
-                      navigate={~p"/artists/#{track.artist.musicbrainz_id}"}
+                      navigate={~p"/artists/#{artist_id}"}
                     >
                       {track.artist.name}
                     </.link>
@@ -407,19 +424,14 @@
                     {gettext("No MB ID")}
                   </.badge>
                   <.link
-                    :if={
+                    :if={collected_record_id}
-                      record_id = tracked_record?(@collected_releases, track.album.musicbrainz_id)
+                    navigate={~p"/collection/#{collected_record_id}"}
-                    }
-                    navigate={~p"/collection/#{record_id}"}
                   >
                     <.badge color="success">{gettext("Collected")}</.badge>
                   </.link>
                   <.link
-                    :if={
+                    :if={wishlisted_record_id}
-                      record_id =
+                    navigate={~p"/wishlist/#{wishlisted_record_id}"}
-                        tracked_record?(@wishlisted_releases, track.album.musicbrainz_id)
-                    }
-                    navigate={~p"/wishlist/#{record_id}"}
                   >
                     <.badge color="warning">{gettext("Wishlisted")}</.badge>
                   </.link>
@@ -427,11 +439,8 @@
 
                 <.dropdown
                   :if={
-                    track.album.musicbrainz_id !== "" and
+                    track.album.musicbrainz_id !== "" and !collected_record_id and
-                      !tracked_record?(
+                      !wishlisted_record_id
-                        @collected_releases ++ @wishlisted_releases,
-                        track.album.musicbrainz_id
-                      )
                   }
                   id={"actions-#{track.scrobbled_at_uts}-tracks"}
                   placement="bottom-end"