From b35f4aed1d93126f319587305b911c22f6dcf6b8 Mon Sep 17 00:00:00 2001 From: Claudio Ortolina Date: Sun, 1 Mar 2026 08:22:15 +0000 Subject: [PATCH] EXP: enrich artist info data via Last.fm and produce better embeddings --- lib/last_fm.ex | 17 +++ lib/last_fm/api.ex | 42 ++++++ lib/music_library/artists.ex | 21 +++ lib/music_library/artists/artist_info.ex | 10 ++ lib/music_library/records/similarity.ex | 137 ++++++++++++++++-- .../worker/fetch_artist_last_fm_data.ex | 11 ++ ...225023_add_lastfm_data_to_artist_infos.exs | 9 ++ .../worker/fetch_artist_last_fm_data_test.exs | 78 ++++++++++ 8 files changed, 309 insertions(+), 16 deletions(-) create mode 100644 lib/music_library/worker/fetch_artist_last_fm_data.ex create mode 100644 priv/repo/migrations/20260228225023_add_lastfm_data_to_artist_infos.exs create mode 100644 test/music_library/worker/fetch_artist_last_fm_data_test.exs diff --git a/lib/last_fm.ex b/lib/last_fm.ex index 5f6e460d..a76780f2 100644 --- a/lib/last_fm.ex +++ b/lib/last_fm.ex @@ -57,6 +57,23 @@ defmodule LastFm do end end + def get_artist_tags(musicbrainz_id, name) do + last_fm_config = last_fm_config() + + case API.get_artist_tags({:musicbrainz_id, musicbrainz_id}, last_fm_config) do + {:ok, tags} -> + {:ok, tags} + + {:error, :invalid_parameters} -> + # Sometimes the artist cannot be identified with the MusicBrainz ID, + # because Last.fm doesn't have that information. In that case, we try again with the artist name. + API.get_artist_tags({:name, name}, last_fm_config) + + error -> + error + end + end + def get_session(token) do last_fm_config = last_fm_config() diff --git a/lib/last_fm/api.ex b/lib/last_fm/api.ex index a2202af3..94419d8c 100644 --- a/lib/last_fm/api.ex +++ b/lib/last_fm/api.ex @@ -97,6 +97,20 @@ defmodule LastFm.API do |> get_request() end + def get_artist_tags(id_or_name_option, config) do + params = + config + |> base_params() + |> Keyword.put(:method, "artist.getTopTags") + |> put_musicbrainz_id_or_name(id_or_name_option) + + config + |> new_request() + |> Req.merge(url: "/", params: params) + |> Req.Request.append_response_steps(parse_tags: &parse_artist_tags/1) + |> get_request() + end + defp put_musicbrainz_id_or_name(params, {:musicbrainz_id, musicbrainz_id}) do Keyword.put(params, :mbid, musicbrainz_id) end @@ -220,6 +234,34 @@ defmodule LastFm.API do {request, Map.put(response, :body, artists)} end + defp parse_artist_tags({request, response}) do + tags = + case get_in(response.body, ["toptags", "tag"]) do + nil -> + [] + + tags when is_list(tags) -> + tags + |> Enum.map(fn tag -> {tag["name"], parse_tag_count(tag["count"])} end) + |> Enum.filter(fn {_name, count} -> count >= 2 end) + |> Enum.sort_by(&elem(&1, 1), :desc) + |> Enum.take(15) + end + + {request, Map.put(response, :body, tags)} + end + + defp parse_tag_count(count) when is_integer(count), do: count + + defp parse_tag_count(count) when is_binary(count) do + case Integer.parse(count) do + {n, _} -> n + :error -> 0 + end + end + + defp parse_tag_count(_), do: 0 + defp sanitize_url(url, api_key) do String.replace(url, api_key, "") end diff --git a/lib/music_library/artists.ex b/lib/music_library/artists.ex index 57c2d9b7..a82f5263 100644 --- a/lib/music_library/artists.ex +++ b/lib/music_library/artists.ex @@ -191,6 +191,27 @@ defmodule MusicLibrary.Artists do end end + def fetch_lastfm_data(artist_id) do + artist_info = get_artist_info!(artist_id) + name = get_in(artist_info.musicbrainz_data, ["name"]) || "" + + case LastFm.get_artist_tags(artist_id, name) do + {:ok, tags} -> + tag_names = Enum.map(tags, fn {tag_name, _count} -> tag_name end) + + artist_info + |> ArtistInfo.changeset(%{lastfm_data: %{"tags" => tag_names}}) + |> Repo.update() + + {:error, _reason} -> + {:ok, artist_info} + end + end + + def fetch_lastfm_data_async(artist_id) do + enqueue_worker(Worker.FetchArtistLastFmData, %{"id" => artist_id}) + end + def fetch_artist_info_async(artist_id) do enqueue_worker(Worker.FetchArtistInfo, %{"id" => artist_id}) end diff --git a/lib/music_library/artists/artist_info.ex b/lib/music_library/artists/artist_info.ex index 986b8a5f..6a49b8cb 100644 --- a/lib/music_library/artists/artist_info.ex +++ b/lib/music_library/artists/artist_info.ex @@ -11,6 +11,7 @@ defmodule MusicLibrary.Artists.ArtistInfo do field :musicbrainz_data, :map, default: %{} field :discogs_data, :map, default: %{} field :wikipedia_data, :map, default: %{} + field :lastfm_data, :map, default: %{} field :image_data_hash, :string has_one :note, Note, foreign_key: :musicbrainz_id @@ -25,6 +26,7 @@ defmodule MusicLibrary.Artists.ArtistInfo do :musicbrainz_data, :discogs_data, :wikipedia_data, + :lastfm_data, :image_data_hash ]) |> validate_required([:musicbrainz_data]) @@ -107,4 +109,12 @@ defmodule MusicLibrary.Artists.ArtistInfo do def wikipedia_description(artist_info) do get_in(artist_info.wikipedia_data, ["description"]) end + + def lastfm_tags(artist_info) do + get_in(artist_info.lastfm_data, ["tags"]) || [] + end + + def lastfm_similar_artists(artist_info) do + get_in(artist_info.lastfm_data, ["similar_artists"]) || [] + end end diff --git a/lib/music_library/records/similarity.ex b/lib/music_library/records/similarity.ex index 97cf4562..8960abf5 100644 --- a/lib/music_library/records/similarity.ex +++ b/lib/music_library/records/similarity.ex @@ -8,6 +8,7 @@ defmodule MusicLibrary.Records.Similarity do alias MusicLibrary.Artists alias MusicLibrary.Artists.ArtistInfo + alias MusicLibrary.Notes alias MusicLibrary.Records alias MusicLibrary.Records.{Record, RecordEmbedding} alias MusicLibrary.Repo @@ -24,18 +25,23 @@ defmodule MusicLibrary.Records.Similarity do - Genres - Release year - Type (album, EP, etc.) - - Artist musical style summaries (from Wikipedia, falling back to Discogs) + - Per-artist blocks: name, country, disambiguation, Wikipedia summary (500 chars), + Discogs profile excerpt (complementing Wikipedia when both available) + - Last.fm community tags and similar artists (when stored) + - User-written record notes (when present) """ def text_representation(%Record{} = record) do - artist_infos = + artist_infos_map = record.artists |> Enum.map(& &1.musicbrainz_id) |> Artists.get_artist_infos() + |> Map.new(&{&1.id, &1}) artist_names = Record.artist_names(record) genres = Enum.join(record.genres, ", ") year = extract_year(record.release_date) type = humanize_type(record.type) + note_text = record_note_text(record.musicbrainz_id) """ Album: #{record.title} @@ -44,30 +50,77 @@ defmodule MusicLibrary.Records.Similarity do Released: #{year} Type: #{type} - #{artist_infos_summary(artist_infos)} + #{artist_blocks_summary(record.artists, artist_infos_map)} """ |> String.trim() + |> Kernel.<>(note_text) end - defp artist_infos_summary([]), do: "" + defp artist_blocks_summary([], _artist_infos_map), do: "" - defp artist_infos_summary(artist_infos) do - artist_infos - |> Enum.map(&artist_info_summary/1) + defp artist_blocks_summary(artists, artist_infos_map) do + artists + |> Enum.map(fn artist -> + artist_info = Map.get(artist_infos_map, artist.musicbrainz_id) + artist_block(artist, artist_info) + end) |> Enum.reject(&(&1 == "")) |> Enum.join("\n\n") end - defp artist_info_summary(artist_info) do - cond do - wikipedia_available?(artist_info) -> - wikipedia_artist_summary(artist_info) + defp artist_block(_artist, nil), do: "" - discogs_available?(artist_info) -> - discogs_artist_summary(artist_info) + defp artist_block(artist, artist_info) do + country = safe_artist_country(artist_info) + disambiguation = non_empty_string(Map.get(artist, :disambiguation)) - true -> - "" + header_extras = [country, disambiguation] |> Enum.reject(&is_nil/1) + + header = + if header_extras == [] do + artist.name + else + "#{artist.name} (#{Enum.join(header_extras, ", ")})" + end + + content = artist_content(artist_info) + + if content == "" do + "" + else + "#{header}:\n#{content}" + end + end + + defp artist_content(artist_info) do + has_wikipedia = wikipedia_available?(artist_info) + has_discogs = discogs_available?(artist_info) + + base_content = + cond do + has_wikipedia && has_discogs -> + wikipedia = wikipedia_artist_summary(artist_info) + discogs_excerpt = discogs_artist_excerpt(artist_info) + [wikipedia, discogs_excerpt] |> Enum.reject(&(&1 == "")) |> Enum.join("\n") + + has_wikipedia -> + wikipedia_artist_summary(artist_info) + + has_discogs -> + discogs_artist_summary(artist_info) + + true -> + "" + end + + tags_line = lastfm_tags_line(artist_info) + similar_line = lastfm_similar_line(artist_info) + extras = [tags_line, similar_line] |> Enum.reject(&(&1 == "")) + + if extras == [] do + base_content + else + [base_content | extras] |> Enum.reject(&(&1 == "")) |> Enum.join("\n") end end @@ -85,7 +138,7 @@ defmodule MusicLibrary.Records.Similarity do defp wikipedia_artist_summary(artist_info) do description = ArtistInfo.wikipedia_description(artist_info) || "" summary = ArtistInfo.wikipedia_summary(artist_info) || "" - truncated_summary = truncate_to_sentence(summary, 200) + truncated_summary = truncate_to_sentence(summary, 500) [description, truncated_summary] |> Enum.reject(&(&1 == "")) @@ -101,6 +154,58 @@ defmodule MusicLibrary.Records.Similarity do truncate_to_sentence(profile, 200) end + defp discogs_artist_excerpt(artist_info) do + profile = + Map.get(artist_info.discogs_data, "profile_plaintext") || + Map.get(artist_info.discogs_data, "profile") || + "" + + truncate_to_sentence(profile, 150) + end + + defp lastfm_tags_line(artist_info) do + tags = ArtistInfo.lastfm_tags(artist_info) + + if tags == [] do + "" + else + "Tags: #{tags |> Enum.take(10) |> Enum.join(", ")}" + end + end + + defp lastfm_similar_line(artist_info) do + similar = ArtistInfo.lastfm_similar_artists(artist_info) + + if similar == [] do + "" + else + "Similar artists: #{similar |> Enum.take(5) |> Enum.join(", ")}" + end + end + + defp safe_artist_country(artist_info) do + case artist_info.musicbrainz_data do + %{"area" => %{"name" => name}} when is_binary(name) -> name + _ -> nil + end + end + + defp record_note_text(nil), do: "" + + defp record_note_text(musicbrainz_id) do + case Notes.get_note(:record, musicbrainz_id) do + %{content: content} when is_binary(content) and content != "" -> + "\n\nNotes: #{content}" + + _ -> + "" + end + end + + defp non_empty_string(nil), do: nil + defp non_empty_string(""), do: nil + defp non_empty_string(s), do: s + @doc false def truncate_to_sentence(text, max_length) when byte_size(text) <= max_length, do: text diff --git a/lib/music_library/worker/fetch_artist_last_fm_data.ex b/lib/music_library/worker/fetch_artist_last_fm_data.ex new file mode 100644 index 00000000..cee6a155 --- /dev/null +++ b/lib/music_library/worker/fetch_artist_last_fm_data.ex @@ -0,0 +1,11 @@ +defmodule MusicLibrary.Worker.FetchArtistLastFmData do + use Oban.Worker, queue: :default, max_attempts: 3 + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"id" => artist_id}}) do + case MusicLibrary.Artists.fetch_lastfm_data(artist_id) do + {:ok, _artist_info} -> :ok + error -> error + end + end +end diff --git a/priv/repo/migrations/20260228225023_add_lastfm_data_to_artist_infos.exs b/priv/repo/migrations/20260228225023_add_lastfm_data_to_artist_infos.exs new file mode 100644 index 00000000..8575bdd7 --- /dev/null +++ b/priv/repo/migrations/20260228225023_add_lastfm_data_to_artist_infos.exs @@ -0,0 +1,9 @@ +defmodule MusicLibrary.Repo.Migrations.AddLastfmDataToArtistInfos do + use Ecto.Migration + + def change do + alter table(:artist_infos) do + add :lastfm_data, :map, default: %{} + end + end +end diff --git a/test/music_library/worker/fetch_artist_last_fm_data_test.exs b/test/music_library/worker/fetch_artist_last_fm_data_test.exs new file mode 100644 index 00000000..37aa575e --- /dev/null +++ b/test/music_library/worker/fetch_artist_last_fm_data_test.exs @@ -0,0 +1,78 @@ +defmodule MusicLibrary.Worker.FetchArtistLastFmDataTest do + use MusicLibrary.DataCase + + alias MusicLibrary.Artists + alias MusicLibrary.Artists.ArtistInfo + alias MusicLibrary.Repo + alias MusicLibrary.Worker.FetchArtistLastFmData + + setup do + artist_id = Ecto.UUID.generate() + + Repo.insert!(%ArtistInfo{ + id: artist_id, + musicbrainz_data: %{"name" => "Steven Wilson"} + }) + + %{artist_id: artist_id} + end + + describe "perform/1" do + test "stores Last.fm tags in lastfm_data", %{artist_id: artist_id} do + Req.Test.stub(LastFm.API, fn conn -> + Req.Test.json(conn, %{ + "toptags" => %{ + "tag" => [ + %{"name" => "progressive rock", "count" => 100}, + %{"name" => "art rock", "count" => 80}, + %{"name" => "psychedelic", "count" => 60} + ] + } + }) + end) + + assert :ok = perform_job(FetchArtistLastFmData, %{"id" => artist_id}) + + artist_info = Artists.get_artist_info!(artist_id) + tags = ArtistInfo.lastfm_tags(artist_info) + + assert "progressive rock" in tags + assert "art rock" in tags + assert "psychedelic" in tags + end + + test "returns ok when Last.fm returns an error", %{artist_id: artist_id} do + Req.Test.stub(LastFm.API, fn conn -> + Req.Test.json(conn, %{"error" => 6, "message" => "Artist not found"}) + end) + + assert :ok = perform_job(FetchArtistLastFmData, %{"id" => artist_id}) + + artist_info = Artists.get_artist_info!(artist_id) + assert ArtistInfo.lastfm_tags(artist_info) == [] + end + + test "filters out tags with count below 2", %{artist_id: artist_id} do + Req.Test.stub(LastFm.API, fn conn -> + Req.Test.json(conn, %{ + "toptags" => %{ + "tag" => [ + %{"name" => "progressive rock", "count" => 50}, + %{"name" => "rare tag", "count" => 1}, + %{"name" => "another rare", "count" => 0} + ] + } + }) + end) + + assert :ok = perform_job(FetchArtistLastFmData, %{"id" => artist_id}) + + artist_info = Artists.get_artist_info!(artist_id) + tags = ArtistInfo.lastfm_tags(artist_info) + + assert "progressive rock" in tags + refute "rare tag" in tags + refute "another rare" in tags + end + end +end