From 1dfb2d2a6fe60bdff472fa8c377454e8ccfe04dd Mon Sep 17 00:00:00 2001 From: Claudio Ortolina Date: Thu, 19 Feb 2026 15:51:12 +0000 Subject: [PATCH] Use Wikipedia data for record similarity and raise similarity threshold --- config/config.exs | 2 + lib/music_library/records/similarity.ex | 88 +++++++++--- .../components/record_components.ex | 2 +- priv/gettext/default.pot | 10 +- priv/gettext/en/LC_MESSAGES/default.po | 10 +- .../music_library/records/similarity_test.exs | 126 +++++++++++++++++- 6 files changed, 205 insertions(+), 33 deletions(-) diff --git a/config/config.exs b/config/config.exs index 8364f3e4..a3e24c41 100644 --- a/config/config.exs +++ b/config/config.exs @@ -17,6 +17,8 @@ config :music_library, config :music_library, default_timezone: "Europe/London" +config :music_library, :similarity, max_distance: 0.45 + config :music_library, :pagination, default_page_size: 20, stats_limit: 30, diff --git a/lib/music_library/records/similarity.ex b/lib/music_library/records/similarity.ex index 5ae620a7..97cf4562 100644 --- a/lib/music_library/records/similarity.ex +++ b/lib/music_library/records/similarity.ex @@ -6,11 +6,15 @@ defmodule MusicLibrary.Records.Similarity do import Ecto.Query import SqliteVec.Ecto.Query - alias MusicLibrary.{Artists, Records} + alias MusicLibrary.Artists + alias MusicLibrary.Artists.ArtistInfo + alias MusicLibrary.Records alias MusicLibrary.Records.{Record, RecordEmbedding} alias MusicLibrary.Repo alias MusicLibrary.Worker.GenerateRecordEmbedding + @max_distance Application.compile_env!(:music_library, :similarity)[:max_distance] + @doc """ Generates a text representation of a record for embedding generation. @@ -20,6 +24,7 @@ defmodule MusicLibrary.Records.Similarity do - Genres - Release year - Type (album, EP, etc.) + - Artist musical style summaries (from Wikipedia, falling back to Discogs) """ def text_representation(%Record{} = record) do artist_infos = @@ -47,32 +52,75 @@ defmodule MusicLibrary.Records.Similarity do defp artist_infos_summary([]), do: "" defp artist_infos_summary(artist_infos) do - Enum.map_join(artist_infos, "\n\n", &artist_info_summary/1) + artist_infos + |> Enum.map(&artist_info_summary/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join("\n\n") end - defp artist_info_summary(artist_info) when is_nil(artist_info.discogs_data), do: "" + defp artist_info_summary(artist_info) do + cond do + wikipedia_available?(artist_info) -> + wikipedia_artist_summary(artist_info) - defp artist_info_summary(%{discogs_data: discogs_data}) do - # if available, use the plain text version of the profile to avoid pollution by BBTags - profile_data = Map.get(discogs_data, "profile_plaintext") || Map.get(discogs_data, "profile") + discogs_available?(artist_info) -> + discogs_artist_summary(artist_info) - profile_section = - if profile_data do - "Profile for #{discogs_data["name"]}: #{profile_data}" - else + true -> "" - end + end + end - members_section = - if members = Map.get(discogs_data, "members") do - members_list = Enum.map_join(members, ", ", fn member -> member["name"] end) + defp wikipedia_available?(artist_info) do + ArtistInfo.wikipedia_description(artist_info) != nil || + ArtistInfo.wikipedia_summary(artist_info) != nil + end - "Members (past and present): #{members_list}" - else + defp discogs_available?(artist_info) do + artist_info.discogs_data != nil && + (Map.get(artist_info.discogs_data, "profile_plaintext") != nil || + Map.get(artist_info.discogs_data, "profile") != nil) + end + + defp wikipedia_artist_summary(artist_info) do + description = ArtistInfo.wikipedia_description(artist_info) || "" + summary = ArtistInfo.wikipedia_summary(artist_info) || "" + truncated_summary = truncate_to_sentence(summary, 200) + + [description, truncated_summary] + |> Enum.reject(&(&1 == "")) + |> Enum.join(". ") + end + + defp discogs_artist_summary(artist_info) do + profile = + Map.get(artist_info.discogs_data, "profile_plaintext") || + Map.get(artist_info.discogs_data, "profile") || "" - end - Enum.join([profile_section, members_section], "\n") + truncate_to_sentence(profile, 200) + end + + @doc false + def truncate_to_sentence(text, max_length) when byte_size(text) <= max_length, do: text + + def truncate_to_sentence(text, max_length) do + truncated = String.slice(text, 0, max_length) + + case String.split(truncated, ~r/[.!?]\s/, include_captures: true) |> Enum.count() do + count when count > 1 -> + # Find the last sentence boundary within the limit + truncated + |> String.replace(~r/[^.!?]*$/, "") + |> String.trim() + |> case do + "" -> String.trim(truncated) + result -> result + end + + _ -> + String.trim(truncated) + end end @doc """ @@ -82,6 +130,8 @@ defmodule MusicLibrary.Records.Similarity do - `:limit` - Maximum number of similar records to return (default: 10) - `:scope` - Filter by :collection or :wishlist (default: no filter) + - `:max_distance` - Maximum cosine distance threshold (default: #{@max_distance}). + Results with distance above this are excluded. ## Examples @@ -94,6 +144,7 @@ defmodule MusicLibrary.Records.Similarity do def find_similar(record_id, opts \\ []) do limit = Keyword.get(opts, :limit, 10) scope = Keyword.get(opts, :scope) + max_distance = Keyword.get(opts, :max_distance, @max_distance) record = Records.get_record!(record_id) record_musicbrainz_id = record.musicbrainz_id @@ -113,6 +164,7 @@ defmodule MusicLibrary.Records.Similarity do |> selected_as(:similarity) }, group_by: r.musicbrainz_id, + having: vec_distance_cosine(re.embedding, vec_f32(source_embedding)) <= ^max_distance, limit: ^limit query = apply_scope_filter(query, scope) diff --git a/lib/music_library_web/components/record_components.ex b/lib/music_library_web/components/record_components.ex index 940ed69d..30a53fac 100644 --- a/lib/music_library_web/components/record_components.ex +++ b/lib/music_library_web/components/record_components.ex @@ -472,7 +472,7 @@ defmodule MusicLibraryWeb.RecordComponents do {gettext("Similar Records")} - {gettext("Based on genres, artists, and release year")} + {gettext("Based on genres, artists, and musical style")} diff --git a/priv/gettext/default.pot b/priv/gettext/default.pot index 9d46252f..9d0876b8 100644 --- a/priv/gettext/default.pot +++ b/priv/gettext/default.pot @@ -1446,11 +1446,6 @@ msgstr "" msgid "External Links" msgstr "" -#: lib/music_library_web/components/record_components.ex -#, elixir-autogen, elixir-format -msgid "Based on genres, artists, and release year" -msgstr "" - #: lib/music_library_web/components/record_components.ex #, elixir-autogen, elixir-format msgid "Similar Records" @@ -1957,3 +1952,8 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Retry" msgstr "" + +#: lib/music_library_web/components/record_components.ex +#, elixir-autogen, elixir-format +msgid "Based on genres, artists, and musical style" +msgstr "" diff --git a/priv/gettext/en/LC_MESSAGES/default.po b/priv/gettext/en/LC_MESSAGES/default.po index af89cf99..647f889b 100644 --- a/priv/gettext/en/LC_MESSAGES/default.po +++ b/priv/gettext/en/LC_MESSAGES/default.po @@ -1446,11 +1446,6 @@ msgstr "" msgid "External Links" msgstr "" -#: lib/music_library_web/components/record_components.ex -#, elixir-autogen, elixir-format -msgid "Based on genres, artists, and release year" -msgstr "" - #: lib/music_library_web/components/record_components.ex #, elixir-autogen, elixir-format msgid "Similar Records" @@ -1957,3 +1952,8 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Retry" msgstr "" + +#: lib/music_library_web/components/record_components.ex +#, elixir-autogen, elixir-format, fuzzy +msgid "Based on genres, artists, and musical style" +msgstr "" diff --git a/test/music_library/records/similarity_test.exs b/test/music_library/records/similarity_test.exs index 6acd8c42..092e7a90 100644 --- a/test/music_library/records/similarity_test.exs +++ b/test/music_library/records/similarity_test.exs @@ -3,7 +3,9 @@ defmodule MusicLibrary.Records.SimilarityTest do import MusicLibrary.Fixtures.Records + alias MusicLibrary.Artists.ArtistInfo alias MusicLibrary.Records.{Record, Similarity} + alias MusicLibrary.Repo describe "text_representation/1" do test "generates text representation for a record" do @@ -32,6 +34,78 @@ defmodule MusicLibrary.Records.SimilarityTest do assert text =~ "Type: Album" end + test "includes Wikipedia data when available" do + artist_id = Ecto.UUID.generate() + + Repo.insert!(%ArtistInfo{ + id: artist_id, + musicbrainz_data: %{"name" => "Radiohead"}, + wikipedia_data: %{ + "description" => "English rock band", + "extract" => + "Radiohead are an English rock band formed in Abingdon. They are known for experimental music." + } + }) + + record = %Record{ + title: "OK Computer", + artists: [ + %{ + name: "Radiohead", + sort_name: "Radiohead", + musicbrainz_id: artist_id, + disambiguation: "", + joinphrase: "" + } + ], + genres: ["alternative rock"], + release_date: "1997-05-21", + type: :album + } + + text = Similarity.text_representation(record) + + assert text =~ "English rock band" + assert text =~ "experimental music" + refute text =~ "Members" + end + + test "falls back to truncated Discogs profile when Wikipedia unavailable" do + artist_id = Ecto.UUID.generate() + + Repo.insert!(%ArtistInfo{ + id: artist_id, + musicbrainz_data: %{"name" => "Some Artist"}, + discogs_data: %{ + "name" => "Some Artist", + "profile_plaintext" => "Some Artist is a funk band. They have released many albums.", + "members" => [%{"name" => "Member One"}, %{"name" => "Member Two"}] + } + }) + + record = %Record{ + title: "Funky Album", + artists: [ + %{ + name: "Some Artist", + sort_name: "Some Artist", + musicbrainz_id: artist_id, + disambiguation: "", + joinphrase: "" + } + ], + genres: ["funk"], + release_date: "2000", + type: :album + } + + text = Similarity.text_representation(record) + + assert text =~ "funk band" + refute text =~ "Members" + refute text =~ "Member One" + end + test "handles records with no release date" do record = %Record{ title: "Unknown Album", @@ -107,6 +181,40 @@ defmodule MusicLibrary.Records.SimilarityTest do end end + describe "truncate_to_sentence/2" do + test "returns text unchanged when within limit" do + assert Similarity.truncate_to_sentence("Short text.", 200) == "Short text." + end + + test "truncates at sentence boundary" do + text = "First sentence. Second sentence. Third sentence that is very long." + + result = Similarity.truncate_to_sentence(text, 40) + + assert result == "First sentence. Second sentence." + end + + test "truncates at character limit when no sentence boundary" do + text = String.duplicate("a", 300) + + result = Similarity.truncate_to_sentence(text, 200) + + assert byte_size(result) <= 200 + end + + test "handles empty string" do + assert Similarity.truncate_to_sentence("", 200) == "" + end + + test "handles text with exclamation and question marks" do + text = "What a band! They play great music? Yes indeed. More text that goes over the limit." + + result = Similarity.truncate_to_sentence(text, 55) + + assert result == "What a band! They play great music? Yes indeed." + end + end + describe "store_embedding/3 and get_embedding/1" do test "stores and retrieves an embedding" do record = record() @@ -158,7 +266,7 @@ defmodule MusicLibrary.Records.SimilarityTest do end test "finds similar records", %{record1: record1, record2: record2} do - similar = Similarity.find_similar(record1.id, limit: 5) + similar = Similarity.find_similar(record1.id, limit: 5, max_distance: 1.0) refute Enum.empty?(similar) # record2 should be most similar to record1 @@ -168,7 +276,7 @@ defmodule MusicLibrary.Records.SimilarityTest do end test "respects limit option", %{record1: record1} do - similar = Similarity.find_similar(record1.id, limit: 1) + similar = Similarity.find_similar(record1.id, limit: 1, max_distance: 1.0) assert length(similar) == 1 end @@ -185,11 +293,21 @@ defmodule MusicLibrary.Records.SimilarityTest do # Mark record2 as purchased {:ok, _} = MusicLibrary.Records.update_record(record2, %{purchased_at: DateTime.utc_now()}) - similar = Similarity.find_similar(record1.id, scope: :wishlist) + similar = Similarity.find_similar(record1.id, scope: :wishlist, max_distance: 1.0) # Should not include record2 since it's in collection - record_ids = Enum.map(similar, fn {record, _} -> record.id end) + record_ids = Enum.map(similar, fn %{record: record} -> record.id end) refute record2.id in record_ids end + + test "filters results by max_distance threshold", %{record1: record1} do + # With a very low threshold, only very similar records should be returned + similar_strict = Similarity.find_similar(record1.id, max_distance: 0.001) + + # With a very high threshold, all records should be returned + similar_loose = Similarity.find_similar(record1.id, max_distance: 2.0) + + assert length(similar_strict) <= length(similar_loose) + end end end