cloud/music_library
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use Wikipedia data for record similarity and raise similarity threshold

Browse Source
This commit is contained in:
Claudio Ortolina
2026-02-19 15:51:12 +00:00
parent e1aa1d371c
commit 1dfb2d2a6f
6 changed files with 205 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/config.exs
+2
View File
@@ -17,6 +17,8 @@ config :music_library,
config :music_library, default_timezone: "Europe/London"
config :music_library, :similarity, max_distance: 0.45
config :music_library, :pagination,
default_page_size: 20,
stats_limit: 30,
lib/music_library/records/similarity.ex
+71 -19
View File
@@ -6,11 +6,15 @@ defmodule MusicLibrary.Records.Similarity do
import Ecto.Query
import SqliteVec.Ecto.Query
alias MusicLibrary.{Artists, Records}
alias MusicLibrary.Artists
alias MusicLibrary.Artists.ArtistInfo
alias MusicLibrary.Records
alias MusicLibrary.Records.{Record, RecordEmbedding}
alias MusicLibrary.Repo
alias MusicLibrary.Worker.GenerateRecordEmbedding
@max_distance Application.compile_env!(:music_library, :similarity)[:max_distance]
@doc """
Generates a text representation of a record for embedding generation.
@@ -20,6 +24,7 @@ defmodule MusicLibrary.Records.Similarity do
- Genres
- Release year
- Type (album, EP, etc.)
- Artist musical style summaries (from Wikipedia, falling back to Discogs)
"""
def text_representation(%Record{} = record) do
artist_infos =
@@ -47,32 +52,75 @@ defmodule MusicLibrary.Records.Similarity do
defp artist_infos_summary([]), do: ""
defp artist_infos_summary(artist_infos) do
Enum.map_join(artist_infos, "\n\n", &artist_info_summary/1)
artist_infos
|> Enum.map(&artist_info_summary/1)
|> Enum.reject(&(&1 == ""))
|> Enum.join("\n\n")
end
defp artist_info_summary(artist_info) when is_nil(artist_info.discogs_data), do: ""
defp artist_info_summary(artist_info) do
cond do
wikipedia_available?(artist_info) ->
wikipedia_artist_summary(artist_info)
defp artist_info_summary(%{discogs_data: discogs_data}) do
# if available, use the plain text version of the profile to avoid pollution by BBTags
profile_data = Map.get(discogs_data, "profile_plaintext") || Map.get(discogs_data, "profile")
discogs_available?(artist_info) ->
discogs_artist_summary(artist_info)
profile_section =
if profile_data do
"Profile for #{discogs_data["name"]}: #{profile_data}"
else
true ->
""
end
members_section =
if members = Map.get(discogs_data, "members") do
members_list = Enum.map_join(members, ", ", fn member -> member["name"] end)
"Members (past and present): #{members_list}"
else
""
end
Enum.join([profile_section, members_section], "\n")
defp wikipedia_available?(artist_info) do
ArtistInfo.wikipedia_description(artist_info) != nil ||
ArtistInfo.wikipedia_summary(artist_info) != nil
end
defp discogs_available?(artist_info) do
artist_info.discogs_data != nil &&
(Map.get(artist_info.discogs_data, "profile_plaintext") != nil ||
Map.get(artist_info.discogs_data, "profile") != nil)
end
defp wikipedia_artist_summary(artist_info) do
description = ArtistInfo.wikipedia_description(artist_info) || ""
summary = ArtistInfo.wikipedia_summary(artist_info) || ""
truncated_summary = truncate_to_sentence(summary, 200)
[description, truncated_summary]
|> Enum.reject(&(&1 == ""))
|> Enum.join(". ")
end
defp discogs_artist_summary(artist_info) do
profile =
Map.get(artist_info.discogs_data, "profile_plaintext") ||
Map.get(artist_info.discogs_data, "profile") ||
""
truncate_to_sentence(profile, 200)
end
@doc false
def truncate_to_sentence(text, max_length) when byte_size(text) <= max_length, do: text
def truncate_to_sentence(text, max_length) do
truncated = String.slice(text, 0, max_length)
case String.split(truncated, ~r/[.!?]\s/, include_captures: true) |> Enum.count() do
count when count > 1 ->
# Find the last sentence boundary within the limit
truncated
|> String.replace(~r/[^.!?]*$/, "")
|> String.trim()
|> case do
"" -> String.trim(truncated)
result -> result
end
_ ->
String.trim(truncated)
end
end
@doc """
@@ -82,6 +130,8 @@ defmodule MusicLibrary.Records.Similarity do
- `:limit` - Maximum number of similar records to return (default: 10)
- `:scope` - Filter by :collection or :wishlist (default: no filter)
- `:max_distance` - Maximum cosine distance threshold (default: #{@max_distance}).
Results with distance above this are excluded.
## Examples
@@ -94,6 +144,7 @@ defmodule MusicLibrary.Records.Similarity do
def find_similar(record_id, opts \\ []) do
limit = Keyword.get(opts, :limit, 10)
scope = Keyword.get(opts, :scope)
max_distance = Keyword.get(opts, :max_distance, @max_distance)
record = Records.get_record!(record_id)
record_musicbrainz_id = record.musicbrainz_id
@@ -113,6 +164,7 @@ defmodule MusicLibrary.Records.Similarity do
|> selected_as(:similarity)
},
group_by: r.musicbrainz_id,
having: vec_distance_cosine(re.embedding, vec_f32(source_embedding)) <= ^max_distance,
limit: ^limit
query = apply_scope_filter(query, scope)
lib/music_library_web/components/record_components.ex
+1 -1
View File
@@ -472,7 +472,7 @@ defmodule MusicLibraryWeb.RecordComponents do
{gettext("Similar Records")}
</h2>
<span class="ml-2 text-xs font-normal text-zinc-500 dark:text-zinc-400">
{gettext("Based on genres, artists, and release year")}
{gettext("Based on genres, artists, and musical style")}
</span>
</header>
priv/gettext/default.pot
+5 -5
View File
@@ -1446,11 +1446,6 @@ msgstr ""
msgid "External Links"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format
msgid "Based on genres, artists, and release year"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format
msgid "Similar Records"
@@ -1957,3 +1952,8 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "Retry"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format
msgid "Based on genres, artists, and musical style"
msgstr ""
priv/gettext/en/LC_MESSAGES/default.po
+5 -5
View File
@@ -1446,11 +1446,6 @@ msgstr ""
msgid "External Links"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format
msgid "Based on genres, artists, and release year"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format
msgid "Similar Records"
@@ -1957,3 +1952,8 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "Retry"
msgstr ""
#: lib/music_library_web/components/record_components.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Based on genres, artists, and musical style"
msgstr ""
test/music_library/records/similarity_test.exs
+122 -4
View File
@@ -3,7 +3,9 @@ defmodule MusicLibrary.Records.SimilarityTest do
import MusicLibrary.Fixtures.Records
alias MusicLibrary.Artists.ArtistInfo
alias MusicLibrary.Records.{Record, Similarity}
alias MusicLibrary.Repo
describe "text_representation/1" do
test "generates text representation for a record" do
@@ -32,6 +34,78 @@ defmodule MusicLibrary.Records.SimilarityTest do
assert text =~ "Type: Album"
end
test "includes Wikipedia data when available" do
artist_id = Ecto.UUID.generate()
Repo.insert!(%ArtistInfo{
id: artist_id,
musicbrainz_data: %{"name" => "Radiohead"},
wikipedia_data: %{
"description" => "English rock band",
"extract" =>
"Radiohead are an English rock band formed in Abingdon. They are known for experimental music."
}
})
record = %Record{
title: "OK Computer",
artists: [
%{
name: "Radiohead",
sort_name: "Radiohead",
musicbrainz_id: artist_id,
disambiguation: "",
joinphrase: ""
}
],
genres: ["alternative rock"],
release_date: "1997-05-21",
type: :album
}
text = Similarity.text_representation(record)
assert text =~ "English rock band"
assert text =~ "experimental music"
refute text =~ "Members"
end
test "falls back to truncated Discogs profile when Wikipedia unavailable" do
artist_id = Ecto.UUID.generate()
Repo.insert!(%ArtistInfo{
id: artist_id,
musicbrainz_data: %{"name" => "Some Artist"},
discogs_data: %{
"name" => "Some Artist",
"profile_plaintext" => "Some Artist is a funk band. They have released many albums.",
"members" => [%{"name" => "Member One"}, %{"name" => "Member Two"}]
}
})
record = %Record{
title: "Funky Album",
artists: [
%{
name: "Some Artist",
sort_name: "Some Artist",
musicbrainz_id: artist_id,
disambiguation: "",
joinphrase: ""
}
],
genres: ["funk"],
release_date: "2000",
type: :album
}
text = Similarity.text_representation(record)
assert text =~ "funk band"
refute text =~ "Members"
refute text =~ "Member One"
end
test "handles records with no release date" do
record = %Record{
title: "Unknown Album",
@@ -107,6 +181,40 @@ defmodule MusicLibrary.Records.SimilarityTest do
end
end
describe "truncate_to_sentence/2" do
test "returns text unchanged when within limit" do
assert Similarity.truncate_to_sentence("Short text.", 200) == "Short text."
end
test "truncates at sentence boundary" do
text = "First sentence. Second sentence. Third sentence that is very long."
result = Similarity.truncate_to_sentence(text, 40)
assert result == "First sentence. Second sentence."
end
test "truncates at character limit when no sentence boundary" do
text = String.duplicate("a", 300)
result = Similarity.truncate_to_sentence(text, 200)
assert byte_size(result) <= 200
end
test "handles empty string" do
assert Similarity.truncate_to_sentence("", 200) == ""
end
test "handles text with exclamation and question marks" do
text = "What a band! They play great music? Yes indeed. More text that goes over the limit."
result = Similarity.truncate_to_sentence(text, 55)
assert result == "What a band! They play great music? Yes indeed."
end
end
describe "store_embedding/3 and get_embedding/1" do
test "stores and retrieves an embedding" do
record = record()
@@ -158,7 +266,7 @@ defmodule MusicLibrary.Records.SimilarityTest do
end
test "finds similar records", %{record1: record1, record2: record2} do
similar = Similarity.find_similar(record1.id, limit: 5)
similar = Similarity.find_similar(record1.id, limit: 5, max_distance: 1.0)
refute Enum.empty?(similar)
# record2 should be most similar to record1
@@ -168,7 +276,7 @@ defmodule MusicLibrary.Records.SimilarityTest do
end
test "respects limit option", %{record1: record1} do
similar = Similarity.find_similar(record1.id, limit: 1)
similar = Similarity.find_similar(record1.id, limit: 1, max_distance: 1.0)
assert length(similar) == 1
end
@@ -185,11 +293,21 @@ defmodule MusicLibrary.Records.SimilarityTest do
# Mark record2 as purchased
{:ok, _} = MusicLibrary.Records.update_record(record2, %{purchased_at: DateTime.utc_now()})
similar = Similarity.find_similar(record1.id, scope: :wishlist)
similar = Similarity.find_similar(record1.id, scope: :wishlist, max_distance: 1.0)
# Should not include record2 since it's in collection
record_ids = Enum.map(similar, fn {record, _} -> record.id end)
record_ids = Enum.map(similar, fn %{record: record} -> record.id end)
refute record2.id in record_ids
end
test "filters results by max_distance threshold", %{record1: record1} do
# With a very low threshold, only very similar records should be returned
similar_strict = Similarity.find_similar(record1.id, max_distance: 0.001)
# With a very high threshold, all records should be returned
similar_loose = Similarity.find_similar(record1.id, max_distance: 2.0)
assert length(similar_strict) <= length(similar_loose)
end
end
end